@polycode-projects/the-mechanical-code-talker 6.0.12 → 6.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/adapters/memory/core.mjs +277 -61
- package/src/adapters/memory/shacl.mjs +2 -2
- package/src/domain/memory/trust.mjs +78 -17
- package/src/services/chat.mjs +18 -10
- package/src/services/extract-facts.mjs +36 -11
- package/src/services/news.mjs +46 -9
- package/src/surfaces/web/memory-ask-browser.bundle.js +122 -122
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@polycode-projects/the-mechanical-code-talker",
|
|
3
|
-
"version": "6.0.
|
|
3
|
+
"version": "6.0.14",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "The Mechanical Code Talker (tmct) — a tolerant, offline, $0 chat surface that guides you toward precision queries about a software repository. ELIZA/PARRY-style but domain-obsessed with code. No model calls; indexes a repo on request (tmct index) or reads any producer's graph.",
|
|
@@ -766,12 +766,22 @@ const ROW_NODE_ID_KEY = "nodeId";
|
|
|
766
766
|
* `wrapRowBackendOverSqliteSeed`, the entry point that takes one.
|
|
767
767
|
* `sqliteSeedOverlayRows` are rows that layer over that seed and under the
|
|
768
768
|
* session's own (a turn's retrieved corpus subgraph), read-only exactly as
|
|
769
|
-
* the seed is.
|
|
769
|
+
* the seed is.
|
|
770
|
+
*
|
|
771
|
+
* `copyOnRead: false` hands every reader the assembled payload itself instead
|
|
772
|
+
* of a copy of it. Copying a seed-sized payload is the most expensive thing
|
|
773
|
+
* this backend does — over a second per read at 60k facts — and a caller that
|
|
774
|
+
* drives the whole session through this one handle (a worker cycle, a request
|
|
775
|
+
* handler) has nobody to protect the payload from. Anyone sharing a handle
|
|
776
|
+
* across independent readers leaves the default alone: without the copy, a
|
|
777
|
+
* reader that mutates what it read has changed the store's own view of
|
|
778
|
+
* itself. */
|
|
770
779
|
export function wrapRowBackend(impl, {
|
|
771
780
|
basePayload = null,
|
|
772
781
|
sqliteSeedStore = null,
|
|
773
782
|
sqliteSeedOverlayRows = null,
|
|
774
783
|
onOversizedRow = "throw",
|
|
784
|
+
copyOnRead = true,
|
|
775
785
|
log = undefined,
|
|
776
786
|
} = {}) {
|
|
777
787
|
const problems = rowBackendProblems(impl);
|
|
@@ -795,6 +805,7 @@ export function wrapRowBackend(impl, {
|
|
|
795
805
|
baseRows: null,
|
|
796
806
|
storedRows: null,
|
|
797
807
|
onOversizedRow,
|
|
808
|
+
copyOnRead,
|
|
798
809
|
log,
|
|
799
810
|
};
|
|
800
811
|
}
|
|
@@ -805,8 +816,8 @@ export function wrapRowBackend(impl, {
|
|
|
805
816
|
* never materialized as a row array: the assembled payload is the only copy
|
|
806
817
|
* of it this process holds. `overlayRows` layer over the seed and under the
|
|
807
818
|
* session's own rows. */
|
|
808
|
-
export function wrapRowBackendOverSqliteSeed(impl, sqliteSeedStore, { overlayRows = null, onOversizedRow = "throw", log = undefined } = {}) {
|
|
809
|
-
return wrapRowBackend(impl, { sqliteSeedStore, sqliteSeedOverlayRows: overlayRows, onOversizedRow, log });
|
|
819
|
+
export function wrapRowBackendOverSqliteSeed(impl, sqliteSeedStore, { overlayRows = null, onOversizedRow = "throw", copyOnRead = true, log = undefined } = {}) {
|
|
820
|
+
return wrapRowBackend(impl, { sqliteSeedStore, sqliteSeedOverlayRows: overlayRows, onOversizedRow, copyOnRead, log });
|
|
810
821
|
}
|
|
811
822
|
|
|
812
823
|
/** Drain whatever `readRows()` returned: the contract allows an array or an
|
|
@@ -863,7 +874,7 @@ async function ensureRowPayload(handle) {
|
|
|
863
874
|
prefixes: await readRowMeta(handle, ROW_META_PREFIXES_KEY, seedScalar(handle, "prefixes", empty.prefixes)),
|
|
864
875
|
};
|
|
865
876
|
if (handle.sqliteSeedStore) {
|
|
866
|
-
handle.cachedPayload = assembleSqliteSeededPayload(handle, meta);
|
|
877
|
+
handle.cachedPayload = migrateStoredMemory(assembleSqliteSeededPayload(handle, meta));
|
|
867
878
|
} else {
|
|
868
879
|
// "keep": the base overlay's own rows never reach the wire (persistRowPayload
|
|
869
880
|
// excludes every seed key from every write via seedOnlyKeys below), so the
|
|
@@ -872,16 +883,49 @@ async function ensureRowPayload(handle) {
|
|
|
872
883
|
// band's own high-fan-out property (one edge per fact is normal, not a
|
|
873
884
|
// pathology) and break every read that depends on it.
|
|
874
885
|
handle.baseRows = payloadToRows(handle.basePayload || empty, { onOversizedRow: "keep" });
|
|
875
|
-
handle.cachedPayload = rowsToPayload(overlayRows(handle.baseRows, handle.storedRows), { meta });
|
|
886
|
+
handle.cachedPayload = migrateStoredMemory(rowsToPayload(overlayRows(handle.baseRows, handle.storedRows), { meta }));
|
|
876
887
|
}
|
|
877
888
|
}
|
|
878
889
|
return handle.cachedPayload;
|
|
879
890
|
}
|
|
880
891
|
|
|
881
892
|
/** loadMemory's read for Backend D: one `readRows()` per cold open, then a
|
|
882
|
-
* clone of the assembled payload per call
|
|
893
|
+
* clone of the assembled payload per call — unless the handle was opened
|
|
894
|
+
* `copyOnRead: false`, where the reader gets the assembled payload itself. */
|
|
883
895
|
async function readRowPayload(handle) {
|
|
884
|
-
|
|
896
|
+
const payload = await ensureRowPayload(handle);
|
|
897
|
+
return handle.copyOnRead === false ? payload : cloneJson(payload);
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
/** A payload a mutation can work on without any of it reaching the one it was
|
|
901
|
+
* copied from. Every container a write reaches into is its own: the individuals
|
|
902
|
+
* array and each individual, the edge groups and each group's example list.
|
|
903
|
+
* What stays shared is what a write never changes in place — an individual's
|
|
904
|
+
* `attributes` array is REPLACED by `setAttr`, never pushed onto, and an edge
|
|
905
|
+
* is replaced rather than edited, so both sides keep reading their own.
|
|
906
|
+
*
|
|
907
|
+
* This is `structuredClone`'s job done at the granularity writes actually use.
|
|
908
|
+
* At seed scale the deep copy runs well over a second and a cycle pays it on
|
|
909
|
+
* every fact it grounds; this is a few milliseconds of pointer copying. */
|
|
910
|
+
function mutablePayloadCopy(payload) {
|
|
911
|
+
return {
|
|
912
|
+
...payload,
|
|
913
|
+
individuals: (payload.individuals || []).map((ind) => ({ ...ind })),
|
|
914
|
+
objectProperties: (payload.objectProperties || []).map((group) => ({
|
|
915
|
+
...group, examples: [...(group.examples || [])],
|
|
916
|
+
})),
|
|
917
|
+
};
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
/** Forget a row handle's assembled payload, so the next read rebuilds it from
|
|
921
|
+
* the store. The one thing that must happen after a mutation dies part-way
|
|
922
|
+
* through: the payload it was changing is neither what the store holds nor a
|
|
923
|
+
* coherent graph. */
|
|
924
|
+
function dropAssembledRowPayload(handle) {
|
|
925
|
+
if (!isRowHandle(handle)) return;
|
|
926
|
+
handle.cachedPayload = null;
|
|
927
|
+
handle.storedRows = null;
|
|
928
|
+
handle.baseRows = null;
|
|
885
929
|
}
|
|
886
930
|
|
|
887
931
|
/** A record with its audit stamp removed. `mgx:updatedAt` moves on every
|
|
@@ -935,9 +979,7 @@ async function persistRowPayload(handle, payload) {
|
|
|
935
979
|
await handle.impl.putMeta(ROW_META_MEMORY_KEY, JSON.stringify(payload.memory ?? emptyMemory().memory));
|
|
936
980
|
await handle.impl.putMeta(ROW_META_PREFIXES_KEY, JSON.stringify(payload.prefixes ?? emptyMemory().prefixes));
|
|
937
981
|
} catch (e) {
|
|
938
|
-
handle
|
|
939
|
-
handle.storedRows = null;
|
|
940
|
-
handle.baseRows = null;
|
|
982
|
+
dropAssembledRowPayload(handle);
|
|
941
983
|
throw e;
|
|
942
984
|
}
|
|
943
985
|
const removed = new Set(removals);
|
|
@@ -949,13 +991,13 @@ async function persistRowPayload(handle, payload) {
|
|
|
949
991
|
patchAssembledPayload(handle.cachedPayload, meta, writes);
|
|
950
992
|
} else if (handle.sqliteSeedStore) {
|
|
951
993
|
handle.cachedPayload = null;
|
|
952
|
-
handle.cachedPayload = assembleSqliteSeededPayload(handle, meta);
|
|
994
|
+
handle.cachedPayload = migrateStoredMemory(assembleSqliteSeededPayload(handle, meta));
|
|
953
995
|
} else {
|
|
954
996
|
// Dropped before the rebuild, not after it: the payload this replaces is the
|
|
955
997
|
// largest object the handle holds, and keeping it reachable while the next
|
|
956
998
|
// one assembles doubles the peak for no reason.
|
|
957
999
|
handle.cachedPayload = null;
|
|
958
|
-
handle.cachedPayload = rowsToPayload(overlayRows(handle.baseRows, handle.storedRows), { meta });
|
|
1000
|
+
handle.cachedPayload = migrateStoredMemory(rowsToPayload(overlayRows(handle.baseRows, handle.storedRows), { meta }));
|
|
959
1001
|
}
|
|
960
1002
|
}
|
|
961
1003
|
|
|
@@ -1951,7 +1993,9 @@ export async function snapshotMemory(dir, { retentionVersions } = {}) {
|
|
|
1951
1993
|
export async function loadMemory(dir) {
|
|
1952
1994
|
if (isMemoryHandle(dir)) return migrateStoredMemory(dir.payload);
|
|
1953
1995
|
if (isSqliteHandle(dir)) return migrateStoredMemory(readSqlitePayload(dir));
|
|
1954
|
-
|
|
1996
|
+
// Not migrated here: a row handle migrates the payload once, as it assembles
|
|
1997
|
+
// it, so every read after the first is spared two walks of the whole graph.
|
|
1998
|
+
if (isRowHandle(dir)) return readRowPayload(dir);
|
|
1955
1999
|
let text;
|
|
1956
2000
|
try {
|
|
1957
2001
|
text = await readFile(memoryGraphFile(dir), "utf8");
|
|
@@ -2315,15 +2359,31 @@ function retractionsFor(payload, groupId) {
|
|
|
2315
2359
|
* linear scan in that case. */
|
|
2316
2360
|
const memoryIndexOf = (payload) => payload?.[MEMORY_INDEX] || null;
|
|
2317
2361
|
|
|
2362
|
+
/** Load, change, persist. A row handle always works on a copy here, even one
|
|
2363
|
+
* opened `copyOnRead: false`: a write drops every change that lands on a
|
|
2364
|
+
* seed-owned row, so a mutation applied straight to the assembled payload
|
|
2365
|
+
* would leave the handle holding changes the store refused. The copy is what
|
|
2366
|
+
* keeps "what this handle reads" and "what a fresh handle would assemble" the
|
|
2367
|
+
* same thing.
|
|
2368
|
+
*
|
|
2369
|
+
* A row handle does skip the prose index built here: `persistRowPayload`
|
|
2370
|
+
* re-derives every derived structure from the rows it just wrote, so building
|
|
2371
|
+
* one now builds it twice over the whole graph. */
|
|
2318
2372
|
async function mutateMemory(dir, fn) {
|
|
2319
|
-
const
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2373
|
+
const overRowHandle = isRowHandle(dir);
|
|
2374
|
+
const payload = overRowHandle ? mutablePayloadCopy(await ensureRowPayload(dir)) : await loadMemory(dir);
|
|
2375
|
+
try {
|
|
2376
|
+
buildMemoryIndex(payload);
|
|
2377
|
+
const out = (await fn(payload)) ?? payload;
|
|
2378
|
+
migrateLegacyProvenance(out);
|
|
2379
|
+
recomputeSourceReliability(out);
|
|
2380
|
+
if (!overRowHandle) out.proseIndex = buildProseIndex(out.individuals);
|
|
2381
|
+
await persistMemory(dir, out);
|
|
2382
|
+
return overRowHandle ? dir.cachedPayload : out;
|
|
2383
|
+
} catch (e) {
|
|
2384
|
+
dropAssembledRowPayload(dir);
|
|
2385
|
+
throw e;
|
|
2386
|
+
}
|
|
2327
2387
|
}
|
|
2328
2388
|
|
|
2329
2389
|
const labelOf = (text) => (text.length > LABEL_CAP ? text.slice(0, LABEL_CAP - 1) + "…" : text);
|
|
@@ -2361,12 +2421,14 @@ function sourceIdFor(desc) {
|
|
|
2361
2421
|
case "provider": return { id: `src:provider:${desc.name}`, type: "provider" };
|
|
2362
2422
|
case "corpus": return { id: `src:corpus:${desc.name}`, type: "corpus" };
|
|
2363
2423
|
case "corpusWeak": return { id: `src:corpus-weak:${desc.name}`, type: "corpusWeak" };
|
|
2364
|
-
// One Source per
|
|
2365
|
-
//
|
|
2366
|
-
|
|
2367
|
-
//
|
|
2424
|
+
// One Source per reference WORK, not per article: Simple English Wikipedia
|
|
2425
|
+
// is one party however many of its pages get read, so two of its articles
|
|
2426
|
+
// stating the same triple corroborate nothing. Which article said it stays
|
|
2427
|
+
// on the fact's own provenance tag, where the audit trail belongs.
|
|
2428
|
+
case "reference": return { id: `src:reference:${desc.pack}`, type: "reference" };
|
|
2429
|
+
// The live-Wikipedia pack: same per-work Source id, but a lower trust
|
|
2368
2430
|
// type so a live lookup ranks below the curated revision-pinned pack.
|
|
2369
|
-
case "referenceLive": return { id: `src:reference:${desc.pack}
|
|
2431
|
+
case "referenceLive": return { id: `src:reference:${desc.pack}`, type: "referenceLive" };
|
|
2370
2432
|
// One Source per source-file basename, not per extraction run.
|
|
2371
2433
|
case "extracted": return { id: `src:extracted:${desc.name}`, type: "extracted" };
|
|
2372
2434
|
// The fuzzy tier's candidates: one low-trust Source per source label.
|
|
@@ -2540,6 +2602,49 @@ const isSessionScopedSourceId = (id) =>
|
|
|
2540
2602
|
|| id.startsWith(`${TEACH_SOURCE_ID}:`)
|
|
2541
2603
|
|| id.startsWith(`${TEACH_NODE_SOURCE_ID}:`));
|
|
2542
2604
|
|
|
2605
|
+
/**
|
|
2606
|
+
* The slice of the graph a reliability pass can possibly read: the triples a
|
|
2607
|
+
* session-scoped Source stated, and every (subject, predicate) those triples
|
|
2608
|
+
* sit under so each one's disagreeing siblings come along. Null when nothing
|
|
2609
|
+
* in the store has a track record to keep, which is the whole of what a
|
|
2610
|
+
* seed-only graph or a store whose writers are all documents ever needs.
|
|
2611
|
+
*
|
|
2612
|
+
* A pure narrowing, not a cache: the answer is still folded from the fact set
|
|
2613
|
+
* on every call, and every group left out is one whose row could not have
|
|
2614
|
+
* changed a number in the tally. A Source nobody stated anything for keeps no
|
|
2615
|
+
* reliability attribute either way, so leaving it out is what the whole-graph
|
|
2616
|
+
* pass does too.
|
|
2617
|
+
*/
|
|
2618
|
+
function sessionScopedFoldScope(payload) {
|
|
2619
|
+
// The Source list is a handful of individuals where the edge list is one per
|
|
2620
|
+
// fact record, so "is there an actor here at all" is asked of the Sources.
|
|
2621
|
+
const idx = memoryIndexOf(payload);
|
|
2622
|
+
if (idx) {
|
|
2623
|
+
let anyActor = false;
|
|
2624
|
+
for (const id of idx.sourcesById.keys()) if (isSessionScopedSourceId(id)) { anyActor = true; break; }
|
|
2625
|
+
if (!anyActor) return null;
|
|
2626
|
+
}
|
|
2627
|
+
const statedGroup = payload.objectProperties.find((g) => g?.prop === STATED_BY_PROP);
|
|
2628
|
+
const statedRecordIds = new Set();
|
|
2629
|
+
for (const e of statedGroup?.examples || []) {
|
|
2630
|
+
if (isSessionScopedSourceId(e?.object)) statedRecordIds.add(e.subject);
|
|
2631
|
+
}
|
|
2632
|
+
const scopedGroups = new Set();
|
|
2633
|
+
const pairs = new Set();
|
|
2634
|
+
for (const ind of payload.individuals) {
|
|
2635
|
+
if (ind?.class !== FACT_CLASS) continue;
|
|
2636
|
+
// A summary that absorbed an actor's record still votes for it, so the
|
|
2637
|
+
// group it stands in is in scope exactly as the record itself would be.
|
|
2638
|
+
const stated = statedRecordIds.has(ind.id)
|
|
2639
|
+
|| (isHeadRollupId(ind.id) && absorbedSourceIds(ind).some(isSessionScopedSourceId));
|
|
2640
|
+
if (!stated) continue;
|
|
2641
|
+
scopedGroups.add(factGroupId(ind.id));
|
|
2642
|
+
const subject = individualKey(ind, "subject");
|
|
2643
|
+
if (subject) pairs.add(subjectPredicateKey(subject, individualKey(ind, "predicate")));
|
|
2644
|
+
}
|
|
2645
|
+
return scopedGroups.size ? { pairs, scopedGroups } : null;
|
|
2646
|
+
}
|
|
2647
|
+
|
|
2543
2648
|
/**
|
|
2544
2649
|
* Recompute + materialise mgx:sourceReliability on every session-scoped
|
|
2545
2650
|
* operator/teach Source: count facts stated vs. contradicted
|
|
@@ -2550,9 +2655,18 @@ const isSessionScopedSourceId = (id) =>
|
|
|
2550
2655
|
*/
|
|
2551
2656
|
function recomputeSourceReliability(payload) {
|
|
2552
2657
|
if (!Array.isArray(payload?.individuals) || !Array.isArray(payload?.objectProperties)) return;
|
|
2553
|
-
const
|
|
2658
|
+
const scope = sessionScopedFoldScope(payload);
|
|
2659
|
+
if (!scope) return;
|
|
2660
|
+
|
|
2661
|
+
// Fact-only — contradiction accounting is inherently Fact-shaped. Scoped to
|
|
2662
|
+
// the triples a scorable actor actually stated, plus every sibling object
|
|
2663
|
+
// those triples could be contradicted by: no other group can put a number in
|
|
2664
|
+
// the tally below, so folding the rest of the graph only costs time.
|
|
2665
|
+
const rows = foldFactRows(payload, factFoldContext(payload, scope));
|
|
2554
2666
|
const contradictedFactIds = new Set();
|
|
2555
|
-
|
|
2667
|
+
// The fold above is the same one findContradictions would take for itself,
|
|
2668
|
+
// and nothing changes the payload between the two, so it goes across.
|
|
2669
|
+
for (const group of findContradictions(payload, { factRows: rows })) for (const r of group) contradictedFactIds.add(r.id);
|
|
2556
2670
|
|
|
2557
2671
|
const bySource = new Map(); // sessionSourceId -> { factsAsserted, factsContradicted }
|
|
2558
2672
|
for (const row of rows) {
|
|
@@ -3750,7 +3864,14 @@ export async function resolveRelationChaseReverse(memory, name, objectTerm, help
|
|
|
3750
3864
|
* answers "what did this source used to say", never "what do I trust now".
|
|
3751
3865
|
*/
|
|
3752
3866
|
export function readFactRows(memory, opts = {}) {
|
|
3753
|
-
|
|
3867
|
+
return foldFactRows(memory, factFoldContext(memory), opts);
|
|
3868
|
+
}
|
|
3869
|
+
|
|
3870
|
+
/** The fold itself, over whatever slice of the graph a context was built for.
|
|
3871
|
+
* `readFactRows` hands it the whole graph; a caller that only needs certain
|
|
3872
|
+
* (subject, predicate) pairs hands it a scoped context and gets exactly the
|
|
3873
|
+
* rows a whole-graph fold would have produced for those pairs. */
|
|
3874
|
+
function foldFactRows(memory, ctx, opts = {}) {
|
|
3754
3875
|
// A materialised head, when the backend keeps one, replaces the group's own
|
|
3755
3876
|
// fold with the audit trail that fold was last built from — the same records,
|
|
3756
3877
|
// read back instead of re-derived. It carries no recency by construction, so
|
|
@@ -3777,17 +3898,47 @@ export function readFactRows(memory, opts = {}) {
|
|
|
3777
3898
|
* Shared by the read fold and by the head materialisation below, deliberately:
|
|
3778
3899
|
* a stored aggregate and a computed one folded from different inputs is the
|
|
3779
3900
|
* failure a materialised table invites, and one shared builder is what keeps
|
|
3780
|
-
* the two from ever drifting apart.
|
|
3781
|
-
|
|
3901
|
+
* the two from ever drifting apart.
|
|
3902
|
+
*
|
|
3903
|
+
* `pairs` narrows the context to the groups a given set of (subject,
|
|
3904
|
+
* predicate) keys carries, with `scopedGroups` naming groups to keep outright
|
|
3905
|
+
* whatever their records read as. Every group of a kept pair is kept, whoever
|
|
3906
|
+
* stated it, so a scoped fold reads each of those triples exactly as a
|
|
3907
|
+
* whole-graph fold does — and a caller asking about one party's facts still
|
|
3908
|
+
* sees every sibling object that party's claim could be contradicted by. */
|
|
3909
|
+
function factFoldContext(memory, { pairs = null, scopedGroups = new Set() } = {}) {
|
|
3782
3910
|
const individuals = memory?.individuals || [];
|
|
3783
|
-
const
|
|
3784
|
-
const
|
|
3785
|
-
|
|
3786
|
-
|
|
3787
|
-
|
|
3788
|
-
|
|
3911
|
+
const idx = memoryIndexOf(memory);
|
|
3912
|
+
const sourcesById = idx
|
|
3913
|
+
? idx.sourcesById
|
|
3914
|
+
: new Map(individuals.filter((i) => i?.class === SOURCE_CLASS).map((i) => [i.id, i]));
|
|
3915
|
+
// The live index already keeps this map, edge for edge, and rebuilding it
|
|
3916
|
+
// allocates one array per fact record over the whole graph.
|
|
3917
|
+
let statedByRecord = idx?.statedByBySubject;
|
|
3918
|
+
if (!statedByRecord) {
|
|
3919
|
+
const statedGroup = (memory?.objectProperties || []).find((g) => g?.prop === STATED_BY_PROP);
|
|
3920
|
+
statedByRecord = new Map();
|
|
3921
|
+
for (const e of statedGroup?.examples || []) {
|
|
3922
|
+
if (!statedByRecord.has(e.subject)) statedByRecord.set(e.subject, []);
|
|
3923
|
+
statedByRecord.get(e.subject).push(e.object);
|
|
3924
|
+
}
|
|
3789
3925
|
}
|
|
3790
3926
|
|
|
3927
|
+
// Subject first, predicate only on a subject hit: reading both attributes off
|
|
3928
|
+
// every fact record in the graph is the whole cost of a scoped pass, and the
|
|
3929
|
+
// subject rules out nearly all of them on one lookup.
|
|
3930
|
+
const wantedSubjects = pairs ? new Set([...pairs].map((key) => key.slice(0, key.indexOf("")))) : null;
|
|
3931
|
+
const outsideScope = (ind) => {
|
|
3932
|
+
if (!pairs) return false;
|
|
3933
|
+
// A group the caller named outright is in whatever its records read as —
|
|
3934
|
+
// a summary standing for absorbed records carries only a copied template,
|
|
3935
|
+
// so its own attributes are not what places it.
|
|
3936
|
+
if (scopedGroups.has(factGroupId(ind.id))) return false;
|
|
3937
|
+
const subject = individualKey(ind, "subject");
|
|
3938
|
+
if (!wantedSubjects.has(subject)) return true;
|
|
3939
|
+
return !pairs.has(subjectPredicateKey(subject, individualKey(ind, "predicate")));
|
|
3940
|
+
};
|
|
3941
|
+
|
|
3791
3942
|
const groups = new Map();
|
|
3792
3943
|
const retractionsByGroup = new Map();
|
|
3793
3944
|
for (const ind of individuals) {
|
|
@@ -3801,6 +3952,7 @@ function factFoldContext(memory) {
|
|
|
3801
3952
|
if (ind?.class !== FACT_CLASS) continue;
|
|
3802
3953
|
if ((ind.attributes || []).some((a) => a?.prop === SUPERSEDED_BY_PROP)) continue; // a demoted leaf, not a head
|
|
3803
3954
|
if (isChainRollupId(ind.id)) continue; // a summary of one source's demoted history, which was never a vote
|
|
3955
|
+
if (outsideScope(ind)) continue;
|
|
3804
3956
|
const groupId = factGroupId(ind.id);
|
|
3805
3957
|
const group = groups.get(groupId);
|
|
3806
3958
|
if (group) group.push(ind);
|
|
@@ -3825,26 +3977,89 @@ function factFoldContext(memory) {
|
|
|
3825
3977
|
else groups.delete(groupId);
|
|
3826
3978
|
}
|
|
3827
3979
|
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
|
|
3831
|
-
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
|
|
3837
|
-
|
|
3838
|
-
|
|
3980
|
+
// Codepoint order on the record id, which sorts by source key — the same
|
|
3981
|
+
// locale-free determinism the P2P layer's own sort insists on, so two peers
|
|
3982
|
+
// holding the same records read the same row.
|
|
3983
|
+
for (const members of groups.values()) members.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
3984
|
+
|
|
3985
|
+
// One entry per distinct Source, not one attribute scan per record: a seed's
|
|
3986
|
+
// 60,000 facts name a handful of Sources between them.
|
|
3987
|
+
const typeBySource = new Map();
|
|
3988
|
+
const sourceTypeOf = (id) => {
|
|
3989
|
+
let type = typeBySource.get(id);
|
|
3990
|
+
if (type === undefined) {
|
|
3991
|
+
type = (sourcesById.get(id)?.attributes || []).find((a) => a?.prop === "mgx:sourceType")?.value || "";
|
|
3992
|
+
typeBySource.set(id, type);
|
|
3993
|
+
}
|
|
3994
|
+
return type;
|
|
3995
|
+
};
|
|
3996
|
+
|
|
3997
|
+
// Likewise for the timestamp a provenance string embeds: a corpus band writes
|
|
3998
|
+
// one tag over tens of thousands of records, and parsing it is the same
|
|
3999
|
+
// answer every time.
|
|
4000
|
+
const embeddedBySource = new Map();
|
|
4001
|
+
const embeddedTimestampOf = (provenance) => {
|
|
4002
|
+
let ts = embeddedBySource.get(provenance);
|
|
4003
|
+
if (ts === undefined) {
|
|
4004
|
+
ts = embeddedTagTimestamp(provenance.split(" | ").filter(Boolean));
|
|
4005
|
+
embeddedBySource.set(provenance, ts);
|
|
4006
|
+
}
|
|
4007
|
+
return ts;
|
|
4008
|
+
};
|
|
4009
|
+
|
|
4010
|
+
// Only the sqlite head materialisation walks siblings by pair, and building
|
|
4011
|
+
// the index costs two attribute reads per group — so it is built when asked
|
|
4012
|
+
// for and not before.
|
|
4013
|
+
let groupsByPair = null;
|
|
3839
4014
|
|
|
3840
4015
|
return {
|
|
3841
4016
|
groups,
|
|
3842
|
-
groupsByPair
|
|
4017
|
+
get groupsByPair() {
|
|
4018
|
+
if (groupsByPair) return groupsByPair;
|
|
4019
|
+
groupsByPair = new Map();
|
|
4020
|
+
for (const [groupId, members] of groups) {
|
|
4021
|
+
const key = subjectPredicateKey(individualKey(members[0], "subject"), individualKey(members[0], "predicate"));
|
|
4022
|
+
const held = groupsByPair.get(key);
|
|
4023
|
+
if (held) held.push(groupId);
|
|
4024
|
+
else groupsByPair.set(key, [groupId]);
|
|
4025
|
+
}
|
|
4026
|
+
return groupsByPair;
|
|
4027
|
+
},
|
|
3843
4028
|
statedByRecord,
|
|
3844
|
-
sourceTypeOf
|
|
4029
|
+
sourceTypeOf,
|
|
4030
|
+
embeddedTimestampOf,
|
|
3845
4031
|
};
|
|
3846
4032
|
}
|
|
3847
4033
|
|
|
4034
|
+
/** Everything the fold reads off one live head, gathered in ONE walk of its
|
|
4035
|
+
* attributes. Ten `.find()` scans of the same short array, once per record, is
|
|
4036
|
+
* the single most expensive thing a whole-graph fold does — the work is all in
|
|
4037
|
+
* the scanning, not in the reading. */
|
|
4038
|
+
function foldHeadFields(head) {
|
|
4039
|
+
let provenance = "";
|
|
4040
|
+
let createdAt = "";
|
|
4041
|
+
let observedAt = "";
|
|
4042
|
+
let extraction = "";
|
|
4043
|
+
let trustScore = "";
|
|
4044
|
+
let sourceId = "";
|
|
4045
|
+
let quantifier = "";
|
|
4046
|
+
let justification = "";
|
|
4047
|
+
for (const a of head?.attributes || []) {
|
|
4048
|
+
switch (a?.prop) {
|
|
4049
|
+
case "mgx:factProvenance": provenance = a.value || ""; continue;
|
|
4050
|
+
case CREATED_AT_PROP: createdAt = a.value || ""; continue;
|
|
4051
|
+
case OBSERVED_AT_PROP: observedAt = a.value || ""; continue;
|
|
4052
|
+
case EXTRACTION_FINDING_PROP: extraction = a.value || ""; continue;
|
|
4053
|
+
case TRUST_SCORE_PROP: trustScore = a.value || ""; continue;
|
|
4054
|
+
case SOURCE_ID_PROP: sourceId = a.value || ""; continue;
|
|
4055
|
+
default: break;
|
|
4056
|
+
}
|
|
4057
|
+
if (a?.key === "quantifier") quantifier = a.value || "";
|
|
4058
|
+
else if (a?.key === "justification") justification = a.value || "";
|
|
4059
|
+
}
|
|
4060
|
+
return { provenance, createdAt, observedAt, extraction, trustScore, sourceId, quantifier, justification };
|
|
4061
|
+
}
|
|
4062
|
+
|
|
3848
4063
|
/** One triple group folded into its row, minus the aggregate trust — that is
|
|
3849
4064
|
* the caller's, because it is the only part that depends on when you ask. */
|
|
3850
4065
|
function foldFactGroup(id, heads, ctx) {
|
|
@@ -3890,10 +4105,11 @@ function foldFactGroup(id, heads, ctx) {
|
|
|
3890
4105
|
});
|
|
3891
4106
|
continue;
|
|
3892
4107
|
}
|
|
3893
|
-
const
|
|
4108
|
+
const field = foldHeadFields(head);
|
|
4109
|
+
const headTags = field.provenance.split(" | ").filter(Boolean);
|
|
3894
4110
|
for (const tag of headTags) tags.add(tag);
|
|
3895
4111
|
const [statedBy] = statedByRecord.get(head.id) || [];
|
|
3896
|
-
const sourceId = statedBy ||
|
|
4112
|
+
const sourceId = statedBy || field.sourceId;
|
|
3897
4113
|
const sourceType = sourceTypeOf(sourceId);
|
|
3898
4114
|
// src:none stands for "no Source at all", so it stays out of the union a
|
|
3899
4115
|
// reader renders and out of the corroboration count, exactly as an
|
|
@@ -3902,9 +4118,9 @@ function foldFactGroup(id, heads, ctx) {
|
|
|
3902
4118
|
sourceIds.push(statedBy);
|
|
3903
4119
|
if (sourceType) sourceTypes.push(sourceType);
|
|
3904
4120
|
}
|
|
3905
|
-
const createdAt =
|
|
3906
|
-
const observedAt =
|
|
3907
|
-
const extraction =
|
|
4121
|
+
const createdAt = field.createdAt;
|
|
4122
|
+
const observedAt = field.observedAt;
|
|
4123
|
+
const extraction = field.extraction ? field.extraction.split(" ").filter(Boolean) : [];
|
|
3908
4124
|
for (const finding of extraction) findings.add(finding);
|
|
3909
4125
|
assertions.push({
|
|
3910
4126
|
id: head.id, sourceId, sourceType,
|
|
@@ -3912,13 +4128,13 @@ function foldFactGroup(id, heads, ctx) {
|
|
|
3912
4128
|
createdAt,
|
|
3913
4129
|
...(observedAt ? { observedAt } : {}),
|
|
3914
4130
|
...(extraction.length ? { extraction } : {}),
|
|
3915
|
-
ownTrust: Number(
|
|
3916
|
-
assertedAt:
|
|
4131
|
+
ownTrust: Number(field.trustScore) || 0,
|
|
4132
|
+
assertedAt: ctx.embeddedTimestampOf(field.provenance) || (Number.isFinite(Date.parse(createdAt)) ? createdAt : ""),
|
|
3917
4133
|
});
|
|
3918
|
-
quantifier = quantifier ||
|
|
4134
|
+
quantifier = quantifier || field.quantifier;
|
|
3919
4135
|
// ' | '-separated environments, one premise-id list per independent
|
|
3920
4136
|
// derivation; a legacy value with no ' | ' parses as one environment.
|
|
3921
|
-
for (const chunk of
|
|
4137
|
+
for (const chunk of field.justification.split(" | ")) {
|
|
3922
4138
|
const env = chunk.split(" ").filter(Boolean);
|
|
3923
4139
|
if (!env.length) continue;
|
|
3924
4140
|
const key = env.join(" ");
|
|
@@ -4195,8 +4411,8 @@ export const MULTI_VALUED_PREDICATES = MERGE_PREDICATES;
|
|
|
4195
4411
|
* only what its own clock could not order (the resolver's trust and codepoint
|
|
4196
4412
|
* tie-breaks — see resolveSiblingGroups), so ordinary succession stops reading
|
|
4197
4413
|
* as disagreement; every other predicate keeps the full keep-both contract. */
|
|
4198
|
-
export function findContradictions(memory, { floor = CONTRADICTION_TRUST_FLOOR } = {}) {
|
|
4199
|
-
const rows = readFactRows(memory).filter((r) => r.trust >= floor);
|
|
4414
|
+
export function findContradictions(memory, { floor = CONTRADICTION_TRUST_FLOOR, factRows = null } = {}) {
|
|
4415
|
+
const rows = (factRows || readFactRows(memory)).filter((r) => r.trust >= floor);
|
|
4200
4416
|
const byKey = new Map();
|
|
4201
4417
|
for (const r of rows) {
|
|
4202
4418
|
if (resolutionStrategyFor(r.predicate) === RESOLUTION_MERGE) continue;
|
|
@@ -59,8 +59,8 @@ const SPANS_A_SENTENCE_BOUNDARY_RE = /[.!?]\s+\w/;
|
|
|
59
59
|
// never for the group part: a hand-built individual with a short opaque id is a
|
|
60
60
|
// legitimate sparse write, and rejecting it is exactly the false positive this
|
|
61
61
|
// gate must never produce. The source suffix is matched loosely on purpose — a
|
|
62
|
-
// Source id legitimately carries colons
|
|
63
|
-
// ("src:
|
|
62
|
+
// Source id legitimately carries colons and spaces of its own
|
|
63
|
+
// ("src:corpus:mud:amber fox").
|
|
64
64
|
const FACT_RECORD_ID_RE = /^[^@]+@(.+?)(#v[1-9][0-9]*)?$/;
|
|
65
65
|
const looksLikeFactRecordId = (id) => id.includes("@") || /#v\d/.test(id);
|
|
66
66
|
|