@polycode-projects/the-mechanical-code-talker 6.0.18 → 6.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -23
- package/bin/tmct.mjs +16 -33
- package/corpus/LICENSES.json +0 -21
- package/corpus/README.md +10 -13
- package/corpus/reference/manifest.json +19 -19
- package/corpus/reference/shards/ref-01.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-04.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-08.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-10.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-11.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-17.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-20.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-25.jsonl.gz +0 -0
- package/corpus/reference/shards/ref-2c.jsonl.gz +0 -0
- package/corpus/tier2/generate.mjs +6 -142
- package/corpus/tier2/manifest.json +0 -42
- package/package.json +6 -4
- package/src/adapters/corpus/child-seed.mjs +74 -0
- package/src/adapters/corpus/conceptnet.mjs +45 -26
- package/src/adapters/corpus/research-source.mjs +6 -2
- package/src/adapters/corpus/wikidata-live.mjs +92 -51
- package/src/adapters/memory/blocks.mjs +7 -1
- package/src/adapters/memory/core.mjs +505 -107
- package/src/adapters/memory/corpus-bands.mjs +27 -10
- package/src/adapters/memory/inspect.mjs +24 -5
- package/src/adapters/memory/rows.mjs +359 -30
- package/src/adapters/memory/shacl.mjs +10 -3
- package/src/domain/ask.mjs +27 -10
- package/src/domain/cli-verbs.mjs +3 -4
- package/src/domain/completions/group.mjs +8 -3
- package/src/domain/completions/infer.mjs +7 -2
- package/src/domain/completions/prune.mjs +5 -1
- package/src/domain/completions/rank.mjs +7 -2
- package/src/domain/digest/compose.mjs +5 -1
- package/src/domain/digest/select.mjs +12 -6
- package/src/domain/domain.mjs +15 -8
- package/src/domain/el-classify.mjs +11 -2
- package/src/domain/fact-phrase.mjs +86 -4
- package/src/domain/hash.mjs +9 -0
- package/src/domain/memory/bias.mjs +8 -4
- package/src/domain/memory/capability.mjs +12 -6
- package/src/domain/memory/fact-order.mjs +29 -0
- package/src/domain/memory/resolution.mjs +3 -0
- package/src/domain/news-feed.mjs +862 -92
- package/src/domain/reference-pack.mjs +5 -0
- package/src/domain/sense-gate.mjs +220 -0
- package/src/domain/sense-scope.mjs +116 -0
- package/src/domain/sense-split.mjs +1 -1
- package/src/domain/syllogise.mjs +60 -21
- package/src/domain/tableau.mjs +23 -14
- package/src/domain/term-ledger.mjs +16 -1
- package/src/domain/worlds-pack.mjs +5 -1
- package/src/services/adventure-autoplay.mjs +6 -1
- package/src/services/adventure-editor.mjs +43 -21
- package/src/services/adventure-viz.mjs +26 -9
- package/src/services/adventure.mjs +40 -10
- package/src/services/chat.mjs +270 -125
- package/src/services/extensions.mjs +51 -58
- package/src/services/extract-facts.mjs +906 -66
- package/src/services/init.mjs +4 -4
- package/src/services/ledger-viz.mjs +9 -4
- package/src/services/memory-panel-viz.mjs +4 -5
- package/src/services/mud-editor.mjs +40 -16
- package/src/services/mud-viz.mjs +8 -2
- package/src/services/mudiii-turn.mjs +5 -3
- package/src/services/mudiii-viz.mjs +8 -2
- package/src/services/news.mjs +306 -21
- package/src/services/research-viz.mjs +1 -1
- package/src/services/sprite-catalog-viz.mjs +10 -5
- package/src/surfaces/web/adventure-browser-entry.mjs +6 -12
- package/src/surfaces/web/memory-ask-browser.bundle.js +152 -151
- package/src/surfaces/web/mud-browser-entry.mjs +7 -11
- package/src/surfaces/web/research-browser-entry.mjs +5 -2
- package/corpus/tier2/aws.jsonl +0 -39
- package/corpus/tier2/java.jsonl +0 -31
- package/corpus/tier2/python.jsonl +0 -30
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
|
|
17
17
|
import { access, mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
|
|
18
18
|
import { dirname, join } from "node:path";
|
|
19
|
-
import { proseTokensFor
|
|
19
|
+
import { proseTokensFor } from "../../domain/prose.mjs";
|
|
20
20
|
import { fnv1aHex, normText, normFactTerm, normFactPredicate, factIdFor, factIdForTriple } from "../../domain/hash.mjs";
|
|
21
21
|
|
|
22
22
|
// Fact identity (normalization + id derivation) lives in hash.mjs — the one
|
|
@@ -39,6 +39,8 @@ import {
|
|
|
39
39
|
// consumers keep one import site.
|
|
40
40
|
export { CREATED_AT_PROP, UPDATED_AT_PROP, provenanceTagToSource } from "../../domain/memory/trust.mjs";
|
|
41
41
|
import { NEG_PREDICATE_PREFIX, negatedPredicate } from "../../domain/memory/capability.mjs";
|
|
42
|
+
import { factOrderKey } from "../../domain/memory/fact-order.mjs";
|
|
43
|
+
import { partitionAttributions } from "../../domain/news-feed.mjs";
|
|
42
44
|
import {
|
|
43
45
|
planHeadRollup, planChainRollup, mergeRollups,
|
|
44
46
|
isHeadRollupId, isChainRollupId, isRollupId, headRollupTypeOf,
|
|
@@ -68,7 +70,10 @@ export {
|
|
|
68
70
|
// fine as long as neither side READS an imported binding while the other's
|
|
69
71
|
// body is still running — rows.mjs builds its class map on first use, and
|
|
70
72
|
// every use here is inside a function.
|
|
71
|
-
import {
|
|
73
|
+
import {
|
|
74
|
+
payloadToRows, rowsToPayload, diffRows, renormalizeAssembledPayload, renormalizeProseIndex,
|
|
75
|
+
canDropAssembledIndividuals, dropAssembledIndividuals, appendAssembledRowOrder, dropAssembledRowOrder,
|
|
76
|
+
} from "./rows.mjs";
|
|
72
77
|
|
|
73
78
|
// The rollup vocabulary and its tuning constants live with the compaction
|
|
74
79
|
// layer; re-exported here so store consumers keep one import site.
|
|
@@ -139,13 +144,13 @@ const MEMORY_VOCABULARY = [
|
|
|
139
144
|
{ prop: "mgx:factProvenance", note: "LEGACY COMPAT SHIM: the ' | '-joined provenance tag string a fact came from; the source-of-truth is now the mgx:statedBy edges derived from it" },
|
|
140
145
|
{ prop: "mgx:sourceId", note: "the assertion key a Fact record is filed under — the Source id of the ONE party asserting it, which is also the @-suffix of the record's own id. `src:none` when no tag names a Source, so every record has a key rather than a hole" },
|
|
141
146
|
{ prop: "mgx:observedAt", note: "OPTIONAL valid time: when the asserting party WITNESSED the claim, as against mgx:createdAt's transaction time (when this store recorded it). A stale article read today loses to an eyewitness report from yesterday. Stored only when a caller supplies one — never fabricated, never backfilled" },
|
|
142
|
-
{ prop: "mgx:extractionFinding", note: "OPTIONAL: the space-joined structural findings the extractor recorded about how THIS assertion's sentence was read, from the closed vocabulary identifier-token | clause-fallback | pronoun-carry | definitional-frame. Per assertion, never per triple: a later clean re-assertion of the same triple carries none. Absence means no findings were recorded, never that the sentence was checked and read cleanly" },
|
|
147
|
+
{ prop: "mgx:extractionFinding", note: "OPTIONAL: the space-joined structural findings the extractor recorded about how THIS assertion's sentence was read, from the closed vocabulary identifier-token | clause-fallback | pronoun-carry | definitional-frame | reported-speech. Per assertion, never per triple: a later clean re-assertion of the same triple carries none. Absence means no findings were recorded, never that the sentence was checked and read cleanly" },
|
|
143
148
|
{ prop: "mgx:supersedes", note: "the record id(s) this one replaced when its own source re-asserted the triple with a newer embedded timestamp. A space-joined LIST; absent, never empty, until the first supersession" },
|
|
144
149
|
{ prop: "mgx:supersededBy", note: "the record id(s) that replaced this one. Its presence is what makes a record a demoted leaf rather than a live head, and the group fold skips it: a source's past belief is not a second vote for the present one. A LIST, because one source with two live replicas can fork before they sync" },
|
|
145
150
|
{ prop: "mgx:factQuantifier", note: "OPTIONAL: the quantifier word a plural class-membership teach used ('every'/'some'/'a few'), for literal recall by 'how many Xs are Ys' — never real cardinality counting" },
|
|
146
151
|
{ prop: "mgx:factJustification", note: "an entailed Fact's supporting premise fact ids: ' | '-separated environments, one space-separated premise-id list per independent derivation, capped by syllogise's maxEnvironments knob; a value with no ' | ' is a single environment" },
|
|
147
152
|
{ prop: "mgx:ruleName", note: "a taught Rule's own name (e.g. 'grandparent') — the query-dispatcher's lookup key, PLAN_TAUGHT_RELATIONS.md §2/§3" },
|
|
148
|
-
{ prop: "mgx:ruleKind", note: "a taught Rule's SHAPE tag — the closed vocabulary compose2 | filter | recursive (structural, like 'Fact'/'Rule' themselves, never a domain word)" },
|
|
153
|
+
{ prop: "mgx:ruleKind", note: "a taught Rule's SHAPE tag — the closed vocabulary compose2 | filter | recursive | action-signature | action-precond | action-effect | action-constraint (structural, like 'Fact'/'Rule' themselves, never a domain word)" },
|
|
149
154
|
{ prop: "mgx:ruleBase1", note: "compose2: the first hop's base relation name; filter: the base rule/relation being filtered (same 'base relation' role in both kinds, so the name is shared)" },
|
|
150
155
|
{ prop: "mgx:ruleBase2", note: "compose2 only: the second hop's base relation name" },
|
|
151
156
|
{ prop: "mgx:ruleFilterProperty", note: "filter only: the property literal candidates are filtered by (an mgx:hasProperty-shaped Fact lookup)" },
|
|
@@ -235,6 +240,14 @@ export function isMemoryOrSqliteHandle(dir) {
|
|
|
235
240
|
return isMemoryHandle(dir) || isSqliteHandle(dir) || isRowHandle(dir);
|
|
236
241
|
}
|
|
237
242
|
|
|
243
|
+
/** Move a store handle's write stamp on. `foldedFactRows` keys the fold it
|
|
244
|
+
* holds to this number, so anything that changes what the store would fold to
|
|
245
|
+
* — a landed write, a payload about to be mutated in place, a dropped
|
|
246
|
+
* assembly, a seed assigned over the top — stamps here first. */
|
|
247
|
+
function stampStoreWrite(dir) {
|
|
248
|
+
if (isMemoryOrSqliteHandle(dir)) dir.storeWrites = (dir.storeWrites || 0) + 1;
|
|
249
|
+
}
|
|
250
|
+
|
|
238
251
|
/** Backend B — pure in-memory store: `{ backend: "memory", payload }` held by
|
|
239
252
|
* the caller (never module-global). Zero file I/O; distinct from
|
|
240
253
|
* `--ephemeral`, which still round-trips a throwaway temp dir. */
|
|
@@ -250,7 +263,12 @@ export function createInMemoryStore() {
|
|
|
250
263
|
* `seedPayload` is null/undefined — a browser session with nothing to seed
|
|
251
264
|
* keeps its own fresh empty payload untouched. */
|
|
252
265
|
export function applySeedPayload(memoryDir, seedPayload) {
|
|
253
|
-
if (seedPayload)
|
|
266
|
+
if (!seedPayload) return;
|
|
267
|
+
memoryDir.payload = { ...memoryDir.payload, ...seedPayload };
|
|
268
|
+
// A seed arrives from anywhere, so whatever an earlier load settled about the
|
|
269
|
+
// ids this store holds no longer describes it.
|
|
270
|
+
memoryDir.storeMigrationsSettled = false;
|
|
271
|
+
stampStoreWrite(memoryDir);
|
|
254
272
|
}
|
|
255
273
|
|
|
256
274
|
/** A structurally independent copy of a memory payload — `structuredClone`
|
|
@@ -502,6 +520,18 @@ function backfillFactsProjection(db) {
|
|
|
502
520
|
|
|
503
521
|
const individualAttr = (ind, prop) => (ind?.attributes || []).find((a) => a?.prop === prop)?.value || "";
|
|
504
522
|
const individualKey = (ind, key) => (ind?.attributes || []).find((a) => a?.key === key)?.value || "";
|
|
523
|
+
|
|
524
|
+
/** Order two strings by codepoint, never by locale. Every listing this store
|
|
525
|
+
* hands out is read on whatever machine holds it, and two locales sorting one
|
|
526
|
+
* set differently is the same broken promise arrival order would be: a read
|
|
527
|
+
* over the fact store answers to the set, and to nothing about where it ran.
|
|
528
|
+
* memory/fact-order.mjs states the rule; inspect.mjs keeps its own copy of
|
|
529
|
+
* this comparator for the text it renders. */
|
|
530
|
+
const byCodepoint = (a, b) => {
|
|
531
|
+
const ka = String(a ?? "");
|
|
532
|
+
const kb = String(b ?? "");
|
|
533
|
+
return ka < kb ? -1 : ka > kb ? 1 : 0;
|
|
534
|
+
};
|
|
505
535
|
const subjectPredicateKey = (subject, predicate) => `${subject}\u0000${predicate}`;
|
|
506
536
|
|
|
507
537
|
// ---- Derived-local tables: `fact_heads` and `fact_object_supersessions` -----
|
|
@@ -798,11 +828,13 @@ export function wrapRowBackend(impl, {
|
|
|
798
828
|
backend: BACKEND_ROW,
|
|
799
829
|
impl,
|
|
800
830
|
cachedPayload: null,
|
|
831
|
+
cachedIndex: null,
|
|
801
832
|
basePayload: cloneMemoryPayload(basePayload),
|
|
802
833
|
sqliteSeedStore,
|
|
803
834
|
sqliteSeedOverlayRows: sqliteSeedOverlayRows ? [...sqliteSeedOverlayRows] : null,
|
|
804
835
|
sqliteSeedKeyOrds: null,
|
|
805
836
|
baseRows: null,
|
|
837
|
+
baseKeyOrds: null,
|
|
806
838
|
storedRows: null,
|
|
807
839
|
onOversizedRow,
|
|
808
840
|
copyOnRead,
|
|
@@ -843,11 +875,8 @@ function overlayRows(baseRows, sessionRows) {
|
|
|
843
875
|
* session store does not already hold a row for. */
|
|
844
876
|
function seedOnlyKeys(handle) {
|
|
845
877
|
const sessionKeys = new Set(handle.storedRows.map((r) => r.rowKey));
|
|
846
|
-
const baseKeys = handle.sqliteSeedStore
|
|
847
|
-
? sqliteSeedKeyOrds(handle).keys()
|
|
848
|
-
: handle.baseRows.map((r) => r.rowKey);
|
|
849
878
|
const seedOnly = new Set();
|
|
850
|
-
for (const key of
|
|
879
|
+
for (const key of readOnlyLayerKeyOrds(handle).keys()) if (!sessionKeys.has(key)) seedOnly.add(key);
|
|
851
880
|
return seedOnly;
|
|
852
881
|
}
|
|
853
882
|
|
|
@@ -923,9 +952,31 @@ function mutablePayloadCopy(payload) {
|
|
|
923
952
|
* coherent graph. */
|
|
924
953
|
function dropAssembledRowPayload(handle) {
|
|
925
954
|
if (!isRowHandle(handle)) return;
|
|
955
|
+
stampStoreWrite(handle);
|
|
926
956
|
handle.cachedPayload = null;
|
|
957
|
+
handle.cachedIndex = null;
|
|
927
958
|
handle.storedRows = null;
|
|
928
959
|
handle.baseRows = null;
|
|
960
|
+
handle.baseKeyOrds = null;
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
/** Every key a READ-ONLY layer under this handle holds, with the ord each one
|
|
964
|
+
* carries — the sqlite seed and its overlay, or the base payload's own
|
|
965
|
+
* projection. A session row keyed the same as one of these SHADOWS it, so
|
|
966
|
+
* deleting that session row brings the read-only row back rather than dropping
|
|
967
|
+
* the individual: a rebuild is the only honest answer to that delete, and this
|
|
968
|
+
* is what tells the write path so.
|
|
969
|
+
*
|
|
970
|
+
* The ords are what let a write project the SESSION's rows alone and still put
|
|
971
|
+
* a newly-keyed row past everything the read-only layer already holds. A key
|
|
972
|
+
* and an integer per row is small where the rows themselves are not, so this is
|
|
973
|
+
* held for the handle's life once a write has asked for it. */
|
|
974
|
+
function readOnlyLayerKeyOrds(handle) {
|
|
975
|
+
if (handle.sqliteSeedStore) return sqliteSeedKeyOrds(handle);
|
|
976
|
+
if (!handle.baseKeyOrds) {
|
|
977
|
+
handle.baseKeyOrds = new Map((handle.baseRows || []).map((row) => [row.rowKey, ordOfRow(row)]));
|
|
978
|
+
}
|
|
979
|
+
return handle.baseKeyOrds;
|
|
929
980
|
}
|
|
930
981
|
|
|
931
982
|
/** A record with its audit stamp removed. `mgx:updatedAt` moves on every
|
|
@@ -958,13 +1009,17 @@ function movedBeyondAuditStamp(beforeRow, row) {
|
|
|
958
1009
|
async function persistRowPayload(handle, payload) {
|
|
959
1010
|
await ensureRowPayload(handle);
|
|
960
1011
|
const seedKeys = seedOnlyKeys(handle);
|
|
961
|
-
//
|
|
962
|
-
//
|
|
1012
|
+
// The seed is out of BOTH sides of the diff, whichever layer holds it: those
|
|
1013
|
+
// keys are exactly the ones no write may touch, so projecting them would
|
|
963
1014
|
// materialize the whole seed only to filter every row of it back out.
|
|
964
|
-
const before = handle.
|
|
1015
|
+
const before = handle.storedRows;
|
|
965
1016
|
const beforeByKey = new Map(before.map((row) => [row.rowKey, row]));
|
|
966
|
-
const after = payloadToRows(
|
|
967
|
-
priorRows:
|
|
1017
|
+
const after = payloadToRows(payloadWithoutRowKeys(payload, seedKeys), {
|
|
1018
|
+
priorRows: before,
|
|
1019
|
+
// The seed's ords are handed over as the map the handle already holds. The
|
|
1020
|
+
// session's own rows come first, so a key both layers hold keeps the ord
|
|
1021
|
+
// assembly gives it.
|
|
1022
|
+
priorOrds: readOnlyLayerKeyOrds(handle),
|
|
968
1023
|
onOversizedRow: handle.onOversizedRow,
|
|
969
1024
|
...(handle.log ? { log: handle.log } : {}),
|
|
970
1025
|
});
|
|
@@ -987,18 +1042,19 @@ async function persistRowPayload(handle, payload) {
|
|
|
987
1042
|
for (const row of writes) next.set(row.rowKey, row);
|
|
988
1043
|
handle.storedRows = [...next.values()];
|
|
989
1044
|
const meta = { memory: payload.memory, prefixes: payload.prefixes };
|
|
990
|
-
if (handle.
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
handle.cachedPayload = null;
|
|
994
|
-
handle.cachedPayload = migrateStoredMemory(assembleSqliteSeededPayload(handle, meta));
|
|
995
|
-
} else {
|
|
996
|
-
// Dropped before the rebuild, not after it: the payload this replaces is the
|
|
997
|
-
// largest object the handle holds, and keeping it reachable while the next
|
|
998
|
-
// one assembles doubles the peak for no reason.
|
|
999
|
-
handle.cachedPayload = null;
|
|
1000
|
-
handle.cachedPayload = migrateStoredMemory(rowsToPayload(overlayRows(handle.baseRows, handle.storedRows), { meta }));
|
|
1045
|
+
if (patchAssembledPayload(handle.cachedPayload, meta, writes, removals, readOnlyLayerKeyOrds(handle))) {
|
|
1046
|
+
patchStoreWideIndex(handle, writes, removals);
|
|
1047
|
+
return;
|
|
1001
1048
|
}
|
|
1049
|
+
// Dropped before the rebuild, not after it: the payload this replaces is the
|
|
1050
|
+
// largest object the handle holds, and keeping it reachable while the next
|
|
1051
|
+
// one assembles doubles the peak for no reason. The index goes with it for
|
|
1052
|
+
// the same reason — it names the arrays that payload owned.
|
|
1053
|
+
handle.cachedPayload = null;
|
|
1054
|
+
handle.cachedIndex = null;
|
|
1055
|
+
handle.cachedPayload = migrateStoredMemory(handle.sqliteSeedStore
|
|
1056
|
+
? assembleSqliteSeededPayload(handle, meta)
|
|
1057
|
+
: rowsToPayload(overlayRows(handle.baseRows, handle.storedRows), { meta }));
|
|
1002
1058
|
}
|
|
1003
1059
|
|
|
1004
1060
|
/** The payload minus a set of row keys, for a projection that must not spend
|
|
@@ -1082,15 +1138,6 @@ function sqliteSeedKeyOrds(handle) {
|
|
|
1082
1138
|
return ords;
|
|
1083
1139
|
}
|
|
1084
1140
|
|
|
1085
|
-
/** What `payloadToRows` reads off prior rows: the ord each key already carries,
|
|
1086
|
-
* in the smallest row shape that carries one. The session's own rows come
|
|
1087
|
-
* last, so a key both layers hold keeps the session's ord — the precedence
|
|
1088
|
-
* assembly gives it. */
|
|
1089
|
-
function* sqliteSeedPriorRows(handle) {
|
|
1090
|
-
for (const [rowKey, ord] of sqliteSeedKeyOrds(handle)) yield { rowKey, json: `{"ord":${ord}}` };
|
|
1091
|
-
yield* handle.storedRows;
|
|
1092
|
-
}
|
|
1093
|
-
|
|
1094
1141
|
function* chainedRows(...sources) {
|
|
1095
1142
|
for (const source of sources) yield* source;
|
|
1096
1143
|
}
|
|
@@ -1107,28 +1154,50 @@ function assembleSqliteSeededPayload(handle, meta) {
|
|
|
1107
1154
|
);
|
|
1108
1155
|
}
|
|
1109
1156
|
|
|
1110
|
-
/** The assembled payload after a write
|
|
1111
|
-
*
|
|
1157
|
+
/** The assembled payload after a write, brought up to date without reading the
|
|
1158
|
+
* seed again. Returns true when it did, false when this write is not patchable
|
|
1159
|
+
* and the caller has to rebuild — decided before anything is touched, so a
|
|
1160
|
+
* `false` leaves the payload exactly as it found it.
|
|
1161
|
+
*
|
|
1162
|
+
* Why the positions work out for an ADDED or REWRITTEN row: a write only ever
|
|
1163
|
+
* carries session rows, and `payloadToRows` gives a newly-keyed row an ord past
|
|
1164
|
+
* every ord already assembled, so a new individual belongs at the tail and a
|
|
1165
|
+
* rewritten one belongs exactly where it already sits.
|
|
1166
|
+
*
|
|
1167
|
+
* Why they work out for a REMOVED one: dropping an individual from the
|
|
1168
|
+
* assembled array would drop it from wherever the fact ordering moved it to,
|
|
1169
|
+
* which is a different slot from the one it held in row order — so the drop
|
|
1170
|
+
* goes through the row order the assembly carries (`dropAssembledIndividuals`)
|
|
1171
|
+
* and the array is refilled in that order first. The slots that survive are
|
|
1172
|
+
* then the ones the surviving ROWS own, which is what a rebuild would leave.
|
|
1173
|
+
*
|
|
1174
|
+
* Either way everything the payload derives goes through
|
|
1175
|
+
* `renormalizeAssembledPayload`, the same function `rowsToPayload` itself ends
|
|
1176
|
+
* on, so a patched payload and a rebuilt one cannot drift apart.
|
|
1112
1177
|
*
|
|
1113
|
-
*
|
|
1114
|
-
*
|
|
1115
|
-
*
|
|
1116
|
-
* belongs exactly where it already sits. Both are where a rebuild from the
|
|
1117
|
-
* same rows would put them, and the fact set is unchanged or grown, so the
|
|
1118
|
-
* fact ordering `renormalizeAssembledPayload` reapplies lands the same way.
|
|
1119
|
-
* Everything the payload derives goes through that one function, the same one
|
|
1120
|
-
* `rowsToPayload` itself ends on, so a patched payload and a rebuilt one
|
|
1121
|
-
* cannot drift apart.
|
|
1178
|
+
* `readOnlyKeys` are the keys a layer under the session holds: deleting a
|
|
1179
|
+
* session row that SHADOWS one of them uncovers the read-only row rather than
|
|
1180
|
+
* dropping the individual, which only a rebuild can work out.
|
|
1122
1181
|
*
|
|
1123
|
-
*
|
|
1124
|
-
*
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1182
|
+
* On `false` the caller replaces this payload with a fresh assembly, so
|
|
1183
|
+
* whatever this got part-way through applying is discarded with it. */
|
|
1184
|
+
function patchAssembledPayload(payload, meta, writes, removals, readOnlyKeys) {
|
|
1185
|
+
const droppedGroups = new Set();
|
|
1186
|
+
const droppedIndividuals = new Set();
|
|
1187
|
+
for (const key of removals) {
|
|
1188
|
+
const rowKey = String(key);
|
|
1189
|
+
if (readOnlyKeys.has(rowKey)) return false;
|
|
1190
|
+
if (rowKey.startsWith(EDGE_GROUP_KEY_PREFIX)) droppedGroups.add(rowKey.slice(EDGE_GROUP_KEY_PREFIX.length));
|
|
1191
|
+
else droppedIndividuals.add(rowKey);
|
|
1192
|
+
}
|
|
1193
|
+
if (droppedIndividuals.size && !canDropAssembledIndividuals(payload, droppedIndividuals)) return false;
|
|
1194
|
+
|
|
1129
1195
|
payload.memory = meta.memory;
|
|
1130
1196
|
payload.prefixes = meta.prefixes;
|
|
1131
|
-
if (!writes.length
|
|
1197
|
+
if (!writes.length && !removals.length) {
|
|
1198
|
+
renormalizeAssembledPayload(payload);
|
|
1199
|
+
return true;
|
|
1200
|
+
}
|
|
1132
1201
|
|
|
1133
1202
|
const individualPositions = new Map();
|
|
1134
1203
|
for (let i = 0; i < payload.individuals.length; i += 1) individualPositions.set(payload.individuals[i]?.id, i);
|
|
@@ -1154,9 +1223,16 @@ function patchAssembledPayload(payload, meta, writes) {
|
|
|
1154
1223
|
if (at === undefined) {
|
|
1155
1224
|
individualPositions.set(individual.id, payload.individuals.length);
|
|
1156
1225
|
payload.individuals.push(individual);
|
|
1226
|
+
appendAssembledRowOrder(payload, individual.id);
|
|
1157
1227
|
} else payload.individuals[at] = individual;
|
|
1158
1228
|
}
|
|
1159
|
-
|
|
1229
|
+
|
|
1230
|
+
if (droppedGroups.size) {
|
|
1231
|
+
payload.objectProperties = payload.objectProperties.filter((group) => !droppedGroups.has(group?.prop));
|
|
1232
|
+
}
|
|
1233
|
+
if (droppedIndividuals.size && !dropAssembledIndividuals(payload, droppedIndividuals)) return false;
|
|
1234
|
+
renormalizeAssembledPayload(payload);
|
|
1235
|
+
return true;
|
|
1160
1236
|
}
|
|
1161
1237
|
|
|
1162
1238
|
/** Every distinct subject and object a seed store's facts carry, off the
|
|
@@ -1271,7 +1347,15 @@ const cloneJson = (v) => (v === undefined ? v : structuredClone(v));
|
|
|
1271
1347
|
* persist would delete them as absent-from-payload. */
|
|
1272
1348
|
function readSqlitePayload(handle) {
|
|
1273
1349
|
const dataVersion = handle.db.prepare("PRAGMA data_version").get()?.data_version;
|
|
1274
|
-
if (handle.cachedPayload && handle.cachedDataVersion !== dataVersion)
|
|
1350
|
+
if (handle.cachedPayload && handle.cachedDataVersion !== dataVersion) {
|
|
1351
|
+
// Another connection committed, so the fold this handle holds describes a
|
|
1352
|
+
// store that no longer exists — it goes with the payload it was taken of.
|
|
1353
|
+
// So does what an earlier load settled about the ids the store carries: the
|
|
1354
|
+
// other writer is the one path that can put a shape there this one wouldn't.
|
|
1355
|
+
stampStoreWrite(handle);
|
|
1356
|
+
handle.cachedPayload = null;
|
|
1357
|
+
handle.storeMigrationsSettled = false;
|
|
1358
|
+
}
|
|
1275
1359
|
if (!handle.cachedPayload) {
|
|
1276
1360
|
handle.cachedPayload = buildSqlitePayloadFromRows(handle);
|
|
1277
1361
|
// The head index rides the same cache lifecycle as the payload it indexes,
|
|
@@ -1386,8 +1470,10 @@ function cacheDropGroups(cache, droppedProps) {
|
|
|
1386
1470
|
const EDGE_GROUP_ROW_CLASS = "edge-group";
|
|
1387
1471
|
const EDGE_GROUP_KEY_PREFIX = "edge-group:";
|
|
1388
1472
|
|
|
1473
|
+
const FACT_ROW_CLASS = "fact";
|
|
1474
|
+
|
|
1389
1475
|
const ROW_CLASS_BY_INDIVIDUAL_CLASS = new Map([
|
|
1390
|
-
[FACT_CLASS,
|
|
1476
|
+
[FACT_CLASS, FACT_ROW_CLASS],
|
|
1391
1477
|
[SOURCE_CLASS, "source"],
|
|
1392
1478
|
[UTTERANCE_CLASS, "utterance"],
|
|
1393
1479
|
[MEMORY_SESSION_CLASS, "session"],
|
|
@@ -1991,8 +2077,8 @@ export async function snapshotMemory(dir, { retentionVersions } = {}) {
|
|
|
1991
2077
|
* append creates the file). The result is a raw entities payload;
|
|
1992
2078
|
* parseEntities() loads it. */
|
|
1993
2079
|
export async function loadMemory(dir) {
|
|
1994
|
-
if (isMemoryHandle(dir)) return
|
|
1995
|
-
if (isSqliteHandle(dir)) return
|
|
2080
|
+
if (isMemoryHandle(dir)) return migrateStoredMemoryOnce(dir, dir.payload);
|
|
2081
|
+
if (isSqliteHandle(dir)) return migrateStoredMemoryOnce(dir, readSqlitePayload(dir));
|
|
1996
2082
|
// Not migrated here: a row handle migrates the payload once, as it assembles
|
|
1997
2083
|
// it, so every read after the first is spared two walks of the whole graph.
|
|
1998
2084
|
if (isRowHandle(dir)) return readRowPayload(dir);
|
|
@@ -2011,6 +2097,30 @@ export async function loadMemory(dir) {
|
|
|
2011
2097
|
* Both are pure payload transforms and both converge to no-ops. */
|
|
2012
2098
|
const migrateStoredMemory = (payload) => migrateFactAssertionKeys(migrateLegacyFactIds(payload));
|
|
2013
2099
|
|
|
2100
|
+
/** True when some Fact still carries an id from before one of the two
|
|
2101
|
+
* migrations above — a pre-widening 32-bit id, or a pre-assertion-model id
|
|
2102
|
+
* keyed on the triple alone. Both lack the `@` a record id carries, so one
|
|
2103
|
+
* pass over the individuals answers it, allocating nothing. */
|
|
2104
|
+
function carriesPreMigrationFactIds(payload) {
|
|
2105
|
+
for (const ind of payload?.individuals || []) {
|
|
2106
|
+
if (ind?.class === FACT_CLASS && !String(ind.id || "").includes("@")) return true;
|
|
2107
|
+
}
|
|
2108
|
+
return false;
|
|
2109
|
+
}
|
|
2110
|
+
|
|
2111
|
+
/** `migrateStoredMemory` for a handle that hands back the same store on every
|
|
2112
|
+
* read. A store whose Facts are all on current ids cannot acquire an old one —
|
|
2113
|
+
* every write mints through `factIdFor` — so the first load that finds nothing
|
|
2114
|
+
* to migrate settles the question for this handle and later loads skip even
|
|
2115
|
+
* the scan. A store that IS on old ids keeps migrating on every read until a
|
|
2116
|
+
* write persists the healed form, exactly as before. */
|
|
2117
|
+
function migrateStoredMemoryOnce(handle, payload) {
|
|
2118
|
+
if (handle.storeMigrationsSettled) return payload;
|
|
2119
|
+
if (carriesPreMigrationFactIds(payload)) return migrateStoredMemory(payload);
|
|
2120
|
+
handle.storeMigrationsSettled = true;
|
|
2121
|
+
return payload;
|
|
2122
|
+
}
|
|
2123
|
+
|
|
2014
2124
|
// A Fact id written before factIdFor widened to 64 bits — `fact:` + exactly 8
|
|
2015
2125
|
// hex. A current id is 16 hex, so this anchored test never matches one, and a
|
|
2016
2126
|
// migrated store pays only string checks with no rehash on load.
|
|
@@ -2038,6 +2148,9 @@ function migrateLegacyFactIds(payload) {
|
|
|
2038
2148
|
ind.id = currentId;
|
|
2039
2149
|
}
|
|
2040
2150
|
if (!remap.size) return payload;
|
|
2151
|
+
// The row order an assembly carries names the ids the rows arrived under, and
|
|
2152
|
+
// this just moved every legacy Fact off one of them.
|
|
2153
|
+
dropAssembledRowOrder(payload);
|
|
2041
2154
|
const remapId = (id) => remap.get(id) || id;
|
|
2042
2155
|
for (const group of payload.objectProperties || []) {
|
|
2043
2156
|
for (const e of group.examples || []) {
|
|
@@ -2198,6 +2311,7 @@ function migrateFactAssertionKeys(payload) {
|
|
|
2198
2311
|
* SQL write (Backend C, persistSqlitePayload), or a diffed row write into an
|
|
2199
2312
|
* injected store (Backend D, persistRowPayload). */
|
|
2200
2313
|
async function persistMemory(dir, payload) {
|
|
2314
|
+
stampStoreWrite(dir);
|
|
2201
2315
|
if (isMemoryHandle(dir)) { dir.payload = payload; return; }
|
|
2202
2316
|
if (isSqliteHandle(dir)) { persistSqlitePayload(dir, payload); return; }
|
|
2203
2317
|
if (isRowHandle(dir)) { await persistRowPayload(dir, payload); return; }
|
|
@@ -2292,49 +2406,200 @@ export async function saveNodeId(dir, nodeId) {
|
|
|
2292
2406
|
* append goes through here, including the lazy legacy-provenance migration
|
|
2293
2407
|
* and actor-level Source reliability recompute. `fn` may be async (the
|
|
2294
2408
|
* SHACL ingest gate awaits validation before ever mutating `payload`). */
|
|
2295
|
-
// Per-call lookup index (
|
|
2296
|
-
//
|
|
2297
|
-
//
|
|
2298
|
-
//
|
|
2409
|
+
// Per-call lookup index, attached to payload under a Symbol key (skipped by
|
|
2410
|
+
// JSON.stringify) so upsertIndividual/upsertSource/upsertEdge/appendFacts get
|
|
2411
|
+
// O(1) lookups instead of re-scanning; discarded when mutateMemory returns. On
|
|
2412
|
+
// a row handle the two id -> id-list halves read through a base the handle
|
|
2413
|
+
// keeps across writes (storeWideIndexOf), so only the three that hold
|
|
2414
|
+
// individuals are built per call.
|
|
2299
2415
|
const MEMORY_INDEX = Symbol("mutateMemory lookup index");
|
|
2300
2416
|
|
|
2301
|
-
/** Build the
|
|
2302
|
-
* under MEMORY_INDEX.
|
|
2303
|
-
|
|
2417
|
+
/** Build the five lookup Maps from the just-loaded payload and attach them
|
|
2418
|
+
* under MEMORY_INDEX.
|
|
2419
|
+
*
|
|
2420
|
+
* `base` is a store-wide index a row handle keeps across writes
|
|
2421
|
+
* (`storeWideIndexOf`). Its two maps are layered rather than rebuilt: this
|
|
2422
|
+
* mutation reads through them and writes only into its own layer, so the
|
|
2423
|
+
* handle's copy still describes the store when the mutation is over — or when
|
|
2424
|
+
* it throws. The other three hold INDIVIDUALS, and every individual this
|
|
2425
|
+
* mutation can reach is a fresh copy `mutablePayloadCopy` just made, so they
|
|
2426
|
+
* are always this payload's own. */
|
|
2427
|
+
function buildMemoryIndex(payload, base = null) {
|
|
2304
2428
|
const individualsById = new Map();
|
|
2305
2429
|
const sourcesById = new Map();
|
|
2306
|
-
const statedByBySubject = new Map();
|
|
2307
|
-
// groupId -> the record ids asserting that triple, so a write can ask "is
|
|
2308
|
-
// anyone asserting this yet" and an edge can resolve a group id to the real
|
|
2309
|
-
// nodes behind it, both without a scan.
|
|
2310
|
-
const factRecordsByGroup = new Map();
|
|
2311
2430
|
// groupId -> the retraction records standing over that triple, so the write
|
|
2312
2431
|
// path can ask "was this source's assertion retracted" without a scan. Almost
|
|
2313
2432
|
// always empty, which is why it is read before anything more expensive.
|
|
2314
2433
|
const retractionsByGroup = new Map();
|
|
2434
|
+
// groupId -> the record ids asserting that triple, so a write can ask "is
|
|
2435
|
+
// anyone asserting this yet" and an edge can resolve a group id to the real
|
|
2436
|
+
// nodes behind it, both without a scan.
|
|
2437
|
+
const factRecordsByGroup = base ? new LayeredIdListMap(base.factRecordsByGroup) : new Map();
|
|
2438
|
+
const statedByBySubject = base ? new LayeredIdListMap(base.statedByBySubject) : new Map();
|
|
2315
2439
|
for (const ind of payload.individuals || []) {
|
|
2316
2440
|
if (!ind?.id) continue;
|
|
2317
2441
|
individualsById.set(ind.id, ind);
|
|
2318
2442
|
if (ind.class === SOURCE_CLASS) sourcesById.set(ind.id, ind);
|
|
2319
2443
|
if (ind.class === RETRACTION_CLASS) indexRetraction(retractionsByGroup, ind);
|
|
2320
|
-
if (ind.class === FACT_CLASS) {
|
|
2444
|
+
if (!base && ind.class === FACT_CLASS) {
|
|
2321
2445
|
const groupId = factGroupId(ind.id);
|
|
2322
2446
|
const held = factRecordsByGroup.get(groupId);
|
|
2323
2447
|
if (held) held.push(ind.id);
|
|
2324
2448
|
else factRecordsByGroup.set(groupId, [ind.id]);
|
|
2325
2449
|
}
|
|
2326
2450
|
}
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2451
|
+
if (!base) {
|
|
2452
|
+
const statedGroup = (payload.objectProperties || []).find((g) => g?.prop === STATED_BY_PROP);
|
|
2453
|
+
for (const e of statedGroup?.examples || []) {
|
|
2454
|
+
if (!e?.subject) continue;
|
|
2455
|
+
const list = statedByBySubject.get(e.subject);
|
|
2456
|
+
if (list) list.push(e.object);
|
|
2457
|
+
else statedByBySubject.set(e.subject, [e.object]);
|
|
2458
|
+
}
|
|
2333
2459
|
}
|
|
2334
2460
|
payload[MEMORY_INDEX] = { individualsById, sourcesById, statedByBySubject, factRecordsByGroup, retractionsByGroup };
|
|
2335
2461
|
return payload[MEMORY_INDEX];
|
|
2336
2462
|
}
|
|
2337
2463
|
|
|
2464
|
+
/** File one id under a key in an id -> id-list map the WRITE path holds, plain
|
|
2465
|
+
* or layered. Idempotent, so a map answering from a layer below cannot start
|
|
2466
|
+
* disagreeing with one answering from its own entry. */
|
|
2467
|
+
function fileIdUnderKey(map, key, id) {
|
|
2468
|
+
if (map.push) { map.push(key, id); return; }
|
|
2469
|
+
const held = map.get(key);
|
|
2470
|
+
if (!held) map.set(key, [id]);
|
|
2471
|
+
else if (!held.includes(id)) held.push(id);
|
|
2472
|
+
}
|
|
2473
|
+
|
|
2474
|
+
/** An id -> id-list map read through a base map this layer may never change.
|
|
2475
|
+
* A key this layer has touched answers from its own entry, everything else
|
|
2476
|
+
* from the base, and `push` copies the base's list before adding to it — so
|
|
2477
|
+
* the base keeps the lists it had however the mutation ends. Copy-on-write at
|
|
2478
|
+
* the one granularity the write path mutates.
|
|
2479
|
+
*
|
|
2480
|
+
* Only `get` and the three writers exist. Nothing folds either of the two maps
|
|
2481
|
+
* this covers, and a layered map cannot answer `size` or an iteration without
|
|
2482
|
+
* merging both layers, which is the walk the layering exists to avoid. */
|
|
2483
|
+
class LayeredIdListMap {
|
|
2484
|
+
#base;
|
|
2485
|
+
#own = new Map();
|
|
2486
|
+
|
|
2487
|
+
constructor(base) { this.#base = base; }
|
|
2488
|
+
|
|
2489
|
+
get(key) {
|
|
2490
|
+
const own = this.#own.get(key);
|
|
2491
|
+
if (own !== undefined) return own === null ? undefined : own;
|
|
2492
|
+
return this.#base.get(key);
|
|
2493
|
+
}
|
|
2494
|
+
|
|
2495
|
+
set(key, ids) { this.#own.set(key, ids); }
|
|
2496
|
+
|
|
2497
|
+
/** A tombstone rather than a removal: the base still holds the key, and this
|
|
2498
|
+
* layer is what says the mutation dropped it. */
|
|
2499
|
+
delete(key) { this.#own.set(key, null); }
|
|
2500
|
+
|
|
2501
|
+
push(key, id) {
|
|
2502
|
+
const held = this.get(key);
|
|
2503
|
+
if (!held) this.#own.set(key, [id]);
|
|
2504
|
+
else if (!held.includes(id)) this.#own.set(key, [...held, id]);
|
|
2505
|
+
}
|
|
2506
|
+
}
|
|
2507
|
+
|
|
2508
|
+
/** The two halves of the lookup index a ROW handle keeps across writes, beside
|
|
2509
|
+
* the assembled payload they describe. Both map an id to a list of ids, so
|
|
2510
|
+
* neither can hand a mutation an individual the cache also holds — that is
|
|
2511
|
+
* what makes them reusable where `individualsById` is not, since
|
|
2512
|
+
* `mutablePayloadCopy` gives every individual a new identity per mutation.
|
|
2513
|
+
*
|
|
2514
|
+
* Each half is guarded on the identity of the container it was read from. An
|
|
2515
|
+
* individuals array or a statedBy examples list that was REPLACED describes a
|
|
2516
|
+
* different graph, so the half built from it is built again. That covers every
|
|
2517
|
+
* rebuild path at once: a fresh assembly, a payload dropped after a failed
|
|
2518
|
+
* write, and a patch that had to fall back to reassembling from rows.
|
|
2519
|
+
*
|
|
2520
|
+
* The fact half also survives a write that keeps the same array, because
|
|
2521
|
+
* `patchStoreWideIndex` applies that write's own row delta to it. The statedBy
|
|
2522
|
+
* half needs no such patch: a write that changes an edge group rewrites the
|
|
2523
|
+
* group row, and the assembled payload takes a fresh examples array from it,
|
|
2524
|
+
* which the guard sees. */
|
|
2525
|
+
function storeWideIndexOf(handle) {
|
|
2526
|
+
const payload = handle.cachedPayload;
|
|
2527
|
+
if (!payload) return null;
|
|
2528
|
+
let held = handle.cachedIndex;
|
|
2529
|
+
if (!held) {
|
|
2530
|
+
held = { individuals: null, factRecordsByGroup: null, statedByExamples: null, statedByBySubject: null };
|
|
2531
|
+
handle.cachedIndex = held;
|
|
2532
|
+
}
|
|
2533
|
+
if (!held.factRecordsByGroup || held.individuals !== payload.individuals) {
|
|
2534
|
+
const factRecordsByGroup = new Map();
|
|
2535
|
+
for (const ind of payload.individuals || []) {
|
|
2536
|
+
if (!ind?.id || ind.class !== FACT_CLASS) continue;
|
|
2537
|
+
const groupId = factGroupId(ind.id);
|
|
2538
|
+
const group = factRecordsByGroup.get(groupId);
|
|
2539
|
+
if (group) group.push(ind.id);
|
|
2540
|
+
else factRecordsByGroup.set(groupId, [ind.id]);
|
|
2541
|
+
}
|
|
2542
|
+
held.factRecordsByGroup = factRecordsByGroup;
|
|
2543
|
+
held.individuals = payload.individuals;
|
|
2544
|
+
}
|
|
2545
|
+
const examples = (payload.objectProperties || []).find((g) => g?.prop === STATED_BY_PROP)?.examples || null;
|
|
2546
|
+
if (!held.statedByBySubject || held.statedByExamples !== examples) {
|
|
2547
|
+
const statedByBySubject = new Map();
|
|
2548
|
+
for (const e of examples || []) {
|
|
2549
|
+
if (!e?.subject) continue;
|
|
2550
|
+
const stated = statedByBySubject.get(e.subject);
|
|
2551
|
+
if (stated) stated.push(e.object);
|
|
2552
|
+
else statedByBySubject.set(e.subject, [e.object]);
|
|
2553
|
+
}
|
|
2554
|
+
held.statedByBySubject = statedByBySubject;
|
|
2555
|
+
held.statedByExamples = examples;
|
|
2556
|
+
}
|
|
2557
|
+
return held;
|
|
2558
|
+
}
|
|
2559
|
+
|
|
2560
|
+
/** Bring the store-wide fact half up to the write that just landed, from the
|
|
2561
|
+
* row delta the write itself carried. A row key IS the individual's id and the
|
|
2562
|
+
* row class says whether it is a Fact, so this reads the same delta
|
|
2563
|
+
* `patchAssembledPayload` applied to the payload and parses none of it.
|
|
2564
|
+
*
|
|
2565
|
+
* Only called where that patch SUCCEEDED. A write that had to rebuild replaces
|
|
2566
|
+
* the individuals array, and the guard in `storeWideIndexOf` builds the half
|
|
2567
|
+
* again rather than trusting this. */
|
|
2568
|
+
function patchStoreWideIndex(handle, writes, removals) {
|
|
2569
|
+
const cached = handle.cachedIndex;
|
|
2570
|
+
if (!cached?.factRecordsByGroup) return;
|
|
2571
|
+
const factRecordsByGroup = cached.factRecordsByGroup;
|
|
2572
|
+
for (const key of removals) {
|
|
2573
|
+
const id = String(key);
|
|
2574
|
+
if (id.startsWith(EDGE_GROUP_KEY_PREFIX)) continue;
|
|
2575
|
+
const groupId = factGroupId(id);
|
|
2576
|
+
const kept = (factRecordsByGroup.get(groupId) || []).filter((recordId) => recordId !== id);
|
|
2577
|
+
if (kept.length) factRecordsByGroup.set(groupId, kept);
|
|
2578
|
+
else factRecordsByGroup.delete(groupId);
|
|
2579
|
+
}
|
|
2580
|
+
for (const row of writes) {
|
|
2581
|
+
if (row.rowClass !== FACT_ROW_CLASS) continue;
|
|
2582
|
+
const id = String(row.rowKey);
|
|
2583
|
+
fileRecordIdInGroupOrder(factRecordsByGroup, factGroupId(id), id);
|
|
2584
|
+
}
|
|
2585
|
+
cached.individuals = handle.cachedPayload.individuals;
|
|
2586
|
+
}
|
|
2587
|
+
|
|
2588
|
+
/** File a record id under its group where a REBUILD would put it. A group's
|
|
2589
|
+
* records sit in codepoint order on their ids in the assembled payload —
|
|
2590
|
+
* `sortFactIndividualsById` puts them there — so a build over that payload
|
|
2591
|
+
* reads them in that order. Appending in arrival order instead would leave the
|
|
2592
|
+
* carried map holding one order and a rebuild another, and every reader of the
|
|
2593
|
+
* list would then answer by which write came first. */
|
|
2594
|
+
function fileRecordIdInGroupOrder(factRecordsByGroup, groupId, id) {
|
|
2595
|
+
const held = factRecordsByGroup.get(groupId);
|
|
2596
|
+
if (!held) { factRecordsByGroup.set(groupId, [id]); return; }
|
|
2597
|
+
if (held.includes(id)) return;
|
|
2598
|
+
let at = 0;
|
|
2599
|
+
while (at < held.length && held[at] < id) at += 1;
|
|
2600
|
+
held.splice(at, 0, id);
|
|
2601
|
+
}
|
|
2602
|
+
|
|
2338
2603
|
/** File one retraction record under the triple it stands over, replacing any
|
|
2339
2604
|
* earlier reference to the same id — upsertIndividual merges in place, so the
|
|
2340
2605
|
* index must hold the record the payload holds, not a stale copy of it. */
|
|
@@ -2372,12 +2637,20 @@ const memoryIndexOf = (payload) => payload?.[MEMORY_INDEX] || null;
|
|
|
2372
2637
|
async function mutateMemory(dir, fn) {
|
|
2373
2638
|
const overRowHandle = isRowHandle(dir);
|
|
2374
2639
|
const payload = overRowHandle ? mutablePayloadCopy(await ensureRowPayload(dir)) : await loadMemory(dir);
|
|
2640
|
+
// Stamped before `fn` runs as well as after it lands: Backend B's own
|
|
2641
|
+
// `loadMemory` hands back the live payload, so the store stops matching any
|
|
2642
|
+
// fold taken of it the moment `fn` starts changing it.
|
|
2643
|
+
stampStoreWrite(dir);
|
|
2375
2644
|
try {
|
|
2376
|
-
buildMemoryIndex(payload);
|
|
2645
|
+
const layered = buildMemoryIndex(payload, overRowHandle ? storeWideIndexOf(dir) : null);
|
|
2377
2646
|
const out = (await fn(payload)) ?? payload;
|
|
2378
2647
|
migrateLegacyProvenance(out);
|
|
2648
|
+
// The legacy-provenance migration re-keys records wholesale and indexes the
|
|
2649
|
+
// result from scratch, so the ids the handle's own index names are no longer
|
|
2650
|
+
// the store's. It goes rather than being reconciled against a re-key.
|
|
2651
|
+
if (overRowHandle && memoryIndexOf(out) !== layered) dir.cachedIndex = null;
|
|
2379
2652
|
recomputeSourceReliability(out);
|
|
2380
|
-
if (!overRowHandle)
|
|
2653
|
+
if (!overRowHandle) renormalizeProseIndex(out);
|
|
2381
2654
|
await persistMemory(dir, out);
|
|
2382
2655
|
return overRowHandle ? dir.cachedPayload : out;
|
|
2383
2656
|
} catch (e) {
|
|
@@ -2741,12 +3014,7 @@ function upsertIndividual(payload, ind) {
|
|
|
2741
3014
|
payload.individuals.push(ind);
|
|
2742
3015
|
idx.individualsById.set(ind.id, ind);
|
|
2743
3016
|
if (ind.class === RETRACTION_CLASS) indexRetraction(idx.retractionsByGroup, ind);
|
|
2744
|
-
if (ind.class === FACT_CLASS)
|
|
2745
|
-
const groupId = factGroupId(ind.id);
|
|
2746
|
-
const held = idx.factRecordsByGroup.get(groupId);
|
|
2747
|
-
if (held) held.push(ind.id);
|
|
2748
|
-
else idx.factRecordsByGroup.set(groupId, [ind.id]);
|
|
2749
|
-
}
|
|
3017
|
+
if (ind.class === FACT_CLASS) fileIdUnderKey(idx.factRecordsByGroup, factGroupId(ind.id), ind.id);
|
|
2750
3018
|
return ind;
|
|
2751
3019
|
}
|
|
2752
3020
|
const i = payload.individuals.findIndex((x) => x?.id === ind.id);
|
|
@@ -2782,8 +3050,7 @@ function upsertEdge(payload, { predicate, prop }, edge) {
|
|
|
2782
3050
|
if (!existing || !existing.includes(edge.object)) {
|
|
2783
3051
|
group.examples.push({ ...edge, createdAt: edge.createdAt || nowIso() });
|
|
2784
3052
|
group.count = group.examples.length;
|
|
2785
|
-
|
|
2786
|
-
else idx.statedByBySubject.set(edge.subject, [edge.object]);
|
|
3053
|
+
fileIdUnderKey(idx.statedByBySubject, edge.subject, edge.object);
|
|
2787
3054
|
return;
|
|
2788
3055
|
}
|
|
2789
3056
|
// Rare re-assert of the exact same (subject,object) pair — fall through
|
|
@@ -2800,11 +3067,7 @@ function upsertEdge(payload, { predicate, prop }, edge) {
|
|
|
2800
3067
|
);
|
|
2801
3068
|
group.examples.push({ ...edge, createdAt });
|
|
2802
3069
|
group.count = group.examples.length;
|
|
2803
|
-
if (idx)
|
|
2804
|
-
const list = idx.statedByBySubject.get(edge.subject) || [];
|
|
2805
|
-
if (!list.includes(edge.object)) list.push(edge.object);
|
|
2806
|
-
idx.statedByBySubject.set(edge.subject, list);
|
|
2807
|
-
}
|
|
3070
|
+
if (idx) fileIdUnderKey(idx.statedByBySubject, edge.subject, edge.object);
|
|
2808
3071
|
}
|
|
2809
3072
|
|
|
2810
3073
|
/** Recount `classes[]` from the individuals — every memory class stays counted
|
|
@@ -3617,7 +3880,7 @@ export function findRulesByName(memory, name) {
|
|
|
3617
3880
|
return (memory?.individuals || [])
|
|
3618
3881
|
.filter((i) => i?.class === RULE_CLASS
|
|
3619
3882
|
&& (i.attributes || []).find((a) => a?.prop === RULE_NAME_PROP)?.value === n)
|
|
3620
|
-
.sort((a, b) => kindOf(a)
|
|
3883
|
+
.sort((a, b) => byCodepoint(kindOf(a), kindOf(b)) || byCodepoint(a.id, b.id));
|
|
3621
3884
|
}
|
|
3622
3885
|
|
|
3623
3886
|
/** Every taught Rule as a plain row {id, name, kind, slots, provenance} —
|
|
@@ -3642,8 +3905,7 @@ export function readRuleRows(memory) {
|
|
|
3642
3905
|
provenance: attr("mgx:factProvenance") || "",
|
|
3643
3906
|
});
|
|
3644
3907
|
}
|
|
3645
|
-
rows.sort((a, b) => a.name.
|
|
3646
|
-
|| a.kind.localeCompare(b.kind) || String(a.id).localeCompare(String(b.id)));
|
|
3908
|
+
rows.sort((a, b) => byCodepoint(a.name, b.name) || byCodepoint(a.kind, b.kind) || byCodepoint(a.id, b.id));
|
|
3647
3909
|
return rows;
|
|
3648
3910
|
}
|
|
3649
3911
|
|
|
@@ -3867,10 +4129,45 @@ export function readFactRows(memory, opts = {}) {
|
|
|
3867
4129
|
return foldFactRows(memory, factFoldContext(memory), opts);
|
|
3868
4130
|
}
|
|
3869
4131
|
|
|
4132
|
+
/** `readFactRows` over a whole store, held between writes.
|
|
4133
|
+
*
|
|
4134
|
+
* Folding the graph is the most expensive read tmct does, and the fold is a
|
|
4135
|
+
* pure function of the payload — so between two writes every caller asking for
|
|
4136
|
+
* it is asking the same question. This answers it once. `stampStoreWrite`
|
|
4137
|
+
* moves the stamp the held fold is keyed to, at both ends of `mutateMemory`
|
|
4138
|
+
* (the single seam every backend's writes pass through) and wherever else a
|
|
4139
|
+
* handle's payload is replaced or dropped, so a fold taken before a write can
|
|
4140
|
+
* never be served after one.
|
|
4141
|
+
*
|
|
4142
|
+
* A repo-path dir has no handle to hold anything on, and another process can
|
|
4143
|
+
* write its file between two reads, so it folds fresh every call exactly as
|
|
4144
|
+
* before. */
|
|
4145
|
+
export async function foldedFactRows(dir) {
|
|
4146
|
+
if (!isMemoryOrSqliteHandle(dir)) return readFactRows(await loadMemory(dir));
|
|
4147
|
+
const stamp = dir.storeWrites || 0;
|
|
4148
|
+
if (dir.heldFactRows && dir.heldFactRowsStamp === stamp) return dir.heldFactRows;
|
|
4149
|
+
const rows = readFactRows(await loadMemory(dir));
|
|
4150
|
+
// A write that landed while this fold was running has already moved the
|
|
4151
|
+
// stamp; holding these rows would serve that write's own reader stale ones.
|
|
4152
|
+
if ((dir.storeWrites || 0) === stamp) {
|
|
4153
|
+
dir.heldFactRows = rows;
|
|
4154
|
+
dir.heldFactRowsStamp = stamp;
|
|
4155
|
+
}
|
|
4156
|
+
return rows;
|
|
4157
|
+
}
|
|
4158
|
+
|
|
3870
4159
|
/** The fold itself, over whatever slice of the graph a context was built for.
|
|
3871
4160
|
* `readFactRows` hands it the whole graph; a caller that only needs certain
|
|
3872
4161
|
* (subject, predicate) pairs hands it a scoped context and gets exactly the
|
|
3873
|
-
* rows a whole-graph fold would have produced for those pairs.
|
|
4162
|
+
* rows a whole-graph fold would have produced for those pairs.
|
|
4163
|
+
*
|
|
4164
|
+
* The rows come out in content order, not in the order the payload happened to
|
|
4165
|
+
* hold them. A reader that takes the first of several equally-ranked rows, or
|
|
4166
|
+
* sorts by a key that ties, otherwise answers by arrival order — and two peers
|
|
4167
|
+
* holding one fact set arrive at it differently. Sorting here is what lets a
|
|
4168
|
+
* reader inherit the guarantee instead of re-earning it: the fold is the one
|
|
4169
|
+
* place every fact read passes through. p2p-room.mjs's sortFactIndividualsById
|
|
4170
|
+
* does the same job one level down, over the stored records. */
|
|
3874
4171
|
function foldFactRows(memory, ctx, opts = {}) {
|
|
3875
4172
|
// A materialised head, when the backend keeps one, replaces the group's own
|
|
3876
4173
|
// fold with the audit trail that fold was last built from — the same records,
|
|
@@ -3888,6 +4185,54 @@ function foldFactRows(memory, ctx, opts = {}) {
|
|
|
3888
4185
|
row.trust = computeAssertionGroupTrust(head ? head.inputs : row.assertions, opts).score;
|
|
3889
4186
|
rows.push(row);
|
|
3890
4187
|
}
|
|
4188
|
+
// Each row's key is built once rather than on every comparison. Handing
|
|
4189
|
+
// compareFactsByContent straight to sort rebuilds both keys per comparison,
|
|
4190
|
+
// and a sort makes O(n log n) of them, which on a large store costs several
|
|
4191
|
+
// times what building one key per row does — and this is the fold every fact
|
|
4192
|
+
// read goes through. The order is the one that comparator defines, and a test
|
|
4193
|
+
// holds the two to the same answer.
|
|
4194
|
+
const keyed = rows.map((row) => ({ key: factOrderKey(row), row }));
|
|
4195
|
+
keyed.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
|
|
4196
|
+
return attachSpeakers(keyed.map((entry) => entry.row));
|
|
4197
|
+
}
|
|
4198
|
+
|
|
4199
|
+
// What every fact id starts with, and nothing a source ever writes as a term:
|
|
4200
|
+
// the prefix factIdFor mints and normFactTerm's carve-out keeps whole.
|
|
4201
|
+
const FACT_ID_PREFIX = "fact:";
|
|
4202
|
+
|
|
4203
|
+
/** Hangs each attributed claim's speakers on the claim's own row, as
|
|
4204
|
+
* `attributedTo` — absent, never empty, the same way `extraction` is.
|
|
4205
|
+
*
|
|
4206
|
+
* A report's claim and the speaker it was attributed to are two rows: the
|
|
4207
|
+
* claim, and `fact:<claimId> | mgx:attributedTo | <speaker>` beside it. A
|
|
4208
|
+
* surface that cannot render the attribution must not render the claim, so
|
|
4209
|
+
* resolving the pair belongs here rather than in each reader — the fold is the
|
|
4210
|
+
* one place every fact read passes through, and a reader that renders a row
|
|
4211
|
+
* inherits its speaker without asking for it.
|
|
4212
|
+
*
|
|
4213
|
+
* news-feed.mjs's partitionAttributions owns which rows are attributions and
|
|
4214
|
+
* how one claim's speakers are ordered; only its map is wanted here, because a
|
|
4215
|
+
* fold hides no row from its readers — the news card suppresses attributions
|
|
4216
|
+
* from its own lanes, the store still holds and reads them.
|
|
4217
|
+
*
|
|
4218
|
+
* Pure: the speakers come back sorted, so a claim two outlets attributed reads
|
|
4219
|
+
* the same whichever order the attributions arrived in, and an attribution
|
|
4220
|
+
* whose claim the fold never saw simply hangs on nothing — which is the case
|
|
4221
|
+
* every time rows arrive over p2p out of order. */
|
|
4222
|
+
function attachSpeakers(rows) {
|
|
4223
|
+
// An attribution names its claim as its SUBJECT, and a claim's own subject is
|
|
4224
|
+
// a term, so the prefix rules nearly every row out on one comparison. The
|
|
4225
|
+
// fold is the hottest fact read there is, and the full test costs two
|
|
4226
|
+
// lowercased copies and two regexes per row — on a store no report has ever
|
|
4227
|
+
// written to, that is the whole price of a feature it does not use.
|
|
4228
|
+
const referring = rows.filter((row) => row.subject.startsWith(FACT_ID_PREFIX));
|
|
4229
|
+
if (!referring.length) return rows;
|
|
4230
|
+
const { speakersByClaimId } = partitionAttributions(referring);
|
|
4231
|
+
if (!speakersByClaimId.size) return rows;
|
|
4232
|
+
for (const row of rows) {
|
|
4233
|
+
const speakers = speakersByClaimId.get(row.id);
|
|
4234
|
+
if (speakers?.length) row.attributedTo = speakers;
|
|
4235
|
+
}
|
|
3891
4236
|
return rows;
|
|
3892
4237
|
}
|
|
3893
4238
|
|
|
@@ -4192,6 +4537,26 @@ function recordSourceIdOf(record) {
|
|
|
4192
4537
|
return hash < 0 ? rest : rest.slice(0, hash);
|
|
4193
4538
|
}
|
|
4194
4539
|
|
|
4540
|
+
/** The triples that name any of `goneGroupIds` in a TERM rather than through an
|
|
4541
|
+
* edge, as group ids — what a retraction has to take with it, because an
|
|
4542
|
+
* objectProperties scrub cannot see a fact id sitting in an attribute value.
|
|
4543
|
+
*
|
|
4544
|
+
* The whole group comes back, not the matching record: an attribution two
|
|
4545
|
+
* outlets both wrote is one triple with two records, and retracting the claim
|
|
4546
|
+
* under it leaves neither of them anything to be about. A term is only ever a
|
|
4547
|
+
* reference when it is a fact id, so membership in the gone set IS the test —
|
|
4548
|
+
* no term a source writes can collide with one. */
|
|
4549
|
+
function factGroupsReferencing(payload, goneGroupIds) {
|
|
4550
|
+
const groups = new Set();
|
|
4551
|
+
for (const ind of payload?.individuals || []) {
|
|
4552
|
+
if (ind?.class !== FACT_CLASS) continue;
|
|
4553
|
+
const subject = individualAttr(ind, "rdf:subject");
|
|
4554
|
+
const object = individualAttr(ind, "rdf:object");
|
|
4555
|
+
if (goneGroupIds.has(subject) || goneGroupIds.has(object)) groups.add(factGroupId(ind.id));
|
|
4556
|
+
}
|
|
4557
|
+
return groups;
|
|
4558
|
+
}
|
|
4559
|
+
|
|
4195
4560
|
/** Retract facts by id — a real DELETE (syllogise.mjs's retractability
|
|
4196
4561
|
* mechanism). A GROUP id retracts the triple: every source's record for it,
|
|
4197
4562
|
* demoted leaves included, since retracting "dogs bark" cannot leave half its
|
|
@@ -4206,9 +4571,16 @@ function recordSourceIdOf(record) {
|
|
|
4206
4571
|
* keeps the retraction on record rather than erasing the fact that something
|
|
4207
4572
|
* was asserted at all. A retraction record is never itself removed here.
|
|
4208
4573
|
*
|
|
4574
|
+
* A triple retracted WHOLE takes the triples that name it with it, through
|
|
4575
|
+
* factGroupsReferencing above — the reified attribution beside a report's
|
|
4576
|
+
* claim is the live case, and the edge scrub cannot reach it. While any
|
|
4577
|
+
* source's record for a triple still stands the claim does too, so nothing
|
|
4578
|
+
* cascades off a single retracted record.
|
|
4579
|
+
*
|
|
4209
4580
|
* Returns { removed, records } — `removed` the ids asked for that matched, so
|
|
4210
4581
|
* it may be smaller than the input and is never longer than it; `records` the
|
|
4211
|
-
* concrete record ids that went, which is what the
|
|
4582
|
+
* concrete record ids that went, cascaded ones included, which is what the
|
|
4583
|
+
* retraction absorbed. */
|
|
4212
4584
|
export async function removeFacts(dir, ids, { provenance = "", retractedAt = "" } = {}) {
|
|
4213
4585
|
const idSet = new Set((ids || []).filter(Boolean));
|
|
4214
4586
|
const removed = [];
|
|
@@ -4219,12 +4591,7 @@ export async function removeFacts(dir, ids, { provenance = "", retractedAt = ""
|
|
|
4219
4591
|
const removedSet = new Set();
|
|
4220
4592
|
const matched = new Set();
|
|
4221
4593
|
const retiredByGroupAndSource = new Map(); // `${groupId}${sourceId}` -> { groupId, sourceId, ids, template }
|
|
4222
|
-
|
|
4223
|
-
if (ind?.class !== FACT_CLASS) return true;
|
|
4224
|
-
const groupId = factGroupId(ind.id);
|
|
4225
|
-
const asked = idSet.has(ind.id) ? ind.id : (idSet.has(groupId) ? groupId : "");
|
|
4226
|
-
if (!asked) return true;
|
|
4227
|
-
matched.add(asked);
|
|
4594
|
+
const retire = (ind, groupId) => {
|
|
4228
4595
|
removedSet.add(ind.id);
|
|
4229
4596
|
const sourceId = recordSourceIdOf(ind);
|
|
4230
4597
|
const key = `${groupId}${sourceId}`;
|
|
@@ -4251,8 +4618,36 @@ export async function removeFacts(dir, ids, { provenance = "", retractedAt = ""
|
|
|
4251
4618
|
},
|
|
4252
4619
|
});
|
|
4253
4620
|
}
|
|
4254
|
-
|
|
4255
|
-
|
|
4621
|
+
};
|
|
4622
|
+
|
|
4623
|
+
// Round one takes the ids asked for; every round after it takes whatever
|
|
4624
|
+
// the last one left pointing at a triple that is now gone. A reference to a
|
|
4625
|
+
// reference would need a third round, so the loop runs until a round finds
|
|
4626
|
+
// nothing rather than assuming one hop.
|
|
4627
|
+
let asking = idSet;
|
|
4628
|
+
let cascading = false;
|
|
4629
|
+
while (asking.size) {
|
|
4630
|
+
const standingGroups = new Set();
|
|
4631
|
+
const emptiedGroups = new Set();
|
|
4632
|
+
payload.individuals = (payload.individuals || []).filter((ind) => {
|
|
4633
|
+
if (ind?.class !== FACT_CLASS) return true;
|
|
4634
|
+
const groupId = factGroupId(ind.id);
|
|
4635
|
+
const asked = asking.has(ind.id) ? ind.id : (asking.has(groupId) ? groupId : "");
|
|
4636
|
+
if (!asked) {
|
|
4637
|
+
standingGroups.add(groupId);
|
|
4638
|
+
return true;
|
|
4639
|
+
}
|
|
4640
|
+
// Only the caller's own ids answer for what it asked; a cascade is this
|
|
4641
|
+
// call's consequence, not part of the request.
|
|
4642
|
+
if (!cascading) matched.add(asked);
|
|
4643
|
+
emptiedGroups.add(groupId);
|
|
4644
|
+
retire(ind, groupId);
|
|
4645
|
+
return false;
|
|
4646
|
+
});
|
|
4647
|
+
const gone = [...emptiedGroups].filter((groupId) => !standingGroups.has(groupId));
|
|
4648
|
+
asking = gone.length ? factGroupsReferencing(payload, new Set(gone)) : new Set();
|
|
4649
|
+
cascading = true;
|
|
4650
|
+
}
|
|
4256
4651
|
for (const id of matched) removed.push(id);
|
|
4257
4652
|
if (!removed.length) return; // honest no-op — nothing matched, no write needed beyond this
|
|
4258
4653
|
for (const id of removedSet) records.push(id);
|
|
@@ -4416,7 +4811,7 @@ export function findContradictions(memory, { floor = CONTRADICTION_TRUST_FLOOR,
|
|
|
4416
4811
|
const byKey = new Map();
|
|
4417
4812
|
for (const r of rows) {
|
|
4418
4813
|
if (resolutionStrategyFor(r.predicate) === RESOLUTION_MERGE) continue;
|
|
4419
|
-
const key =
|
|
4814
|
+
const key = subjectPredicateKey(r.subject, r.predicate);
|
|
4420
4815
|
if (!byKey.has(key)) byKey.set(key, []);
|
|
4421
4816
|
byKey.get(key).push(r);
|
|
4422
4817
|
}
|
|
@@ -4425,7 +4820,10 @@ export function findContradictions(memory, { floor = CONTRADICTION_TRUST_FLOOR,
|
|
|
4425
4820
|
if (new Set(group.map((r) => r.object)).size < 2) continue;
|
|
4426
4821
|
const strategy = resolutionStrategyFor(group[0].predicate);
|
|
4427
4822
|
if (strategy !== RESOLUTION_CONTRADICTION && !resolveSiblingGroups(group, strategy).contested) continue;
|
|
4428
|
-
out.push(group.slice().sort((a, b) => b.trust - a.trust || a.object
|
|
4823
|
+
out.push(group.slice().sort((a, b) => b.trust - a.trust || byCodepoint(a.object, b.object)));
|
|
4429
4824
|
}
|
|
4430
|
-
return out.sort((a, b) =>
|
|
4825
|
+
return out.sort((a, b) => byCodepoint(
|
|
4826
|
+
subjectPredicateKey(a[0].subject, a[0].predicate),
|
|
4827
|
+
subjectPredicateKey(b[0].subject, b[0].predicate),
|
|
4828
|
+
));
|
|
4431
4829
|
}
|