@polycode-projects/the-mechanical-code-talker 6.0.18 → 6.0.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/README.md +20 -23
  2. package/bin/tmct.mjs +16 -33
  3. package/corpus/LICENSES.json +0 -21
  4. package/corpus/README.md +10 -13
  5. package/corpus/reference/manifest.json +19 -19
  6. package/corpus/reference/shards/ref-01.jsonl.gz +0 -0
  7. package/corpus/reference/shards/ref-04.jsonl.gz +0 -0
  8. package/corpus/reference/shards/ref-08.jsonl.gz +0 -0
  9. package/corpus/reference/shards/ref-10.jsonl.gz +0 -0
  10. package/corpus/reference/shards/ref-11.jsonl.gz +0 -0
  11. package/corpus/reference/shards/ref-17.jsonl.gz +0 -0
  12. package/corpus/reference/shards/ref-20.jsonl.gz +0 -0
  13. package/corpus/reference/shards/ref-25.jsonl.gz +0 -0
  14. package/corpus/reference/shards/ref-2c.jsonl.gz +0 -0
  15. package/corpus/tier2/generate.mjs +6 -142
  16. package/corpus/tier2/manifest.json +0 -42
  17. package/package.json +6 -4
  18. package/src/adapters/corpus/child-seed.mjs +74 -0
  19. package/src/adapters/corpus/conceptnet.mjs +45 -26
  20. package/src/adapters/corpus/research-source.mjs +6 -2
  21. package/src/adapters/corpus/wikidata-live.mjs +92 -51
  22. package/src/adapters/memory/blocks.mjs +7 -1
  23. package/src/adapters/memory/core.mjs +505 -107
  24. package/src/adapters/memory/corpus-bands.mjs +27 -10
  25. package/src/adapters/memory/inspect.mjs +24 -5
  26. package/src/adapters/memory/rows.mjs +359 -30
  27. package/src/adapters/memory/shacl.mjs +10 -3
  28. package/src/domain/ask.mjs +27 -10
  29. package/src/domain/cli-verbs.mjs +3 -4
  30. package/src/domain/completions/group.mjs +8 -3
  31. package/src/domain/completions/infer.mjs +7 -2
  32. package/src/domain/completions/prune.mjs +5 -1
  33. package/src/domain/completions/rank.mjs +7 -2
  34. package/src/domain/digest/compose.mjs +5 -1
  35. package/src/domain/digest/select.mjs +12 -6
  36. package/src/domain/domain.mjs +15 -8
  37. package/src/domain/el-classify.mjs +11 -2
  38. package/src/domain/fact-phrase.mjs +86 -4
  39. package/src/domain/hash.mjs +9 -0
  40. package/src/domain/memory/bias.mjs +8 -4
  41. package/src/domain/memory/capability.mjs +12 -6
  42. package/src/domain/memory/fact-order.mjs +29 -0
  43. package/src/domain/memory/resolution.mjs +3 -0
  44. package/src/domain/news-feed.mjs +862 -92
  45. package/src/domain/reference-pack.mjs +5 -0
  46. package/src/domain/sense-gate.mjs +220 -0
  47. package/src/domain/sense-scope.mjs +116 -0
  48. package/src/domain/sense-split.mjs +1 -1
  49. package/src/domain/syllogise.mjs +60 -21
  50. package/src/domain/tableau.mjs +23 -14
  51. package/src/domain/term-ledger.mjs +16 -1
  52. package/src/domain/worlds-pack.mjs +5 -1
  53. package/src/services/adventure-autoplay.mjs +6 -1
  54. package/src/services/adventure-editor.mjs +43 -21
  55. package/src/services/adventure-viz.mjs +26 -9
  56. package/src/services/adventure.mjs +40 -10
  57. package/src/services/chat.mjs +270 -125
  58. package/src/services/extensions.mjs +51 -58
  59. package/src/services/extract-facts.mjs +906 -66
  60. package/src/services/init.mjs +4 -4
  61. package/src/services/ledger-viz.mjs +9 -4
  62. package/src/services/memory-panel-viz.mjs +4 -5
  63. package/src/services/mud-editor.mjs +40 -16
  64. package/src/services/mud-viz.mjs +8 -2
  65. package/src/services/mudiii-turn.mjs +5 -3
  66. package/src/services/mudiii-viz.mjs +8 -2
  67. package/src/services/news.mjs +306 -21
  68. package/src/services/research-viz.mjs +1 -1
  69. package/src/services/sprite-catalog-viz.mjs +10 -5
  70. package/src/surfaces/web/adventure-browser-entry.mjs +6 -12
  71. package/src/surfaces/web/memory-ask-browser.bundle.js +152 -151
  72. package/src/surfaces/web/mud-browser-entry.mjs +7 -11
  73. package/src/surfaces/web/research-browser-entry.mjs +5 -2
  74. package/corpus/tier2/aws.jsonl +0 -39
  75. package/corpus/tier2/java.jsonl +0 -31
  76. package/corpus/tier2/python.jsonl +0 -30
@@ -16,7 +16,7 @@
16
16
 
17
17
  import { access, mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
18
18
  import { dirname, join } from "node:path";
19
- import { proseTokensFor, buildProseIndex } from "../../domain/prose.mjs";
19
+ import { proseTokensFor } from "../../domain/prose.mjs";
20
20
  import { fnv1aHex, normText, normFactTerm, normFactPredicate, factIdFor, factIdForTriple } from "../../domain/hash.mjs";
21
21
 
22
22
  // Fact identity (normalization + id derivation) lives in hash.mjs — the one
@@ -39,6 +39,8 @@ import {
39
39
  // consumers keep one import site.
40
40
  export { CREATED_AT_PROP, UPDATED_AT_PROP, provenanceTagToSource } from "../../domain/memory/trust.mjs";
41
41
  import { NEG_PREDICATE_PREFIX, negatedPredicate } from "../../domain/memory/capability.mjs";
42
+ import { factOrderKey } from "../../domain/memory/fact-order.mjs";
43
+ import { partitionAttributions } from "../../domain/news-feed.mjs";
42
44
  import {
43
45
  planHeadRollup, planChainRollup, mergeRollups,
44
46
  isHeadRollupId, isChainRollupId, isRollupId, headRollupTypeOf,
@@ -68,7 +70,10 @@ export {
68
70
  // fine as long as neither side READS an imported binding while the other's
69
71
  // body is still running — rows.mjs builds its class map on first use, and
70
72
  // every use here is inside a function.
71
- import { payloadToRows, rowsToPayload, diffRows, renormalizeAssembledPayload } from "./rows.mjs";
73
+ import {
74
+ payloadToRows, rowsToPayload, diffRows, renormalizeAssembledPayload, renormalizeProseIndex,
75
+ canDropAssembledIndividuals, dropAssembledIndividuals, appendAssembledRowOrder, dropAssembledRowOrder,
76
+ } from "./rows.mjs";
72
77
 
73
78
  // The rollup vocabulary and its tuning constants live with the compaction
74
79
  // layer; re-exported here so store consumers keep one import site.
@@ -139,13 +144,13 @@ const MEMORY_VOCABULARY = [
139
144
  { prop: "mgx:factProvenance", note: "LEGACY COMPAT SHIM: the ' | '-joined provenance tag string a fact came from; the source-of-truth is now the mgx:statedBy edges derived from it" },
140
145
  { prop: "mgx:sourceId", note: "the assertion key a Fact record is filed under — the Source id of the ONE party asserting it, which is also the @-suffix of the record's own id. `src:none` when no tag names a Source, so every record has a key rather than a hole" },
141
146
  { prop: "mgx:observedAt", note: "OPTIONAL valid time: when the asserting party WITNESSED the claim, as against mgx:createdAt's transaction time (when this store recorded it). A stale article read today loses to an eyewitness report from yesterday. Stored only when a caller supplies one — never fabricated, never backfilled" },
142
- { prop: "mgx:extractionFinding", note: "OPTIONAL: the space-joined structural findings the extractor recorded about how THIS assertion's sentence was read, from the closed vocabulary identifier-token | clause-fallback | pronoun-carry | definitional-frame. Per assertion, never per triple: a later clean re-assertion of the same triple carries none. Absence means no findings were recorded, never that the sentence was checked and read cleanly" },
147
+ { prop: "mgx:extractionFinding", note: "OPTIONAL: the space-joined structural findings the extractor recorded about how THIS assertion's sentence was read, from the closed vocabulary identifier-token | clause-fallback | pronoun-carry | definitional-frame | reported-speech. Per assertion, never per triple: a later clean re-assertion of the same triple carries none. Absence means no findings were recorded, never that the sentence was checked and read cleanly" },
143
148
  { prop: "mgx:supersedes", note: "the record id(s) this one replaced when its own source re-asserted the triple with a newer embedded timestamp. A space-joined LIST; absent, never empty, until the first supersession" },
144
149
  { prop: "mgx:supersededBy", note: "the record id(s) that replaced this one. Its presence is what makes a record a demoted leaf rather than a live head, and the group fold skips it: a source's past belief is not a second vote for the present one. A LIST, because one source with two live replicas can fork before they sync" },
145
150
  { prop: "mgx:factQuantifier", note: "OPTIONAL: the quantifier word a plural class-membership teach used ('every'/'some'/'a few'), for literal recall by 'how many Xs are Ys' — never real cardinality counting" },
146
151
  { prop: "mgx:factJustification", note: "an entailed Fact's supporting premise fact ids: ' | '-separated environments, one space-separated premise-id list per independent derivation, capped by syllogise's maxEnvironments knob; a value with no ' | ' is a single environment" },
147
152
  { prop: "mgx:ruleName", note: "a taught Rule's own name (e.g. 'grandparent') — the query-dispatcher's lookup key, PLAN_TAUGHT_RELATIONS.md §2/§3" },
148
- { prop: "mgx:ruleKind", note: "a taught Rule's SHAPE tag — the closed vocabulary compose2 | filter | recursive (structural, like 'Fact'/'Rule' themselves, never a domain word)" },
153
+ { prop: "mgx:ruleKind", note: "a taught Rule's SHAPE tag — the closed vocabulary compose2 | filter | recursive | action-signature | action-precond | action-effect | action-constraint (structural, like 'Fact'/'Rule' themselves, never a domain word)" },
149
154
  { prop: "mgx:ruleBase1", note: "compose2: the first hop's base relation name; filter: the base rule/relation being filtered (same 'base relation' role in both kinds, so the name is shared)" },
150
155
  { prop: "mgx:ruleBase2", note: "compose2 only: the second hop's base relation name" },
151
156
  { prop: "mgx:ruleFilterProperty", note: "filter only: the property literal candidates are filtered by (an mgx:hasProperty-shaped Fact lookup)" },
@@ -235,6 +240,14 @@ export function isMemoryOrSqliteHandle(dir) {
235
240
  return isMemoryHandle(dir) || isSqliteHandle(dir) || isRowHandle(dir);
236
241
  }
237
242
 
243
+ /** Move a store handle's write stamp on. `foldedFactRows` keys the fold it
244
+ * holds to this number, so anything that changes what the store would fold to
245
+ * — a landed write, a payload about to be mutated in place, a dropped
246
+ * assembly, a seed assigned over the top — stamps here first. */
247
+ function stampStoreWrite(dir) {
248
+ if (isMemoryOrSqliteHandle(dir)) dir.storeWrites = (dir.storeWrites || 0) + 1;
249
+ }
250
+
238
251
  /** Backend B — pure in-memory store: `{ backend: "memory", payload }` held by
239
252
  * the caller (never module-global). Zero file I/O; distinct from
240
253
  * `--ephemeral`, which still round-trips a throwaway temp dir. */
@@ -250,7 +263,12 @@ export function createInMemoryStore() {
250
263
  * `seedPayload` is null/undefined — a browser session with nothing to seed
251
264
  * keeps its own fresh empty payload untouched. */
252
265
  export function applySeedPayload(memoryDir, seedPayload) {
253
- if (seedPayload) memoryDir.payload = { ...memoryDir.payload, ...seedPayload };
266
+ if (!seedPayload) return;
267
+ memoryDir.payload = { ...memoryDir.payload, ...seedPayload };
268
+ // A seed arrives from anywhere, so whatever an earlier load settled about the
269
+ // ids this store holds no longer describes it.
270
+ memoryDir.storeMigrationsSettled = false;
271
+ stampStoreWrite(memoryDir);
254
272
  }
255
273
 
256
274
  /** A structurally independent copy of a memory payload — `structuredClone`
@@ -502,6 +520,18 @@ function backfillFactsProjection(db) {
502
520
 
503
521
  const individualAttr = (ind, prop) => (ind?.attributes || []).find((a) => a?.prop === prop)?.value || "";
504
522
  const individualKey = (ind, key) => (ind?.attributes || []).find((a) => a?.key === key)?.value || "";
523
+
524
+ /** Order two strings by codepoint, never by locale. Every listing this store
525
+ * hands out is read on whatever machine holds it, and two locales sorting one
526
+ * set differently is the same broken promise arrival order would be: a read
527
+ * over the fact store answers to the set, and to nothing about where it ran.
528
+ * memory/fact-order.mjs states the rule; inspect.mjs keeps its own copy of
529
+ * this comparator for the text it renders. */
530
+ const byCodepoint = (a, b) => {
531
+ const ka = String(a ?? "");
532
+ const kb = String(b ?? "");
533
+ return ka < kb ? -1 : ka > kb ? 1 : 0;
534
+ };
505
535
  const subjectPredicateKey = (subject, predicate) => `${subject}\u0000${predicate}`;
506
536
 
507
537
  // ---- Derived-local tables: `fact_heads` and `fact_object_supersessions` -----
@@ -798,11 +828,13 @@ export function wrapRowBackend(impl, {
798
828
  backend: BACKEND_ROW,
799
829
  impl,
800
830
  cachedPayload: null,
831
+ cachedIndex: null,
801
832
  basePayload: cloneMemoryPayload(basePayload),
802
833
  sqliteSeedStore,
803
834
  sqliteSeedOverlayRows: sqliteSeedOverlayRows ? [...sqliteSeedOverlayRows] : null,
804
835
  sqliteSeedKeyOrds: null,
805
836
  baseRows: null,
837
+ baseKeyOrds: null,
806
838
  storedRows: null,
807
839
  onOversizedRow,
808
840
  copyOnRead,
@@ -843,11 +875,8 @@ function overlayRows(baseRows, sessionRows) {
843
875
  * session store does not already hold a row for. */
844
876
  function seedOnlyKeys(handle) {
845
877
  const sessionKeys = new Set(handle.storedRows.map((r) => r.rowKey));
846
- const baseKeys = handle.sqliteSeedStore
847
- ? sqliteSeedKeyOrds(handle).keys()
848
- : handle.baseRows.map((r) => r.rowKey);
849
878
  const seedOnly = new Set();
850
- for (const key of baseKeys) if (!sessionKeys.has(key)) seedOnly.add(key);
879
+ for (const key of readOnlyLayerKeyOrds(handle).keys()) if (!sessionKeys.has(key)) seedOnly.add(key);
851
880
  return seedOnly;
852
881
  }
853
882
 
@@ -923,9 +952,31 @@ function mutablePayloadCopy(payload) {
923
952
  * coherent graph. */
924
953
  function dropAssembledRowPayload(handle) {
925
954
  if (!isRowHandle(handle)) return;
955
+ stampStoreWrite(handle);
926
956
  handle.cachedPayload = null;
957
+ handle.cachedIndex = null;
927
958
  handle.storedRows = null;
928
959
  handle.baseRows = null;
960
+ handle.baseKeyOrds = null;
961
+ }
962
+
963
+ /** Every key a READ-ONLY layer under this handle holds, with the ord each one
964
+ * carries — the sqlite seed and its overlay, or the base payload's own
965
+ * projection. A session row keyed the same as one of these SHADOWS it, so
966
+ * deleting that session row brings the read-only row back rather than dropping
967
+ * the individual: a rebuild is the only honest answer to that delete, and this
968
+ * is what tells the write path so.
969
+ *
970
+ * The ords are what let a write project the SESSION's rows alone and still put
971
+ * a newly-keyed row past everything the read-only layer already holds. A key
972
+ * and an integer per row is small where the rows themselves are not, so this is
973
+ * held for the handle's life once a write has asked for it. */
974
+ function readOnlyLayerKeyOrds(handle) {
975
+ if (handle.sqliteSeedStore) return sqliteSeedKeyOrds(handle);
976
+ if (!handle.baseKeyOrds) {
977
+ handle.baseKeyOrds = new Map((handle.baseRows || []).map((row) => [row.rowKey, ordOfRow(row)]));
978
+ }
979
+ return handle.baseKeyOrds;
929
980
  }
930
981
 
931
982
  /** A record with its audit stamp removed. `mgx:updatedAt` moves on every
@@ -958,13 +1009,17 @@ function movedBeyondAuditStamp(beforeRow, row) {
958
1009
  async function persistRowPayload(handle, payload) {
959
1010
  await ensureRowPayload(handle);
960
1011
  const seedKeys = seedOnlyKeys(handle);
961
- // On the sqlite path the seed is out of BOTH sides of the diff: the rows it
962
- // holds are exactly the keys no write may touch, so projecting them would
1012
+ // The seed is out of BOTH sides of the diff, whichever layer holds it: those
1013
+ // keys are exactly the ones no write may touch, so projecting them would
963
1014
  // materialize the whole seed only to filter every row of it back out.
964
- const before = handle.sqliteSeedStore ? handle.storedRows : overlayRows(handle.baseRows, handle.storedRows);
1015
+ const before = handle.storedRows;
965
1016
  const beforeByKey = new Map(before.map((row) => [row.rowKey, row]));
966
- const after = payloadToRows(handle.sqliteSeedStore ? payloadWithoutRowKeys(payload, seedKeys) : payload, {
967
- priorRows: handle.sqliteSeedStore ? sqliteSeedPriorRows(handle) : before,
1017
+ const after = payloadToRows(payloadWithoutRowKeys(payload, seedKeys), {
1018
+ priorRows: before,
1019
+ // The seed's ords are handed over as the map the handle already holds. The
1020
+ // session's own rows come first, so a key both layers hold keeps the ord
1021
+ // assembly gives it.
1022
+ priorOrds: readOnlyLayerKeyOrds(handle),
968
1023
  onOversizedRow: handle.onOversizedRow,
969
1024
  ...(handle.log ? { log: handle.log } : {}),
970
1025
  });
@@ -987,18 +1042,19 @@ async function persistRowPayload(handle, payload) {
987
1042
  for (const row of writes) next.set(row.rowKey, row);
988
1043
  handle.storedRows = [...next.values()];
989
1044
  const meta = { memory: payload.memory, prefixes: payload.prefixes };
990
- if (handle.sqliteSeedStore && !removals.length) {
991
- patchAssembledPayload(handle.cachedPayload, meta, writes);
992
- } else if (handle.sqliteSeedStore) {
993
- handle.cachedPayload = null;
994
- handle.cachedPayload = migrateStoredMemory(assembleSqliteSeededPayload(handle, meta));
995
- } else {
996
- // Dropped before the rebuild, not after it: the payload this replaces is the
997
- // largest object the handle holds, and keeping it reachable while the next
998
- // one assembles doubles the peak for no reason.
999
- handle.cachedPayload = null;
1000
- handle.cachedPayload = migrateStoredMemory(rowsToPayload(overlayRows(handle.baseRows, handle.storedRows), { meta }));
1045
+ if (patchAssembledPayload(handle.cachedPayload, meta, writes, removals, readOnlyLayerKeyOrds(handle))) {
1046
+ patchStoreWideIndex(handle, writes, removals);
1047
+ return;
1001
1048
  }
1049
+ // Dropped before the rebuild, not after it: the payload this replaces is the
1050
+ // largest object the handle holds, and keeping it reachable while the next
1051
+ // one assembles doubles the peak for no reason. The index goes with it for
1052
+ // the same reason — it names the arrays that payload owned.
1053
+ handle.cachedPayload = null;
1054
+ handle.cachedIndex = null;
1055
+ handle.cachedPayload = migrateStoredMemory(handle.sqliteSeedStore
1056
+ ? assembleSqliteSeededPayload(handle, meta)
1057
+ : rowsToPayload(overlayRows(handle.baseRows, handle.storedRows), { meta }));
1002
1058
  }
1003
1059
 
1004
1060
  /** The payload minus a set of row keys, for a projection that must not spend
@@ -1082,15 +1138,6 @@ function sqliteSeedKeyOrds(handle) {
1082
1138
  return ords;
1083
1139
  }
1084
1140
 
1085
- /** What `payloadToRows` reads off prior rows: the ord each key already carries,
1086
- * in the smallest row shape that carries one. The session's own rows come
1087
- * last, so a key both layers hold keeps the session's ord — the precedence
1088
- * assembly gives it. */
1089
- function* sqliteSeedPriorRows(handle) {
1090
- for (const [rowKey, ord] of sqliteSeedKeyOrds(handle)) yield { rowKey, json: `{"ord":${ord}}` };
1091
- yield* handle.storedRows;
1092
- }
1093
-
1094
1141
  function* chainedRows(...sources) {
1095
1142
  for (const source of sources) yield* source;
1096
1143
  }
@@ -1107,28 +1154,50 @@ function assembleSqliteSeededPayload(handle, meta) {
1107
1154
  );
1108
1155
  }
1109
1156
 
1110
- /** The assembled payload after a write that only added or rewrote rows, brought
1111
- * up to date without reading the seed again.
1157
+ /** The assembled payload after a write, brought up to date without reading the
1158
+ * seed again. Returns true when it did, false when this write is not patchable
1159
+ * and the caller has to rebuild — decided before anything is touched, so a
1160
+ * `false` leaves the payload exactly as it found it.
1161
+ *
1162
+ * Why the positions work out for an ADDED or REWRITTEN row: a write only ever
1163
+ * carries session rows, and `payloadToRows` gives a newly-keyed row an ord past
1164
+ * every ord already assembled, so a new individual belongs at the tail and a
1165
+ * rewritten one belongs exactly where it already sits.
1166
+ *
1167
+ * Why they work out for a REMOVED one: dropping an individual from the
1168
+ * assembled array would drop it from wherever the fact ordering moved it to,
1169
+ * which is a different slot from the one it held in row order — so the drop
1170
+ * goes through the row order the assembly carries (`dropAssembledIndividuals`)
1171
+ * and the array is refilled in that order first. The slots that survive are
1172
+ * then the ones the surviving ROWS own, which is what a rebuild would leave.
1173
+ *
1174
+ * Either way everything the payload derives goes through
1175
+ * `renormalizeAssembledPayload`, the same function `rowsToPayload` itself ends
1176
+ * on, so a patched payload and a rebuilt one cannot drift apart.
1112
1177
  *
1113
- * Why the positions work out: a write only ever carries session rows, and
1114
- * `payloadToRows` gives a newly-keyed row an ord past every ord already
1115
- * assembled, so a new individual belongs at the tail and a rewritten one
1116
- * belongs exactly where it already sits. Both are where a rebuild from the
1117
- * same rows would put them, and the fact set is unchanged or grown, so the
1118
- * fact ordering `renormalizeAssembledPayload` reapplies lands the same way.
1119
- * Everything the payload derives goes through that one function, the same one
1120
- * `rowsToPayload` itself ends on, so a patched payload and a rebuilt one
1121
- * cannot drift apart.
1178
+ * `readOnlyKeys` are the keys a layer under the session holds: deleting a
1179
+ * session row that SHADOWS one of them uncovers the read-only row rather than
1180
+ * dropping the individual, which only a rebuild can work out.
1122
1181
  *
1123
- * A REMOVAL is not patchable this way and does not come here: dropping an
1124
- * individual from the assembled array drops it from wherever the fact ordering
1125
- * moved it to, which is a different position from the one it occupied in row
1126
- * order, and every fact after it shifts. `persistRowPayload` rebuilds for those.
1127
- * Mutates and returns `payload`. */
1128
- function patchAssembledPayload(payload, meta, writes) {
1182
+ * On `false` the caller replaces this payload with a fresh assembly, so
1183
+ * whatever this got part-way through applying is discarded with it. */
1184
+ function patchAssembledPayload(payload, meta, writes, removals, readOnlyKeys) {
1185
+ const droppedGroups = new Set();
1186
+ const droppedIndividuals = new Set();
1187
+ for (const key of removals) {
1188
+ const rowKey = String(key);
1189
+ if (readOnlyKeys.has(rowKey)) return false;
1190
+ if (rowKey.startsWith(EDGE_GROUP_KEY_PREFIX)) droppedGroups.add(rowKey.slice(EDGE_GROUP_KEY_PREFIX.length));
1191
+ else droppedIndividuals.add(rowKey);
1192
+ }
1193
+ if (droppedIndividuals.size && !canDropAssembledIndividuals(payload, droppedIndividuals)) return false;
1194
+
1129
1195
  payload.memory = meta.memory;
1130
1196
  payload.prefixes = meta.prefixes;
1131
- if (!writes.length) return renormalizeAssembledPayload(payload);
1197
+ if (!writes.length && !removals.length) {
1198
+ renormalizeAssembledPayload(payload);
1199
+ return true;
1200
+ }
1132
1201
 
1133
1202
  const individualPositions = new Map();
1134
1203
  for (let i = 0; i < payload.individuals.length; i += 1) individualPositions.set(payload.individuals[i]?.id, i);
@@ -1154,9 +1223,16 @@ function patchAssembledPayload(payload, meta, writes) {
1154
1223
  if (at === undefined) {
1155
1224
  individualPositions.set(individual.id, payload.individuals.length);
1156
1225
  payload.individuals.push(individual);
1226
+ appendAssembledRowOrder(payload, individual.id);
1157
1227
  } else payload.individuals[at] = individual;
1158
1228
  }
1159
- return renormalizeAssembledPayload(payload);
1229
+
1230
+ if (droppedGroups.size) {
1231
+ payload.objectProperties = payload.objectProperties.filter((group) => !droppedGroups.has(group?.prop));
1232
+ }
1233
+ if (droppedIndividuals.size && !dropAssembledIndividuals(payload, droppedIndividuals)) return false;
1234
+ renormalizeAssembledPayload(payload);
1235
+ return true;
1160
1236
  }
1161
1237
 
1162
1238
  /** Every distinct subject and object a seed store's facts carry, off the
@@ -1271,7 +1347,15 @@ const cloneJson = (v) => (v === undefined ? v : structuredClone(v));
1271
1347
  * persist would delete them as absent-from-payload. */
1272
1348
  function readSqlitePayload(handle) {
1273
1349
  const dataVersion = handle.db.prepare("PRAGMA data_version").get()?.data_version;
1274
- if (handle.cachedPayload && handle.cachedDataVersion !== dataVersion) handle.cachedPayload = null;
1350
+ if (handle.cachedPayload && handle.cachedDataVersion !== dataVersion) {
1351
+ // Another connection committed, so the fold this handle holds describes a
1352
+ // store that no longer exists — it goes with the payload it was taken of.
1353
+ // So does what an earlier load settled about the ids the store carries: the
1354
+ // other writer is the one path that can put a shape there this one wouldn't.
1355
+ stampStoreWrite(handle);
1356
+ handle.cachedPayload = null;
1357
+ handle.storeMigrationsSettled = false;
1358
+ }
1275
1359
  if (!handle.cachedPayload) {
1276
1360
  handle.cachedPayload = buildSqlitePayloadFromRows(handle);
1277
1361
  // The head index rides the same cache lifecycle as the payload it indexes,
@@ -1386,8 +1470,10 @@ function cacheDropGroups(cache, droppedProps) {
1386
1470
  const EDGE_GROUP_ROW_CLASS = "edge-group";
1387
1471
  const EDGE_GROUP_KEY_PREFIX = "edge-group:";
1388
1472
 
1473
+ const FACT_ROW_CLASS = "fact";
1474
+
1389
1475
  const ROW_CLASS_BY_INDIVIDUAL_CLASS = new Map([
1390
- [FACT_CLASS, "fact"],
1476
+ [FACT_CLASS, FACT_ROW_CLASS],
1391
1477
  [SOURCE_CLASS, "source"],
1392
1478
  [UTTERANCE_CLASS, "utterance"],
1393
1479
  [MEMORY_SESSION_CLASS, "session"],
@@ -1991,8 +2077,8 @@ export async function snapshotMemory(dir, { retentionVersions } = {}) {
1991
2077
  * append creates the file). The result is a raw entities payload;
1992
2078
  * parseEntities() loads it. */
1993
2079
  export async function loadMemory(dir) {
1994
- if (isMemoryHandle(dir)) return migrateStoredMemory(dir.payload);
1995
- if (isSqliteHandle(dir)) return migrateStoredMemory(readSqlitePayload(dir));
2080
+ if (isMemoryHandle(dir)) return migrateStoredMemoryOnce(dir, dir.payload);
2081
+ if (isSqliteHandle(dir)) return migrateStoredMemoryOnce(dir, readSqlitePayload(dir));
1996
2082
  // Not migrated here: a row handle migrates the payload once, as it assembles
1997
2083
  // it, so every read after the first is spared two walks of the whole graph.
1998
2084
  if (isRowHandle(dir)) return readRowPayload(dir);
@@ -2011,6 +2097,30 @@ export async function loadMemory(dir) {
2011
2097
  * Both are pure payload transforms and both converge to no-ops. */
2012
2098
  const migrateStoredMemory = (payload) => migrateFactAssertionKeys(migrateLegacyFactIds(payload));
2013
2099
 
2100
+ /** True when some Fact still carries an id from before one of the two
2101
+ * migrations above — a pre-widening 32-bit id, or a pre-assertion-model id
2102
+ * keyed on the triple alone. Both lack the `@` a record id carries, so one
2103
+ * pass over the individuals answers it, allocating nothing. */
2104
+ function carriesPreMigrationFactIds(payload) {
2105
+ for (const ind of payload?.individuals || []) {
2106
+ if (ind?.class === FACT_CLASS && !String(ind.id || "").includes("@")) return true;
2107
+ }
2108
+ return false;
2109
+ }
2110
+
2111
+ /** `migrateStoredMemory` for a handle that hands back the same store on every
2112
+ * read. A store whose Facts are all on current ids cannot acquire an old one —
2113
+ * every write mints through `factIdFor` — so the first load that finds nothing
2114
+ * to migrate settles the question for this handle and later loads skip even
2115
+ * the scan. A store that IS on old ids keeps migrating on every read until a
2116
+ * write persists the healed form, exactly as before. */
2117
+ function migrateStoredMemoryOnce(handle, payload) {
2118
+ if (handle.storeMigrationsSettled) return payload;
2119
+ if (carriesPreMigrationFactIds(payload)) return migrateStoredMemory(payload);
2120
+ handle.storeMigrationsSettled = true;
2121
+ return payload;
2122
+ }
2123
+
2014
2124
  // A Fact id written before factIdFor widened to 64 bits — `fact:` + exactly 8
2015
2125
  // hex. A current id is 16 hex, so this anchored test never matches one, and a
2016
2126
  // migrated store pays only string checks with no rehash on load.
@@ -2038,6 +2148,9 @@ function migrateLegacyFactIds(payload) {
2038
2148
  ind.id = currentId;
2039
2149
  }
2040
2150
  if (!remap.size) return payload;
2151
+ // The row order an assembly carries names the ids the rows arrived under, and
2152
+ // this just moved every legacy Fact off one of them.
2153
+ dropAssembledRowOrder(payload);
2041
2154
  const remapId = (id) => remap.get(id) || id;
2042
2155
  for (const group of payload.objectProperties || []) {
2043
2156
  for (const e of group.examples || []) {
@@ -2198,6 +2311,7 @@ function migrateFactAssertionKeys(payload) {
2198
2311
  * SQL write (Backend C, persistSqlitePayload), or a diffed row write into an
2199
2312
  * injected store (Backend D, persistRowPayload). */
2200
2313
  async function persistMemory(dir, payload) {
2314
+ stampStoreWrite(dir);
2201
2315
  if (isMemoryHandle(dir)) { dir.payload = payload; return; }
2202
2316
  if (isSqliteHandle(dir)) { persistSqlitePayload(dir, payload); return; }
2203
2317
  if (isRowHandle(dir)) { await persistRowPayload(dir, payload); return; }
@@ -2292,49 +2406,200 @@ export async function saveNodeId(dir, nodeId) {
2292
2406
  * append goes through here, including the lazy legacy-provenance migration
2293
2407
  * and actor-level Source reliability recompute. `fn` may be async (the
2294
2408
  * SHACL ingest gate awaits validation before ever mutating `payload`). */
2295
- // Per-call lookup index (individualsById/sourcesById/statedByBySubject),
2296
- // attached to payload under a Symbol key (skipped by JSON.stringify) so
2297
- // upsertIndividual/upsertSource/upsertEdge/appendFacts get O(1) lookups
2298
- // instead of re-scanning; discarded when mutateMemory returns.
2409
+ // Per-call lookup index, attached to payload under a Symbol key (skipped by
2410
+ // JSON.stringify) so upsertIndividual/upsertSource/upsertEdge/appendFacts get
2411
+ // O(1) lookups instead of re-scanning; discarded when mutateMemory returns. On
2412
+ // a row handle the two id -> id-list halves read through a base the handle
2413
+ // keeps across writes (storeWideIndexOf), so only the three that hold
2414
+ // individuals are built per call.
2299
2415
  const MEMORY_INDEX = Symbol("mutateMemory lookup index");
2300
2416
 
2301
- /** Build the three lookup Maps from the just-loaded payload and attach them
2302
- * under MEMORY_INDEX. */
2303
- function buildMemoryIndex(payload) {
2417
+ /** Build the five lookup Maps from the just-loaded payload and attach them
2418
+ * under MEMORY_INDEX.
2419
+ *
2420
+ * `base` is a store-wide index a row handle keeps across writes
2421
+ * (`storeWideIndexOf`). Its two maps are layered rather than rebuilt: this
2422
+ * mutation reads through them and writes only into its own layer, so the
2423
+ * handle's copy still describes the store when the mutation is over — or when
2424
+ * it throws. The other three hold INDIVIDUALS, and every individual this
2425
+ * mutation can reach is a fresh copy `mutablePayloadCopy` just made, so they
2426
+ * are always this payload's own. */
2427
+ function buildMemoryIndex(payload, base = null) {
2304
2428
  const individualsById = new Map();
2305
2429
  const sourcesById = new Map();
2306
- const statedByBySubject = new Map();
2307
- // groupId -> the record ids asserting that triple, so a write can ask "is
2308
- // anyone asserting this yet" and an edge can resolve a group id to the real
2309
- // nodes behind it, both without a scan.
2310
- const factRecordsByGroup = new Map();
2311
2430
  // groupId -> the retraction records standing over that triple, so the write
2312
2431
  // path can ask "was this source's assertion retracted" without a scan. Almost
2313
2432
  // always empty, which is why it is read before anything more expensive.
2314
2433
  const retractionsByGroup = new Map();
2434
+ // groupId -> the record ids asserting that triple, so a write can ask "is
2435
+ // anyone asserting this yet" and an edge can resolve a group id to the real
2436
+ // nodes behind it, both without a scan.
2437
+ const factRecordsByGroup = base ? new LayeredIdListMap(base.factRecordsByGroup) : new Map();
2438
+ const statedByBySubject = base ? new LayeredIdListMap(base.statedByBySubject) : new Map();
2315
2439
  for (const ind of payload.individuals || []) {
2316
2440
  if (!ind?.id) continue;
2317
2441
  individualsById.set(ind.id, ind);
2318
2442
  if (ind.class === SOURCE_CLASS) sourcesById.set(ind.id, ind);
2319
2443
  if (ind.class === RETRACTION_CLASS) indexRetraction(retractionsByGroup, ind);
2320
- if (ind.class === FACT_CLASS) {
2444
+ if (!base && ind.class === FACT_CLASS) {
2321
2445
  const groupId = factGroupId(ind.id);
2322
2446
  const held = factRecordsByGroup.get(groupId);
2323
2447
  if (held) held.push(ind.id);
2324
2448
  else factRecordsByGroup.set(groupId, [ind.id]);
2325
2449
  }
2326
2450
  }
2327
- const statedGroup = (payload.objectProperties || []).find((g) => g?.prop === STATED_BY_PROP);
2328
- for (const e of statedGroup?.examples || []) {
2329
- if (!e?.subject) continue;
2330
- const list = statedByBySubject.get(e.subject);
2331
- if (list) list.push(e.object);
2332
- else statedByBySubject.set(e.subject, [e.object]);
2451
+ if (!base) {
2452
+ const statedGroup = (payload.objectProperties || []).find((g) => g?.prop === STATED_BY_PROP);
2453
+ for (const e of statedGroup?.examples || []) {
2454
+ if (!e?.subject) continue;
2455
+ const list = statedByBySubject.get(e.subject);
2456
+ if (list) list.push(e.object);
2457
+ else statedByBySubject.set(e.subject, [e.object]);
2458
+ }
2333
2459
  }
2334
2460
  payload[MEMORY_INDEX] = { individualsById, sourcesById, statedByBySubject, factRecordsByGroup, retractionsByGroup };
2335
2461
  return payload[MEMORY_INDEX];
2336
2462
  }
2337
2463
 
2464
+ /** File one id under a key in an id -> id-list map the WRITE path holds, plain
2465
+ * or layered. Idempotent, so a map answering from a layer below cannot start
2466
+ * disagreeing with one answering from its own entry. */
2467
+ function fileIdUnderKey(map, key, id) {
2468
+ if (map.push) { map.push(key, id); return; }
2469
+ const held = map.get(key);
2470
+ if (!held) map.set(key, [id]);
2471
+ else if (!held.includes(id)) held.push(id);
2472
+ }
2473
+
2474
+ /** An id -> id-list map read through a base map this layer may never change.
2475
+ * A key this layer has touched answers from its own entry, everything else
2476
+ * from the base, and `push` copies the base's list before adding to it — so
2477
+ * the base keeps the lists it had however the mutation ends. Copy-on-write at
2478
+ * the one granularity the write path mutates.
2479
+ *
2480
+ * Only `get` and the three writers exist. Nothing folds either of the two maps
2481
+ * this covers, and a layered map cannot answer `size` or an iteration without
2482
+ * merging both layers, which is the walk the layering exists to avoid. */
2483
+ class LayeredIdListMap {
2484
+ #base;
2485
+ #own = new Map();
2486
+
2487
+ constructor(base) { this.#base = base; }
2488
+
2489
+ get(key) {
2490
+ const own = this.#own.get(key);
2491
+ if (own !== undefined) return own === null ? undefined : own;
2492
+ return this.#base.get(key);
2493
+ }
2494
+
2495
+ set(key, ids) { this.#own.set(key, ids); }
2496
+
2497
+ /** A tombstone rather than a removal: the base still holds the key, and this
2498
+ * layer is what says the mutation dropped it. */
2499
+ delete(key) { this.#own.set(key, null); }
2500
+
2501
+ push(key, id) {
2502
+ const held = this.get(key);
2503
+ if (!held) this.#own.set(key, [id]);
2504
+ else if (!held.includes(id)) this.#own.set(key, [...held, id]);
2505
+ }
2506
+ }
2507
+
2508
+ /** The two halves of the lookup index a ROW handle keeps across writes, beside
2509
+ * the assembled payload they describe. Both map an id to a list of ids, so
2510
+ * neither can hand a mutation an individual the cache also holds — that is
2511
+ * what makes them reusable where `individualsById` is not, since
2512
+ * `mutablePayloadCopy` gives every individual a new identity per mutation.
2513
+ *
2514
+ * Each half is guarded on the identity of the container it was read from. An
2515
+ * individuals array or a statedBy examples list that was REPLACED describes a
2516
+ * different graph, so the half built from it is built again. That covers every
2517
+ * rebuild path at once: a fresh assembly, a payload dropped after a failed
2518
+ * write, and a patch that had to fall back to reassembling from rows.
2519
+ *
2520
+ * The fact half also survives a write that keeps the same array, because
2521
+ * `patchStoreWideIndex` applies that write's own row delta to it. The statedBy
2522
+ * half needs no such patch: a write that changes an edge group rewrites the
2523
+ * group row, and the assembled payload takes a fresh examples array from it,
2524
+ * which the guard sees. */
2525
+ function storeWideIndexOf(handle) {
2526
+ const payload = handle.cachedPayload;
2527
+ if (!payload) return null;
2528
+ let held = handle.cachedIndex;
2529
+ if (!held) {
2530
+ held = { individuals: null, factRecordsByGroup: null, statedByExamples: null, statedByBySubject: null };
2531
+ handle.cachedIndex = held;
2532
+ }
2533
+ if (!held.factRecordsByGroup || held.individuals !== payload.individuals) {
2534
+ const factRecordsByGroup = new Map();
2535
+ for (const ind of payload.individuals || []) {
2536
+ if (!ind?.id || ind.class !== FACT_CLASS) continue;
2537
+ const groupId = factGroupId(ind.id);
2538
+ const group = factRecordsByGroup.get(groupId);
2539
+ if (group) group.push(ind.id);
2540
+ else factRecordsByGroup.set(groupId, [ind.id]);
2541
+ }
2542
+ held.factRecordsByGroup = factRecordsByGroup;
2543
+ held.individuals = payload.individuals;
2544
+ }
2545
+ const examples = (payload.objectProperties || []).find((g) => g?.prop === STATED_BY_PROP)?.examples || null;
2546
+ if (!held.statedByBySubject || held.statedByExamples !== examples) {
2547
+ const statedByBySubject = new Map();
2548
+ for (const e of examples || []) {
2549
+ if (!e?.subject) continue;
2550
+ const stated = statedByBySubject.get(e.subject);
2551
+ if (stated) stated.push(e.object);
2552
+ else statedByBySubject.set(e.subject, [e.object]);
2553
+ }
2554
+ held.statedByBySubject = statedByBySubject;
2555
+ held.statedByExamples = examples;
2556
+ }
2557
+ return held;
2558
+ }
2559
+
2560
+ /** Bring the store-wide fact half up to the write that just landed, from the
2561
+ * row delta the write itself carried. A row key IS the individual's id and the
2562
+ * row class says whether it is a Fact, so this reads the same delta
2563
+ * `patchAssembledPayload` applied to the payload and parses none of it.
2564
+ *
2565
+ * Only called where that patch SUCCEEDED. A write that had to rebuild replaces
2566
+ * the individuals array, and the guard in `storeWideIndexOf` builds the half
2567
+ * again rather than trusting this. */
2568
+ function patchStoreWideIndex(handle, writes, removals) {
2569
+ const cached = handle.cachedIndex;
2570
+ if (!cached?.factRecordsByGroup) return;
2571
+ const factRecordsByGroup = cached.factRecordsByGroup;
2572
+ for (const key of removals) {
2573
+ const id = String(key);
2574
+ if (id.startsWith(EDGE_GROUP_KEY_PREFIX)) continue;
2575
+ const groupId = factGroupId(id);
2576
+ const kept = (factRecordsByGroup.get(groupId) || []).filter((recordId) => recordId !== id);
2577
+ if (kept.length) factRecordsByGroup.set(groupId, kept);
2578
+ else factRecordsByGroup.delete(groupId);
2579
+ }
2580
+ for (const row of writes) {
2581
+ if (row.rowClass !== FACT_ROW_CLASS) continue;
2582
+ const id = String(row.rowKey);
2583
+ fileRecordIdInGroupOrder(factRecordsByGroup, factGroupId(id), id);
2584
+ }
2585
+ cached.individuals = handle.cachedPayload.individuals;
2586
+ }
2587
+
2588
+ /** File a record id under its group where a REBUILD would put it. A group's
2589
+ * records sit in codepoint order on their ids in the assembled payload —
2590
+ * `sortFactIndividualsById` puts them there — so a build over that payload
2591
+ * reads them in that order. Appending in arrival order instead would leave the
2592
+ * carried map holding one order and a rebuild another, and every reader of the
2593
+ * list would then answer by which write came first. */
2594
+ function fileRecordIdInGroupOrder(factRecordsByGroup, groupId, id) {
2595
+ const held = factRecordsByGroup.get(groupId);
2596
+ if (!held) { factRecordsByGroup.set(groupId, [id]); return; }
2597
+ if (held.includes(id)) return;
2598
+ let at = 0;
2599
+ while (at < held.length && held[at] < id) at += 1;
2600
+ held.splice(at, 0, id);
2601
+ }
2602
+
2338
2603
  /** File one retraction record under the triple it stands over, replacing any
2339
2604
  * earlier reference to the same id — upsertIndividual merges in place, so the
2340
2605
  * index must hold the record the payload holds, not a stale copy of it. */
@@ -2372,12 +2637,20 @@ const memoryIndexOf = (payload) => payload?.[MEMORY_INDEX] || null;
2372
2637
  async function mutateMemory(dir, fn) {
2373
2638
  const overRowHandle = isRowHandle(dir);
2374
2639
  const payload = overRowHandle ? mutablePayloadCopy(await ensureRowPayload(dir)) : await loadMemory(dir);
2640
+ // Stamped before `fn` runs as well as after it lands: Backend B's own
2641
+ // `loadMemory` hands back the live payload, so the store stops matching any
2642
+ // fold taken of it the moment `fn` starts changing it.
2643
+ stampStoreWrite(dir);
2375
2644
  try {
2376
- buildMemoryIndex(payload);
2645
+ const layered = buildMemoryIndex(payload, overRowHandle ? storeWideIndexOf(dir) : null);
2377
2646
  const out = (await fn(payload)) ?? payload;
2378
2647
  migrateLegacyProvenance(out);
2648
+ // The legacy-provenance migration re-keys records wholesale and indexes the
2649
+ // result from scratch, so the ids the handle's own index names are no longer
2650
+ // the store's. It goes rather than being reconciled against a re-key.
2651
+ if (overRowHandle && memoryIndexOf(out) !== layered) dir.cachedIndex = null;
2379
2652
  recomputeSourceReliability(out);
2380
- if (!overRowHandle) out.proseIndex = buildProseIndex(out.individuals);
2653
+ if (!overRowHandle) renormalizeProseIndex(out);
2381
2654
  await persistMemory(dir, out);
2382
2655
  return overRowHandle ? dir.cachedPayload : out;
2383
2656
  } catch (e) {
@@ -2741,12 +3014,7 @@ function upsertIndividual(payload, ind) {
2741
3014
  payload.individuals.push(ind);
2742
3015
  idx.individualsById.set(ind.id, ind);
2743
3016
  if (ind.class === RETRACTION_CLASS) indexRetraction(idx.retractionsByGroup, ind);
2744
- if (ind.class === FACT_CLASS) {
2745
- const groupId = factGroupId(ind.id);
2746
- const held = idx.factRecordsByGroup.get(groupId);
2747
- if (held) held.push(ind.id);
2748
- else idx.factRecordsByGroup.set(groupId, [ind.id]);
2749
- }
3017
+ if (ind.class === FACT_CLASS) fileIdUnderKey(idx.factRecordsByGroup, factGroupId(ind.id), ind.id);
2750
3018
  return ind;
2751
3019
  }
2752
3020
  const i = payload.individuals.findIndex((x) => x?.id === ind.id);
@@ -2782,8 +3050,7 @@ function upsertEdge(payload, { predicate, prop }, edge) {
2782
3050
  if (!existing || !existing.includes(edge.object)) {
2783
3051
  group.examples.push({ ...edge, createdAt: edge.createdAt || nowIso() });
2784
3052
  group.count = group.examples.length;
2785
- if (existing) existing.push(edge.object);
2786
- else idx.statedByBySubject.set(edge.subject, [edge.object]);
3053
+ fileIdUnderKey(idx.statedByBySubject, edge.subject, edge.object);
2787
3054
  return;
2788
3055
  }
2789
3056
  // Rare re-assert of the exact same (subject,object) pair — fall through
@@ -2800,11 +3067,7 @@ function upsertEdge(payload, { predicate, prop }, edge) {
2800
3067
  );
2801
3068
  group.examples.push({ ...edge, createdAt });
2802
3069
  group.count = group.examples.length;
2803
- if (idx) {
2804
- const list = idx.statedByBySubject.get(edge.subject) || [];
2805
- if (!list.includes(edge.object)) list.push(edge.object);
2806
- idx.statedByBySubject.set(edge.subject, list);
2807
- }
3070
+ if (idx) fileIdUnderKey(idx.statedByBySubject, edge.subject, edge.object);
2808
3071
  }
2809
3072
 
2810
3073
  /** Recount `classes[]` from the individuals — every memory class stays counted
@@ -3617,7 +3880,7 @@ export function findRulesByName(memory, name) {
3617
3880
  return (memory?.individuals || [])
3618
3881
  .filter((i) => i?.class === RULE_CLASS
3619
3882
  && (i.attributes || []).find((a) => a?.prop === RULE_NAME_PROP)?.value === n)
3620
- .sort((a, b) => kindOf(a).localeCompare(kindOf(b)) || String(a.id).localeCompare(String(b.id)));
3883
+ .sort((a, b) => byCodepoint(kindOf(a), kindOf(b)) || byCodepoint(a.id, b.id));
3621
3884
  }
3622
3885
 
3623
3886
  /** Every taught Rule as a plain row {id, name, kind, slots, provenance} —
@@ -3642,8 +3905,7 @@ export function readRuleRows(memory) {
3642
3905
  provenance: attr("mgx:factProvenance") || "",
3643
3906
  });
3644
3907
  }
3645
- rows.sort((a, b) => a.name.localeCompare(b.name)
3646
- || a.kind.localeCompare(b.kind) || String(a.id).localeCompare(String(b.id)));
3908
+ rows.sort((a, b) => byCodepoint(a.name, b.name) || byCodepoint(a.kind, b.kind) || byCodepoint(a.id, b.id));
3647
3909
  return rows;
3648
3910
  }
3649
3911
 
@@ -3867,10 +4129,45 @@ export function readFactRows(memory, opts = {}) {
3867
4129
  return foldFactRows(memory, factFoldContext(memory), opts);
3868
4130
  }
3869
4131
 
4132
+ /** `readFactRows` over a whole store, held between writes.
4133
+ *
4134
+ * Folding the graph is the most expensive read tmct does, and the fold is a
4135
+ * pure function of the payload — so between two writes every caller asking for
4136
+ * it is asking the same question. This answers it once. `stampStoreWrite`
4137
+ * moves the stamp the held fold is keyed to, at both ends of `mutateMemory`
4138
+ * (the single seam every backend's writes pass through) and wherever else a
4139
+ * handle's payload is replaced or dropped, so a fold taken before a write can
4140
+ * never be served after one.
4141
+ *
4142
+ * A repo-path dir has no handle to hold anything on, and another process can
4143
+ * write its file between two reads, so it folds fresh every call exactly as
4144
+ * before. */
4145
+ export async function foldedFactRows(dir) {
4146
+ if (!isMemoryOrSqliteHandle(dir)) return readFactRows(await loadMemory(dir));
4147
+ const stamp = dir.storeWrites || 0;
4148
+ if (dir.heldFactRows && dir.heldFactRowsStamp === stamp) return dir.heldFactRows;
4149
+ const rows = readFactRows(await loadMemory(dir));
4150
+ // A write that landed while this fold was running has already moved the
4151
+ // stamp; holding these rows would serve that write's own reader stale ones.
4152
+ if ((dir.storeWrites || 0) === stamp) {
4153
+ dir.heldFactRows = rows;
4154
+ dir.heldFactRowsStamp = stamp;
4155
+ }
4156
+ return rows;
4157
+ }
4158
+
3870
4159
  /** The fold itself, over whatever slice of the graph a context was built for.
3871
4160
  * `readFactRows` hands it the whole graph; a caller that only needs certain
3872
4161
  * (subject, predicate) pairs hands it a scoped context and gets exactly the
3873
- * rows a whole-graph fold would have produced for those pairs. */
4162
+ * rows a whole-graph fold would have produced for those pairs.
4163
+ *
4164
+ * The rows come out in content order, not in the order the payload happened to
4165
+ * hold them. A reader that takes the first of several equally-ranked rows, or
4166
+ * sorts by a key that ties, otherwise answers by arrival order — and two peers
4167
+ * holding one fact set arrive at it differently. Sorting here is what lets a
4168
+ * reader inherit the guarantee instead of re-earning it: the fold is the one
4169
+ * place every fact read passes through. p2p-room.mjs's sortFactIndividualsById
4170
+ * does the same job one level down, over the stored records. */
3874
4171
  function foldFactRows(memory, ctx, opts = {}) {
3875
4172
  // A materialised head, when the backend keeps one, replaces the group's own
3876
4173
  // fold with the audit trail that fold was last built from — the same records,
@@ -3888,6 +4185,54 @@ function foldFactRows(memory, ctx, opts = {}) {
3888
4185
  row.trust = computeAssertionGroupTrust(head ? head.inputs : row.assertions, opts).score;
3889
4186
  rows.push(row);
3890
4187
  }
4188
+ // Each row's key is built once rather than on every comparison. Handing
4189
+ // compareFactsByContent straight to sort rebuilds both keys per comparison,
4190
+ // and a sort makes O(n log n) of them, which on a large store costs several
4191
+ // times what building one key per row does — and this is the fold every fact
4192
+ // read goes through. The order is the one that comparator defines, and a test
4193
+ // holds the two to the same answer.
4194
+ const keyed = rows.map((row) => ({ key: factOrderKey(row), row }));
4195
+ keyed.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
4196
+ return attachSpeakers(keyed.map((entry) => entry.row));
4197
+ }
4198
+
4199
+ // What every fact id starts with, and nothing a source ever writes as a term:
4200
+ // the prefix factIdFor mints and normFactTerm's carve-out keeps whole.
4201
+ const FACT_ID_PREFIX = "fact:";
4202
+
4203
+ /** Hangs each attributed claim's speakers on the claim's own row, as
4204
+ * `attributedTo` — absent, never empty, the same way `extraction` is.
4205
+ *
4206
+ * A report's claim and the speaker it was attributed to are two rows: the
4207
+ * claim, and `fact:<claimId> | mgx:attributedTo | <speaker>` beside it. A
4208
+ * surface that cannot render the attribution must not render the claim, so
4209
+ * resolving the pair belongs here rather than in each reader — the fold is the
4210
+ * one place every fact read passes through, and a reader that renders a row
4211
+ * inherits its speaker without asking for it.
4212
+ *
4213
+ * news-feed.mjs's partitionAttributions owns which rows are attributions and
4214
+ * how one claim's speakers are ordered; only its map is wanted here, because a
4215
+ * fold hides no row from its readers — the news card suppresses attributions
4216
+ * from its own lanes, the store still holds and reads them.
4217
+ *
4218
+ * Pure: the speakers come back sorted, so a claim two outlets attributed reads
4219
+ * the same whichever order the attributions arrived in, and an attribution
4220
+ * whose claim the fold never saw simply hangs on nothing — which is the case
4221
+ * every time rows arrive over p2p out of order. */
4222
+ function attachSpeakers(rows) {
4223
+ // An attribution names its claim as its SUBJECT, and a claim's own subject is
4224
+ // a term, so the prefix rules nearly every row out on one comparison. The
4225
+ // fold is the hottest fact read there is, and the full test costs two
4226
+ // lowercased copies and two regexes per row — on a store no report has ever
4227
+ // written to, that is the whole price of a feature it does not use.
4228
+ const referring = rows.filter((row) => row.subject.startsWith(FACT_ID_PREFIX));
4229
+ if (!referring.length) return rows;
4230
+ const { speakersByClaimId } = partitionAttributions(referring);
4231
+ if (!speakersByClaimId.size) return rows;
4232
+ for (const row of rows) {
4233
+ const speakers = speakersByClaimId.get(row.id);
4234
+ if (speakers?.length) row.attributedTo = speakers;
4235
+ }
3891
4236
  return rows;
3892
4237
  }
3893
4238
 
@@ -4192,6 +4537,26 @@ function recordSourceIdOf(record) {
4192
4537
  return hash < 0 ? rest : rest.slice(0, hash);
4193
4538
  }
4194
4539
 
4540
+ /** The triples that name any of `goneGroupIds` in a TERM rather than through an
4541
+ * edge, as group ids — what a retraction has to take with it, because an
4542
+ * objectProperties scrub cannot see a fact id sitting in an attribute value.
4543
+ *
4544
+ * The whole group comes back, not the matching record: an attribution two
4545
+ * outlets both wrote is one triple with two records, and retracting the claim
4546
+ * under it leaves neither of them anything to be about. A term is only ever a
4547
+ * reference when it is a fact id, so membership in the gone set IS the test —
4548
+ * no term a source writes can collide with one. */
4549
+ function factGroupsReferencing(payload, goneGroupIds) {
4550
+ const groups = new Set();
4551
+ for (const ind of payload?.individuals || []) {
4552
+ if (ind?.class !== FACT_CLASS) continue;
4553
+ const subject = individualAttr(ind, "rdf:subject");
4554
+ const object = individualAttr(ind, "rdf:object");
4555
+ if (goneGroupIds.has(subject) || goneGroupIds.has(object)) groups.add(factGroupId(ind.id));
4556
+ }
4557
+ return groups;
4558
+ }
4559
+
4195
4560
  /** Retract facts by id — a real DELETE (syllogise.mjs's retractability
4196
4561
  * mechanism). A GROUP id retracts the triple: every source's record for it,
4197
4562
  * demoted leaves included, since retracting "dogs bark" cannot leave half its
@@ -4206,9 +4571,16 @@ function recordSourceIdOf(record) {
4206
4571
  * keeps the retraction on record rather than erasing the fact that something
4207
4572
  * was asserted at all. A retraction record is never itself removed here.
4208
4573
  *
4574
+ * A triple retracted WHOLE takes the triples that name it with it, through
4575
+ * factGroupsReferencing above — the reified attribution beside a report's
4576
+ * claim is the live case, and the edge scrub cannot reach it. While any
4577
+ * source's record for a triple still stands the claim does too, so nothing
4578
+ * cascades off a single retracted record.
4579
+ *
4209
4580
  * Returns { removed, records } — `removed` the ids asked for that matched, so
4210
4581
  * it may be smaller than the input and is never longer than it; `records` the
4211
- * concrete record ids that went, which is what the retraction absorbed. */
4582
+ * concrete record ids that went, cascaded ones included, which is what the
4583
+ * retraction absorbed. */
4212
4584
  export async function removeFacts(dir, ids, { provenance = "", retractedAt = "" } = {}) {
4213
4585
  const idSet = new Set((ids || []).filter(Boolean));
4214
4586
  const removed = [];
@@ -4219,12 +4591,7 @@ export async function removeFacts(dir, ids, { provenance = "", retractedAt = ""
4219
4591
  const removedSet = new Set();
4220
4592
  const matched = new Set();
4221
4593
  const retiredByGroupAndSource = new Map(); // `${groupId}${sourceId}` -> { groupId, sourceId, ids, template }
4222
- payload.individuals = (payload.individuals || []).filter((ind) => {
4223
- if (ind?.class !== FACT_CLASS) return true;
4224
- const groupId = factGroupId(ind.id);
4225
- const asked = idSet.has(ind.id) ? ind.id : (idSet.has(groupId) ? groupId : "");
4226
- if (!asked) return true;
4227
- matched.add(asked);
4594
+ const retire = (ind, groupId) => {
4228
4595
  removedSet.add(ind.id);
4229
4596
  const sourceId = recordSourceIdOf(ind);
4230
4597
  const key = `${groupId}${sourceId}`;
@@ -4251,8 +4618,36 @@ export async function removeFacts(dir, ids, { provenance = "", retractedAt = ""
4251
4618
  },
4252
4619
  });
4253
4620
  }
4254
- return false;
4255
- });
4621
+ };
4622
+
4623
+ // Round one takes the ids asked for; every round after it takes whatever
4624
+ // the last one left pointing at a triple that is now gone. A reference to a
4625
+ // reference would need a third round, so the loop runs until a round finds
4626
+ // nothing rather than assuming one hop.
4627
+ let asking = idSet;
4628
+ let cascading = false;
4629
+ while (asking.size) {
4630
+ const standingGroups = new Set();
4631
+ const emptiedGroups = new Set();
4632
+ payload.individuals = (payload.individuals || []).filter((ind) => {
4633
+ if (ind?.class !== FACT_CLASS) return true;
4634
+ const groupId = factGroupId(ind.id);
4635
+ const asked = asking.has(ind.id) ? ind.id : (asking.has(groupId) ? groupId : "");
4636
+ if (!asked) {
4637
+ standingGroups.add(groupId);
4638
+ return true;
4639
+ }
4640
+ // Only the caller's own ids answer for what it asked; a cascade is this
4641
+ // call's consequence, not part of the request.
4642
+ if (!cascading) matched.add(asked);
4643
+ emptiedGroups.add(groupId);
4644
+ retire(ind, groupId);
4645
+ return false;
4646
+ });
4647
+ const gone = [...emptiedGroups].filter((groupId) => !standingGroups.has(groupId));
4648
+ asking = gone.length ? factGroupsReferencing(payload, new Set(gone)) : new Set();
4649
+ cascading = true;
4650
+ }
4256
4651
  for (const id of matched) removed.push(id);
4257
4652
  if (!removed.length) return; // honest no-op — nothing matched, no write needed beyond this
4258
4653
  for (const id of removedSet) records.push(id);
@@ -4416,7 +4811,7 @@ export function findContradictions(memory, { floor = CONTRADICTION_TRUST_FLOOR,
4416
4811
  const byKey = new Map();
4417
4812
  for (const r of rows) {
4418
4813
  if (resolutionStrategyFor(r.predicate) === RESOLUTION_MERGE) continue;
4419
- const key = `${r.subject} ${r.predicate}`;
4814
+ const key = subjectPredicateKey(r.subject, r.predicate);
4420
4815
  if (!byKey.has(key)) byKey.set(key, []);
4421
4816
  byKey.get(key).push(r);
4422
4817
  }
@@ -4425,7 +4820,10 @@ export function findContradictions(memory, { floor = CONTRADICTION_TRUST_FLOOR,
4425
4820
  if (new Set(group.map((r) => r.object)).size < 2) continue;
4426
4821
  const strategy = resolutionStrategyFor(group[0].predicate);
4427
4822
  if (strategy !== RESOLUTION_CONTRADICTION && !resolveSiblingGroups(group, strategy).contested) continue;
4428
- out.push(group.slice().sort((a, b) => b.trust - a.trust || a.object.localeCompare(b.object)));
4823
+ out.push(group.slice().sort((a, b) => b.trust - a.trust || byCodepoint(a.object, b.object)));
4429
4824
  }
4430
- return out.sort((a, b) => `${a[0].subject} ${a[0].predicate}`.localeCompare(`${b[0].subject} ${b[0].predicate}`));
4825
+ return out.sort((a, b) => byCodepoint(
4826
+ subjectPredicateKey(a[0].subject, a[0].predicate),
4827
+ subjectPredicateKey(b[0].subject, b[0].predicate),
4828
+ ));
4431
4829
  }