@polycode-projects/the-mechanical-code-talker 6.0.18 → 6.0.20

package/src/adapters/memory/corpus-bands.mjs

@@ -27,30 +27,22 @@ import { assertValidRow } from "./row-backend.mjs";
 export const BAND_PARTITION_PREFIX = "corpus:";
 export const MANIFEST_SORT_KEY = "manifest";
 
-/** The three bands this plan ships a build pipeline and a loader for. */
+/** The bands this repo ships a build pipeline and a loader for. */
 export const FIRST_CLASS_BANDS = Object.freeze([
-  "wikidata-slice",
   "wordnet-complete",
-  "conceptnet-full",
 ]);
 
-// Reserved so the partition name is spoken for; no pipeline for it ships
-// here. Its own design doc covers the band when it lands.
-export const RESERVED_BAND_NAMES = Object.freeze(["simplewiki-derived"]);
-
 /** The licence and attribution notice each first-class band's content
  *  carries — a property of the band's identity, not of any one load, so the
  *  loader reads it from here rather than taking it as a per-invocation flag.
  *  `notice` is a repo-relative path to the human-readable attribution file;
  *  null when the licence carries no attribution burden. */
 export const BAND_LICENSES = Object.freeze({
-  "wikidata-slice": Object.freeze({ license: "CC0-1.0", notice: null }),
   "wordnet-complete": Object.freeze({ license: "CC-BY-4.0", notice: "corpus/wordnet/LICENSE-NOTICE" }),
-  "conceptnet-full": Object.freeze({ license: "CC-BY-SA-4.0", notice: "corpus/conceptnet/conceptnet-full.NOTICE" }),
 });
 
 /** `{license, notice}` for `band`, or both null when the band carries no
- *  entry here (a consumer's own band, outside the three first-class ones). */
+ *  entry here (a consumer's own band, outside the first-class ones). */
 export function bandLicenseInfo(band) {
   return BAND_LICENSES[band] ?? { license: null, notice: null };
 }
@@ -153,3 +145,28 @@ export function bandFactRow({ subject, predicate, object, provenance = "", band,
   };
   return assertValidRow(row, { provenance });
 }
+
+/** The inverse of `bandFactRow`: the `{subject, predicate, object,
+ *  provenance}` triple a band's own wire row carries, read back off its
+ *  `json` field. A caller that grounds a term straight from a band Query (no
+ *  article, no extraction) hands this triple to `appendFacts` unchanged, so
+ *  the fact lands under the SAME content-addressed id and the SAME
+ *  `corpus:<band> ...` provenance the band shipped it with — trust reads it
+ *  as a corpus source, not a live research one.
+ *
+ *  Null when `json` fails to parse or the individual carries no
+ *  rdf:subject/predicate/object triple — a malformed row degrades to
+ *  "nothing to fold in" rather than a throw, since a caller reads a whole
+ *  page of rows and one bad one should not cost the rest. */
+export function factFromBandRow(row) {
+  let parsed;
+  try { parsed = JSON.parse(row?.json ?? ""); } catch { return null; }
+  const attributes = parsed?.individual?.attributes;
+  if (!Array.isArray(attributes)) return null;
+  const valueOf = (prop) => attributes.find((a) => a?.prop === prop)?.value;
+  const subject = valueOf("rdf:subject");
+  const predicate = valueOf("rdf:predicate");
+  const object = valueOf("rdf:object");
+  if (!subject || !predicate || !object) return null;
+  return { subject, predicate, object, provenance: valueOf("mgx:factProvenance") || "" };
+}

package/src/adapters/memory/inspect.mjs

@@ -5,6 +5,7 @@
 
 import { loadMemory, UTTERANCE_CLASS, IN_REPLY_TO_PROP, readFactRows, findContradictions } from "./core.mjs";
 import { loadBlockIndex } from "./blocks.mjs";
+import { compareFactsByContent } from "../../domain/memory/fact-order.mjs";
 import { buildTableauKb, findTableauViolations } from "../../domain/tableau.mjs";
 import { elUnsatisfiableClasses } from "../../domain/el-classify.mjs";
 
@@ -33,6 +34,23 @@ export function balancedSample(items, k) {
 }
 
 const attrOf = (ind, key) => (ind?.attributes || []).find((a) => a.key === key)?.value || "";
+
+/** Codepoint order, never localeCompare — this text is read on whatever machine
+ *  holds the store, and two locales have to land on the same lines. */
+const byCodepoint = (a, b) => {
+  const ka = String(a ?? "");
+  const kb = String(b ?? "");
+  return ka < kb ? -1 : ka > kb ? 1 : 0;
+};
+
+/** Order two individuals by their own id, in codepoint order. Every id here is
+ *  derived from content — a Fact hashes its triple, an Utterance reads
+ *  `utt:<session>#<ts>#<role>`, a Source keys on its provenance tag — so this
+ *  is a pure function of the stored set, and the utterance case still comes out
+ *  in the order the conversation happened. Without it the class samples below
+ *  span the payload in the order the individuals were written, which is arrival
+ *  order: two peers holding one store would sample different rows. */
+const byIndividualId = (a, b) => byCodepoint(a?.id, b?.id);
 const truncate = (s, cap) => {
   const t = String(s ?? "").replace(/\s+/g, " ").trim();
   return t.length > cap ? `${t.slice(0, cap - 1)}…` : t;
@@ -55,7 +73,8 @@ export function renderMemory({ memory, blocks }, { verbose = false } = {}) {
       if (!byClass.has(cls)) byClass.set(cls, []);
       byClass.get(cls).push(ind);
     }
-    const classes = [...byClass.entries()].sort((a, b) => b[1].length - a[1].length);
+    for (const members of byClass.values()) members.sort(byIndividualId);
+    const classes = [...byClass.entries()].sort((a, b) => b[1].length - a[1].length || byCodepoint(a[0], b[0]));
     lines.push(`memory — ${individuals.length} individuals: ${classes.map(([c, of]) => `${of.length} ${c}`).join(", ")}.`);
     for (const [cls, of] of classes) {
       const k = sampleSize(of.length, { verbose });
@@ -68,7 +87,7 @@ export function renderMemory({ memory, blocks }, { verbose = false } = {}) {
       .filter((r) => r.sourceIds.length || r.provenance)
       .sort((a, b) => b.trust - a.trust
         || b.sourceIds.length - a.sourceIds.length
-        || `${a.subject} ${a.predicate} ${a.object}`.localeCompare(`${b.subject} ${b.predicate} ${b.object}`));
+        || compareFactsByContent(a, b));
     if (ranked.length) {
       lines.push("", "top facts by trust:");
       for (const r of ranked.slice(0, verbose ? 8 : 3)) {
@@ -137,7 +156,7 @@ export function renderMemory({ memory, blocks }, { verbose = false } = {}) {
     const pairs = (replyGroup?.examples || [])
       .map((e) => ({ a: byId.get(e.subject), q: byId.get(e.object) }))
       .filter((p) => p.a && p.q && p.a.class === UTTERANCE_CLASS)
-      .sort((x, y) => attrOf(y.a, "ts").localeCompare(attrOf(x.a, "ts")));
+      .sort((x, y) => byCodepoint(attrOf(y.a, "ts"), attrOf(x.a, "ts")) || byIndividualId(x.a, y.a));
     if (pairs.length) {
       lines.push("", `recent Q→A pairs (${pairs.length} recorded):`);
       for (const p of pairs.slice(0, verbose ? 8 : 3)) {
@@ -153,7 +172,7 @@ export function renderMemory({ memory, blocks }, { verbose = false } = {}) {
     const tokens = entries.reduce((n, [, b]) => n + (b.tokens?.length || 0), 0);
     const top = entries
       .slice()
-      .sort((a, b) => (b[1].rank ?? 0) - (a[1].rank ?? 0) || a[0].localeCompare(b[0]))
+      .sort((a, b) => (b[1].rank ?? 0) - (a[1].rank ?? 0) || byCodepoint(a[0], b[0]))
       .slice(0, verbose ? 8 : 3);
     lines.push("", `blocks — ${entries.length} folded session block${entries.length === 1 ? "" : "s"}, ${tokens} indexed tokens.`);
     lines.push(`  top by rank: ${top.map(([id, b]) => `${String(id).slice(0, 8)} (${(b.rank ?? 0).toFixed(3)})`).join(", ")}`);
@@ -169,7 +188,7 @@ export function renderMemory({ memory, blocks }, { verbose = false } = {}) {
     const freq = new Map();
     for (const f of facts) if (clean(f.object)) freq.set(f.object, (freq.get(f.object) || 0) + 1);
     const terms = [...freq.entries()]
-      .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+      .sort((a, b) => b[1] - a[1] || byCodepoint(a[0], b[0]))
       .map(([t]) => t)
       .slice(0, 3);
     const sample = facts.find((f) => clean(f.subject) && terms.includes(f.object));

package/src/adapters/memory/rows.mjs

@@ -89,8 +89,13 @@ function storedIndividualForm(ind) {
 /** The ord each row key carries. An existing key keeps the ord it already had
  *  and a new one takes the next free number, matching how a store keeps a row's
  *  sort position across an update. Without prior rows the ords are the payload's
- *  own array order. */
-function ordAssigner(priorRows) {
+ *  own array order.
+ *
+ *  `baseOrds` is a read-only layer's key -> ord map, consulted after the prior
+ *  rows so a key both layers hold keeps the row's own ord. It is read in place:
+ *  a caller holding one for the life of its handle hands it over rather than
+ *  projecting a row per key and parsing each back. */
+function ordAssigner(priorRows, baseOrds) {
   const priorOrds = new Map();
   let next = 0;
   for (const row of priorRows || []) {
@@ -100,8 +105,9 @@ function ordAssigner(priorRows) {
     priorOrds.set(row.rowKey, ord);
     if (ord >= next) next = ord + 1;
   }
+  for (const ord of baseOrds?.values() || []) if (Number.isFinite(ord) && ord >= next) next = ord + 1;
   return (rowKey) => {
-    const prior = priorOrds.get(rowKey);
+    const prior = priorOrds.get(rowKey) ?? baseOrds?.get(rowKey);
     if (prior !== undefined) return prior;
     const ord = next;
     next += 1;
@@ -139,14 +145,16 @@ const OVERSIZED_ROW_POSTURES = new Set(["throw", "drop", "keep"]);
  *
  *  `priorRows` are the rows this payload was last projected as; pass them and
  *  every unchanged row comes back byte-identical, so `diffRows` writes only
- *  what actually moved. `onOversizedRow` is "throw" (default), "drop", or
- *  "keep" (rows that never reach the wire — see `admitRow`), and `log` takes
- *  the drop notices. */
-export function payloadToRows(payload, { priorRows = null, onOversizedRow = "throw", log = warnToConsole } = {}) {
+ *  what actually moved. `priorOrds` is the same information for a read-only
+ *  layer that already holds it as a key -> ord map (a sqlite seed's own key
+ *  columns), read under the prior rows. `onOversizedRow` is "throw" (default),
+ *  "drop", or "keep" (rows that never reach the wire — see `admitRow`), and
+ *  `log` takes the drop notices. */
+export function payloadToRows(payload, { priorRows = null, priorOrds = null, onOversizedRow = "throw", log = warnToConsole } = {}) {
   if (!OVERSIZED_ROW_POSTURES.has(onOversizedRow)) {
     throw new TypeError(`onOversizedRow must be "throw", "drop", or "keep", got ${JSON.stringify(onOversizedRow)}`);
   }
-  const ordFor = ordAssigner(priorRows);
+  const ordFor = ordAssigner(priorRows, priorOrds);
   const posture = { onOversizedRow, log };
   const rows = [];
 
@@ -206,6 +214,168 @@ function sortFactIndividualsById(individuals) {
   for (let i = 0; i < slots.length; i += 1) individuals[slots[i]] = facts[i];
 }
 
+/** The prose tokens, forward supersession list and backward-pointer presence one
+ *  individual carries, read in a single pass over its attributes. First match
+ *  per field, which is what the `find`-based readers elsewhere in this file
+ *  take. */
+function derivationInputsOf(ind) {
+  let proseTokens = "";
+  let supersedes = "";
+  let carriesSupersededBy = false;
+  let seenProseTokens = false;
+  let seenSupersedes = false;
+  for (const a of ind?.attributes || []) {
+    if (!seenProseTokens && a?.key === "prose_tokens") { proseTokens = a.value || ""; seenProseTokens = true; }
+    if (!seenSupersedes && a?.prop === SUPERSEDES_PROP) { supersedes = a.value || ""; seenSupersedes = true; }
+    if (a?.prop === SUPERSEDED_BY_PROP) carriesSupersededBy = true;
+  }
+  return { proseTokens, supersedes, carriesSupersededBy };
+}
+
+// What a payload carries forward so the next write reconciles its derived
+// structures instead of building them again: what each individual last put into
+// the prose index, what each record last named as superseded, and the reverse of
+// that. Non-enumerable and symbol-keyed, so no copy of the payload inherits
+// state that describes a different array of individuals, and a payload without
+// it (a fresh assembly, a clone, a hand-built fixture) derives from scratch.
+const CARRIED_DERIVATIONS = Symbol("tmct.carriedDerivations");
+
+function carryForward(payload, carried) {
+  carried.index = payload.proseIndex;
+  Object.defineProperty(payload, CARRIED_DERIVATIONS, {
+    value: carried, writable: true, configurable: true, enumerable: false,
+  });
+}
+
+// The ids of the assembled individuals in ROW order, kept beside the assembled
+// array. `sortFactIndividualsById` lifts the Facts out of the slots row order
+// put them in, so the array itself no longer says which slot each Fact came
+// from — and a removal needs exactly that, because dropping a Fact drops one of
+// those slots, not the position the sort moved it to. Non-enumerable and
+// symbol-keyed for the same reason as the derivations above; the `individuals`
+// reference is the guard, so a copy or a rebuilt array is never reconciled
+// against an order that describes a different one.
+const ASSEMBLED_ROW_ORDER = Symbol("tmct.assembledRowOrder");
+
+function carryRowOrder(payload, ids) {
+  Object.defineProperty(payload, ASSEMBLED_ROW_ORDER, {
+    value: { individuals: payload.individuals, ids },
+    writable: true, configurable: true, enumerable: false,
+  });
+}
+
+/** The row order this payload was assembled in, or null when it carries none
+ *  or the one it carries describes a different individuals array. */
+function carriedRowOrder(payload) {
+  const carried = payload?.[ASSEMBLED_ROW_ORDER];
+  if (!carried || carried.individuals !== payload.individuals) return null;
+  return carried;
+}
+
+/** True when `removedIds` can be dropped from this payload's individuals
+ *  incrementally — every id is one the assembly actually holds, and the payload
+ *  still carries the row order that says which slot each one occupies. Asked
+ *  before anything is mutated, so a caller that gets `false` can rebuild from a
+ *  payload nothing has touched. */
+export function canDropAssembledIndividuals(payload, removedIds) {
+  const carried = carriedRowOrder(payload);
+  if (!carried) return false;
+  const held = new Set(carried.ids);
+  for (const id of removedIds) if (!held.has(String(id))) return false;
+  return true;
+}
+
+/** Drop `removedIds` from an assembled payload's individuals, in place, leaving
+ *  the array a rebuild from the remaining rows would have produced. Reads the
+ *  row order back, filters the dropped ids out of it, and refills the array in
+ *  that order — so the fact slots that survive are the ones the surviving ROWS
+ *  own, which is what `sortFactIndividualsById` then sorts into. Returns false
+ *  when the payload has no usable row order, having changed nothing. */
+export function dropAssembledIndividuals(payload, removedIds) {
+  const carried = carriedRowOrder(payload);
+  if (!carried) return false;
+  const dropped = new Set([...removedIds].map((id) => String(id)));
+  const byId = new Map();
+  for (const ind of payload.individuals || []) if (ind?.id) byId.set(String(ind.id), ind);
+  const keptIds = [];
+  const kept = [];
+  for (const id of carried.ids) {
+    if (dropped.has(id)) continue;
+    const ind = byId.get(id);
+    if (!ind) return false;
+    keptIds.push(id);
+    kept.push(ind);
+  }
+  payload.individuals = kept;
+  carried.individuals = kept;
+  carried.ids = keptIds;
+  return true;
+}
+
+/** Forget the row order this payload carries, for a caller that rewrote the ids
+ *  the order names (the load-time legacy-fact-id heal). The next removal
+ *  rebuilds from rows instead of patching. */
+export function dropAssembledRowOrder(payload) {
+  if (payload && payload[ASSEMBLED_ROW_ORDER]) payload[ASSEMBLED_ROW_ORDER] = null;
+}
+
+/** Record one newly-assembled individual's id at the tail of the row order —
+ *  where a row keyed for the first time lands, since `payloadToRows` gives it an
+ *  ord past every ord already assembled. A payload carrying no row order stays
+ *  that way, and the next removal rebuilds instead of patching. */
+export function appendAssembledRowOrder(payload, id) {
+  const carried = carriedRowOrder(payload);
+  if (carried) carried.ids.push(String(id));
+}
+
+/** What this payload can reconcile against, or null when it must derive
+ *  instead. `index` identity is the guard: state that describes some other
+ *  prose index cannot be applied to this one. `needSupersessions` narrows to
+ *  the callers that maintain the backward pointers too. */
+function carriedDerivationsOf(payload, { needSupersessions } = {}) {
+  const carried = payload[CARRIED_DERIVATIONS];
+  if (!carried || carried.index !== payload.proseIndex || !payload.proseIndex) return null;
+  if (needSupersessions && !carried.successorsById) return null;
+  return carried;
+}
+
+/** Where `id` sits in a sorted posting list, and whether it is there at all. */
+function postingSlot(list, id) {
+  let low = 0;
+  let high = list.length;
+  while (low < high) {
+    const mid = (low + high) >> 1;
+    if (list[mid] < id) low = mid + 1;
+    else high = mid;
+  }
+  return low;
+}
+
+/** Move one individual's contribution to the prose index from `before` to
+ *  `after`, in place.
+ *
+ *  `buildProseIndex` produces, per word, the multiset of ids that named it —
+ *  one entry per (individual, occurrence of the word in its token string) —
+ *  sorted, with a word nobody names absent altogether. Dropping one entry per
+ *  old occurrence and inserting one per new occurrence lands on exactly that
+ *  multiset, and inserting in sorted position keeps exactly that order, so a
+ *  reconciled index and a rebuilt one are the same object graph. */
+function moveProseTokens(index, id, before, after) {
+  if (before === after) return;
+  for (const word of before ? before.split(" ") : []) {
+    const list = index[word];
+    if (!list) continue;
+    const at = postingSlot(list, id);
+    if (list[at] !== id) continue;
+    list.splice(at, 1);
+    if (!list.length) delete index[word];
+  }
+  for (const word of after ? after.split(" ") : []) {
+    const list = index[word] || (index[word] = []);
+    list.splice(postingSlot(list, id), 0, id);
+  }
+}
+
 /** Re-derive each record's backward supersession pointer from the union of the
  *  forward ones. Two turns that superseded the same record concurrently both
  *  land, because each wrote its own row and neither touched the record they
@@ -215,36 +385,143 @@ function sortFactIndividualsById(individuals) {
  *  used to carry. Rows never carry one (`storedIndividualForm` strips it), so
  *  that arm is dead when this runs over a fresh projection and live when it
  *  runs again over individuals it already derived. */
-function applyDerivedSupersessions(individuals) {
+function writeSupersededBy(ind, successors) {
+  const carried = (ind?.attributes || []).some((a) => a?.prop === SUPERSEDED_BY_PROP);
+  if (!successors && !carried) return;
+  const rest = (ind.attributes || []).filter((a) => a?.prop !== SUPERSEDED_BY_PROP);
+  ind.attributes = successors
+    ? [...rest, { prop: SUPERSEDED_BY_PROP, key: "supersededBy", value: [...successors].sort().join(" ") }]
+    : rest;
+}
+
+/** The forward supersession list one individual states, as the derivation reads
+ *  it: a Fact's own `mgx:supersedes` value, and nothing at all from anything
+ *  else, which is the same narrowing the from-scratch pass applies. */
+const supersedesStatedBy = (ind, supersedes) => (ind?.class === FACT_CLASS ? supersedes : "");
+
+/** Both derived structures over one payload, built from nothing and recorded so
+ *  the next write can reconcile rather than repeat this. */
+function deriveProseAndSupersessions(payload, individuals) {
   const successorsById = new Map();
+  const tokensById = new Map();
+  const supersedesById = new Map();
   for (const ind of individuals) {
-    if (ind?.class !== FACT_CLASS) continue;
-    for (const replaced of attrValue(ind, SUPERSEDES_PROP).split(" ").filter(Boolean)) {
+    const { proseTokens, supersedes } = derivationInputsOf(ind);
+    if (proseTokens) tokensById.set(ind.id, proseTokens);
+    const stated = supersedesStatedBy(ind, supersedes);
+    if (!stated) continue;
+    supersedesById.set(ind.id, stated);
+    for (const replaced of stated.split(" ").filter(Boolean)) {
       const successors = successorsById.get(replaced) || new Set();
       successors.add(ind.id);
       successorsById.set(replaced, successors);
     }
   }
+  for (const ind of individuals) writeSupersededBy(ind, successorsById.get(ind?.id));
+  // Released before the replacement is built, not after: over a seed-sized
+  // store the prose index is the largest derived structure here, and holding
+  // the outgoing one while the incoming one grows doubles it for no reason.
+  payload.proseIndex = null;
+  payload.proseIndex = buildProseIndex(individuals);
+  carryForward(payload, { tokensById, supersedesById, successorsById });
+}
+
+/** Both derived structures brought up to date from the ones this payload
+ *  already carries. One pass over the individuals reads what each contributes
+ *  now; only what disagrees with what it contributed last time is applied.
+ *
+ *  A write touches a handful of records out of a seed's worth, so this pays for
+ *  the walk and the handful, where the from-scratch build pays for every token
+ *  in the store and sorts every posting list it produced. */
+function reconcileProseAndSupersessions(payload, individuals, carried) {
+  const { tokensById, supersedesById, successorsById } = carried;
+  const index = payload.proseIndex;
+  const resettled = new Set();
+  const carriesPointer = new Set();
+  let tokenBearers = 0;
+  let supersedingRecords = 0;
+
+  const forgetSupersedes = (id, replaced) => {
+    const successors = successorsById.get(replaced);
+    if (!successors?.delete(id)) return;
+    if (!successors.size) successorsById.delete(replaced);
+    resettled.add(replaced);
+  };
+  const recordSupersedes = (id, replaced) => {
+    const successors = successorsById.get(replaced) || new Set();
+    if (successors.has(id)) return;
+    successors.add(id);
+    successorsById.set(replaced, successors);
+    resettled.add(replaced);
+  };
+
+  for (const ind of individuals) {
+    const id = ind?.id;
+    const { proseTokens, supersedes, carriesSupersededBy } = derivationInputsOf(ind);
+    if (carriesSupersededBy) carriesPointer.add(id);
+
+    if (proseTokens) tokenBearers += 1;
+    const wasTokens = tokensById.get(id) || "";
+    if (proseTokens !== wasTokens) {
+      moveProseTokens(index, id, wasTokens, proseTokens);
+      if (proseTokens) tokensById.set(id, proseTokens);
+      else tokensById.delete(id);
+    }
+
+    const stated = supersedesStatedBy(ind, supersedes);
+    if (stated) supersedingRecords += 1;
+    const wasStated = supersedesById.get(id) || "";
+    if (stated !== wasStated) {
+      for (const replaced of wasStated.split(" ").filter(Boolean)) forgetSupersedes(id, replaced);
+      for (const replaced of stated.split(" ").filter(Boolean)) recordSupersedes(id, replaced);
+      if (stated) supersedesById.set(id, stated);
+      else supersedesById.delete(id);
+    }
+  }
+
+  // A caller that only added or rewrote individuals leaves both maps holding
+  // exactly what the walk above just saw, and the counts say so. They disagree
+  // only when an individual left the payload, which costs one more walk to
+  // find and is the rarer write by far.
+  if (tokensById.size !== tokenBearers || supersedesById.size !== supersedingRecords) {
+    const present = new Set(individuals.map((ind) => ind?.id));
+    for (const [id, tokens] of tokensById) {
+      if (present.has(id)) continue;
+      moveProseTokens(index, id, tokens, "");
+      tokensById.delete(id);
+    }
+    for (const [id, stated] of supersedesById) {
+      if (present.has(id)) continue;
+      for (const replaced of stated.split(" ").filter(Boolean)) forgetSupersedes(id, replaced);
+      supersedesById.delete(id);
+    }
+  }
+
+  // Exactly the records the from-scratch pass rewrites: the ones with
+  // successors and the ones already carrying a pointer. Both sets are bounded
+  // by how many supersessions the store holds, which is a handful beside its
+  // individuals, so this rewrites the same records for the same cost and leaves
+  // the rest of the store alone.
+  for (const id of carriesPointer) resettled.add(id);
+  for (const id of successorsById.keys()) resettled.add(id);
+  if (!resettled.size) return;
   for (const ind of individuals) {
-    const successors = successorsById.get(ind?.id);
-    const carried = (ind?.attributes || []).some((a) => a?.prop === SUPERSEDED_BY_PROP);
-    if (!successors && !carried) continue;
-    const rest = (ind.attributes || []).filter((a) => a?.prop !== SUPERSEDED_BY_PROP);
-    ind.attributes = successors
-      ? [...rest, { prop: SUPERSEDED_BY_PROP, key: "supersededBy", value: [...successors].sort().join(" ") }]
-      : rest;
+    if (resettled.has(ind?.id)) writeSupersededBy(ind, successorsById.get(ind?.id));
   }
 }
 
 /** Recount `classes[]` from the assembled individuals, the same count-and-
  *  sample shape the store keeps. */
 function recountedClasses(individuals) {
-  const classes = [];
-  for (const name of [MEMORY_SESSION_CLASS, UTTERANCE_CLASS, FACT_CLASS, SOURCE_CLASS, RULE_CLASS]) {
-    const of = individuals.filter((i) => i?.class === name);
-    if (of.length) classes.push({ name, count: of.length, sample: of.slice(0, 3).map((i) => i.label) });
+  const order = [MEMORY_SESSION_CLASS, UTTERANCE_CLASS, FACT_CLASS, SOURCE_CLASS, RULE_CLASS];
+  const counted = new Map(order.map((name) => [name, { name, count: 0, sample: [] }]));
+  for (const ind of individuals) {
+    const row = counted.get(ind?.class);
+    if (!row) continue;
+    row.count += 1;
+    if (row.sample.length < 3) row.sample.push(ind.label);
   }
-  return classes;
+  return order.map((name) => counted.get(name)).filter((row) => row.count);
 }
 
 /** The store's `generated_at`: the latest utterance timestamp it holds, which
@@ -281,8 +558,16 @@ export function rowsToPayload(rows, { meta = null } = {}) {
   individualEntries.sort(byOrdThenRowKey);
   groupEntries.sort(byOrdThenRowKey);
 
-  payload.individuals = individualEntries.map((e) => e.individual).filter(Boolean);
+  const rowOrderIds = [];
+  const individuals = [];
+  for (const entry of individualEntries) {
+    if (!entry.individual) continue;
+    individuals.push(entry.individual);
+    rowOrderIds.push(String(entry.individual.id ?? ""));
+  }
+  payload.individuals = individuals;
   payload.objectProperties = groupEntries.map((e) => e.group).filter(Boolean);
+  carryRowOrder(payload, rowOrderIds);
   return renormalizeAssembledPayload(payload);
 }
 
@@ -296,17 +581,61 @@ export function rowsToPayload(rows, { meta = null } = {}) {
 export function renormalizeAssembledPayload(payload) {
   const individuals = payload.individuals || [];
   sortFactIndividualsById(individuals);
-  applyDerivedSupersessions(individuals);
+  const carried = carriedDerivationsOf(payload, { needSupersessions: true });
+  if (carried) reconcileProseAndSupersessions(payload, individuals, carried);
+  else deriveProseAndSupersessions(payload, individuals);
   payload.classes = recountedClasses(individuals);
-  // Released before the replacement is built, not after: over a seed-sized
-  // store the prose index is the largest derived structure here, and holding
-  // the outgoing one while the incoming one grows doubles it for no reason.
-  payload.proseIndex = null;
-  payload.proseIndex = buildProseIndex(individuals);
   payload.generated_at = latestUtteranceTimestamp(individuals);
   return payload;
 }
 
+/** Only the prose index, reconciled the same way — for a caller that derives
+ *  everything else itself and whose payload is not a row assembly
+ *  (`mutateMemory` over a non-row backend). A payload with nothing to reconcile
+ *  against gets the full build, and carries the state on so the next write
+ *  reconciles. Mutates and returns `payload`. */
+export function renormalizeProseIndex(payload) {
+  const individuals = payload.individuals || [];
+  const carried = carriedDerivationsOf(payload);
+  if (!carried) {
+    const tokensById = new Map();
+    for (const ind of individuals) {
+      const { proseTokens } = derivationInputsOf(ind);
+      if (proseTokens) tokensById.set(ind.id, proseTokens);
+    }
+    payload.proseIndex = null;
+    payload.proseIndex = buildProseIndex(individuals);
+    carryForward(payload, { tokensById, supersedesById: null, successorsById: null });
+    return payload;
+  }
+  const { tokensById } = carried;
+  // This pass maintains the index and nothing else, so any supersession state
+  // beside it stops describing the individuals it claims to and goes now,
+  // rather than being reconciled against later.
+  carried.supersedesById = null;
+  carried.successorsById = null;
+  const index = payload.proseIndex;
+  let tokenBearers = 0;
+  for (const ind of individuals) {
+    const { proseTokens } = derivationInputsOf(ind);
+    if (proseTokens) tokenBearers += 1;
+    const wasTokens = tokensById.get(ind?.id) || "";
+    if (proseTokens === wasTokens) continue;
+    moveProseTokens(index, ind?.id, wasTokens, proseTokens);
+    if (proseTokens) tokensById.set(ind.id, proseTokens);
+    else tokensById.delete(ind?.id);
+  }
+  if (tokensById.size !== tokenBearers) {
+    const present = new Set(individuals.map((ind) => ind?.id));
+    for (const [id, tokens] of tokensById) {
+      if (present.has(id)) continue;
+      moveProseTokens(index, id, tokens, "");
+      tokensById.delete(id);
+    }
+  }
+  return payload;
+}
+
 /** The rows to write and the row keys to delete to turn `before` into `after`.
  *  A row whose stored bytes did not change is not in either list, so a store
  *  pays only for what actually moved. */

package/src/adapters/memory/shacl.mjs

@@ -18,12 +18,19 @@ export const EXTRACTION_FINDINGS = Object.freeze([
   "clause-fallback",    // the row grounded from a clause fragment after the whole sentence declined
   "pronoun-carry",      // the subject was substituted from the paragraph's pronoun carry
   "definitional-frame", // the row came from a definitional copula frame ("X is the name for Y")
+  "reported-speech",    // the row was read from a reported-speech clause, its speaker attributed separately
 ]);
 const EXTRACTION_FINDING_SET = new Set(EXTRACTION_FINDINGS);
-const RULE_KINDS = new Set([
+
+// The closed vocabulary of a taught Rule's SHAPE tag — three structural
+// kinds plus the action-* family shared by one taught sentence's sibling
+// Rules. Exported (like EXTRACTION_FINDINGS above) so an estate guard can
+// cross-check it against its own prose mirrors in the ontology files.
+export const RULE_KINDS = Object.freeze([
   "compose2", "filter", "recursive",
   "action-signature", "action-precond", "action-effect", "action-constraint",
 ]);
+const RULE_KIND_SET = new Set(RULE_KINDS);
 
 // Mirrors the REQUIRED subset of core.mjs's own (unexported) RULE_SLOT_SPEC —
 // kept in sync by hand. Two kinds there also carry optional slots
@@ -140,8 +147,8 @@ function checkFact(ind, violations) {
 function checkRule(ind, violations) {
   if (!nonEmpty(attrValue(ind, "mgx:ruleName"))) violations.push("a Rule needs a non-empty mgx:ruleName");
   const kind = attrValue(ind, "mgx:ruleKind");
-  if (!kind || !RULE_KINDS.has(kind)) {
-    violations.push(`a Rule's mgx:ruleKind must be one of ${[...RULE_KINDS].join(" | ")} (got ${JSON.stringify(kind)})`);
+  if (!kind || !RULE_KIND_SET.has(kind)) {
+    violations.push(`a Rule's mgx:ruleKind must be one of ${RULE_KINDS.join(" | ")} (got ${JSON.stringify(kind)})`);
     return; // no declared kind to check slots against
   }
   for (const prop of RULE_SLOT_PROPS[kind]) {