@polycode-projects/the-mechanical-code-talker 4.1.1 → 4.1.3

package/src/domain/memory/compaction.mjs

@@ -0,0 +1,284 @@
+// memory/compaction.mjs — bounding fact-record growth along its two separate
+// axes, with a separate rollup for each.
+//
+// A triple's records grow two ways, and the ways mean different things:
+//
+//   - MANY SOURCES each asserting it once. Every one is a live head and a live
+//     vote in the group fold, so a summary that absorbed them has to carry
+//     their trust contribution forward or the compacted answer silently
+//     under-trusts. That is `mgx:rollupPrior`.
+//   - ONE SOURCE re-asserting it many times. Every record it replaced is a
+//     demoted leaf that already contributes nothing (the fold reads heads
+//     only), so absorbing them moves no number at all — it only shortens how
+//     far back a walk can reach. A chain summary therefore carries NO prior,
+//     and a compacted chain segment must not start contributing on the way out.
+//
+// Rolling both into one "keep newest K" pool would rank a chatty source's
+// recent demoted leaves above a quiet source's still-live head, and compacting
+// that head away is exactly the mistake the split exists to avoid. So: two
+// pools per (triple, source type), each with its own trigger and its own id.
+//
+// Both summaries replicate, and deleting from a replicated grow-only set is not
+// a G-Set operation — an uncoordinated delete comes back on the next sync. What
+// closes that hole is making the summary carry the ids it absorbed, so
+// absorption itself replicates: two summaries at one id merge by UNION of those
+// ids, which is a join, so peers that compacted at different moments converge.
+// Everything else on a summary (its count, its bounds, its prior) is derived
+// from the union, so the whole record is a pure function of it — which is what
+// makes the merge idempotent and order-independent rather than merely usually
+// right.
+//
+// Pure: this module plans and merges summary RECORDS. core.mjs owns the payload.
+
+const FACT_CLASS = "Fact";
+
+// Starting guesses, not measurements — revisit against real store data once
+// `tmct inspect`'s widest-group metric has something to report.
+export const GROUP_ROLLUP_THRESHOLD = 64; // live heads of one type before pool 1 fires
+export const ROLLUP_KEEP_PER_TYPE = 8;    // heads of that type kept intact
+export const CHAIN_ROLLUP_THRESHOLD = 8;  // demoted leaves of one chain before pool 2 fires
+export const CHAIN_KEEP_DEPTH = 2;        // leaves of that chain kept intact
+
+export const ROLLUP_SOURCE_IDS_PROP = "mgx:rollupSourceIds"; // pool 1: the SOURCE ids absorbed
+export const ROLLUP_RECORD_IDS_PROP = "mgx:rollupRecordIds"; // pool 2: the RECORD ids absorbed
+export const ROLLUP_COUNT_PROP = "mgx:rollupCount";
+export const ROLLUP_EARLIEST_PROP = "mgx:rollupEarliest";
+export const ROLLUP_LATEST_PROP = "mgx:rollupLatest";
+export const ROLLUP_PRIOR_PROP = "mgx:rollupPrior";
+
+const HEAD_ROLLUP_MARKER = "@rollup:";
+const CHAIN_ROLLUP_SUFFIX = "#rollup";
+
+export const headRollupIdFor = (groupId, sourceType) => `${groupId}${HEAD_ROLLUP_MARKER}${sourceType}`;
+export const chainRollupIdFor = (groupId, sourceId) => `${groupId}@${sourceId}${CHAIN_ROLLUP_SUFFIX}`;
+
+export const isHeadRollupId = (id) => String(id || "").includes(HEAD_ROLLUP_MARKER);
+export const isChainRollupId = (id) => String(id || "").endsWith(CHAIN_ROLLUP_SUFFIX);
+export const isRollupId = (id) => isHeadRollupId(id) || isChainRollupId(id);
+
+/** The source type a pool-1 summary covers, read back off the id that carries
+ *  it. A summary is one type by construction, which is what keeps type priors
+ *  and per-type ceilings computable over a compacted group. */
+export function headRollupTypeOf(recordId) {
+  const id = String(recordId || "");
+  const at = id.indexOf(HEAD_ROLLUP_MARKER);
+  return at < 0 ? "" : id.slice(at + HEAD_ROLLUP_MARKER.length);
+}
+
+const attrValue = (ind, prop) => (ind?.attributes || []).find((a) => a?.prop === prop)?.value || "";
+const idList = (value) => String(value || "").split(" ").filter(Boolean);
+
+export const absorbedSourceIds = (ind) => idList(attrValue(ind, ROLLUP_SOURCE_IDS_PROP));
+export const absorbedRecordIds = (ind) => idList(attrValue(ind, ROLLUP_RECORD_IDS_PROP));
+
+const clamp01 = (n) => Math.max(0, Math.min(1, Number(n) || 0));
+const round6 = (n) => Number(n.toFixed(6));
+
+/** The noisy-OR base over a set of effective priors — the same corroboration
+ *  math the group fold runs, with recency deliberately absent. Recency is a
+ *  function of the reading moment, so a summary bakes in only what it knows at
+ *  write time and the fold applies decay off `rollupLatest` on read. */
+export function noisyOr(priors) {
+  let complement = 1;
+  let counted = 0;
+  for (const p of priors) { complement *= 1 - clamp01(p); counted += 1; }
+  return counted ? round6(Math.min(1, 1 - complement)) : 0;
+}
+
+/** Pick the earlier/later of two instants, tolerating "" and unparseable input
+ *  on either side. Min-of-earliest and max-of-latest are both joins, which is
+ *  why two summaries can merge in either order and agree. */
+function pickInstant(a, b, pick) {
+  const at = Date.parse(a);
+  const bt = Date.parse(b);
+  if (!Number.isFinite(at)) return Number.isFinite(bt) ? String(b) : "";
+  if (!Number.isFinite(bt)) return String(a);
+  return pick(at, bt) === at ? String(a) : String(b);
+}
+const earlierOf = (a, b) => pickInstant(a, b, Math.min);
+const laterOf = (a, b) => pickInstant(a, b, Math.max);
+
+/** Newest first, ties broken on the record id in codepoint order — the same
+ *  locale-free determinism the rest of the fact layer sorts by, so two peers
+ *  holding the same records choose the same keep-window. */
+function byNewestFirst(a, b) {
+  const at = Date.parse(a.assertedAt);
+  const bt = Date.parse(b.assertedAt);
+  const av = Number.isFinite(at) ? at : -Infinity;
+  const bv = Number.isFinite(bt) ? bt : -Infinity;
+  if (av !== bv) return bv - av;
+  return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
+}
+
+function boundsOver(records, seedEarliest = "", seedLatest = "") {
+  let earliest = seedEarliest;
+  let latest = seedLatest;
+  for (const r of records) {
+    earliest = earlierOf(earliest, r.assertedAt);
+    latest = laterOf(latest, r.assertedAt);
+  }
+  return { earliest, latest };
+}
+
+/**
+ * Build a summary record from the ONLY things that define one: which ids it
+ * absorbed, the span they covered, and (pool 1 only) how to price a source.
+ * Both the compaction path and the merge path go through here, so a summary
+ * built by absorbing and the same summary reached by merging two halves come
+ * out byte-identical — which is the property the whole design rests on.
+ */
+function buildRollup({ id, sourceKey, template, idsProp, ids, earliest, latest, priorFor }) {
+  const sorted = [...new Set(ids)].filter(Boolean).sort();
+  const prior = priorFor ? noisyOr(sorted.map((sid) => priorFor(sid))) : undefined;
+  return {
+    id,
+    label: template?.label || "",
+    class: FACT_CLASS,
+    derived_from: [],
+    mentions: [],
+    attributes: [
+      { prop: "rdf:type", key: "type", value: "rdf:Statement" },
+      { prop: "rdf:subject", key: "subject", value: template?.subject || "" },
+      { prop: "rdf:predicate", key: "predicate", value: template?.predicate || "" },
+      { prop: "rdf:object", key: "object", value: template?.object || "" },
+      { prop: "mgx:createdAt", key: "createdAt", value: earliest },
+      { prop: "mgx:sourceId", key: "sourceId", value: sourceKey },
+      { prop: idsProp, key: idsProp === ROLLUP_SOURCE_IDS_PROP ? "rollupSourceIds" : "rollupRecordIds", value: sorted.join(" ") },
+      { prop: ROLLUP_COUNT_PROP, key: "rollupCount", value: String(sorted.length) },
+      { prop: ROLLUP_EARLIEST_PROP, key: "rollupEarliest", value: earliest },
+      { prop: ROLLUP_LATEST_PROP, key: "rollupLatest", value: latest },
+      ...(prior === undefined ? [] : [{ prop: ROLLUP_PRIOR_PROP, key: "rollupPrior", value: String(prior) }]),
+    ],
+  };
+}
+
+const templateOf = (record) => ({
+  label: record?.label || "",
+  subject: attrValue(record, "rdf:subject"),
+  predicate: attrValue(record, "rdf:predicate"),
+  object: attrValue(record, "rdf:object"),
+});
+
+/**
+ * Pool 1. Given every LIVE HEAD of one type on one triple, decide whether that
+ * type has grown past its threshold and, if so, which heads to absorb.
+ *
+ * `heads` are `{ id, sourceId, assertedAt, record }`. `existing` is the type's
+ * current summary, if it already has one — its absorbed ids come along, so
+ * compacting twice keeps one summary rather than growing a chain of them.
+ * `priorFor(sourceId)` prices one absorbed source.
+ *
+ * Null when nothing should happen, which is the answer for very nearly every
+ * fact in a store: under the threshold, no type at all, or nothing older than
+ * the keep window.
+ */
+export function planHeadRollup({ groupId, sourceType, heads = [], existing = null, priorFor }) {
+  if (!groupId || !sourceType) return null;            // never roll up across types, or an untyped record
+  if (heads.length < GROUP_ROLLUP_THRESHOLD) return null;
+  if (heads.length <= ROLLUP_KEEP_PER_TYPE) return null; // never the sole records of a type
+  const ordered = heads.slice().sort(byNewestFirst);
+  const absorb = ordered.slice(ROLLUP_KEEP_PER_TYPE);
+  if (!absorb.length) return null;
+  const { earliest, latest } = boundsOver(
+    absorb,
+    attrValue(existing, ROLLUP_EARLIEST_PROP),
+    attrValue(existing, ROLLUP_LATEST_PROP),
+  );
+  const rollup = buildRollup({
+    id: headRollupIdFor(groupId, sourceType),
+    sourceKey: `rollup:${sourceType}`,
+    template: templateOf(absorb[0].record || existing),
+    idsProp: ROLLUP_SOURCE_IDS_PROP,
+    ids: [...absorbedSourceIds(existing), ...absorb.map((h) => h.sourceId)],
+    earliest,
+    latest,
+    priorFor,
+  });
+  return { rollup, absorbed: absorb.map((h) => h.id), absorbedSourceIds: absorb.map((h) => h.sourceId) };
+}
+
+/**
+ * Pool 2. Given every DEMOTED LEAF of one source's own chain on one triple,
+ * decide whether that chain has grown past its threshold and which leaves to
+ * absorb. Triggers far more eagerly than pool 1 because nothing here is
+ * trust-sensitive: no prior, no type-ceiling interaction, no read-time trust to
+ * recompute.
+ *
+ * `leaves` are `{ id, assertedAt, record }`. The summary keeps the absorbed
+ * span's BOUNDS rather than any exact instant, so a walk that lands inside it
+ * gets "sometime in this window" and never a fabricated moment.
+ */
+export function planChainRollup({ groupId, sourceId, leaves = [], existing = null }) {
+  if (!groupId || !sourceId) return null;
+  if (leaves.length < CHAIN_ROLLUP_THRESHOLD) return null;
+  if (leaves.length <= CHAIN_KEEP_DEPTH) return null;   // never the sole leaf of a chain
+  const ordered = leaves.slice().sort(byNewestFirst);
+  const keep = ordered.slice(0, CHAIN_KEEP_DEPTH);
+  const absorb = ordered.slice(CHAIN_KEEP_DEPTH);
+  if (!absorb.length) return null;
+  const { earliest, latest } = boundsOver(
+    absorb,
+    attrValue(existing, ROLLUP_EARLIEST_PROP),
+    attrValue(existing, ROLLUP_LATEST_PROP),
+  );
+  const rollup = buildRollup({
+    id: chainRollupIdFor(groupId, sourceId),
+    sourceKey: sourceId,
+    template: templateOf(absorb[0].record || existing),
+    idsProp: ROLLUP_RECORD_IDS_PROP,
+    ids: [...absorbedRecordIds(existing), ...absorb.map((l) => l.id)],
+    earliest,
+    latest,
+  });
+  return { rollup, absorbed: absorb.map((l) => l.id), rewire: keep[keep.length - 1]?.id || "" };
+}
+
+/**
+ * Join two summaries that share an id: union the absorbed ids, then re-derive
+ * everything else from that union. Commutative, associative and idempotent
+ * because union, min and max all are, and because count and prior are functions
+ * of the union rather than independently accumulated state.
+ *
+ * The same call handles both pools; `priorFor` is supplied for pool 1 and
+ * omitted for pool 2, which is what keeps a compacted chain segment from
+ * starting to contribute trust on the way out.
+ */
+export function mergeRollups(existing, incoming, { priorFor } = {}) {
+  const id = existing?.id || incoming?.id;
+  const headPool = isHeadRollupId(id);
+  const idsProp = headPool ? ROLLUP_SOURCE_IDS_PROP : ROLLUP_RECORD_IDS_PROP;
+  const read = (ind) => idList(attrValue(ind, idsProp));
+  return buildRollup({
+    id,
+    sourceKey: attrValue(existing, "mgx:sourceId") || attrValue(incoming, "mgx:sourceId"),
+    template: templateOf(attrValue(existing, "rdf:subject") ? existing : incoming),
+    idsProp,
+    ids: [...read(existing), ...read(incoming)],
+    earliest: earlierOf(attrValue(existing, ROLLUP_EARLIEST_PROP), attrValue(incoming, ROLLUP_EARLIEST_PROP)),
+    latest: laterOf(attrValue(existing, ROLLUP_LATEST_PROP), attrValue(incoming, ROLLUP_LATEST_PROP)),
+    priorFor: headPool ? priorFor : undefined,
+  });
+}
+
+/** Is this source's assertion already absorbed into one of the group's pool-1
+ *  summaries? A late or re-synced copy of an absorbed assertion must stay
+ *  absorbed rather than reappearing as a live head — without this the next sync
+ *  resurrects everything compaction just folded away, which is the failure mode
+ *  that makes deleting from a replicated set hard in the first place. */
+export function isAbsorbedSource(rollups, sourceId) {
+  if (!sourceId) return false;
+  for (const rollup of rollups || []) {
+    if (absorbedSourceIds(rollup).includes(sourceId)) return true;
+  }
+  return false;
+}
+
+/** The same check for one source's own chain: a leaf id already inside the
+ *  chain summary merges as a no-op re-absorption, never a re-insertion. */
+export function isAbsorbedRecord(rollups, recordId) {
+  if (!recordId) return false;
+  for (const rollup of rollups || []) {
+    if (absorbedRecordIds(rollup).includes(recordId)) return true;
+  }
+  return false;
+}

package/src/domain/memory/resolution.mjs

@@ -0,0 +1,171 @@
+// memory/resolution.mjs — how sibling facts sharing one (subject, predicate)
+// but disagreeing on the OBJECT resolve. A fact is stored one record per
+// asserting source, so two kinds of plurality exist: records inside one triple
+// group (corroboration — same claim, different mouths, handled by the group
+// fold in core.mjs) and different objects across groups. Only the second one
+// needs a decision, and which decision it needs is a property of the PREDICATE.
+//
+// The table below is a closed enum keyed by predicate, matching this codebase's
+// standing preference for closed vocabularies over general rules — SOURCE_PRIOR
+// and the ISA set are the same shape. Four strategies:
+//
+//   merge                   many objects at once are all true; a second object
+//                           is a second fact, never a disagreement
+//   latest-observation-wins the object is a current state; successive objects
+//                           are successive states, newest observation renders
+//   first-claim-wins        objects race for a registration; the oldest claim
+//                           takes it
+//   contradiction           the default: a real disagreement, both kept, both
+//                           reported, never silently resolved
+//
+// Pure and import-free of core.mjs, exactly like trust.mjs and capability.mjs
+// beside it.
+
+import { negatedPredicate } from "./capability.mjs";
+import { AGENT_SOURCE_TYPES } from "./trust.mjs";
+
+export const RESOLUTION_MERGE = "merge";
+export const RESOLUTION_LATEST_OBSERVATION_WINS = "latest-observation-wins";
+export const RESOLUTION_FIRST_CLAIM_WINS = "first-claim-wins";
+export const RESOLUTION_CONTRADICTION = "contradiction";
+
+/** Predicates whose real-world semantics allow many objects at once — a wheel
+ *  is part of a car AND of a bike, a dog is a mammal AND a pet, you are LIKELY
+ *  to find a dog in a kennel AND in a park. The ConceptNet surface templates
+ *  say "typically", not "uniquely", so the associative and lexical relations
+ *  all sit here. Each entry's negative twin joins it: "cannot fly" and "cannot
+ *  sing" are two claims, not a self-contradiction. */
+const MERGE_PREDICATE_STEMS = [
+  "mgx:hasA", "mgx:capableOf",
+  "rdfs:subClassOf", "rdf:type", "owl:disjointWith", "mgx:partOf",
+  "mgx:usedFor", "mgx:receivesAction", "mgx:causes", "mgx:causesDesire",
+  "mgx:hasSubevent", "mgx:hasPrerequisite", "mgx:motivatedByGoal", "mgx:obstructedBy",
+  "mgx:desires", "mgx:hasProperty", "mgx:madeOf", "mgx:atLocation", "mgx:locatedNear",
+  "mgx:createdBy",
+  "mgx:mannerOf", "mgx:relatedTo", "mgx:synonym", "mgx:antonym", "mgx:similarTo", "mgx:symbolOf",
+  "mgx:knows-about",
+];
+
+export const MERGE_PREDICATES = new Set(
+  MERGE_PREDICATE_STEMS.flatMap((p) => [p, negatedPredicate(p)]),
+);
+
+/** Predicates whose object is the CURRENT state of something: a placement, a
+ *  position, a mutable property, a name. Two different objects are two moments,
+ *  not two opinions, so the newest observation renders and no contradiction is
+ *  reported — unless observation time cannot order them at all (see
+ *  resolveSiblingGroups). */
+const LATEST_OBSERVATION_PREDICATES = new Set([
+  "mgx:currently-in", "mgx:located-in", "mgx:fixed-in", "mgx:stands-locked-in", "mgx:hidden-in",
+  "mgx:on-top-of", "mgx:on-plane", "mgx:under",
+  "mgx:is-open", "mgx:hasMass", "mgx:feels", "mgx:faces", "mgx:pose",
+  "mgx:display-name", "mgx:nodeName", "mgx:worldName",
+]);
+
+/** The `mgx:has-exit-<direction>` family, which digging rewires: one more
+ *  state predicate, generated per direction rather than enumerated. */
+const EXIT_PREDICATE_RE = /^mgx:has-exit-([a-z]+)$/;
+
+/** Registrations: different objects race for one slot and the OLDEST claim
+ *  takes it. `mgx:playedBy` already ships this semantic ("first claim wins by
+ *  timestamp"); the table is where the rule lives now, instead of per-caller
+ *  lore. */
+const FIRST_CLAIM_PREDICATES = new Set(["mgx:playedBy"]);
+
+/** How sibling objects on `predicate` resolve. `contradiction` is the default,
+ *  so a predicate nobody has classified keeps the full keep-both contract. */
+export function resolutionStrategyFor(predicate) {
+  const p = String(predicate || "");
+  if (MERGE_PREDICATES.has(p)) return RESOLUTION_MERGE;
+  if (LATEST_OBSERVATION_PREDICATES.has(p) || EXIT_PREDICATE_RE.test(p)) return RESOLUTION_LATEST_OBSERVATION_WINS;
+  if (FIRST_CLAIM_PREDICATES.has(p)) return RESOLUTION_FIRST_CLAIM_WINS;
+  return RESOLUTION_CONTRADICTION;
+}
+
+/**
+ * When the asserting party WITNESSED this record's claim — valid time, as
+ * against mgx:createdAt's transaction time (when this store recorded it). The
+ * split is what makes a stale newspaper read today lose to an eyewitness report
+ * from yesterday; ingestion order gets that exactly backwards.
+ *
+ * The chain, in order:
+ *   1. the stored mgx:observedAt, when a caller supplied a parseable one;
+ *   2. for AGENT-kind sources only, the record's assertion time (its tag's own
+ *      embedded timestamp, else its createdAt) — a live agent asserting now is
+ *      witnessing now;
+ *   3. otherwise undefined.
+ *
+ * Document-kind sources (corpus, reference, web, extracted) and activity-kind
+ * (entailed) never fall back to createdAt: their createdAt is ingestion time
+ * and says nothing about observation. That is what resolves the newspaper case
+ * — an undated corpus row scores undefined and loses to any dated witness.
+ *
+ * Returns the instant as its ISO string, or undefined. Every returned value
+ * parses, so a caller may Date.parse it without re-checking.
+ */
+export function effectiveObservedAt(record) {
+  const stored = record?.observedAt;
+  if (stored && Number.isFinite(Date.parse(stored))) return String(stored);
+  if (!AGENT_SOURCE_TYPES.has(record?.sourceType)) return undefined;
+  for (const candidate of [record?.assertedAt, record?.createdAt]) {
+    if (candidate && Number.isFinite(Date.parse(candidate))) return String(candidate);
+  }
+  return undefined;
+}
+
+/** The instant that scores a whole object-group: the newest observation any of
+ *  its records carries for latest-observation-wins, the oldest for
+ *  first-claim-wins. null when no record in the group is dated at all. */
+function groupObservationInstant(row, oldest) {
+  let best = null;
+  for (const assertion of row?.assertions || []) {
+    const at = Date.parse(effectiveObservedAt(assertion) ?? "");
+    if (!Number.isFinite(at)) continue;
+    if (best === null || (oldest ? at < best : at > best)) best = at;
+  }
+  return best;
+}
+
+/**
+ * Resolve the object-groups of ONE (subject, predicate) under a time-ordered
+ * strategy. `rows` are stage-1 rows (one per triple group, so one per distinct
+ * object); `strategy` is latest-observation-wins or first-claim-wins. Returns
+ * null for any other strategy, since merge presents everything and
+ * contradiction resolves nothing.
+ *
+ * The ladder, in order:
+ *   1. score each group by its own observation instant (above);
+ *   2. a dated group beats an undated one, then the wanted extreme wins —
+ *      newest for latest-observation-wins, oldest for first-claim-wins;
+ *   3. tie (equal instants, or every group undated): higher aggregate trust;
+ *   4. still tied: the codepoint-smallest object string — plain `<`, never
+ *      localeCompare, so two peers holding the same records read the same
+ *      winner whatever locale they run under.
+ *
+ * `contested` reports whether the WINNER came down to step 3 or 4. Those are
+ * the cases time could not order, so they still belong in the contradiction
+ * report: the ranking exists so a page always has one deterministic answer to
+ * render, never so a disagreement disappears.
+ */
+export function resolveSiblingGroups(rows, strategy) {
+  const oldest = strategy === RESOLUTION_FIRST_CLAIM_WINS;
+  if (!oldest && strategy !== RESOLUTION_LATEST_OBSERVATION_WINS) return null;
+  const scored = (rows || []).map((row) => ({ row, at: groupObservationInstant(row, oldest) }));
+  scored.sort((a, b) => {
+    if (a.at !== b.at) {
+      if (a.at === null) return 1;
+      if (b.at === null) return -1;
+      return oldest ? a.at - b.at : b.at - a.at;
+    }
+    const trustGap = (b.row.trust || 0) - (a.row.trust || 0);
+    if (trustGap) return trustGap;
+    const [x, y] = [String(a.row.object), String(b.row.object)];
+    return x < y ? -1 : x > y ? 1 : 0;
+  });
+  const ranked = scored.map((s) => s.row);
+  return {
+    winner: ranked[0] || null,
+    ranked,
+    contested: scored.length > 1 && scored[0].at === scored[1].at,
+  };
+}