@polycode-projects/the-mechanical-code-talker 6.0.15 → 6.0.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polycode-projects/the-mechanical-code-talker",
-  "version": "6.0.15",
+  "version": "6.0.16",
   "private": false,
   "type": "module",
   "description": "The Mechanical Code Talker (tmct) — a tolerant, offline, $0 chat surface that guides you toward precision queries about a software repository. ELIZA/PARRY-style but domain-obsessed with code. No model calls; indexes a repo on request (tmct index) or reads any producer's graph.",
@@ -118,6 +118,8 @@
     "gen:screenshots": "node scripts/gen-screenshots.mjs",
     "gen:og-images": "node scripts/gen-og-images.mjs",
     "probe:news-sources": "node scripts/probe-news-sources.mjs",
+    "bench:news": "node scripts/news-bench/run.mjs --seed=xl",
+    "bench:news:fast": "node scripts/news-bench/run.mjs --seed=fixture",
     "check:links": "node scripts/check-links.mjs",
     "check:pii": "node scripts/pii-lint.mjs",
     "check:pack": "node scripts/check-pack-manifest.mjs",

package/src/domain/fact-phrase.mjs

@@ -117,6 +117,41 @@ export function gerundVerbSurface(verb) {
   return /[^aeiou]e$/i.test(base) && base.length > 2 ? `${base.slice(0, -1)}ing` : `${base}ing`;
 }
 
+/** Irregular plural nouns whose surface carries no "-s" at all, so the suffix
+ *  check in isSubjectPlural below would read them as singular. A small closed
+ *  table, not this file's own pluralizer — it answers one question only
+ *  ("is this head noun plural"), never generates a form. */
+const IRREGULAR_PLURAL_NOUNS = new Set([
+  "people", "men", "women", "children", "mice", "geese", "feet", "teeth", "oxen",
+]);
+
+/** Nouns that end in "s" but stay grammatically SINGULAR ("the news
+ *  spreads", not "the news spread") — the suffix check below would otherwise
+ *  misread them as plural. */
+const SINGULAR_NOUNS_ENDING_S = new Set([
+  "news", "physics", "species", "series", "means", "measles", "mathematics", "politics", "economics",
+]);
+
+/**
+ * Is a stored fact's SUBJECT text grammatically plural, for the one thing
+ * this file needs it for: choosing a minted verb's surface form. Reads the
+ * HEAD noun only — the word right after any leading article, before a
+ * trailing "of ..." phrase, so "the group of scientists" agrees on "group"
+ * ("the group of scientists reports"), not "scientists". From there: the
+ * closed irregular table above, then the regular "-s" suffix, same
+ * naive-morphology trade thirdPersonSingularSurface already takes above. A
+ * subject this can't read (empty, or a plural-invariant noun like "sheep")
+ * defaults to singular — English's own unmarked form, and also this file's
+ * pre-existing default for every caller that passes no subject at all.
+ */
+export function isSubjectPlural(subject) {
+  const head = String(subject || "").trim().replace(/^(?:the|a|an)\s+/i, "").split(/\s+/)[0]?.toLowerCase() ?? "";
+  if (!head) return false;
+  if (IRREGULAR_PLURAL_NOUNS.has(head)) return true;
+  if (SINGULAR_NOUNS_ENDING_S.has(head)) return false;
+  return /[a-z]s$/.test(head) && !/ss$/.test(head);
+}
+
 /**
  * How a stored predicate reads in English: a curated table hit, else the
  * mechanical surface form of a minted predicate, else the predicate's local
@@ -124,11 +159,21 @@ export function gerundVerbSurface(verb) {
  * none). The local-name tail is what keeps a namespace tmct itself mints —
  * `tmct:needs`, stamped by the lexicon — out of a public-facing sentence.
  *
+ * `subject` is optional and used ONLY by the minted "mgx:<lemma>" verb fold
+ * (and the do-support it derives under negation): a plural subject reads the
+ * lemma's bare form ("scientists report"), anything else reads its
+ * third-person-singular fold ("an earthquake strikes"). A CURATED phrase is
+ * returned exactly as written regardless of subject — the table is fixed
+ * English on purpose, no morphology applied to it here or anywhere else.
+ * Callers with no subject to offer (most of chat.mjs's call sites, which
+ * render a bare predicate with no sentence around it) get today's singular
+ * default, unchanged.
+ *
  * Self-contained on purpose: the news page runs this exact function in the
- * browser (phraseRendererSource below stringifies it and its two helpers), so
- * it reaches for nothing outside this module.
+ * browser (phraseRendererSource below stringifies it and everything it calls
+ * into), so it reaches for nothing outside this module.
  */
-export function predicatePhrase(predicate) {
+export function predicatePhrase(predicate, subject) {
   if (FACT_PREDICATE_PHRASES[predicate]) return FACT_PREDICATE_PHRASES[predicate];
   const p = String(predicate ?? "");
   // NEGATIVE polarity renders as its own positive phrase, negated — ONE branch
@@ -144,12 +189,13 @@ export function predicatePhrase(predicate) {
   // for the same reason: a modal, a copula and a plain verb take different
   // negations, and nothing else does.
   if (p.startsWith("mgxneg:")) {
-    const phrase = predicatePhrase(`mgx:${p.slice("mgxneg:".length)}`);
+    const phrase = predicatePhrase(`mgx:${p.slice("mgxneg:".length)}`, subject);
     if (phrase === "can") return "cannot";
     if (phrase === "can be") return "cannot be";
     if (phrase === "is" || phrase.startsWith("is ")) return `is not${phrase.slice(2)}`;
     const [head, ...tail] = phrase.split(" ");
-    return ["does not", baseVerbSurface(head), ...tail].join(" ");
+    const doSupport = isSubjectPlural(subject) ? "do not" : "does not";
+    return [doSupport, baseVerbSurface(head), ...tail].join(" ");
   }
   // a comparative renders as its copula surface: mgx:smaller-than ->
   // "is smaller than" (never a 3sg fold — "smallers" isn't a word)
@@ -164,15 +210,23 @@ export function predicatePhrase(predicate) {
   const same = /^mgx:same-([a-z]+)-as$/i.exec(p);
   if (same) return `has the same ${same[1].toLowerCase()} as`;
   // a folded preposition renders back naturally: mgx:rest-on -> "rests on"
+  // (subject-verb agreement lives here alone: mgx:<lemma> is minted from the
+  // verb's OWN base form, so a plural subject reads it bare — "scientists
+  // report" — and anything else takes the 3sg fold — "an earthquake strikes")
   const minted = /^mgx:([a-z]+)(?:-([a-z]+))?$/i.exec(p);
-  if (minted) return `${thirdPersonSingularSurface(minted[1])}${minted[2] ? ` ${minted[2]}` : ""}`;
+  if (minted) {
+    const verb = isSubjectPlural(subject) ? minted[1] : thirdPersonSingularSurface(minted[1]);
+    return `${verb}${minted[2] ? ` ${minted[2]}` : ""}`;
+  }
   const colon = p.indexOf(":");
   return colon === -1 ? p : p.slice(colon + 1);
 }
 
-/** "a heart has a valve" from one { subject, predicate, object } fact row. */
+/** "a heart has a valve" from one { subject, predicate, object } fact row —
+ *  "scientists report a finding" for a plural row.subject, off the same
+ *  agreement rule in predicatePhrase above. */
 export function factSentence(row) {
-  return `${row.subject} ${predicatePhrase(row.predicate)} ${row.object}`;
+  return `${row.subject} ${predicatePhrase(row.predicate, row.subject)} ${row.object}`;
 }
 
 /**
@@ -224,6 +278,9 @@ export function phraseRendererSource() {
     `const TEACH_PARTICIPLE_SRC = ${JSON.stringify(TEACH_PARTICIPLE_SRC)};`,
     `const thirdPersonSingularSurface = ${thirdPersonSingularSurface};`,
     `const baseVerbSurface = ${baseVerbSurface};`,
+    `const IRREGULAR_PLURAL_NOUNS = new Set(${JSON.stringify([...IRREGULAR_PLURAL_NOUNS])});`,
+    `const SINGULAR_NOUNS_ENDING_S = new Set(${JSON.stringify([...SINGULAR_NOUNS_ENDING_S])});`,
+    `const isSubjectPlural = ${isSubjectPlural};`,
     `const predicatePhrase = ${predicatePhrase};`,
     `const factSentence = ${factSentence};`,
   ].join("\n  ");

package/src/domain/news-feed.mjs

@@ -71,6 +71,51 @@ export function newsWindowRows(rows, { now, windowMs }) {
   });
 }
 
+// ---------------------------------------------------------------------------
+// Item identity: what makes two fetched snapshots the same newsworthy item.
+// ---------------------------------------------------------------------------
+
+// Everything a source can respell between two readings of one article —
+// punctuation, capitalisation, entity escapes already stripped upstream, run
+// of spaces — folds away, so the key answers to the item's words alone.
+const CONTENT_KEY_NOISE_RE = /[^a-z0-9]+/g;
+
+function itemContentText(snapshot) {
+  return `${snapshot?.title ?? ""} ${snapshot?.summary ?? ""}`
+    .toLowerCase()
+    .replace(CONTENT_KEY_NOISE_RE, " ")
+    .trim();
+}
+
+/** The content key one snapshot answers to, or "" when it carries no words:
+ *  its source, its own publication stamp and its normalized text. The key a
+ *  source with no stable id of its own de-dupes on, and the second chance at
+ *  recognising an article a source re-issued under a fresh id. The publication
+ *  stamp stays in so two genuinely different events that happen to share a
+ *  headline — two quakes of the same size near the same town — keep separate
+ *  keys. Pure. */
+export function newsItemContentKey(snapshot) {
+  const text = itemContentText(snapshot);
+  if (!text) return "";
+  const sourceId = String(snapshot?.sourceId ?? "");
+  const publishedAt = String(snapshot?.publishedAt ?? "");
+  return `news-text:${sha256HexPrefix(`${sourceId}\0${publishedAt}\0${text}`, 8)}`;
+}
+
+/** Every key a fetched snapshot is the same item under: the id the fetcher
+ *  minted from the source's own identifier (Hacker News story id, USGS event
+ *  id, an RSS guid, a Wikinews page id, a Wikimedia article title) and its
+ *  content key. Two snapshots sharing any key name one item. Pure — a function
+ *  of the snapshot's own fields, never of when or in what order it arrived. */
+export function newsItemKeys(snapshot) {
+  const keys = [];
+  const id = String(snapshot?.id ?? "");
+  if (id) keys.push(id);
+  const contentKey = newsItemContentKey(snapshot);
+  if (contentKey) keys.push(contentKey);
+  return keys;
+}
+
 // ---------------------------------------------------------------------------
 // The newsworthiness gate (PLAN_NEWS_FEED.md section 17). A card reports what
 // a contemporary source said inside the window; everything the graph looked
@@ -580,7 +625,7 @@ export function renderNewsParagraph(hub, subgraphRows, { reportedIds = null } =
       .map((r) => r.object)
       .sort();
     if (!objects.length) continue;
-    sentences.push(`${hub} ${predicatePhrase(predicate)} ${joinObjects(objects)}`);
+    sentences.push(`${hub} ${predicatePhrase(predicate, hub)} ${joinObjects(objects)}`);
   }
 
   // A hub that only ever appears as an OBJECT — the place a quake struck, the

package/src/domain/term-ledger.mjs

@@ -31,8 +31,42 @@ const FUNCTION_WORD_TERMS = new Set([
   "usually", "indeed", "however", "therefore", "thus", "hence", "meanwhile",
 ]);
 
-function isFunctionWordTerm(term) {
-  return FUNCTION_WORD_TERMS.has(term);
+// A bare measurement-unit abbreviation names a quantity, not a subject —
+// "10 km SSW of Ridgecrest" leaks "km" as a standalone term. Closed set of
+// the abbreviations USGS/wire-service distance, weight and speed reports
+// actually use standalone.
+const MEASUREMENT_UNIT_TERMS = new Set([
+  "km", "kms", "m", "mi", "mis", "ft", "kg", "kgs", "mph", "kmh", "kph",
+  "cm", "mm", "nm", "yd", "yds", "lb", "lbs", "oz",
+]);
+
+// A compass abbreviation locates something relative to a place; it never
+// names the place itself — "10 km SSW of Ridgecrest" leaks "ssw" the same
+// way it leaks "km". Closed set: the 16 standard compass-rose points.
+const COMPASS_ABBREVIATION_TERMS = new Set([
+  "n", "s", "e", "w",
+  "ne", "nw", "se", "sw",
+  "nne", "ene", "ese", "sse", "ssw", "wsw", "wnw", "nnw",
+]);
+
+// A bare foreign article/preposition arrives only as a fragment of a title
+// that never got tokenized as a whole name ("Tour de France Femmes" leaking
+// "de"). Closed set, standalone only — this never matches "de" glued inside
+// a multi-word term like "tour de france", since bumpTerms and
+// ledgerFromPayload both key on the full normalized term, not its words.
+const FOREIGN_PARTICLE_TERMS = new Set([
+  "de", "la", "le", "du", "der", "von", "van", "di", "el",
+]);
+
+const NOISE_TERM_SETS = [
+  FUNCTION_WORD_TERMS,
+  MEASUREMENT_UNIT_TERMS,
+  COMPASS_ABBREVIATION_TERMS,
+  FOREIGN_PARTICLE_TERMS,
+];
+
+function isNoiseTerm(term) {
+  return NOISE_TERM_SETS.some((set) => set.has(term));
 }
 
 /** Ledger entry field order fixed once here so `ledgerPayload` serializes
@@ -62,7 +96,7 @@ export function createTermLedger() {
 export function bumpTerms(ledger, termCounts, itemId, now, vocabGroundedByTerm = new Map()) {
   for (const [rawTerm, occurrences] of termCounts) {
     const term = normFactTerm(rawTerm);
-    if (!term || isFunctionWordTerm(term)) continue;
+    if (!term || isNoiseTerm(term)) continue;
     let entry = ledger.terms.get(term);
     if (!entry) {
       const vocabGrounded = vocabGroundedByTerm.has(rawTerm)
@@ -139,7 +173,7 @@ export function ledgerPayload(ledger) {
 export function ledgerFromPayload(payload) {
   const ledger = createTermLedger();
   for (const entry of payload?.terms ?? []) {
-    if (isFunctionWordTerm(entry.term)) continue;
+    if (isNoiseTerm(entry.term)) continue;
     ledger.terms.set(entry.term, { ...entry, itemIds: [...(entry.itemIds ?? [])] });
   }
   return ledger;

package/src/services/news.mjs

@@ -17,11 +17,11 @@
 //              (the same invalidation convention chat.mjs's own caches use).
 //   lexicon    a loaded lexicon (loadLexicon() when absent).
 //   config     resolveNewsConfig()'s shape.
-//   state      the news-store shape (news-store.mjs): { items, ledger
-//              (ledgerPayload form), health, requestLog, metrics, lastPollAt,
-//              lastEnrichAt } — always JSON-plain; a live term ledger is
-//              built from state.ledger via ledgerFromPayload only for the
-//              span of one call, then folded back with ledgerPayload.
+//   state      the news-store shape (news-store.mjs): { items, seenItemKeys,
+//              ledger (ledgerPayload form), health, requestLog, metrics,
+//              lastPollAt, lastEnrichAt } — always JSON-plain; a live term
+//              ledger is built from state.ledger via ledgerFromPayload only
+//              for the span of one call, then folded back with ledgerPayload.
 //   providers  { newsFetchers: Map<sourceId, { id, fetchItems }>,
 //              getResearchProvider({ source }), preflightNewsUrl?(url) } —
 //              every fetcher and provider this session may call, already
@@ -44,7 +44,7 @@
 import { normFactTerm, normFactPredicate, factIdFor } from "../domain/hash.mjs";
 import {
   newsWindowRows, renderNewsParagraph, buildNewsItems, evictNewsFacts,
-  conceptTerms, isQuantityTerm,
+  conceptTerms, isQuantityTerm, newsItemKeys,
 } from "../domain/news-feed.mjs";
 import {
   createTermLedger, bumpTerms, rankedTerms, markTerm, groundedSweep, ledgerPayload, ledgerFromPayload,
@@ -473,38 +473,90 @@ function recordSuccess(health, nowVal, status) {
   health.autoDisabled = false;
 }
 
-/** Merges `incoming` snapshots into `existing` by id (an already-seen id is
- *  never re-added or re-ingested), then enforces `cap` by dropping the
- *  oldest-by-fetchedAt entries. Returns the merged list and the genuinely
- *  new snapshots the caller still needs to ingest. */
-function mergeSnapshotsById(existing, incoming, cap) {
-  const byIdMap = new Map((existing || []).map((s) => [s.id, s]));
-  const added = [];
-  for (const snap of incoming || []) {
-    if (byIdMap.has(snap.id)) continue;
-    byIdMap.set(snap.id, snap);
-    added.push(snap);
-  }
-  let items = [...byIdMap.values()];
-  if (items.length > cap) {
-    items = items
-      .slice()
-      .sort((a, b) => (toMs(a.fetchedAt) - toMs(b.fetchedAt)) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
-      .slice(items.length - cap);
-  }
-  return { items, added };
-}
-
 const fetchedAtMs = (snapshot) => {
   const ms = toMs(snapshot?.fetchedAt);
   return Number.isFinite(ms) ? ms : 0;
 };
 
-/** True once a snapshot has been through a grounding round. `mergeSnapshotsById`
- *  files a snapshot the moment it arrives, so "known" and "grounded" are two
- *  different states and only this one means the facts landed. */
+/** True once a snapshot has been through a grounding round. A snapshot is
+ *  filed the moment it arrives, so "known" and "grounded" are two different
+ *  states and only this one means the facts landed. */
 const isGroundedSnapshot = (snapshot) => (snapshot?.processedRounds || 0) > 0;
 
+const byIdAscending = (a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
+const byFetchedAtThenId = (a, b) => (fetchedAtMs(a) - fetchedAtMs(b)) || byIdAscending(a, b);
+// The item cap's drop order: a snapshot already read is the first to go, since
+// its facts are in the graph and its keys are remembered, while one still
+// waiting to be read would lose its facts for good.
+const alreadyReadFirst = (a, b) => (isGroundedSnapshot(b) - isGroundedSnapshot(a)) || byFetchedAtThenId(a, b);
+
+// How many item keys the de-dupe memory carries. Each grounded item files two
+// (its source id and its content key), so this remembers roughly a thousand
+// articles — many times the item window itself, which is the point: the window
+// forgets an article as soon as newer ones crowd it out, and without this the
+// next poll would read that same article as brand new.
+const SEEN_ITEM_KEY_CAP = 2000;
+
+const seenEntries = (seen) => (Array.isArray(seen) ? seen : []);
+
+/** Every key `snapshots` and `seen` between them name, newest first and capped
+ *  — an entry's `at` is the snapshot's own fetchedAt, never a fresh clock
+ *  reading. Ordered off the entries themselves, so the same items produce the
+ *  same memory whatever order they arrived in. */
+function rememberItemKeys(seen, snapshots) {
+  const atByKey = new Map();
+  const remember = (key, at) => {
+    if (!key) return;
+    const previous = atByKey.get(key);
+    atByKey.set(key, previous === undefined ? at : Math.max(previous, at));
+  };
+  for (const entry of seenEntries(seen)) {
+    const at = Number(entry?.at);
+    remember(String(entry?.key ?? ""), Number.isFinite(at) ? at : 0);
+  }
+  for (const snap of snapshots || []) {
+    for (const key of newsItemKeys(snap)) remember(key, fetchedAtMs(snap));
+  }
+  return [...atByKey.entries()]
+    .map(([key, at]) => ({ key, at }))
+    .sort((a, b) => (b.at - a.at) || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0))
+    .slice(0, SEEN_ITEM_KEY_CAP);
+}
+
+/** Merges `incoming` snapshots into `existing` by item identity: a snapshot
+ *  whose id or content key already sits in the window, or in the `seen` memory
+ *  of what has been grounded, is neither re-added nor re-ingested. Two
+ *  incoming snapshots naming one item collapse to the lower id, so the same
+ *  fetch read in two orders admits the same snapshot. The merged list sorts by
+ *  fetchedAt then id, so the window is a function of which items are in it
+ *  rather than of when each arrived, and `cap` then drops the snapshots that
+ *  have already been read before any that still have facts to contribute.
+ *  Returns the merged list and the genuinely new snapshots the caller still
+ *  needs to ingest. */
+function mergeSnapshots(existing, incoming, { cap, seen } = {}) {
+  const known = new Set();
+  for (const snap of existing || []) for (const key of newsItemKeys(snap)) known.add(key);
+  for (const entry of seenEntries(seen)) if (entry?.key) known.add(String(entry.key));
+
+  const claimed = new Set();
+  const admitted = new Set();
+  for (const snap of [...(incoming || [])].sort(byIdAscending)) {
+    const keys = newsItemKeys(snap);
+    if (!keys.length) continue;
+    if (keys.some((key) => known.has(key) || claimed.has(key))) continue;
+    for (const key of keys) claimed.add(key);
+    admitted.add(snap);
+  }
+
+  const added = (incoming || []).filter((snap) => admitted.has(snap));
+  const items = [...(existing || []), ...added].sort(byFetchedAtThenId);
+  if (items.length <= cap) return { items, added };
+  const dropped = new Set(
+    items.slice().sort(alreadyReadFirst).slice(0, items.length - cap).map((snap) => snap.id),
+  );
+  return { items: items.filter((snap) => !dropped.has(snap.id)), added };
+}
+
 /** One source's fetched-but-not-yet-grounded snapshots, oldest first. A cycle
  *  that ran out of time leaves its backlog here, so the next cycle works
  *  through that before anything newer and the same article is never ingested
@@ -513,7 +565,7 @@ const isGroundedSnapshot = (snapshot) => (snapshot?.processedRounds || 0) > 0;
 function pendingSnapshotsFor(items, sourceId) {
   return (items || [])
     .filter((snap) => snap?.sourceId === sourceId && !isGroundedSnapshot(snap))
-    .sort((a, b) => (fetchedAtMs(a) - fetchedAtMs(b)) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+    .sort(byFetchedAtThenId);
 }
 
 function emptyCycleAccumulator(at) {
@@ -575,7 +627,9 @@ export async function pollNewsSources(ctx) {
       recordSuccess(health, nowVal, "not-modified");
     } else {
       recordSuccess(health, nowVal, "ok");
-      const merged = mergeSnapshotsById(state.items, result.items, config.itemCap);
+      const merged = mergeSnapshots(state.items, result.items, {
+        cap: config.itemCap, seen: state.seenItemKeys,
+      });
       state.items = merged.items;
       added = merged.added;
       newItemsTotal += added.length;
@@ -614,6 +668,15 @@ export async function pollNewsSources(ctx) {
     if (aborted) break;
   }
 
+  // Only a GROUNDED snapshot is remembered. One the item cap dropped before it
+  // was ever read still has its facts to contribute, so the next poll is meant
+  // to pick it up again; one whose facts already landed must never be read a
+  // second time, however long ago the window forgot it.
+  state.seenItemKeys = rememberItemKeys(
+    state.seenItemKeys,
+    (state.items || []).filter(isGroundedSnapshot),
+  );
+
   const memory = await store.loadMemory(memoryDir);
   const rows = store.readFactRows(memory);
   const evictIds = evictNewsFacts(rows, { cap: config.newsFactCap });
@@ -998,7 +1061,7 @@ export function cycleMetrics(before, after, { source } = {}) {
 
 export function createNewsState() {
   return {
-    items: [], ledger: ledgerPayload(createTermLedger()), health: [], requestLog: [], metrics: [],
-    lastPollAt: "", lastEnrichAt: "",
+    items: [], seenItemKeys: [], ledger: ledgerPayload(createTermLedger()), health: [],
+    requestLog: [], metrics: [], lastPollAt: "", lastEnrichAt: "",
   };
 }