@polycode-projects/the-mechanical-code-talker 0.2.0 → 0.3.0

package/src/interpret/strategies/keywords.mjs

@@ -0,0 +1,185 @@
+// interpret/strategies/keywords.mjs — strategy 2: keyword-spotting/decomposition,
+// extracted MOVE-only from ask.mjs (item 13). ELIZA's own mechanism: find the
+// keyword(s) anywhere in the text, decompose around them, tolerate reordering and
+// casual phrasing. Position-independent (no `^...$` anchor), so it tolerates
+// "what calls this" / "who invokes this" / "something executes this, where from"
+// — real phrasings the anchored grammar's fixed shapes don't cover.
+
+import {
+  VERB_TO_KIND, ENTITY_TO_TYPE, MODIFIER_TO_KIND,
+  WHERE_MARKERS, MENTION_MARKERS,
+} from "../../ask-vocab.mjs";
+import { STOPWORDS } from "../normalize.mjs";
+import { VOCAB_WORDS, eligibleForCanon, fuzzyVocabWord } from "../fuzzy.mjs";
+
+/** Find the longest phrase from `table`'s keys that appears as a contiguous
+ *  run of `words` (case already lowercased by the caller). Longest-match-first
+ *  (multi-word phrases before single words) so "co-changes with" isn't
+ *  shadowed by a shorter unrelated word. A span overlapping `consumed` indices
+ *  is skipped: the verb and entity tables now share a surface form ("change"
+ *  is both a touches verb and the Change entity noun), and a word already
+ *  claimed by the verb pass must not double as the entity keyword. Returns
+ *  {kind, start, end} (end exclusive) or null. */
+export function findPhrase(lcWords, table, consumed = null) {
+  const phrases = Object.keys(table).sort((a, b) => b.split(" ").length - a.split(" ").length);
+  for (const p of phrases) {
+    const pWords = p.split(" ");
+    for (let i = 0; i <= lcWords.length - pWords.length; i += 1) {
+      if (consumed && pWords.some((_, j) => consumed.has(i + j))) continue;
+      if (pWords.every((w, j) => lcWords[i + j] === w)) return { kind: table[p], start: i, end: i + pWords.length };
+    }
+  }
+  return null;
+}
+
+/** Strategy 2: scan (already-normalized) text for a verb keyword anywhere,
+ *  plus optional entity/modifier keywords anywhere, then split whatever's
+ *  left (after removing the matched spans + stopwords) into the words BEFORE
+ *  and AFTER the verb. Which side(s) are non-empty decides the shape —
+ *  mirrors the three anchored shapes but by decomposition instead of a fixed
+ *  template, so it tolerates reordering/casual phrasing the anchored regexes
+ *  don't: text on BOTH sides ("does X import Y") -> ask{subject:before,
+ *  object:after}; only AFTER the verb ("what calls this") -> reverse{object:
+ *  after}; only BEFORE it ("what does X import") -> forward{object:before}.
+ *  A lone context pronoun ("this"/"it"/"that"/"here") ending up as a resolved
+ *  term is left as plain text — resolveTermOrContext (traverse-time)
+ *  recognizes it against an optional contextId, so no separate flag is
+ *  needed here. A misparse here costs nothing beyond an honest object-miss
+ *  downstream (resolveObject never guesses).
+ *
+ *  Keyword matching is TIERED (two-level fuzzy work, 2026-07-02) — each lower
+ *  tier fires ONLY when every tier above found no verb phrase at all, so an
+ *  exact curated match can never be displaced:
+ *    1. exact — the words as typed (post-normalization, which already applied
+ *       the curated CONTRACTIONS/MISSPELLINGS/WRONG_WORDS corrections);
+ *    2. lemma (only with the optional Node-side `nlp` adapter) — each eligible
+ *       word is replaced by its wink lemma IF that lemma is itself a vocab word
+ *       ("imported"/"importing" -> "import"), so inflections hit the curated
+ *       phrases without enumerating them. Every verb family already stores its
+ *       lemma form ("import", "call", "touch", "use", …), so a direct
+ *       lemma-in-vocab check is the whole lookup — no reverse index needed;
+ *    3. fuzzy (adapter-free; works in the inlined viewer too) — a word ≥4 chars
+ *       matching nothing exactly may rewrite to a UNIQUE verb/modifier
+ *       constituent within the bounded edit distance (see fuzzyVocabWord; ties
+ *       are refused, entity nouns are never fuzzy targets).
+ *  The canonicalized words drive PHRASE FINDING only — sideText always reads the
+ *  ORIGINAL words, so a correction can never corrupt an object/subject term. */
+export function parseKeywordSpot(text, nlp = null) {
+  // Strip a trailing "?" (mirrors the anchored templates' own `\??$`) and turn commas into
+  // pauses/spaces — but NEVER strip a mid-word ".": object terms are routinely dotted file/module
+  // names ("a.py", "utils.mjs"), and the anchored strategy captures those raw, so keyword-spot
+  // must too or the two strategies would "disagree" over a period that was never part of the intent.
+  const words = text.replace(/\?+\s*$/, "").replace(/,/g, " ").split(/\s+/).filter(Boolean);
+  const lcWords = words.map((w) => w.toLowerCase());
+  // where/mentions shapes (2026-07-02 query families): "where is X [defined]" and
+  // "where is X mentioned" carry NO relation verb, so the verb-driven decomposition
+  // below can never reach them. Routed here by the "where" question word + marker —
+  // but ONLY when no relation verb exists anywhere in the sentence: "something
+  // executes this, where from" (an existing worked phrasing) has a verb, and its
+  // "where" is decorative, not a location question.
+  if (lcWords.includes("where") && !findPhrase(lcWords, VERB_TO_KIND)) {
+    const mention = lcWords.some((w) => MENTION_MARKERS.includes(w));
+    const markers = new Set([...WHERE_MARKERS, ...MENTION_MARKERS]);
+    const objText = words.filter((w, i) => !STOPWORDS.has(lcWords[i]) && !markers.has(lcWords[i])).join(" ").trim();
+    if (objText) {
+      const kind = mention ? "mentions" : "where";
+      return { shape: kind, entityType: null, modifier: "direct", kind, object: objText };
+    }
+  }
+  let canonWords = lcWords;
+  let verbHit = findPhrase(lcWords, VERB_TO_KIND);
+  if (!verbHit && nlp) {
+    // tier 2: lemma (see the tier doc above) — replace only when the lemma is
+    // itself vocabulary, so unknown words ("myfile") pass through untouched.
+    const lemmaWords = lcWords.map((w) => {
+      if (!eligibleForCanon(w)) return w;
+      const l = nlp.lemma(w);
+      return VOCAB_WORDS.has(l) ? l : w;
+    });
+    verbHit = findPhrase(lemmaWords, VERB_TO_KIND);
+    if (verbHit) canonWords = lemmaWords;
+  }
+  if (!verbHit) {
+    // tier 3: bounded-edit-distance rewrite toward verb/modifier keywords only
+    // ("impotr" -> "import"); ≥4-char words only — below that the bound covers
+    // half of English (and "and" is 1 edit from the "land in" constituent).
+    const fuzzyWords = lcWords.map((w) => (w.length >= 4 && eligibleForCanon(w) ? fuzzyVocabWord(w) || w : w));
+    verbHit = findPhrase(fuzzyWords, VERB_TO_KIND);
+    if (verbHit) canonWords = fuzzyWords;
+  }
+  if (!verbHit) return null;
+  // POS consumer (wink adapter, Node-side only): rescue the ONE decomposition this
+  // strategy provably mis-parses — a relation word used as a NOUN in a "the
+  // <imports> of <term>" nominal ("show the imports of walk.mjs" otherwise
+  // decomposes to ask{subject:"show"}; bare "the imports of walk.mjs" to the
+  // reverse shape, both wrong). The wink probe showed "import" is tagged NOUN even
+  // in genuine verb use ("which modules import walk.mjs"), so the POS signal is
+  // deliberately NOT a general verb veto — it only fires inside this exact
+  // det+NOUN+"of" frame, where the nominal reading is grammatically forced.
+  if (nlp && verbHit.end - verbHit.start === 1) {
+    const i = verbHit.start;
+    const det = lcWords[i - 1];
+    if ((det === "the" || det === "these" || det === "those") && lcWords[i + 1] === "of") {
+      const tags = nlp.posTags(words);
+      if (tags[i] === "NOUN") {
+        const objText = words.slice(i + 2).filter((w, j) => !STOPWORDS.has(lcWords[i + 2 + j])).join(" ").trim();
+        if (objText) return { shape: "forward", entityType: null, modifier: "direct", kind: verbHit.kind, object: objText };
+      }
+    }
+  }
+  const consumed = new Set();
+  const mark = (hit) => { if (hit) for (let i = hit.start; i < hit.end; i += 1) consumed.add(i); };
+  mark(verbHit);
+  const entityHit = findPhrase(canonWords, ENTITY_TO_TYPE, consumed);
+  mark(entityHit);
+  const modifierHit = findPhrase(canonWords, MODIFIER_TO_KIND, consumed);
+  mark(modifierHit);
+  const sideText = (from, to) => words
+    .slice(from, to)
+    .filter((_, j) => !consumed.has(from + j) && !STOPWORDS.has(lcWords[from + j]))
+    .join(" ")
+    .trim();
+  const beforeText = sideText(0, verbHit.start);
+  const afterText = sideText(verbHit.end, words.length);
+  const kind = verbHit.kind;
+  // slices read canonWords, not lcWords: the entity/modifier spans were matched
+  // against the canonicalized array, whose word IS the table key.
+  const entityType = entityHit ? ENTITY_TO_TYPE[canonWords.slice(entityHit.start, entityHit.end).join(" ")] : null;
+  const modifier = modifierHit ? MODIFIER_TO_KIND[canonWords.slice(modifierHit.start, modifierHit.end).join(" ")] : "direct";
+
+  // when shape (2026-07-02 query families): "when did X change" / "when was X last
+  // touched" — the "when" question word turns a touches decomposition temporal.
+  // Only touches carries commit dates to answer with; a "when" next to any other
+  // relation verb falls through to the ordinary shapes (and their honest answers).
+  if (kind === "touches" && lcWords.includes("when")) {
+    const objText = beforeText || afterText;
+    if (objText) return { shape: "when", entityType: null, modifier: "direct", kind: "touches", object: objText };
+  }
+
+  if (beforeText && afterText) return { shape: "ask", entityType: null, modifier: "direct", kind, subject: beforeText, object: afterText };
+  if (afterText) return { shape: "reverse", entityType, modifier, kind, object: afterText };
+  // forward keeps the spotted entityType ("which modules did commit <sha> touch" is a
+  // forward decomposition — subject before the verb — whose asked grain would otherwise
+  // be lost); traverse() only consults it for the commit-as-subject grain selection,
+  // so plain forwards behave exactly as before. Modifier stays hardcoded: no forward
+  // closure traversal exists (see ask.mjs's modifierIsWired).
+  if (beforeText) return { shape: "forward", entityType, modifier: "direct", kind, object: beforeText };
+  return null;
+}
+
+/** Pipeline registration (interpret/pipeline.mjs): keyword-spotting as a
+ *  strategy. Class "graph-query" — shared with the anchored grammar, so the two
+ *  merge (agree/disagree) exactly as the legacy two-way merge did. Confidence
+ *  0.7: a decomposition is looser evidence than a full-template match. The
+ *  lemma/POS adapter arrives via ctx.nlp (the pipeline's default is the same
+ *  Node-only wink adapter ask.mjs picks up). */
+export const keywordSpotStrategy = {
+  id: "keyword-spot",
+  class: "graph-query",
+  run(text, ctx = {}) {
+    const parsed = parseKeywordSpot(text, ctx.nlp || null);
+    return parsed
+      ? { strategyId: "keyword-spot", class: "graph-query", candidates: [{ parsed, confidence: 0.7 }] }
+      : null;
+  },
+};

package/src/interpret/strategies/noise-strip.mjs

@@ -0,0 +1,114 @@
+// interpret/strategies/noise-strip.mjs — the item-10 noise-tolerant fallback
+// strategy: strip filler/noise/stop words the closed grammar gives no meaning to,
+// then RE-RUN THE PARSE over what's left (the anchored grammar first, then the
+// keyword-spot decomposition — see the discipline notes). Rationale: the keyword-spot
+// strategy decomposes around the verb and a leading vocative/adverb lands in the
+// SUBJECT slot ("hey man which modules import X" -> ask{subject:"man"}), an
+// unresolvable term the relaxation cascade can only rescue when the true answer
+// is positive (it refuses to relax into a miss) — so a noise-wrapped question
+// whose honest answer is negative/empty dies as "couldn't resolve one of the
+// terms". Stripping the noise FIRST recovers the template parse and the same
+// honest answer the clean phrasing gets.
+//
+// Discipline (no behavior change to anything that already parses):
+//   · fires ONLY when the anchored grammar MISSES the text as-given — if a
+//     template already matches, the grammar owns the sentence, noise and all;
+//   · strips ONLY all-lowercase alphabetic tokens (a Capitalized/dotted/digit
+//     token names something) that are curated noise (FILLER_WORDS + the
+//     cascade's CASCADE_NOISE) or wink-flagged English stop words (ctx.nlp's
+//     isStopWord — the optional Node-only tier), and NEVER a word in KEEP: the
+//     grammar's own vocabulary, question scaffolding, and context pronouns;
+//   · returns a candidate ONLY when the stripped text then parses — first against
+//     the anchored TEMPLATES (the strictest parser), then (cycle-2 robustness,
+//     CHATBENCH_001 L1) against the keyword-spot decomposition over the SAME
+//     stripped text. The second tier exists because the clean phrasing of half
+//     the worked questions ("what calls fnAlpha") is itself a keyword-spot
+//     parse, not a template — so a noise-wrapped variant ("i was wondering what
+//     calls fnAlpha", "hey tmct, what calls fnAlpha thanks") could never be
+//     rescued by the template re-parse alone and died as "couldn't resolve one
+//     of the terms". The keyword-spot re-parse also swallows auxiliary-verb
+//     residue for free ("was what calls fnAlpha" → reverse{fnAlpha}): its
+//     decomposition filters STOPWORDS out of the subject/object sides, which is
+//     exactly where a stripped frame's "was"/"is" residue lands. Same cost
+//     bound as before: only curated-noise/stop-word tokens were removed, and a
+//     wrongly-stripped word can at worst cost an honest object-miss downstream
+//     (resolveObject never guesses), never manufacture an entity.
+//
+// Registered with its own class ("noise-stripped") at confidence 0.75: above
+// keyword-spot (0.7 — over noisy text its decomposition has provably swallowed
+// the noise into a term) and below the anchored grammar (0.9 — which anyway
+// gates this strategy off whenever it fires). A distinct class, so disagreement
+// with keyword-spot is a ranked winner + "if you mean X then …" alternate,
+// never a forced same-class ambiguity over a garbage subject.
+
+import {
+  VERB_TO_KIND, ENTITY_TO_TYPE, MODIFIER_TO_KIND, META_MEANING_VERBS,
+  WHERE_MARKERS, MENTION_MARKERS, RELATIVE_PRONOUNS, PLACEHOLDER_NOUNS,
+  BOOLEAN_CONNECTIVES, QUALIFIERS, AGGREGATE_TRIGGERS, LIST_TRIGGERS,
+  SUPERLATIVE_EXTREMES, EDGE_NOUN_TO_METRIC, ANAPHORA_TRIGGERS,
+  CONTEXT_PRONOUNS, CASCADE_NOISE, CASCADE_SYNONYMS, FILLER_WORDS,
+} from "../../ask-vocab.mjs";
+import { STOPWORDS, splitWords, wordsOf } from "../normalize.mjs";
+import { parseAnchored } from "./grammar.mjs";
+import { parseKeywordSpot } from "./keywords.mjs";
+
+/** Words this strategy may NEVER strip: everything the closed grammar gives
+ *  query meaning to, plus the question/auxiliary scaffolding and the context
+ *  pronouns. Mirrors the cascade's CONTENT_VOCAB ∪ STRUCTURAL_WORDS union —
+ *  wink flags "which"/"what"/"does" as stop words, and stripping those would
+ *  destroy the very templates this strategy re-runs. */
+const KEEP = new Set([
+  ...STOPWORDS, ...wordsOf(CONTEXT_PRONOUNS),
+  ...wordsOf(Object.keys(VERB_TO_KIND)), ...wordsOf(Object.keys(ENTITY_TO_TYPE)),
+  ...wordsOf(Object.keys(MODIFIER_TO_KIND)), ...wordsOf(Object.keys(QUALIFIERS)),
+  ...wordsOf(AGGREGATE_TRIGGERS), ...wordsOf(LIST_TRIGGERS),
+  ...wordsOf(Object.keys(SUPERLATIVE_EXTREMES)), ...wordsOf(Object.keys(EDGE_NOUN_TO_METRIC)),
+  ...wordsOf(Object.keys(BOOLEAN_CONNECTIVES)), ...wordsOf(PLACEHOLDER_NOUNS),
+  ...wordsOf(ANAPHORA_TRIGGERS), ...wordsOf(META_MEANING_VERBS),
+  ...wordsOf(WHERE_MARKERS), ...wordsOf(MENTION_MARKERS), ...wordsOf(RELATIVE_PRONOUNS),
+  ...wordsOf(Object.keys(CASCADE_SYNONYMS)),
+]);
+
+/** The curated noise tier — FILLER_WORDS (normally consumed by normalizeQuery;
+ *  carried here too so the strategy stands alone) + the cascade's noise list. */
+const CURATED_NOISE = new Set([...wordsOf(FILLER_WORDS), ...wordsOf(CASCADE_NOISE)]);
+
+/** Strip the strippable tokens (see the file doc). Returns {text, dropped}. */
+export function stripNoise(text, nlp = null) {
+  const kept = [];
+  const dropped = [];
+  for (const w of splitWords(text)) {
+    const lc = w.toLowerCase();
+    const strippable = /^[a-z]+$/.test(w) && !KEEP.has(lc)
+      && (CURATED_NOISE.has(lc)
+        || (nlp && typeof nlp.isStopWord === "function" && nlp.isStopWord(lc)));
+    if (strippable) dropped.push(w);
+    else kept.push(w);
+  }
+  return { text: kept.join(" "), dropped };
+}
+
+/** Pipeline registration (interpret/pipeline.mjs). */
+export const noiseStripStrategy = {
+  id: "noise-strip",
+  class: "noise-stripped",
+  run(text, ctx = {}) {
+    if (parseAnchored(text)) return null; // the grammar owns the text as-given
+    const { text: stripped, dropped } = stripNoise(text, ctx.nlp || null);
+    if (!dropped.length || !stripped) return null;
+    // tier 1: the anchored templates over the stripped text — the strictest
+    // re-parse, tried first so a template shape is never displaced by a looser
+    // decomposition of the same words. tier 2 (cycle-2, CHATBENCH_001 L1): the
+    // keyword-spot decomposition over the SAME stripped text — the parser the
+    // clean phrasing of non-template questions ("what calls fnAlpha") actually
+    // uses, so their noise-wrapped variants recover the identical honest answer
+    // (see the file doc for the discipline/cost argument).
+    const parsed = parseAnchored(stripped) || parseKeywordSpot(stripped, ctx.nlp || null);
+    if (!parsed) return null;
+    return {
+      strategyId: "noise-strip",
+      class: "noise-stripped",
+      candidates: [{ parsed, confidence: 0.75, note: `noise-stripped to "${stripped}"` }],
+    };
+  },
+};

package/src/memory/blocks.mjs

@@ -0,0 +1,201 @@
+// memory/blocks.mjs — text blocks + the relevance index (ROADMAP item 9).
+//
+// The memory graph (core.mjs) holds STRUCTURE; this module holds TEXT: cleaned
+// session transcripts (fold.mjs writes one block per session) and, later,
+// corpus documents. Storage under <repo>/.tmct/memory/blocks/:
+//   <block-id>.txt   — the plain block text
+//   index.json       — per-block prose tokens (prose.mjs tokenizer) + a static rank
+//
+// Ranking is two signals, combined at query time:
+//   1. STATIC rank — a genuine iterative PageRank over the block-similarity
+//      graph (an undirected edge wherever two blocks share ≥ OVERLAP_MIN
+//      tokens; damping 0.85, 20 iterations — cheap at this scale). A block that
+//      shares vocabulary with many other blocks is "well-connected": it covers
+//      ground the corpus keeps returning to, so it wins ties.
+//   2. QUERY-time IDF match — each query token is weighted by rarity across
+//      blocks (idf = log(1 + N/(1+df)), the codegraph.mjs locate discipline),
+//      so a whole-question query is not dominated by its ubiquitous words.
+// retrieveBlocks() scores idf-sum × (1 + rank): the IDF match decides topic,
+// the static rank breaks ties toward well-connected blocks.
+//
+// All writes are temp+rename atomic; saveBlock is an upsert (same id replaces —
+// what makes fold.mjs's re-fold idempotent).
+
+import { mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { splitIdentifierWords, tokenizeProse } from "../prose.mjs";
+
+export const BLOCKS_DIR_REL = join(".tmct", "memory", "blocks");
+const INDEX_NAME = "index.json";
+
+export const PAGERANK_DAMPING = 0.85;
+export const PAGERANK_ITERATIONS = 20;
+const OVERLAP_MIN = 2;          // shared tokens for a similarity edge
+const MAX_TOKENS_PER_BLOCK = 800; // beyond tokenizeProse's per-doc cap: union over lines
+
+const blocksDir = (dir) => join(dir, BLOCKS_DIR_REL);
+
+/** Only safe, filesystem-friendly block file names (ids are session uuids or
+ *  corpus slugs; anything else is normalized, never trusted into a path). */
+const safeName = (id) => String(id).replace(/[^A-Za-z0-9._-]+/g, "_").slice(0, 120) || "_";
+
+/** Tokenize a whole block: prose tokens + identifier decomposition, unioned per
+ *  line so tokenizeProse's 120-token-per-doc cap can't starve a long transcript;
+ *  bounded overall. Sorted + deduped — the index stays deterministic. */
+export function tokenizeBlock(text) {
+  const set = new Set();
+  for (const line of String(text || "").split("\n")) {
+    for (const t of tokenizeProse(line)) set.add(t);
+    for (const t of splitIdentifierWords(line)) set.add(t);
+    if (set.size >= MAX_TOKENS_PER_BLOCK) break;
+  }
+  return [...set].sort().slice(0, MAX_TOKENS_PER_BLOCK);
+}
+
+async function atomicWrite(file, text) {
+  const tmp = `${file}.tmp-${process.pid}-${Math.random().toString(36).slice(2, 10)}`;
+  await writeFile(tmp, text);
+  await rename(tmp, file);
+}
+
+/** Load the block index ({ blocks: { id: { file, tokens[], rank } } }); a
+ *  missing index is the empty bootstrap. */
+export async function loadBlockIndex(dir) {
+  try {
+    return JSON.parse(await readFile(join(blocksDir(dir), INDEX_NAME), "utf8"));
+  } catch (e) {
+    if (e?.code === "ENOENT") return { blocks: {} };
+    throw e;
+  }
+}
+
+/**
+ * Iterative PageRank over the block-similarity graph. `tokensById` is a plain
+ * { id: tokens[] } map; an undirected edge joins two blocks sharing at least
+ * `overlapMin` tokens. Standard damped iteration (d=0.85, 20 rounds), dangling
+ * mass redistributed evenly, ranks summing to ~1. Pure — returns { id: rank }.
+ */
+export function rankBlocks(tokensById, {
+  damping = PAGERANK_DAMPING, iterations = PAGERANK_ITERATIONS, overlapMin = OVERLAP_MIN,
+} = {}) {
+  const ids = Object.keys(tokensById || {});
+  const N = ids.length;
+  if (!N) return {};
+  const sets = ids.map((id) => new Set(tokensById[id] || []));
+
+  // similarity edges: shared-token count ≥ overlapMin (undirected → both directions)
+  const neighbours = ids.map(() => []);
+  for (let i = 0; i < N; i += 1) {
+    for (let j = i + 1; j < N; j += 1) {
+      const [small, big] = sets[i].size <= sets[j].size ? [sets[i], sets[j]] : [sets[j], sets[i]];
+      let shared = 0;
+      for (const t of small) {
+        if (big.has(t) && (shared += 1) >= overlapMin) break;
+      }
+      if (shared >= overlapMin) {
+        neighbours[i].push(j);
+        neighbours[j].push(i);
+      }
+    }
+  }
+
+  let rank = new Array(N).fill(1 / N);
+  for (let round = 0; round < iterations; round += 1) {
+    const next = new Array(N).fill((1 - damping) / N);
+    let dangling = 0;
+    for (let i = 0; i < N; i += 1) {
+      const out = neighbours[i].length;
+      if (!out) { dangling += rank[i]; continue; }
+      const share = (damping * rank[i]) / out;
+      for (const j of neighbours[i]) next[j] += share;
+    }
+    const danglingShare = (damping * dangling) / N; // dangling mass spread evenly
+    for (let i = 0; i < N; i += 1) next[i] += danglingShare;
+    rank = next;
+  }
+  const out = {};
+  for (let i = 0; i < N; i += 1) out[ids[i]] = rank[i];
+  return out;
+}
+
+/** Re-tokenize nothing, re-rank everything: recompute every block's static rank
+ *  from the tokens already in the index (called by saveBlock/removeBlock). */
+function rerank(index) {
+  const tokensById = {};
+  for (const [id, b] of Object.entries(index.blocks)) tokensById[id] = b.tokens || [];
+  const ranks = rankBlocks(tokensById);
+  for (const [id, b] of Object.entries(index.blocks)) b.rank = ranks[id] ?? 0;
+  return index;
+}
+
+/**
+ * Upsert one text block: write <id>.txt (atomic), tokenize it, update
+ * index.json and recompute the static ranks. Same id → replaced, never
+ * duplicated (fold.mjs's re-fold idempotency rests on this).
+ * Returns the block's index entry { file, tokens, rank }.
+ */
+export async function saveBlock(dir, { id, text }) {
+  if (!id) throw new Error("a block needs an id");
+  const bdir = blocksDir(dir);
+  await mkdir(bdir, { recursive: true });
+  const file = `${safeName(id)}.txt`;
+  await atomicWrite(join(bdir, file), String(text ?? ""));
+  const index = await loadBlockIndex(dir);
+  index.blocks[id] = { file, tokens: tokenizeBlock(text) };
+  rerank(index);
+  await atomicWrite(join(bdir, INDEX_NAME), JSON.stringify(index));
+  return index.blocks[id];
+}
+
+/** Remove a block (id unknown → no-op) and re-rank the survivors. */
+export async function removeBlock(dir, id) {
+  const index = await loadBlockIndex(dir);
+  const entry = index.blocks[id];
+  if (!entry) return false;
+  delete index.blocks[id];
+  rerank(index);
+  await atomicWrite(join(blocksDir(dir), INDEX_NAME), JSON.stringify(index));
+  await rm(join(blocksDir(dir), entry.file), { force: true });
+  return true;
+}
+
+/**
+ * The top-k blocks a question "touches": IDF-weighted token match (rarity-
+ * weighted, so common words can't dominate) combined with the static PageRank
+ * (score × (1 + rank) — on an IDF tie the better-connected block wins).
+ * Returns [{ id, score, rank, file, text }], best first; [] when nothing
+ * matches (never a guessed block).
+ */
+export async function retrieveBlocks(dir, query, k = 3) {
+  const index = await loadBlockIndex(dir);
+  const entries = Object.entries(index.blocks);
+  const N = entries.length;
+  if (!N) return [];
+  const qTokens = [...new Set([...tokenizeProse(query), ...splitIdentifierWords(query)])];
+  if (!qTokens.length) return [];
+
+  const sets = entries.map(([, b]) => new Set(b.tokens || []));
+  // idf per query token, computed once: log(1 + N/(1+df)) — the codegraph.mjs
+  // locate weighting (~0 for ubiquitous tokens, large for rare ones).
+  const idf = new Map(qTokens.map((t) => {
+    let df = 0;
+    for (const s of sets) if (s.has(t)) df += 1;
+    return [t, Math.log(1 + N / (1 + df))];
+  }));
+  const scored = [];
+  for (let i = 0; i < N; i += 1) {
+    const [id, b] = entries[i];
+    let idfSum = 0;
+    for (const t of qTokens) if (sets[i].has(t)) idfSum += idf.get(t);
+    if (idfSum <= 0) continue;
+    const rank = b.rank ?? 0;
+    scored.push({ id, score: idfSum * (1 + rank), rank, file: b.file });
+  }
+  scored.sort((a, b) => b.score - a.score || b.rank - a.rank || a.id.localeCompare(b.id));
+  const top = scored.slice(0, Math.max(1, k));
+  for (const hit of top) {
+    try { hit.text = await readFile(join(blocksDir(dir), hit.file), "utf8"); }
+    catch { hit.text = ""; } // index/file drift — honest empty, never a crash
+  }
+  return top;
+}