@polycode-projects/the-mechanical-code-talker 1.4.0 → 1.5.2

package/src/completions/complete.mjs

@@ -0,0 +1,138 @@
+// completions/complete.mjs — the full PLAN_COMPLETIONS.md pipeline, wired end to end:
+// broadSearch (Stage 1) -> groupHits (Stage 2) -> rankSentences per group + inferRelations
+// across groups (Stage 3+4) -> pruneCompletion's keep/drop (Stage 5) -> assemble the kept
+// sentences into prose -> finish()'s grammar/voice pass (Stage 6). §4's staging table, row 3:
+// "Stage 5+6 — pruning + grammar/voice pass wired end to end... Stage 6 reuses finish.mjs
+// directly... Exit criterion: A full end-to-end completion reads as one consistent voice and
+// every sentence traces to a source span."
+//
+// Extractive, by construction (PLAN_COMPLETIONS.md §3's honest ceiling): the assembled text is
+// the KEPT sentences, in order, joined by a single space — never paraphrased, never reordered
+// beyond what pruning's own group/rank order already decided, never smoothed over a genuine
+// disjointedness in the source material. Stage 6 fixes CAPITALISATION and PUNCTUATION at every
+// sentence boundary (this dispatch's own generalisation of src/finish.mjs's ruleCapitalise/
+// ruleTerminal); it does not invent, reorder, or merge any sentence's WORDS. Where the source
+// material simply doesn't chain into a fluent read, the output reads disjointedly and stays
+// honest about it — that is the plan's own accepted, documented limit, not a bug this module
+// tries to paper over.
+//
+// Traceability: `sourceSpans` is built directly from pruneCompletion()'s own `kept` list (which
+// already carries sourceBlockId + groupId per sentence) BEFORE the grammar pass runs, and the
+// grammar pass never reorders, drops, or merges sentences (only mutates casing/punctuation of
+// prose text) — so sourceSpans stays index-aligned with "one entry per output sentence" even
+// though the FINAL text is produced by finish(), not directly from `kept`. `relations` is
+// infer.mjs's own output, untouched by pruning (relations are never pruned, only the sentences
+// that anchor them are decided) — every relation still carries its own `licensingTest`/
+// `evidence` per infer.mjs's contract.
+//
+// Determinism: every stage this module chains is already deterministic (broadSearch, groupHits,
+// rankSentences, inferRelations, pruneCompletion, finish() are all pure/deterministic functions
+// over their inputs); this module adds no randomness of its own. See
+// test/completions-complete.test.mjs's own double-run diff.
+
+import { broadSearch } from "./search.mjs";
+import { groupHits } from "./group.mjs";
+import { rankSentences } from "./rank.mjs";
+import { inferRelations } from "./infer.mjs";
+import { pruneCompletion } from "./prune.mjs";
+import { loadMemory } from "../memory/core.mjs";
+import { finish, grammarRules } from "../finish.mjs";
+
+/** grammarRules() with sentence-capitalisation FORCE-ENABLED, everything else exactly as the
+ *  live table has it. sentence-capitalisation stays PARKED (enabled=false) in the live chat
+ *  table (grammar-rules.toml's own cycle-006 note: it regresses case-sensitive single-answer
+ *  goldens pinning lowercase openers) — that is a CHAT-voice decision, not a limitation of the
+ *  rule itself. A genuinely multi-sentence extractive completion has no such single-opener
+ *  voice to protect and NEEDS every internal sentence boundary capitalised to read as one
+ *  consistent voice (this pipeline's own exit criterion) — so this module's own Stage 6 call
+ *  overrides just that one rule's enabled flag, via finish()'s ctx.rules seam, without touching
+ *  the cached global table or any other caller's behaviour. */
+function completionGrammarRules() {
+  return grammarRules().map((r) => (r.id === "sentence-capitalisation" ? { ...r, enabled: true } : r));
+}
+
+const DEFAULT_MAX_SENTENCES_PER_GROUP = 3; // see prune.mjs's own file header for the reasoning
+
+/**
+ * The full mechanical-text-generation pipeline, Stage 1 through Stage 6.
+ *
+ * @param {string} dir  repo root (broadSearch's block corpus + loadMemory's fact store)
+ * @param {string} prompt  the broad prompt driving retrieval, grouping-focus ranking, and
+ *   (by default) relation-anchored pruning
+ * @param {object} [opts]
+ * @param {number} [opts.blockK]  broadSearch's block k (default: search.mjs's own default)
+ * @param {object|null} [opts.graphService=null]  optional Repository-Interface graph service,
+ *   passed straight through to broadSearch (see search.mjs)
+ * @param {number} [opts.graphLimit]  broadSearch's graph search() limit
+ * @param {number} [opts.overlapMin]  groupHits' shared-token edge threshold
+ * @param {object} [opts.memory]  an already-loaded memory/core.mjs loadMemory() payload; when
+ *   omitted, this module loads it itself via loadMemory(dir) (a fresh/empty memory is a
+ *   perfectly valid, honest input — inferRelations() simply asserts nothing over it)
+ * @param {string} [opts.query]  the query rankSentences()/pruning focus on; defaults to `prompt`
+ *   itself (query-focused summarization, PLAN_COMPLETIONS.md §1.4's own literature framing) —
+ *   pass `null` explicitly to fall back to self-weighted (LexRank-style) ranking instead
+ * @param {number} [opts.maxSentencesPerGroup=3]  prune.mjs's top-K-per-group cutoff
+ * @param {object} [opts.graph]  optional loaded graph (src/codegraph.mjs parseEntities() shape)
+ *   handed to finish()'s maskSegments so known entity labels inside the assembled text are
+ *   protected during the grammar pass — the same ctx.graph existing finish() call sites pass
+ * @returns {Promise<{
+ *   text: string,
+ *   sourceSpans: Array<{sourceBlockId:string, groupId:string, sentence:string}>,
+ *   relations: Array<object>,
+ *   dropped: Array<{item:object, reason:string}>,
+ *   declined?: boolean,
+ *   reason?: string,
+ * }>}
+ */
+export async function generateCompletion(dir, prompt, opts = {}) {
+  const {
+    blockK, graphService = null, graphLimit, overlapMin,
+    memory: memoryOpt, query = prompt, maxSentencesPerGroup = DEFAULT_MAX_SENTENCES_PER_GROUP,
+    graph,
+  } = opts;
+
+  // Stage 1 — broad search
+  const hits = await broadSearch(dir, prompt, { blockK, graphService, graphLimit });
+
+  // Stage 2 — grouping
+  const groups = groupHits(hits, { overlapMin });
+
+  // Stage 3 — cross-group inference (needs a loaded memory; a fresh/empty one is honest and
+  // simply asserts nothing, never fabricates something to compensate)
+  const memory = memoryOpt || await loadMemory(dir);
+  const relations = groups.length >= 2 ? await inferRelations(groups, memory) : [];
+
+  // Stage 4 — extractive sentence ranking, per group (query-focused unless the caller opted out)
+  const rankedByGroup = {};
+  for (const g of groups) rankedByGroup[g.id] = rankSentences(g, { query });
+
+  // Stage 5 — pruning: decide keep/drop, with an itemized, auditable drop log
+  const { kept, dropped } = pruneCompletion(
+    { hits, groups, relations, rankedByGroup },
+    { maxSentencesPerGroup },
+  );
+
+  if (!kept.length) {
+    // Honest decline (PLAN_COMPLETIONS.md §3): nothing cleared the pruning bar for this prompt
+    // over this corpus — never fabricate a completion to fill the gap.
+    return { text: "", sourceSpans: [], relations, dropped, declined: true, reason: "no source span cleared the pruning bar for this prompt" };
+  }
+
+  // Assemble: kept sentences, in pruneCompletion's own (group-id, rank-order) order, joined by a
+  // single space — purely extractive, no reordering/paraphrasing beyond that (§3's honest
+  // ceiling; see file header).
+  const rawText = kept.map((k) => k.sentence).join(" ");
+
+  // Stage 6 — grammar/voice pass. Reuses finish() verbatim (its maskSegments() masker still
+  // protects any known entity/path/number/receipt/provenance span WITHIN the assembled text,
+  // exactly as it does for a single-answer composed answer); only the rule table differs
+  // (sentence-capitalisation force-enabled — see completionGrammarRules() above).
+  const finished = finish({ answer: rawText, via: "completion" }, { graph, rules: completionGrammarRules() });
+
+  // sourceSpans traces every OUTPUT sentence back to its block id + group id — built from
+  // `kept` (pre-grammar-pass), which stays index-aligned with the final text because finish()'s
+  // grammar rules only mutate casing/punctuation of prose, never reorder/drop/merge sentences.
+  const sourceSpans = kept.map((k) => ({ sourceBlockId: k.sourceBlockId, groupId: k.groupId, sentence: k.sentence }));
+
+  return { text: finished.answer, sourceSpans, relations, dropped };
+}

package/src/completions/group.mjs

@@ -0,0 +1,171 @@
+// completions/group.mjs — Stage 2 ("grouping") of PLAN_COMPLETIONS.md's six-stage
+// mechanical-text-generation pipeline. Stage-0 scope ONLY per the plan's own §4 staging
+// table: cluster search.mjs's broadSearch() hits into topical groups; no cross-group
+// inference (Stage 3), summarization (Stage 4), pruning (Stage 5), or voice pass (Stage 6)
+// happens here.
+//
+// Genuinely greenfield, as the plan's own prior research established: PLAN_COMPLETIONS.md's
+// citation of "insights-panel tag clustering" as prior art turned out to be a different,
+// sibling project's capability, not tmct's — there was no clustering/tag-grouping code
+// anywhere in this repo before this file.
+//
+// Granularity: this module clusters at BLOCK granularity — the exact unit search.mjs's
+// broadSearch() hands it (one node per whole retrieveBlocks hit, or one node per whole
+// graph-search/ask result). PLAN_COMPLETIONS.md's own prose loosely says "spans"; that is
+// resolved explicitly here rather than silently assumed: no sub-block span segmentation is
+// attempted in this build. A finer-grained span-level increment, if the pipeline ever needs
+// it, is future work, out of scope for Stage 0.
+//
+// Algorithm: CONNECTED COMPONENTS over the block-similarity graph (shared-token-overlap
+// edges), reusing memory/blocks.mjs's buildNeighbours()/OVERLAP_MIN — the exact adjacency
+// rankBlocks()/degreeOf() already build for PageRank, exported from blocks.mjs for this
+// purpose rather than re-derived. This is the strategy-advisor's explicit Stage-0
+// recommendation: "the cheapest deterministic starting point" for grouping — genuinely
+// CoRank-adjacent (CoRank = "clustering cum graph ranking", PLAN_COMPLETIONS.md §1.2) since
+// it clusters over the same graph-ranking substrate blocks.mjs's PageRank runs on, and fully
+// deterministic/auditable, unlike any softer/fuzzier clustering. Richer ranking WITHIN a
+// cluster (e.g. running rankBlocks per component, or a true joint clustering+ranking pass)
+// is a future increment if the pipeline needs it later — not attempted here.
+//
+// Labeling: each group carries its member ids AND a human-inspectable label — its top
+// shared-IDF tokens. The only term-frequency machinery this repo ships is the
+// idf = log(1 + N/(1+df)) formula memory/blocks.mjs's retrieveBlocks() already computes
+// inline (there is no standalone exported idf() to import — it lives inline in
+// retrieveBlocks, scoped to a single query's tokens); that exact formula is replicated here,
+// scoped instead to df/N over the hit set passed into groupHits(), rather than inventing a
+// new weighting scheme.
+//
+// Determinism: no randomness anywhere. Same hits in (same ids, same text, same order) ->
+// same groups out (same partition, same member order, same label) — this is exactly what
+// test/completions-stage0.test.mjs's double-run diff asserts as Stage 0's exit criterion.
+
+import { buildNeighbours, OVERLAP_MIN, tokenizeBlock } from "../memory/blocks.mjs";
+import { STOPWORDS } from "../prose.mjs";
+
+const LABEL_TOKEN_COUNT = 5;
+
+// tokenizeBlock unions tokenizeProse(line) (stopword-filtered, punctuation-stripped) with
+// splitIdentifierWords(line) (NOT stopword-filtered — it's built for code identifiers, where
+// a stopword-shaped fragment is rare, and it doesn't strip trailing sentence punctuation
+// either). For prose-similarity clustering that union re-admits filler ("the", "and", "is",
+// "q:", "a:", "do?", …) as if it were real overlap signal — harmless for retrieveBlocks'
+// IDF-weighted single-query scoring (rare query tokens still dominate), but fatal for
+// unweighted shared-token-overlap clustering: almost any two English sentences share ≥2
+// stopwords, which would collapse everything into one giant component. isContentToken()
+// closes that gap for THIS module's own adjacency/labeling use, without changing
+// tokenizeBlock's shipped behavior (still exactly what retrieveBlocks/rankBlocks use).
+const isContentToken = (t) => /^[a-z0-9]+$/.test(t) && !STOPWORDS.has(t);
+
+/** tokenizeBlock(text), narrowed to real content tokens (see isContentToken above) — the
+ *  token set this module actually clusters and labels on. */
+function contentTokens(text) {
+  return tokenizeBlock(text).filter(isContentToken);
+}
+
+/** Plain union-find (path halving, union-by-index) — small N here (a single broad search's
+ *  hit count), so this is deliberately the simplest correct structure, not a fancy one. */
+function unionFind(n) {
+  const parent = Array.from({ length: n }, (_, i) => i);
+  function find(x) {
+    while (parent[x] !== x) { parent[x] = parent[parent[x]]; x = parent[x]; }
+    return x;
+  }
+  function union(a, b) {
+    const ra = find(a);
+    const rb = find(b);
+    if (ra !== rb) parent[Math.max(ra, rb)] = Math.min(ra, rb);
+  }
+  return { find, union };
+}
+
+/**
+ * Stage 2 — grouping. Clusters a flat hit list (search.mjs's broadSearch() output, or any
+ * `{ id, text }` array) into topical groups via connected components over the shared-token-
+ * overlap similarity graph (the same adjacency memory/blocks.mjs's PageRank runs over).
+ *
+ * @param {Array<{id:string, text:string}>} hits
+ * @param {object} [opts]
+ * @param {number} [opts.overlapMin=OVERLAP_MIN]  shared-token threshold for a similarity
+ *   edge — defaults to the exact value memory/blocks.mjs's own PageRank graph uses, so
+ *   grouping and ranking agree on what "related" means unless the caller deliberately
+ *   overrides it.
+ * @returns {Array<{ id: string, members: Array<{id:string, text:string}>, memberIds: string[],
+ *   tokens: string[], label: string }>}
+ *   One entry per connected component (a singleton hit that shares no edge with anything
+ *   else is still a valid, if lonely, one-member group — never dropped). Deterministic
+ *   order: groups sorted by their lowest member id; members within a group sorted by id.
+ *   `id` is the group's own stable id (`"g:" + memberIds[0]`, deterministic from content).
+ */
+export function groupHits(hits, { overlapMin = OVERLAP_MIN } = {}) {
+  const list = Array.isArray(hits) ? hits.filter((h) => h && h.id != null) : [];
+  if (!list.length) return [];
+
+  // dedupe by id — a hit can legitimately appear from more than one source (e.g. a block
+  // and a graph-ask answer both id-tagged distinctly, but a caller re-running broadSearch
+  // for an overlapping prompt set could still hand duplicate ids); first occurrence wins.
+  const seen = new Set();
+  const deduped = [];
+  for (const h of list) {
+    if (seen.has(h.id)) continue;
+    seen.add(h.id);
+    deduped.push(h);
+  }
+
+  const tokensById = {};
+  for (const h of deduped) tokensById[h.id] = contentTokens(h.text || "");
+
+  const { ids, neighbours } = buildNeighbours(tokensById, overlapMin);
+  const { find, union } = unionFind(ids.length);
+  for (let i = 0; i < ids.length; i += 1) {
+    for (const j of neighbours[i]) union(i, j);
+  }
+
+  const byId = new Map(deduped.map((h) => [h.id, h]));
+  const componentIdx = new Map(); // root index -> [member indices]
+  for (let i = 0; i < ids.length; i += 1) {
+    const root = find(i);
+    if (!componentIdx.has(root)) componentIdx.set(root, []);
+    componentIdx.get(root).push(i);
+  }
+
+  // IDF over THIS hit set (df/N here, not the whole corpus) — grouping is scoped to what
+  // Stage 1 actually retrieved for this prompt, same discipline retrieveBlocks applies to a
+  // single query's tokens.
+  const N = ids.length;
+  const df = new Map();
+  for (const id of ids) {
+    for (const t of new Set(tokensById[id])) df.set(t, (df.get(t) || 0) + 1);
+  }
+  const idf = (t) => Math.log(1 + N / (1 + (df.get(t) || 0)));
+
+  const groups = [];
+  for (const memberIdx of componentIdx.values()) {
+    const members = memberIdx
+      .map((i) => byId.get(ids[i]))
+      .sort((a, b) => a.id.localeCompare(b.id));
+    const memberIds = members.map((m) => m.id);
+
+    // label tokens: rank by (a) how many members share the token — full coverage first, so
+    // a group's label prefers what its members have IN COMMON over what one member merely
+    // contains a lot of — then (b) IDF (rarer across the hit set wins ties), then (c) token
+    // text for a fully deterministic order.
+    const coverage = new Map();
+    for (const i of memberIdx) {
+      for (const t of new Set(tokensById[ids[i]])) coverage.set(t, (coverage.get(t) || 0) + 1);
+    }
+    const tokens = [...coverage.keys()]
+      .sort((a, b) => (coverage.get(b) - coverage.get(a)) || (idf(b) - idf(a)) || a.localeCompare(b))
+      .slice(0, LABEL_TOKEN_COUNT);
+
+    groups.push({
+      id: `g:${memberIds[0]}`,
+      members,
+      memberIds,
+      tokens,
+      label: tokens.join(" ") || "(untitled group)",
+    });
+  }
+
+  groups.sort((a, b) => a.memberIds[0].localeCompare(b.memberIds[0]));
+  return groups;
+}