@polycode-projects/the-mechanical-code-talker 1.4.0 → 1.5.2

package/src/completions/rank.mjs

@@ -0,0 +1,154 @@
+// completions/rank.mjs — Stage 4 ("mechanical summarization") of PLAN_COMPLETIONS.md's
+// six-stage mechanical-text-generation pipeline. Stage-2-of-staging scope per the plan's own
+// §4 table: extractive SENTENCE ranking within a group.mjs group — no cross-group inference
+// (Stage 3, §1.3 — separately scoped), pruning (Stage 5), or voice pass (Stage 6) happens
+// here.
+//
+// "Extractive sentence selection over the grouped-and-inferred material, not abstractive
+// rewriting... query-focused multi-document summarization (feature-fusion sentence
+// selection, graph-ranking approaches in the LexRank/TextRank family, clustering-cum-ranking
+// as in CoRank)" — PLAN_COMPLETIONS.md §1.4/§4. The graph-ranking machinery this stage needs
+// already exists and ships: memory/blocks.mjs's rankBlocks() (PageRank, d=0.85, 20
+// iterations, over the shared-token block-similarity graph) and degreeOf() (hub dampening),
+// both generic over any `{ id: tokens[] }` map — NOT block-specific despite the module name.
+// This file reuses rankBlocks()/degreeOf() VERBATIM at sentence granularity (no PageRank
+// reimplementation), combined with the same idf = log(1 + N/(1+df)) formula group.mjs already
+// replicates for group-scoped (not whole-corpus-scoped) weighting, and the same
+// idfSum * (1+rank) / sqrt(1+degree) combination retrieveBlocks() uses to fuse relevance,
+// centrality, and hub-dampening into one score.
+//
+// New in this file: splitSentences() (no sentence-splitter existed anywhere in this repo —
+// grepped for "sentence" across src/ and found none; a simple regex splitter is intentional
+// here per the dispatch's own instruction not to pull in an NLP dependency for this) and the
+// sentence-level id/token/scoring wiring that adapts rankBlocks/degreeOf from block to
+// sentence granularity.
+//
+// Determinism: no randomness anywhere. Same group in (same members, same text, same order)
+// -> same ranked sentence list out (stable score-descending sort with a deterministic
+// sourceBlockId/sentence-text tiebreak) — see test/completions-stage2.test.mjs's double-run
+// diff, the exact discipline test/completions-stage0.test.mjs established for search+group.
+
+import { degreeOf, rankBlocks, tokenizeBlock, OVERLAP_MIN } from "../memory/blocks.mjs";
+import { STOPWORDS } from "../prose.mjs";
+
+// Same content-token filter group.mjs applies to its own adjacency/labeling (not exported
+// from group.mjs, so replicated here rather than reached across files — group.mjs's own
+// header explains why raw tokenizeBlock output is unsuitable for unweighted overlap: it
+// re-admits stopwords/filler that would collapse unrelated sentences into false edges).
+const isContentToken = (t) => /^[a-z0-9]+$/.test(t) && !STOPWORDS.has(t);
+
+/** tokenizeBlock(text), narrowed to real content tokens — see isContentToken above. */
+function contentTokens(text) {
+  return tokenizeBlock(text).filter(isContentToken);
+}
+
+// Sentence boundary: a run of [.!?] followed by whitespace and an uppercase letter or digit —
+// deliberately simple (no abbreviation dictionary, no NLP dependency, per the dispatch's own
+// instruction). Applied per-line (a block's own "Q: ...\nA: ..." shape already gives clean
+// boundaries at the newline; this regex further splits any line that itself holds more than
+// one sentence).
+const SENTENCE_SPLIT_RE = /(?<=[.!?])\s+(?=[A-Z0-9])/;
+
+/**
+ * Split raw block text into trimmed, non-empty sentences (order-preserving, no dedup —
+ * dedup, if ever needed, is the caller's call). Splits on blank lines first (a block's own
+ * natural line structure), then on sentence-ending punctuation within each line.
+ *
+ * @param {string} text
+ * @returns {string[]}
+ */
+export function splitSentences(text) {
+  const out = [];
+  for (const rawLine of String(text || "").split("\n")) {
+    const line = rawLine.trim();
+    if (!line) continue;
+    for (const part of line.split(SENTENCE_SPLIT_RE)) {
+      const s = part.trim();
+      if (s) out.push(s);
+    }
+  }
+  return out;
+}
+
+/**
+ * Stage 4 — extractive sentence ranking within one group.mjs group. Splits every member's
+ * text into sentences, builds a sentence-level shared-token-overlap similarity graph (the
+ * exact adjacency buildNeighbours/rankBlocks/degreeOf already establish, just re-keyed to
+ * sentence ids instead of block ids), runs the same PageRank (rankBlocks) and hub-dampening
+ * (degreeOf) over it, and combines with group-scoped IDF the same way retrieveBlocks() fuses
+ * relevance × centrality × hub-dampening — self-weighted (LexRank-style: a sentence's own
+ * rare/informative tokens) unless `opts.query` is supplied, in which case only tokens
+ * overlapping the query are IDF-summed (query-focused summarization, PLAN_COMPLETIONS.md's
+ * own literature framing) — a sentence with zero query overlap honestly scores 0 rather than
+ * silently falling back to self-weighting, the same "never a guessed match" discipline
+ * retrieveBlocks() applies when idfSum <= 0.
+ *
+ * @param {{members: Array<{id:string, text:string}>}} group  a group.mjs groupHits() entry
+ *   (or any object shaped `{ members: [{id, text}, ...] }`)
+ * @param {object} [opts]
+ * @param {number} [opts.overlapMin=OVERLAP_MIN]  shared content-token threshold for a
+ *   sentence-similarity edge — defaults to the same value group.mjs/blocks.mjs use, so
+ *   grouping, block-ranking, and sentence-ranking all agree on what "related" means unless
+ *   the caller deliberately overrides it.
+ * @param {string|null} [opts.query=null]  optional query text to focus ranking on (see above)
+ * @returns {Array<{sentence:string, score:number, sourceBlockId:string}>} best-first;
+ *   deterministic tiebreak (sourceBlockId, then sentence text) on equal score.
+ */
+export function rankSentences(group, { overlapMin = OVERLAP_MIN, query = null } = {}) {
+  const members = Array.isArray(group?.members) ? group.members : [];
+  if (!members.length) return [];
+
+  // One entry per sentence: a stable id ("<blockId>#<index>") so PageRank/degree/IDF can be
+  // keyed exactly like rankBlocks/degreeOf already key blocks — deterministic since it's
+  // derived purely from member order (group.mjs's members are already id-sorted) and each
+  // block's own sentence order.
+  const sentences = [];
+  for (const m of members) {
+    const parts = splitSentences(m?.text || "");
+    parts.forEach((sentence, i) => {
+      sentences.push({ id: `${m.id}#${i}`, sentence, sourceBlockId: m.id });
+    });
+  }
+  if (!sentences.length) return [];
+
+  const tokensById = {};
+  for (const s of sentences) tokensById[s.id] = contentTokens(s.sentence);
+
+  // Reused verbatim — the exact PageRank/hub-dampening machinery rankBlocks()/degreeOf()
+  // already run for block-level ranking, generic over any id->tokens map.
+  const ranks = rankBlocks(tokensById, { overlapMin });
+  const degrees = degreeOf(tokensById, { overlapMin });
+
+  // IDF scoped to THIS group's sentence set (df/N here, not the whole corpus) — the same
+  // scoping discipline group.mjs applies to its own label tokens.
+  const ids = Object.keys(tokensById);
+  const N = ids.length;
+  const df = new Map();
+  for (const id of ids) {
+    for (const t of new Set(tokensById[id])) df.set(t, (df.get(t) || 0) + 1);
+  }
+  const idf = (t) => Math.log(1 + N / (1 + (df.get(t) || 0)));
+
+  const queryTokens = query ? new Set(contentTokens(query)) : null;
+
+  const scored = sentences.map((s) => {
+    const tokens = tokensById[s.id];
+    const idfTokens = queryTokens ? tokens.filter((t) => queryTokens.has(t)) : tokens;
+    let idfSum = 0;
+    for (const t of idfTokens) idfSum += idf(t);
+    const rank = ranks[s.id] ?? 0;
+    const degree = degrees[s.id] ?? 0;
+    // idfSum * (1 + rank) / sqrt(1 + degree) — retrieveBlocks()'s own combination formula
+    // (memory/blocks.mjs), reused at sentence granularity: IDF-weighted informativeness
+    // decides content, PageRank centrality breaks ties toward well-connected (consensus)
+    // sentences, and the degree divisor dampens a sentence that merely shares vocabulary
+    // with disproportionately many others from winning on inflated rank alone.
+    const score = (idfSum * (1 + rank)) / Math.sqrt(1 + degree);
+    return { sentence: s.sentence, score, sourceBlockId: s.sourceBlockId };
+  });
+
+  scored.sort((a, b) => b.score - a.score
+    || a.sourceBlockId.localeCompare(b.sourceBlockId)
+    || a.sentence.localeCompare(b.sentence));
+  return scored;
+}

package/src/completions/search.mjs

@@ -0,0 +1,85 @@
+// completions/search.mjs — Stage 1 ("broad search") of PLAN_COMPLETIONS.md's six-stage
+// mechanical-text-generation pipeline. This is Stage-0 scope ONLY per the plan's own §4
+// staging table (Stage 1 + Stage 2, no inference/summarization/pruning/voice-pass yet — see
+// group.mjs for Stage 2).
+//
+// Near-total reuse, as the plan's own prior research established: this is a thin composition
+// of retrieveBlocks (src/memory/blocks.mjs's best-first block ranker) and the graph
+// search()/ask() services (src/providers/graph-service.mjs, contracted by
+// src/repository-interface.mjs's SERVICE_GROUPS.search) — a WIDER query shape ("broad
+// search": more hits per source, both sources asked) rather than any new retrieval
+// machinery. Nothing here re-implements ranking, tokenizing, or graph traversal.
+//
+// Granularity note (explicit, per the strategy-advisor's Stage-0 correction): retrieveBlocks
+// only ever returns whole BLOCK text — one node per ~800-token document/transcript chunk, the
+// unit memory/blocks.mjs's index actually stores. PLAN_COMPLETIONS.md's own prose loosely
+// says "spans"; this module does NOT invent sub-block span segmentation. Every hit this
+// module returns is whole-block (or a whole graph-search/ask result) granularity. Finer-grain
+// spans, if the pipeline ever needs them, are a real future increment, out of scope here.
+//
+// Determinism: retrieveBlocks is already deterministic (stable sort with an id tiebreak);
+// the graph service's search()/ask() are pure functions over a fixed graph. Given the same
+// dir/query/graphService, broadSearch() always returns the same array in the same order.
+
+import { retrieveBlocks } from "../memory/blocks.mjs";
+
+const DEFAULT_BLOCK_K = 8;   // "broad" > chat's narrow single-answer k (typically 3)
+const DEFAULT_GRAPH_LIMIT = 8;
+
+/**
+ * Stage 1 — broad search. Runs retrieveBlocks (memory/blocks.mjs) for text-block hits and,
+ * when a Repository-Interface graph service is supplied, its search() and ask() services for
+ * graph hits — merged into one flat, source-tagged hit list.
+ *
+ * @param {string} dir   repo root (retrieveBlocks reads <dir>/.tmct/memory/blocks/)
+ * @param {string} query the broad prompt driving retrieval
+ * @param {object} [opts]
+ * @param {number} [opts.blockK=8]  retrieveBlocks' k (how many blocks to pull)
+ * @param {object|null} [opts.graphService=null]  an optional Repository-Interface service
+ *   (e.g. createGraphService(graph) from src/providers/graph-service.mjs, or any provider
+ *   satisfying SERVICE_GROUPS.search's `search`/`ask`). When supplied, its search() and
+ *   ask() services are queried too. Omitted -> block-only search (still a valid, honest
+ *   broad search; graph access is opt-in, not required — this module never constructs a
+ *   graph service itself, it only calls one it's handed).
+ * @param {number} [opts.graphLimit=8]  graph search()'s result limit
+ * @returns {Promise<Array<{source:"block"|"graph-search"|"graph-ask", id:string, text:string, score:number}>>}
+ *   best-first within each source; blocks first, then graph-search, then graph-ask (stable,
+ *   deterministic order — never shuffled/merged by score across sources, since block scores
+ *   and graph relevance are not on a comparable scale; Stage 2's grouping doesn't need them
+ *   to be).
+ */
+export async function broadSearch(dir, query, {
+  blockK = DEFAULT_BLOCK_K, graphService = null, graphLimit = DEFAULT_GRAPH_LIMIT,
+} = {}) {
+  const q = String(query || "").trim();
+  if (!q) return [];
+
+  const hits = [];
+
+  const blockHits = await retrieveBlocks(dir, q, blockK);
+  for (const b of blockHits) {
+    hits.push({ source: "block", id: b.id, text: b.text || "", score: typeof b.score === "number" ? b.score : 0 });
+  }
+
+  if (graphService) {
+    if (typeof graphService.search === "function") {
+      const res = graphService.search(q, { limit: graphLimit });
+      if (res?.ok) {
+        for (const r of res.value.results || []) {
+          if (!r) continue;
+          hits.push({ source: "graph-search", id: r.id, text: r.label || r.id, score: 0 });
+        }
+      }
+    }
+    if (typeof graphService.ask === "function") {
+      const res = graphService.ask(q);
+      if (res?.ok && res.value && res.value.content) {
+        // one honest hit per prompt — the ask() answer itself, id-tagged with the query so
+        // it never collides with a block/search id and stays traceable to what produced it.
+        hits.push({ source: "graph-ask", id: `ask:${q}`, text: String(res.value.content), score: 0 });
+      }
+    }
+  }
+
+  return hits;
+}

package/src/corpus/conceptnet.mjs

@@ -147,8 +147,27 @@ export function toFacts(assertions, map, provenancePrefix = "corpus:conceptnet")
  *  survivors are written in ONE batched appendFacts call, not a per-fact loop.
  *  Returns { appended, skipped, total }. `provenancePrefix` is threaded through to
  *  toFacts (default "corpus:conceptnet" → byte-identical seed) so a seon/tier-2
- *  corpus can tag its facts "corpus:seon" / "corpus:tier2:<id>". */
-export async function seedMemory(dir, { limit, slicePath = SLICE_FILE, mapPath = MAP_FILE, prefer, provenancePrefix } = {}) {
+ *  corpus can tag its facts "corpus:seon" / "corpus:tier2:<id>".
+ *
+ *  `captureUnknownContext` (default false — every existing call stays
+ *  byte-identical): when true, also runs corpus/unknown-ingest.mjs's
+ *  `ingestUnknownFromAssertions` over this same assertions/map pair — the
+ *  PLAN_AGENTS.md §4 "context-preserving ingestion for unknown words"
+ *  mechanism — so a term that ONLY ever appears in a row `toFacts` silently
+ *  drops (an `ace = "none"` relation like RelatedTo/HasContext) still lands
+ *  in memory, tagged with the passage it was found in, instead of vanishing.
+ *  `unknownContextLimit` bounds how many distinct unknown terms one call
+ *  captures (default 500 — see that module's own doc comment for why an
+ *  unbounded sweep over a wide slice would not be "bounded, not padding").
+ *  The result's `unknown` key is present only when the flag is set. Loaded
+ *  dynamically (not a static import) to avoid a load-time import cycle with
+ *  unknown-ingest.mjs, which itself statically imports `termText` from here —
+ *  the same "avoid the cycle" discipline extensions.mjs's
+ *  seedActiveCorpusEntries already uses for this very module. */
+export async function seedMemory(dir, {
+  limit, slicePath = SLICE_FILE, mapPath = MAP_FILE, prefer, provenancePrefix,
+  captureUnknownContext = false, unknownContextLimit,
+} = {}) {
   const [assertions, map] = await Promise.all([loadSlice(slicePath), loadMap(mapPath)]);
   let facts = toFacts(assertions, map, provenancePrefix);
   if (Array.isArray(prefer) && prefer.length) {
@@ -186,5 +205,19 @@ export async function seedMemory(dir, { limit, slicePath = SLICE_FILE, mapPath =
   // for the 6 k-fact slice). appendFacts also skips any malformed row rather than
   // throwing, so its skipped count folds into the dedup skips here.
   const res = await appendFacts(dir, toWrite);
-  return { appended: res.appended, skipped: skipped + res.skipped, total: facts.length };
+
+  let unknown;
+  if (captureUnknownContext) {
+    const { ingestUnknownFromAssertions } = await import("./unknown-ingest.mjs");
+    unknown = await ingestUnknownFromAssertions(dir, {
+      assertions, map, mappedFacts: facts, memory,
+      provenancePrefix: provenancePrefix ? `${provenancePrefix}-unknown` : undefined,
+      limit: unknownContextLimit,
+    });
+  }
+
+  return {
+    appended: res.appended, skipped: skipped + res.skipped, total: facts.length,
+    ...(unknown ? { unknown } : {}),
+  };
 }

package/src/corpus/unknown-ingest.mjs

@@ -0,0 +1,209 @@
+// corpus/unknown-ingest.mjs — context-preserving ingestion for unknown terms
+// (PLAN_AGENTS.md §4 Phase 1, the "still not built at all" bullet).
+//
+// The problem this closes: `toFacts()` (conceptnet.mjs) only emits a Fact for
+// a relation the ACE-OWL map marks axiom-worthy (`ace != "none"`) — a row
+// whose relation is RelatedTo/Synonym/FormOf/SimilarTo/HasContext/etc is
+// SILENTLY skipped (`if (row.ace === "none") continue;`), on purpose, because
+// the relation itself doesn't fit a clean OWL axiom. That is the right call
+// for the AXIOM graph, but it means a term that ONLY ever shows up in one of
+// those dropped rows — never as the endpoint of a relation tmct actually
+// reifies — has NO anchor in memory at all. A wider seed set (a broader
+// ConceptNet slice, a tier-2 bundle, Phase 4's scraped web content) makes
+// this common: real terms, genuinely mentioned, quietly vanishing.
+//
+// This module does NOT change what counts as an axiom. It adds a SEPARATE,
+// honestly-labelled kind of individual for exactly the terms that would
+// otherwise vanish: a term is "unknown" here if it has never been the
+// subject/object of any reified Fact — not in memory already, and not in the
+// mapped facts this same seeding batch is about to write. For each dropped
+// (ace="none") row that touches an unknown term, the term becomes a Fact
+// tagged with the PASSAGE it was found in (ConceptNet's own `surfaceText`
+// when present, else the map's own `surface` template filled with the row's
+// two endpoints — both are committed, closed-vocabulary text; nothing here
+// ever generates free text), via a dedicated `mgx:contextPassage` predicate.
+// The row's OTHER endpoint (always) plus any already-known term recognizable
+// by exact word/bigram match in the passage (bounded to a handful) are linked
+// to the unknown term via one plain, closed-vocabulary co-occurrence
+// predicate, `mgx:coOccursWith` — deliberately NOT distributional/embedding
+// meaning induction (PLAN_AGENTS.md's own scoping): this buys traceable
+// context, never automatic sense disambiguation.
+//
+// Reuses src/memory/core.mjs's existing appendFacts/loadMemory/normFactTerm
+// machinery unmodified — a captured term is a completely ordinary Fact
+// individual (same trust/provenance/Source pipeline every other fact gets),
+// just carrying two new-but-closed predicates instead of an ACE-OWL one.
+//
+// Wiring: `seedMemory` (conceptnet.mjs) accepts an opt-in
+// `captureUnknownContext: true` (default false — every existing seed call
+// stays byte-identical) that calls `ingestUnknownFromAssertions` after the
+// mapped facts are computed, dynamically imported (the same "avoid a static
+// import cycle" discipline extensions.mjs's seedActiveCorpusEntries already
+// uses for conceptnet.mjs itself).
+//
+// NOT covered here (out of scope for this module, see the caller's report):
+// the LIVE chat teach/miss path (src/chat.mjs) has its own, separate
+// unknown-word moment — a visitor's utterance mentioning a term the grammar
+// can't classify — which has no "assertion batch" or ConceptNet-shaped
+// surfaceText to draw a passage from at all. That needs its own hook (the
+// raw utterance text IS the passage there); this module only ever consumes
+// {start, rel, end, surfaceText?} shaped rows, so it cannot be reused as-is
+// for that path without a chat.mjs-side adapter. See the report for the
+// exact shape that hook would need — deliberately not built here.
+
+import { appendFacts, loadMemory, normFactTerm, FACT_CLASS } from "../memory/core.mjs";
+import { termText } from "./conceptnet.mjs";
+
+/** unknown term -> the passage it was captured from (object = the passage
+ *  text itself, capped by normFactTerm's own TEXT_CAP like any fact term). */
+export const CONTEXT_PASSAGE_PREDICATE = "mgx:contextPassage";
+/** plain, undirected-in-spirit co-occurrence edge: term -> another term seen
+ *  in the SAME passage. Deliberately the ONE relation this module ever
+ *  emits for "these two showed up together" — no relation-type inference. */
+export const CO_OCCURS_PREDICATE = "mgx:coOccursWith";
+
+// Function words filtered out of passage word/bigram scanning — never
+// candidates for a co-occurrence link (a "the"<->term edge would be noise,
+// not context). Closed, small, hand-curated — not a general stopword list.
+const STOPWORDS = new Set([
+  "a", "an", "the", "is", "are", "was", "were", "be", "being", "been",
+  "to", "of", "for", "in", "on", "at", "with", "by", "as", "and", "or",
+  "that", "this", "it", "its", "you", "your", "related", "kind", "used",
+]);
+
+const MAX_EXTRA_LINKS_PER_PASSAGE = 3; // bounded — not an open-ended sweep
+
+/** The set of terms tmct already "recognizes": every subject/object across
+ *  every reified Fact currently in memory, union every term the mapped facts
+ *  THIS batch is about to write introduce. A term with a real Fact anywhere
+ *  already has structured knowledge — only a term that never gets one is a
+ *  candidate for context-only capture. Pure; does not mutate `memory`. */
+export function knownTermsFrom(memory, mappedFacts) {
+  const known = new Set();
+  for (const ind of memory?.individuals || []) {
+    if (ind?.class !== FACT_CLASS) continue;
+    const get = (k) => (ind.attributes || []).find((a) => a?.key === k)?.value;
+    const s = get("subject");
+    const o = get("object");
+    if (s) known.add(normFactTerm(s));
+    if (o) known.add(normFactTerm(o));
+  }
+  for (const f of mappedFacts || []) {
+    if (f?.subject) known.add(normFactTerm(f.subject));
+    if (f?.object) known.add(normFactTerm(f.object));
+  }
+  return known;
+}
+
+/** The human-readable context passage for one DROPPED assertion — ConceptNet's
+ *  own `surfaceText` (bracket-stripped) when present, else the mapping row's
+ *  own `surface` template filled with the two endpoint terms. Both sources
+ *  are committed, closed-vocabulary text — this never generates free text.
+ *  Returns null when neither source is available (never fatal). */
+export function passageFor(assertion, row) {
+  const raw = assertion?.surfaceText;
+  if (typeof raw === "string" && raw.trim()) {
+    return raw.replace(/\[\[([^\]]+)\]\]/g, "$1").replace(/\s+/g, " ").trim();
+  }
+  const s = termText(assertion?.start);
+  const o = termText(assertion?.end);
+  if (row?.surface && s && o) {
+    return row.surface.replace("{start}", s).replace("{end}", o);
+  }
+  return null;
+}
+
+/** Single words + adjacent bigrams in `passage` that are already-known terms
+ *  (present in `known`), excluding anything in `exclude` (the row's own two
+ *  endpoints — already linked directly). Bounded to
+ *  MAX_EXTRA_LINKS_PER_PASSAGE hits; independent of `known`'s size (a Set
+ *  lookup per token/bigram, never a substring sweep over every known term). */
+function knownMentionsIn(passage, known, exclude) {
+  const tokens = String(passage || "").toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter(Boolean);
+  const hits = [];
+  const seen = new Set();
+  for (let i = 0; i < tokens.length && hits.length < MAX_EXTRA_LINKS_PER_PASSAGE; i += 1) {
+    const candidates = [tokens[i]];
+    if (i + 1 < tokens.length) candidates.push(`${tokens[i]} ${tokens[i + 1]}`);
+    for (const c of candidates) {
+      if (STOPWORDS.has(c) || seen.has(c) || exclude.has(c)) continue;
+      if (known.has(c)) {
+        hits.push(c);
+        seen.add(c);
+        if (hits.length >= MAX_EXTRA_LINKS_PER_PASSAGE) break;
+      }
+    }
+  }
+  return hits;
+}
+
+/**
+ * Capture unknown terms out of the assertions a corpus-seeding batch is
+ * about to (or just did) write. Scans only rows whose relation maps to
+ * `ace = "none"` (the genuinely-dropped rows) — a mapped (ace != "none") row
+ * always gets a real reified Fact via toFacts()/appendFacts() already, so it
+ * is never a "silently dropped" case this module needs to rescue.
+ *
+ * { assertions, map, mappedFacts, memory?, provenancePrefix?, limit? }:
+ *   - assertions/map: the SAME loadSlice()/loadMap() results seedMemory has.
+ *   - mappedFacts: the toFacts() output for this same batch (feeds
+ *     knownTermsFrom so a term this batch itself just defined isn't
+ *     re-captured as "unknown").
+ *   - memory: a pre-loaded payload (seedMemory already has one) — loaded
+ *     fresh via loadMemory(dir) when omitted.
+ *   - provenancePrefix: tags the captured facts, default "corpus:unknown".
+ *   - limit: caps how many DISTINCT unknown terms get captured in one call
+ *     (default 500) — a wide slice's RelatedTo rows alone number in the tens
+ *     of thousands; this keeps one run bounded, not a flood.
+ *
+ * Returns { captured, linked, appended, skipped } — `captured` = distinct
+ * unknown terms newly given a contextPassage fact, `linked` = co-occurrence
+ * edges written (the direct pair plus any bounded extra known-term hits).
+ */
+export async function ingestUnknownFromAssertions(dir, {
+  assertions, map, mappedFacts = [], memory, provenancePrefix = "corpus:unknown", limit = 500,
+} = {}) {
+  if (!Array.isArray(assertions) || !map) return { captured: 0, linked: 0, appended: 0, skipped: 0 };
+  const mem = memory || await loadMemory(dir);
+  const known = knownTermsFrom(mem, mappedFacts);
+  const capturedTerms = new Set();
+  const toWrite = [];
+
+  for (const a of assertions) {
+    if (capturedTerms.size >= limit) break;
+    const row = map.get(a.rel);
+    if (!row || row.ace !== "none") continue; // only the genuinely-dropped rows
+    const subject = termText(a.start);
+    const object = termText(a.end);
+    if (!subject || !object) continue; // non-en endpoint — nothing to anchor a passage to
+    const sKey = normFactTerm(subject);
+    const oKey = normFactTerm(object);
+    const subjectUnknown = !known.has(sKey);
+    const objectUnknown = !known.has(oKey);
+    if (!subjectUnknown && !objectUnknown) continue; // both sides already recognized
+
+    const passage = passageFor(a, row);
+    if (!passage) continue;
+
+    const pairs = [[subject, object, subjectUnknown], [object, subject, objectUnknown]];
+    for (const [term, other, isUnknown] of pairs) {
+      if (!isUnknown || capturedTerms.size >= limit) continue;
+      const termKey = normFactTerm(term);
+      if (capturedTerms.has(termKey)) continue; // one context capture per term per run
+      const tag = `${provenancePrefix} ${a.rel}`;
+      toWrite.push({ subject: term, predicate: CONTEXT_PASSAGE_PREDICATE, object: passage, provenance: tag });
+      toWrite.push({ subject: term, predicate: CO_OCCURS_PREDICATE, object: other, provenance: tag });
+      const extra = knownMentionsIn(passage, known, new Set([sKey, oKey]));
+      for (const m of extra) {
+        toWrite.push({ subject: term, predicate: CO_OCCURS_PREDICATE, object: m, provenance: tag });
+      }
+      capturedTerms.add(termKey);
+      known.add(termKey); // now recognized — a captured term is never re-captured
+    }
+  }
+
+  if (!toWrite.length) return { captured: 0, linked: 0, appended: 0, skipped: 0 };
+  const res = await appendFacts(dir, toWrite);
+  const linked = toWrite.filter((f) => f.predicate === CO_OCCURS_PREDICATE).length;
+  return { captured: capturedTerms.size, linked, appended: res.appended, skipped: res.skipped };
+}

package/src/extensions.mjs

@@ -6,10 +6,14 @@
 //   resolveExtensions(repoRoot) → { entries: Map<name, ResolvedEntry>, biasByBundle }
 //
 // BUILTIN_EXTENSIONS ships the exact two bundles chat.mjs's bootstrap has
-// always seeded — `seon` and `conceptnet`, both active — plus three shipped-
-// but-INACTIVE tier-2 bundles (`tier2-aws` / `tier2-python` / `tier2-java`).
-// Activating one is a config-only edit (`tmct init --corpus aws`, or a
-// `[extensions.tier2-aws] active = true` in tmct.toml) — zero code change.
+// always seeded — `seon` and `conceptnet`, both active — plus four shipped-
+// but-INACTIVE tier-2 bundles (`tier2-aws` / `tier2-python` / `tier2-java` /
+// `tier2-general`). Activating one is a config-only edit (`tmct init --corpus
+// aws`, or a `[extensions.tier2-aws] active = true` in tmct.toml) — zero code
+// change. `tier2-general` (PLAN_AGENTS.md Phase 1) is deliberately NOT a
+// language/domain bundle like the other three — everyday-knowledge concepts
+// with zero code-domain framing, the "wider general-knowledge seed set"
+// bullet made real instead of just mechanically activatable.
 //
 // A `tmct.toml` may carry a top-level `[extensions]` table-of-tables
 // (`[extensions.tier2-aws]`, …): a RECOGNIZED name (one of the builtins above)
@@ -87,6 +91,12 @@ function builtinExtensions() {
       corpusPath: join(TIER2_DIR, "java.jsonl"),
       provenancePrefix: "corpus:tier2-java",
     },
+    "tier2-general": {
+      kind: "corpus",
+      active: false,
+      corpusPath: join(TIER2_DIR, "general.jsonl"),
+      provenancePrefix: "corpus:tier2-general",
+    },
   };
 }
 

package/src/finish.mjs

@@ -312,13 +312,46 @@ function ruleAgreement(segments, rule) {
   return out;
 }
 
-// Rule 3 — sentence capitalisation. Only when the answer OPENS on a prose span
-// whose first non-space character is a lowercase letter. An answer that opens on
-// a protected span (a path/entity) is left exactly as grounded.
+// Rule 3 — sentence capitalisation. Capitalises (a) the very first character
+// when the answer OPENS on a prose span (the ORIGINAL single-answer scope,
+// unchanged), (b) every INTERNAL sentence boundary inside a single prose span
+// — a run of terminal punctuation + whitespace followed by a lowercase letter
+// — and (c) a sentence boundary that CROSSES a span boundary: a prose span
+// ends in terminal punctuation (+ optional trailing whitespace) and the next
+// real-content span is itself prose starting lowercase (any purely-whitespace
+// prose spans in between are skipped over). (b) and (c) are the
+// PLAN_COMPLETIONS.md Stage 6 generalisation — a genuinely multi-sentence
+// completion needs every internal boundary capitalised, not just the whole-
+// answer opener. A boundary that lands on a PROTECTED span (path/entity/…) is
+// left exactly as grounded — a protected span's casing is never
+// rule-transformed, the same guard (a) always applied to an answer that opens
+// on one.
 function ruleCapitalise(segments) {
-  if (!segments.length || segments[0].type !== "prose") return segments;
+  if (!segments.length) return segments;
   const out = segments.map((s) => ({ ...s }));
-  out[0].text = out[0].text.replace(/^(\s*)([a-z])/, (m, sp, ch) => sp + ch.toUpperCase());
+
+  // (a) answer-initial capitalisation
+  if (out[0].type === "prose") {
+    out[0].text = out[0].text.replace(/^(\s*)([a-z])/, (m, sp, ch) => sp + ch.toUpperCase());
+  }
+
+  // (b) every internal sentence boundary WITHIN a single prose span
+  for (const seg of out) {
+    if (seg.type !== "prose") continue;
+    seg.text = seg.text.replace(/([.!?])(\s+)([a-z])/g, (m, stop, sp, ch) => stop + sp + ch.toUpperCase());
+  }
+
+  // (c) a sentence boundary that CROSSES a span boundary
+  for (let i = 0; i < out.length; i += 1) {
+    const seg = out[i];
+    if (seg.type !== "prose" || !/[.!?]\s*$/.test(seg.text)) continue;
+    let j = i + 1;
+    while (j < out.length && out[j].type === "prose" && /^\s*$/.test(out[j].text)) j += 1;
+    if (j < out.length && out[j].type === "prose") {
+      out[j].text = out[j].text.replace(/^(\s*)([a-z])/, (m, sp, ch) => sp + ch.toUpperCase());
+    }
+  }
+
   return out;
 }
 
@@ -345,19 +378,18 @@ function ruleList(segments, rule) {
   return out;
 }
 
-// Rule 5 — terminal punctuation. Collapse a trailing run of 2+ sentence stops in
-// the LAST prose span to a single stop ("done.." → "done."). Adds nothing where
-// a fragment/list answer legitimately ends without a stop.
+// Rule 5 — terminal punctuation. Collapses ANY run of 2+ sentence stops within
+// a prose span to a single stop ("done.." → "done."; "Sentence one.. Sentence
+// two!!" → "Sentence one. Sentence two!") — generalised (PLAN_COMPLETIONS.md
+// Stage 6) from the original single-answer scope (only the LAST prose span,
+// only a run anchored at the very end of the answer) to every internal
+// sentence boundary a multi-sentence completion can carry. A legitimate
+// fragment/list answer that ends without a stop is unaffected — there is
+// nothing to collapse.
 function ruleTerminal(segments, rule) {
   const stops = (rule.stops && rule.stops.length ? rule.stops : [".", "!", "?"]).map(escapeRe2).join("");
-  const out = segments.map((s) => ({ ...s }));
-  let idx = -1;
-  for (let i = out.length - 1; i >= 0; i -= 1) if (out[i].type === "prose") { idx = i; break; }
-  if (idx < 0) return out;
-  const re = new RegExp(`([${stops}])(?:\\s*[${stops}])+(\\s*)$`);
-  const m = out[idx].text.match(re);
-  if (m) out[idx].text = out[idx].text.slice(0, m.index) + m[1] + m[2];
-  return out;
+  const re = new RegExp(`([${stops}])(?:\\s*[${stops}])+(\\s*)`, "g");
+  return segments.map((s) => (s.type === "prose" ? { ...s, text: s.text.replace(re, "$1$2") } : { ...s }));
 }
 
 const HANDLERS = {
@@ -421,16 +453,27 @@ export function applyGrammar(segments, rules = grammarRules()) {
 // runTurn, at the `withLast` seam, `result = finish(result, { graph })` so every
 // producer passes through once. Until then finish() is exercised by its unit +
 // golden tests; wiring it changes no fact, only fixes our own generated defects.
+//
+// ctx.rules (optional, additive): a caller-supplied rule table overriding the
+// cached grammarRules() for this call only — e.g. completions/complete.mjs's
+// Stage 6 pass force-enables the sentence-capitalisation rule (PARKED in the
+// live chat table per grammar-rules.toml's own cycle-006 note) because a
+// genuinely multi-sentence extractive completion needs every internal sentence
+// boundary capitalised to read as one voice, without touching the chat
+// pipeline's default live/parked flags. Every EXISTING call site omits
+// ctx.rules and is therefore byte-identical to before this option existed.
 
 /** Finish a turn result: grammar-correct its prose spans, preserving every fact.
  *  Byte-stable when neutral (returns its argument unchanged); rebuilds only on a
- *  genuine fix. Throws if finishing would move any protected span (guard #4). */
+ *  genuine fix. Throws if finishing would move any protected span (guard #4).
+ *  @param {{rules?: object[]}} [ctx.rules] optional rule-table override (see above) */
 export function finish(result, ctx = {}) {
   if (!result || typeof result.answer !== "string") return result;
+  const rules = Array.isArray(ctx.rules) ? ctx.rules : grammarRules();
   const before = Array.isArray(result.segments) && result.segments.length
     ? result.segments
     : maskSegments(result.answer, ctx);
-  const after = applyGrammar(before, grammarRules());
+  const after = applyGrammar(before, rules);
   assertInvariance(before, after);
   const answer = flatten(after);
   if (answer === result.answer) return result; // neutral → byte-stable, same reference