@polycode-projects/the-mechanical-code-talker 1.9.2 → 1.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +441 -202
  2. package/bin/tmct.mjs +126 -1
  3. package/package.json +4 -2
  4. package/src/answer-variants.mjs +8 -36
  5. package/src/ask-browser-entry.mjs +5 -23
  6. package/src/ask-browser.bundle.js +1 -2
  7. package/src/ask-nlp.mjs +9 -23
  8. package/src/ask-vocab.mjs +139 -589
  9. package/src/ask.mjs +627 -1729
  10. package/src/chat.mjs +1684 -2872
  11. package/src/cli-args.mjs +14 -28
  12. package/src/codegraph.mjs +236 -644
  13. package/src/completions/complete.mjs +18 -62
  14. package/src/completions/graph-adapter.mjs +14 -60
  15. package/src/completions/group.mjs +12 -68
  16. package/src/completions/infer.mjs +38 -126
  17. package/src/completions/prune.mjs +17 -70
  18. package/src/completions/rank.mjs +16 -69
  19. package/src/completions/search.mjs +8 -31
  20. package/src/concept.mjs +32 -88
  21. package/src/conformance.mjs +11 -15
  22. package/src/corpus/conceptnet.mjs +31 -89
  23. package/src/corpus/templates.mjs +19 -45
  24. package/src/corpus/unknown-ingest.mjs +31 -92
  25. package/src/embed.mjs +10 -22
  26. package/src/extensions.mjs +50 -154
  27. package/src/finish.mjs +35 -91
  28. package/src/grammar/ace.mjs +16 -40
  29. package/src/grammar/assert.mjs +1 -1
  30. package/src/grammar/lexicon-core.json +1 -1
  31. package/src/grammar/lexicon.mjs +9 -27
  32. package/src/graph-merge.mjs +2 -3
  33. package/src/hash.mjs +6 -14
  34. package/src/index.mjs +6 -10
  35. package/src/init.mjs +38 -125
  36. package/src/interpret/fuzzy.mjs +10 -29
  37. package/src/interpret/merge.mjs +9 -27
  38. package/src/interpret/normalize.mjs +137 -585
  39. package/src/interpret/pipeline.mjs +23 -71
  40. package/src/interpret/strategies/ace.mjs +7 -31
  41. package/src/interpret/strategies/constructions.mjs +14 -41
  42. package/src/interpret/strategies/grammar.mjs +21 -60
  43. package/src/interpret/strategies/keywords.mjs +42 -131
  44. package/src/interpret/strategies/noise-strip.mjs +18 -89
  45. package/src/memory/bias.mjs +11 -54
  46. package/src/memory/blocks.mjs +18 -69
  47. package/src/memory/core.mjs +171 -591
  48. package/src/memory/fold.mjs +0 -0
  49. package/src/memory/inspect.mjs +7 -25
  50. package/src/memory/shacl.mjs +10 -39
  51. package/src/memory/trust.mjs +26 -127
  52. package/src/memory-ask-browser-entry.mjs +7 -30
  53. package/src/memory-ask-browser.bundle.js +1 -1
  54. package/src/paraphrase.mjs +20 -53
  55. package/src/planning.mjs +15 -157
  56. package/src/prose-nlp.mjs +4 -17
  57. package/src/prose.mjs +19 -67
  58. package/src/providers/bootstrap.mjs +1 -2
  59. package/src/providers/fixture.mjs +1 -2
  60. package/src/providers/graph-service.mjs +28 -59
  61. package/src/repository-interface.mjs +6 -8
  62. package/src/router/drive.mjs +183 -0
  63. package/src/router/goal-reasoner.mjs +66 -231
  64. package/src/router/guardrail.mjs +20 -58
  65. package/src/router/planner.mjs +15 -46
  66. package/src/router/registry.mjs +13 -43
  67. package/src/router/resolver.mjs +46 -131
  68. package/src/router/results.mjs +231 -0
  69. package/src/schema-docs.mjs +10 -27
  70. package/src/server-http.mjs +10 -19
  71. package/src/server.mjs +22 -28
  72. package/src/sessions.mjs +15 -30
  73. package/src/source-slice.mjs +5 -7
  74. package/src/source.mjs +10 -20
  75. package/src/syllogise.mjs +187 -575
  76. package/src/telemetry.mjs +3 -3
  77. package/src/toml-config.mjs +4 -4
  78. package/src/tui/app.mjs +9 -19
  79. package/src/viz.mjs +66 -123
  80. package/src/wink-model.mjs +10 -24
@@ -1,34 +1,6 @@
1
- // completions/complete.mjs — the full PLAN_COMPLETIONS.md pipeline, wired end to end:
2
- // broadSearch (Stage 1) -> groupHits (Stage 2) -> rankSentences per group + inferRelations
3
- // across groups (Stage 3+4) -> pruneCompletion's keep/drop (Stage 5) -> assemble the kept
4
- // sentences into prose -> finish()'s grammar/voice pass (Stage 6). §4's staging table, row 3:
5
- // "Stage 5+6 — pruning + grammar/voice pass wired end to end... Stage 6 reuses finish.mjs
6
- // directly... Exit criterion: A full end-to-end completion reads as one consistent voice and
7
- // every sentence traces to a source span."
8
- //
9
- // Extractive, by construction (PLAN_COMPLETIONS.md §3's honest ceiling): the assembled text is
10
- // the KEPT sentences, in order, joined by a single space — never paraphrased, never reordered
11
- // beyond what pruning's own group/rank order already decided, never smoothed over a genuine
12
- // disjointedness in the source material. Stage 6 fixes CAPITALISATION and PUNCTUATION at every
13
- // sentence boundary (this dispatch's own generalisation of src/finish.mjs's ruleCapitalise/
14
- // ruleTerminal); it does not invent, reorder, or merge any sentence's WORDS. Where the source
15
- // material simply doesn't chain into a fluent read, the output reads disjointedly and stays
16
- // honest about it — that is the plan's own accepted, documented limit, not a bug this module
17
- // tries to paper over.
18
- //
19
- // Traceability: `sourceSpans` is built directly from pruneCompletion()'s own `kept` list (which
20
- // already carries sourceBlockId + groupId per sentence) BEFORE the grammar pass runs, and the
21
- // grammar pass never reorders, drops, or merges sentences (only mutates casing/punctuation of
22
- // prose text) — so sourceSpans stays index-aligned with "one entry per output sentence" even
23
- // though the FINAL text is produced by finish(), not directly from `kept`. `relations` is
24
- // infer.mjs's own output, untouched by pruning (relations are never pruned, only the sentences
25
- // that anchor them are decided) — every relation still carries its own `licensingTest`/
26
- // `evidence` per infer.mjs's contract.
27
- //
28
- // Determinism: every stage this module chains is already deterministic (broadSearch, groupHits,
29
- // rankSentences, inferRelations, pruneCompletion, finish() are all pure/deterministic functions
30
- // over their inputs); this module adds no randomness of its own. See
31
- // test/completions-complete.test.mjs's own double-run diff.
1
+ // completions/complete.mjs — the mechanical-text-generation pipeline: broadSearch ->
2
+ // groupHits -> rankSentences + inferRelations -> pruneCompletion -> assemble -> finish()'s
3
+ // grammar pass. Extractive only: output is the kept sentences, joined, never paraphrased.
32
4
 
33
5
  import { broadSearch } from "./search.mjs";
34
6
  import { groupHits } from "./group.mjs";
@@ -38,15 +10,9 @@ import { pruneCompletion } from "./prune.mjs";
38
10
  import { loadMemory } from "../memory/core.mjs";
39
11
  import { finish, grammarRules } from "../finish.mjs";
40
12
 
41
- /** grammarRules() with sentence-capitalisation FORCE-ENABLED, everything else exactly as the
42
- * live table has it. sentence-capitalisation stays PARKED (enabled=false) in the live chat
43
- * table (grammar-rules.toml's own cycle-006 note: it regresses case-sensitive single-answer
44
- * goldens pinning lowercase openers) — that is a CHAT-voice decision, not a limitation of the
45
- * rule itself. A genuinely multi-sentence extractive completion has no such single-opener
46
- * voice to protect and NEEDS every internal sentence boundary capitalised to read as one
47
- * consistent voice (this pipeline's own exit criterion) — so this module's own Stage 6 call
48
- * overrides just that one rule's enabled flag, via finish()'s ctx.rules seam, without touching
49
- * the cached global table or any other caller's behaviour. */
13
+ /** grammarRules() with sentence-capitalisation force-enabled (disabled in live chat only to
14
+ * protect single-answer lowercase-opener goldens, which don't apply to a multi-sentence
15
+ * completion). */
50
16
  function completionGrammarRules() {
51
17
  return grammarRules().map((r) => (r.id === "sentence-capitalisation" ? { ...r, enabled: true } : r));
52
18
  }
@@ -65,16 +31,13 @@ const DEFAULT_MAX_SENTENCES_PER_GROUP = 3; // see prune.mjs's own file header fo
65
31
  * passed straight through to broadSearch (see search.mjs)
66
32
  * @param {number} [opts.graphLimit] broadSearch's graph search() limit
67
33
  * @param {number} [opts.overlapMin] groupHits' shared-token edge threshold
68
- * @param {object} [opts.memory] an already-loaded memory/core.mjs loadMemory() payload; when
69
- * omitted, this module loads it itself via loadMemory(dir) (a fresh/empty memory is a
70
- * perfectly valid, honest input inferRelations() simply asserts nothing over it)
71
- * @param {string} [opts.query] the query rankSentences()/pruning focus on; defaults to `prompt`
72
- * itself (query-focused summarization, PLAN_COMPLETIONS.md §1.4's own literature framing) —
73
- * pass `null` explicitly to fall back to self-weighted (LexRank-style) ranking instead
34
+ * @param {object} [opts.memory] an already-loaded memory/core.mjs loadMemory() payload;
35
+ * defaults to loadMemory(dir)
36
+ * @param {string} [opts.query] the query rankSentences()/pruning focus on; defaults to `prompt`;
37
+ * pass `null` for self-weighted (LexRank-style) ranking instead
74
38
  * @param {number} [opts.maxSentencesPerGroup=3] prune.mjs's top-K-per-group cutoff
75
39
  * @param {object} [opts.graph] optional loaded graph (src/codegraph.mjs parseEntities() shape)
76
- * handed to finish()'s maskSegments so known entity labels inside the assembled text are
77
- * protected during the grammar pass — the same ctx.graph existing finish() call sites pass
40
+ * handed to finish()'s maskSegments to protect known entity labels during the grammar pass
78
41
  * @returns {Promise<{
79
42
  * text: string,
80
43
  * sourceSpans: Array<{sourceBlockId:string, groupId:string, sentence:string}>,
@@ -97,8 +60,7 @@ export async function generateCompletion(dir, prompt, opts = {}) {
97
60
  // Stage 2 — grouping
98
61
  const groups = groupHits(hits, { overlapMin });
99
62
 
100
- // Stage 3 — cross-group inference (needs a loaded memory; a fresh/empty one is honest and
101
- // simply asserts nothing, never fabricates something to compensate)
63
+ // Stage 3 — cross-group inference
102
64
  const memory = memoryOpt || await loadMemory(dir);
103
65
  const relations = groups.length >= 2 ? await inferRelations(groups, memory) : [];
104
66
 
@@ -113,25 +75,19 @@ export async function generateCompletion(dir, prompt, opts = {}) {
113
75
  );
114
76
 
115
77
  if (!kept.length) {
116
- // Honest decline (PLAN_COMPLETIONS.md §3): nothing cleared the pruning bar for this prompt
117
- // over this corpus — never fabricate a completion to fill the gap.
78
+ // Honest decline: nothing cleared the pruning bar for this prompt over this corpus — never
79
+ // fabricate a completion to fill the gap.
118
80
  return { text: "", sourceSpans: [], relations, dropped, declined: true, reason: "no source span cleared the pruning bar for this prompt" };
119
81
  }
120
82
 
121
- // Assemble: kept sentences, in pruneCompletion's own (group-id, rank-order) order, joined by a
122
- // single space — purely extractive, no reordering/paraphrasing beyond that (§3's honest
123
- // ceiling; see file header).
83
+ // Assemble: kept sentences, in pruneCompletion's own order, joined by a single space.
124
84
  const rawText = kept.map((k) => k.sentence).join(" ");
125
85
 
126
- // Stage 6 — grammar/voice pass. Reuses finish() verbatim (its maskSegments() masker still
127
- // protects any known entity/path/number/receipt/provenance span WITHIN the assembled text,
128
- // exactly as it does for a single-answer composed answer); only the rule table differs
129
- // (sentence-capitalisation force-enabled — see completionGrammarRules() above).
86
+ // Stage 6 — grammar/voice pass (reuses finish() verbatim; see completionGrammarRules() above).
130
87
  const finished = finish({ answer: rawText, via: "completion" }, { graph, rules: completionGrammarRules() });
131
88
 
132
- // sourceSpans traces every OUTPUT sentence back to its block id + group id built from
133
- // `kept` (pre-grammar-pass), which stays index-aligned with the final text because finish()'s
134
- // grammar rules only mutate casing/punctuation of prose, never reorder/drop/merge sentences.
89
+ // sourceSpans traces every output sentence back to its block id + group id, from `kept`
90
+ // (pre-grammar-pass) stays index-aligned since finish() only mutates casing/punctuation.
135
91
  const sourceSpans = kept.map((k) => ({ sourceBlockId: k.sourceBlockId, groupId: k.groupId, sentence: k.sentence }));
136
92
 
137
93
  return { text: finished.answer, sourceSpans, relations, dropped };
@@ -1,61 +1,20 @@
1
- // completions/graph-adapter.mjs — HANDOVER.md item 1: the graphService-shaped adapter
2
- // src/completions/search.mjs's broadSearch() was always built to accept its own
3
- // docblock names createGraphService(graph) (src/providers/graph-service.mjs) as the
4
- // reference shape — but until now nothing in live chat ever constructed and passed one
5
- // through (src/chat.mjs's completionsRescueAnswer called generateCompletion() with no
6
- // `graphService` at all). Without it, broadSearch could only ever see memory BLOCKS
7
- // saved via an explicit saveBlock() call — never the already-loaded code graph, and
8
- // never a taught Fact — so "give me a detailed summary of how X works" declined for any
9
- // subject on its first real mention in a session, no matter how much the graph or
10
- // taught Facts actually knew about it.
1
+ // completions/graph-adapter.mjs — a graphService-shaped adapter for broadSearch(), wrapping
2
+ // two already-loaded stores (graph, memory) without re-loading either from disk.
11
3
  //
12
- // createCompletionsGraphAdapter(graph, memory) wraps TWO already-loaded stores (never
13
- // re-loads either from disk both are handed in by the caller, exactly as loaded for
14
- // this turn):
15
- //
16
- // - .search(q, {limit}) delegates straight to createGraphService(graph).search() —
17
- // the same ranked lexical module/symbol search every other Repository-Interface
18
- // consumer uses (src/codegraph.mjs's searchModulesRanked/scoreSymbolsRanked under
19
- // the hood). No new search machinery.
20
- //
21
- // - .ask(q) does NOT delegate to createGraphService(graph).ask() (src/ask.mjs) —
22
- // that engine is a mechanical NATURAL-LANGUAGE QUESTION grammar ("which functions
23
- // call X", "what does X import"), and broadSearch always calls .ask() with the
24
- // bare SUBJECT TERM itself ("TaskController"), not a question. Tried live: that
25
- // produces an honest but useless "couldn't parse this as a graph question"
26
- // rephrase-hint every time — real text, but not about the subject, and it would
27
- // pollute the completion with noise. Instead .ask() here builds real sentences
28
- // from two sources that a bare term CAN resolve against directly:
29
- // 1. resolveSymbol + renderDescribe (src/codegraph.mjs) — the SAME graph-only
30
- // renderer src/server.mjs's own tmct_describe tool uses: real facts (defining
31
- // module, contains, inherits, calls, tests, attributes, …), never invented.
32
- // 2. readFactRows(memory) (src/memory/core.mjs) — any TAUGHT Fact whose subject
33
- // or object mentions the term. This is the one source the pipeline had NO
34
- // path to before at all: Stage 3 (inferRelations) only ever augments groups
35
- // that already exist from Stage 1's hits, so a subject with real taught Facts
36
- // but zero blocks/code-graph hits still surfaced nothing.
37
- // svc.ask() is still tried last, but its content is kept ONLY when it genuinely
38
- // parsed (tmct_ask.miss === false) — e.g. the rare case where the bare term happens
39
- // to also be a real registered question shape — never its own rephrase-hint noise.
40
- //
41
- // Every sentence this adapter returns traces to a real graph edge/attribute or a real
42
- // taught Fact — never invented, matching src/completions/'s extractive-only discipline
43
- // (see complete.mjs's own file header).
4
+ // .ask(q) does not delegate to createGraphService(graph).ask(): that expects a natural-
5
+ // language question, but broadSearch calls .ask() with a bare term, which would only produce
6
+ // a "couldn't parse" rephrase-hint. Instead it builds sentences from resolveSymbol/
7
+ // renderDescribe and readFactRows(memory), falling back to svc.ask() only when it genuinely
8
+ // parsed the term.
44
9
 
45
10
  import { createGraphService } from "../providers/graph-service.mjs";
46
11
  import { resolveSymbol, renderDescribe } from "../codegraph.mjs";
47
12
  import { readFactRows } from "../memory/core.mjs";
48
13
 
49
- /** renderDescribe() renders one LINE per fact (label header, each attribute, each edge
50
- * group) with no terminal punctuation of its own fine for its own "compact plain-text
51
- * description for an agent consumer" purpose, but src/completions/rank.mjs's
52
- * splitSentences() treats each line as its own candidate sentence, and complete.mjs
53
- * joins kept sentences with a single space — so two adjacent kept lines without a
54
- * period between them would otherwise read as one run-on clause. Ensuring every line
55
- * ends in terminal punctuation here (never rewording/reordering the line itself) is
56
- * the cheapest fix that stays entirely inside this adapter, touching neither
57
- * renderDescribe() (server.mjs's tmct_describe tool relies on its current line shape)
58
- * nor rank.mjs/complete.mjs's own join logic. */
14
+ /** renderDescribe() renders one line per fact with no terminal punctuation of its own;
15
+ * rank.mjs's splitSentences() treats each line as its own candidate sentence, and
16
+ * complete.mjs joins kept sentences with a single space, so two adjacent lines without
17
+ * a period between them would otherwise read as one run-on clause. */
59
18
  function withTerminalPunctuation(text) {
60
19
  return String(text || "")
61
20
  .split("\n")
@@ -66,14 +25,9 @@ function withTerminalPunctuation(text) {
66
25
  }
67
26
 
68
27
  /**
69
- * @param {object|null} graph a parseEntities() result (src/codegraph.mjs), or null
70
- * when no code graph is loaded search()/the describe-half of ask() then honestly
71
- * contribute nothing, rather than throwing.
72
- * @param {object|null} [memory=null] a loadMemory() payload (src/memory/core.mjs), or
73
- * null when there's no Fact store to search — the Fact-half of ask() then honestly
74
- * contributes nothing.
75
- * @returns {{search: Function, ask: Function}} a Repository-Interface-shaped
76
- * graphService satisfying src/completions/search.mjs's broadSearch() contract.
28
+ * @param {object|null} graph a parseEntities() result, or null when no code graph is loaded
29
+ * @param {object|null} [memory=null] a loadMemory() payload, or null when there's no Fact store
30
+ * @returns {{search: Function, ask: Function}} a Repository-Interface-shaped graphService
77
31
  */
78
32
  export function createCompletionsGraphAdapter(graph, memory = null) {
79
33
  const svc = graph ? createGraphService(graph) : null;
@@ -1,59 +1,15 @@
1
- // completions/group.mjs — Stage 2 ("grouping") of PLAN_COMPLETIONS.md's six-stage
2
- // mechanical-text-generation pipeline. Stage-0 scope ONLY per the plan's own §4 staging
3
- // table: cluster search.mjs's broadSearch() hits into topical groups; no cross-group
4
- // inference (Stage 3), summarization (Stage 4), pruning (Stage 5), or voice pass (Stage 6)
5
- // happens here.
6
- //
7
- // Genuinely greenfield, as the plan's own prior research established: PLAN_COMPLETIONS.md's
8
- // citation of "insights-panel tag clustering" as prior art turned out to be a different,
9
- // sibling project's capability, not tmct's — there was no clustering/tag-grouping code
10
- // anywhere in this repo before this file.
11
- //
12
- // Granularity: this module clusters at BLOCK granularity — the exact unit search.mjs's
13
- // broadSearch() hands it (one node per whole retrieveBlocks hit, or one node per whole
14
- // graph-search/ask result). PLAN_COMPLETIONS.md's own prose loosely says "spans"; that is
15
- // resolved explicitly here rather than silently assumed: no sub-block span segmentation is
16
- // attempted in this build. A finer-grained span-level increment, if the pipeline ever needs
17
- // it, is future work, out of scope for Stage 0.
18
- //
19
- // Algorithm: CONNECTED COMPONENTS over the block-similarity graph (shared-token-overlap
20
- // edges), reusing memory/blocks.mjs's buildNeighbours()/OVERLAP_MIN — the exact adjacency
21
- // rankBlocks()/degreeOf() already build for PageRank, exported from blocks.mjs for this
22
- // purpose rather than re-derived. This is the strategy-advisor's explicit Stage-0
23
- // recommendation: "the cheapest deterministic starting point" for grouping — genuinely
24
- // CoRank-adjacent (CoRank = "clustering cum graph ranking", PLAN_COMPLETIONS.md §1.2) since
25
- // it clusters over the same graph-ranking substrate blocks.mjs's PageRank runs on, and fully
26
- // deterministic/auditable, unlike any softer/fuzzier clustering. Richer ranking WITHIN a
27
- // cluster (e.g. running rankBlocks per component, or a true joint clustering+ranking pass)
28
- // is a future increment if the pipeline needs it later — not attempted here.
29
- //
30
- // Labeling: each group carries its member ids AND a human-inspectable label — its top
31
- // shared-IDF tokens. The only term-frequency machinery this repo ships is the
32
- // idf = log(1 + N/(1+df)) formula memory/blocks.mjs's retrieveBlocks() already computes
33
- // inline (there is no standalone exported idf() to import — it lives inline in
34
- // retrieveBlocks, scoped to a single query's tokens); that exact formula is replicated here,
35
- // scoped instead to df/N over the hit set passed into groupHits(), rather than inventing a
36
- // new weighting scheme.
37
- //
38
- // Determinism: no randomness anywhere. Same hits in (same ids, same text, same order) ->
39
- // same groups out (same partition, same member order, same label) — this is exactly what
40
- // test/completions-stage0.test.mjs's double-run diff asserts as Stage 0's exit criterion.
1
+ // completions/group.mjs — Stage 2 ("grouping"): clusters search.mjs's broadSearch() hits
2
+ // into topical groups via connected components over a shared-token-overlap graph (reusing
3
+ // memory/blocks.mjs's buildNeighbours()/OVERLAP_MIN). Block granularity, no sub-block spans.
4
+ // Each group's label is its top shared-IDF tokens (df/N over the hit set, not the corpus).
41
5
 
42
6
  import { buildNeighbours, OVERLAP_MIN, tokenizeBlock } from "../memory/blocks.mjs";
43
7
  import { STOPWORDS } from "../prose.mjs";
44
8
 
45
9
  const LABEL_TOKEN_COUNT = 5;
46
10
 
47
- // tokenizeBlock unions tokenizeProse(line) (stopword-filtered, punctuation-stripped) with
48
- // splitIdentifierWords(line) (NOT stopword-filtered it's built for code identifiers, where
49
- // a stopword-shaped fragment is rare, and it doesn't strip trailing sentence punctuation
50
- // either). For prose-similarity clustering that union re-admits filler ("the", "and", "is",
51
- // "q:", "a:", "do?", …) as if it were real overlap signal — harmless for retrieveBlocks'
52
- // IDF-weighted single-query scoring (rare query tokens still dominate), but fatal for
53
- // unweighted shared-token-overlap clustering: almost any two English sentences share ≥2
54
- // stopwords, which would collapse everything into one giant component. isContentToken()
55
- // closes that gap for THIS module's own adjacency/labeling use, without changing
56
- // tokenizeBlock's shipped behavior (still exactly what retrieveBlocks/rankBlocks use).
11
+ // tokenizeBlock's raw output re-admits stopwords, which would collapse almost any two
12
+ // sentences into one giant component under unweighted overlap clustering; filter them here.
57
13
  const isContentToken = (t) => /^[a-z0-9]+$/.test(t) && !STOPWORDS.has(t);
58
14
 
59
15
  /** tokenizeBlock(text), narrowed to real content tokens (see isContentToken above) — the
@@ -85,24 +41,17 @@ function unionFind(n) {
85
41
  *
86
42
  * @param {Array<{id:string, text:string}>} hits
87
43
  * @param {object} [opts]
88
- * @param {number} [opts.overlapMin=OVERLAP_MIN] shared-token threshold for a similarity
89
- * edge — defaults to the exact value memory/blocks.mjs's own PageRank graph uses, so
90
- * grouping and ranking agree on what "related" means unless the caller deliberately
91
- * overrides it.
44
+ * @param {number} [opts.overlapMin=OVERLAP_MIN] shared-token threshold for a similarity edge
92
45
  * @returns {Array<{ id: string, members: Array<{id:string, text:string}>, memberIds: string[],
93
46
  * tokens: string[], label: string }>}
94
- * One entry per connected component (a singleton hit that shares no edge with anything
95
- * else is still a valid, if lonely, one-member group never dropped). Deterministic
96
- * order: groups sorted by their lowest member id; members within a group sorted by id.
97
- * `id` is the group's own stable id (`"g:" + memberIds[0]`, deterministic from content).
47
+ * One entry per connected component (a singleton hit is still a one-member group).
48
+ * Deterministic order: groups by lowest member id; members within a group by id.
98
49
  */
99
50
  export function groupHits(hits, { overlapMin = OVERLAP_MIN } = {}) {
100
51
  const list = Array.isArray(hits) ? hits.filter((h) => h && h.id != null) : [];
101
52
  if (!list.length) return [];
102
53
 
103
- // dedupe by id — a hit can legitimately appear from more than one source (e.g. a block
104
- // and a graph-ask answer both id-tagged distinctly, but a caller re-running broadSearch
105
- // for an overlapping prompt set could still hand duplicate ids); first occurrence wins.
54
+ // dedupe by id — first occurrence wins.
106
55
  const seen = new Set();
107
56
  const deduped = [];
108
57
  for (const h of list) {
@@ -128,9 +77,7 @@ export function groupHits(hits, { overlapMin = OVERLAP_MIN } = {}) {
128
77
  componentIdx.get(root).push(i);
129
78
  }
130
79
 
131
- // IDF over THIS hit set (df/N here, not the whole corpus) — grouping is scoped to what
132
- // Stage 1 actually retrieved for this prompt, same discipline retrieveBlocks applies to a
133
- // single query's tokens.
80
+ // IDF over THIS hit set (df/N), not the whole corpus.
134
81
  const N = ids.length;
135
82
  const df = new Map();
136
83
  for (const id of ids) {
@@ -145,10 +92,7 @@ export function groupHits(hits, { overlapMin = OVERLAP_MIN } = {}) {
145
92
  .sort((a, b) => a.id.localeCompare(b.id));
146
93
  const memberIds = members.map((m) => m.id);
147
94
 
148
- // label tokens: rank by (a) how many members share the token full coverage first, so
149
- // a group's label prefers what its members have IN COMMON over what one member merely
150
- // contains a lot of — then (b) IDF (rarer across the hit set wins ties), then (c) token
151
- // text for a fully deterministic order.
95
+ // label tokens: rank by member coverage, then IDF, then token text (deterministic).
152
96
  const coverage = new Map();
153
97
  for (const i of memberIdx) {
154
98
  for (const t of new Set(tokensById[ids[i]])) coverage.set(t, (coverage.get(t) || 0) + 1);
@@ -1,56 +1,19 @@
1
- // completions/infer.mjs — Stage 3 ("inference between groups") of PLAN_COMPLETIONS.md's
2
- // six-stage mechanical-text-generation pipeline. §4's staging table: "cross-group inference
3
- // wired to the existing entailment/rule-chase machinery, closed inference-relation
4
- // vocabulary... Reuses syllogise.mjs/resolveRelationChase, no new engine... Every asserted
5
- // inference cites a concrete licensing test, zero fabricated relationships on a hand-labeled
6
- // set" — this file, plus test/completions-infer.test.mjs's hand-labeled fixture, is that exit
7
- // criterion made concrete.
1
+ // completions/infer.mjs — Stage 3 ("inference between groups"): applies resolveRelationChase
2
+ // (src/memory/core.mjs) to relationships BETWEEN retrieved text groups, not just graph facts.
3
+ // Four relations (supports/contradicts/elaborates/exemplifies), each with its own named
4
+ // licensing test — see the four test*() functions below. A relation is asserted only when its
5
+ // test concretely licenses it, never from prose similarity.
8
6
  //
9
- // "Apply tmct's existing entailment machinery... not just to graph facts, but to
10
- // relationships BETWEEN retrieved text groups... a closed, small inference-relation
11
- // vocabulary, deliberately mirroring marginalia's own TYPED_EDGES closed set... A relationship
12
- // between two groups is only asserted when a concrete, named test licenses it... never
13
- // inferred by prose similarity alone" (PLAN_COMPLETIONS.md §1.3). Four relations, each with
14
- // its own mechanical, named licensing test — see the four test*() functions below, one per
15
- // relation, each documented at its own definition:
16
- //
17
- // supports — resolveRelationChase (src/memory/core.mjs, PLAN_COMPLETIONS.md Stage 1's
18
- // own prerequisite extraction) confirms a taught relation fact between two
19
- // entities both groups' text share.
20
- // contradicts — the two groups' text carry OPPOSITE negation polarity around the SAME
21
- // shared graph-known entity + shared content token (token-level, closed
22
- // negation-marker set — no graph fact required).
23
- // elaborates — one group's graph-known entity set is a PROPER SUBSET of the other's (the
24
- // wider group elaborates the narrower one).
25
- // exemplifies — one group names a class-level term (something else is taught
26
- // rdfs:subClassOf/rdf:type it — checkable via the SAME memory/core.mjs-loaded
27
- // fact rows), and the other group names a taught INSTANCE of that class.
28
- //
29
- // "Entity" grounding: a group's raw content tokens (tokenizeBlock, the same tokenizer
30
- // group.mjs/rank.mjs use, filtered the same isContentToken way those two files already
31
- // establish) are narrowed to GRAPH-KNOWN terms only — tokens that normFactTerm-match some
32
- // fact's subject or object in the loaded memory. This is the concrete grounding that keeps
33
- // "group A and group B share an entity" a checkable graph fact, not a prose-similarity guess:
34
- // two groups merely using the same English word never licenses anything on its own unless
35
- // that word is itself a taught term.
36
- //
37
- // Determinism: no randomness anywhere. Groups are processed in a fixed (id-sorted) pairwise
38
- // order; every internal token/entity set is turned into a sorted array before use; every
39
- // per-relation test returns at most one hit per (group pair, relation), picked by that fixed
40
- // order — never "first of an unordered Set/Map iteration". See
41
- // test/completions-infer.test.mjs's own double-run diff test, the same discipline
42
- // test/completions-stage0.test.mjs and test/completions-stage2.test.mjs already apply to
43
- // search+group and to sentence ranking.
7
+ // Entities are a group's content tokens narrowed to graph-known terms (normFactTerm-matched
8
+ // against loaded facts) sharing an English word alone never licenses a relation.
44
9
 
45
10
  import { normFactTerm, readFactRows, resolveRelationChase } from "../memory/core.mjs";
46
11
  import { tokenizeBlock } from "../memory/blocks.mjs";
47
12
  import { splitSentences } from "./rank.mjs";
48
13
  import { STOPWORDS } from "../prose.mjs";
49
14
 
50
- // Same content-token filter group.mjs/rank.mjs each apply to their own adjacency/ranking (not
51
- // exported from either, so replicated here rather than reached across files — see either
52
- // file's own header for why raw tokenizeBlock output, which re-admits stopword-shaped filler,
53
- // is unsuitable for unweighted set operations like the ones this file runs).
15
+ // Same content-token filter group.mjs/rank.mjs apply to their own adjacency/ranking; not
16
+ // exported from either, so replicated here rather than reached across files.
54
17
  const isContentToken = (t) => /^[a-z0-9]+$/.test(t) && !STOPWORDS.has(t);
55
18
 
56
19
  /** tokenizeBlock(text), narrowed to real content tokens — see isContentToken above. */
@@ -58,25 +21,15 @@ function contentTokens(text) {
58
21
  return tokenizeBlock(text).filter(isContentToken);
59
22
  }
60
23
 
61
- // This file's own local copy of chat.mjs's private HAS_PROPERTY_PREDICATE constant (same
62
- // literal string, "mgx:hasProperty") not exported from memory/core.mjs or chat.mjs, so
63
- // resolveRelationChase's own unit tests (test/memory-core.test.mjs) establish the precedent of
64
- // a caller supplying its own minimal, self-contained copy in its `helpers` bag rather than
65
- // reaching into chat.mjs (out of scope for this dispatch) for the shared constant.
24
+ // Local copy of chat.mjs's private HAS_PROPERTY_PREDICATE constant — not exported, so callers
25
+ // supply their own copy in the `helpers` bag resolveRelationChase expects.
66
26
  const HAS_PROPERTY_PREDICATE = "mgx:hasProperty";
67
27
 
68
- // The taught ISA-family predicates (chat.mjs's own MINT_ISA_PREDICATES/ISA_PREDICATES sets
69
- // this same pair, elsewhere) — the exemplifies test's "checkable via the taught IsA/subClassOf
70
- // graph" per PLAN_COMPLETIONS.md §1.3, read directly off readFactRows() rows rather than via
71
- // syllogise.mjs's fuller OWL 2 RL machinery (out of scope for this dispatch; this file only
72
- // needs the STORED isa edges, not their transitive closure).
28
+ // The taught ISA-family predicates (stored edges only, not their transitive closure).
73
29
  const ISA_PREDICATES = new Set(["rdfs:subClassOf", "rdf:type"]);
74
30
 
75
- // The contradicts test's closed negation-marker vocabulary (PLAN_COMPLETIONS.md §1.3's own
76
- // suggestion: "a simpler token-level polarity check... a small closed negation-word set").
77
- // Deliberately checked against the RAW sentence text, not contentTokens()'s output — prose.mjs's
78
- // STOPWORDS (which isContentToken filters through) already strips "not"/"no" as filler for
79
- // clustering/ranking purposes, which would silently erase the exact signal this test needs.
31
+ // The contradicts test's closed negation-marker vocabulary, checked against RAW sentence
32
+ // text (contentTokens() strips "not"/"no" as stopword filler, which would erase this signal).
80
33
  const NEGATION_MARKERS = new Set([
81
34
  "not", "no", "never", "cannot", "none", "nobody", "nothing", "neither", "nor", "without",
82
35
  ]);
@@ -127,30 +80,21 @@ function buildGraphTerms(rows) {
127
80
  return set;
128
81
  }
129
82
 
130
- /** A group's GRAPH-GROUNDED entities: its own content-token vocabulary, narrowed to tokens
131
- * that are themselves graph-known terms (buildGraphTerms' universe) — sorted for determinism.
132
- * This is the concrete grounding test/completions-infer.test.mjs's fixture exercises: two
133
- * groups merely sharing an English word (e.g. both saying "abstraction") never counts unless
134
- * that word is itself a taught fact term. */
83
+ /** A group's GRAPH-GROUNDED entities: content tokens narrowed to graph-known terms, sorted. */
135
84
  function entitiesOf(group, graphTerms) {
136
85
  return [...groupContentTokenSet(group)].filter((t) => graphTerms.has(t)).sort();
137
86
  }
138
87
 
139
- /** A minimal, self-contained relationFactsFor(name) direct-predicate match only
140
- * (`mgx:${name}`), no alias/subClassOf-over-relation-names chase. Mirrors
141
- * test/memory-core.test.mjs's own testRelationFactsFor precedent exactly: resolveRelationChase
142
- * never calls the alias substrate itself, that lives entirely inside whatever relationFactsFor
143
- * the caller supplies, and a direct-only implementation is an honest, valid, simpler one — no
144
- * alias-chase claim is made or needed for the supports test's own licensing standard. */
88
+ /** A minimal relationFactsFor(name): direct-predicate match only (`mgx:${name}`), no
89
+ * alias/subClassOf chase — sufficient for this test's licensing standard. */
145
90
  function makeRelationFactsFor(rows) {
146
91
  return (name) => rows
147
92
  .filter((f) => f.predicate === `mgx:${name}`)
148
93
  .map((f) => ({ fact: f, aliasFacts: [] }));
149
94
  }
150
95
 
151
- /** The resolveRelationChase/resolveRelationChaseReverse `helpers` bag this file supplies
152
- * same shape test/memory-core.test.mjs's own direct unit tests use, built once per
153
- * inferRelations() call over the loaded memory's fact rows. */
96
+ /** The resolveRelationChase/resolveRelationChaseReverse `helpers` bag this file supplies,
97
+ * built once per inferRelations() call over the loaded memory's fact rows. */
154
98
  function makeHelpers(rows) {
155
99
  return {
156
100
  relationFactsFor: makeRelationFactsFor(rows),
@@ -163,10 +107,8 @@ function makeHelpers(rows) {
163
107
  };
164
108
  }
165
109
 
166
- /** Every distinct relation NAME resolveRelationChase can plausibly be asked about: every
167
- * `mgx:<name>` predicate actually present among the loaded facts, minus HAS_PROPERTY_PREDICATE
168
- * (a property-literal marker, not a relation name) — a closed, corpus-derived candidate list,
169
- * never an open-ended guess at what "a relation" might be named. Sorted for determinism. */
110
+ /** Every distinct relation NAME present among the loaded facts (`mgx:<name>` predicates,
111
+ * minus HAS_PROPERTY_PREDICATE, a property marker not a relation name). Sorted. */
170
112
  function relationNameCandidates(rows) {
171
113
  const names = new Set();
172
114
  for (const r of rows) {
@@ -177,15 +119,9 @@ function relationNameCandidates(rows) {
177
119
  return [...names].sort();
178
120
  }
179
121
 
180
- /**
181
- * SUPPORTS group A and group B share at least two graph-grounded entities, AND a taught
182
- * relation fact (resolveRelationChase, src/memory/core.mjs this file's direct tie-in to
183
- * PLAN_COMPLETIONS.md Stage 1's own prerequisite extraction) confirms a claim connecting two of
184
- * those shared entities. Tries every (subject, object) ordered pair drawn from the shared
185
- * entity set, against every candidate relation name, in fixed sorted order; the first hit
186
- * (deterministic given fixed inputs) is the one asserted. Returns
187
- * `{ licensingTest, evidence }` or null.
188
- */
122
+ /** SUPPORTS — groups A/B share >=2 graph-grounded entities AND a taught relation fact
123
+ * connects two of them. Tries every (subject, object) pair against every candidate relation
124
+ * name in fixed sorted order; first hit wins. Returns `{ licensingTest, evidence }` or null. */
189
125
  async function testSupports(a, b, memory, helpers, relationNames, graphTerms) {
190
126
  const entitiesA = entitiesOf(a, graphTerms);
191
127
  const entitiesB = entitiesOf(b, graphTerms);
@@ -209,14 +145,9 @@ async function testSupports(a, b, memory, helpers, relationNames, graphTerms) {
209
145
  return null;
210
146
  }
211
147
 
212
- /**
213
- * CONTRADICTS group A and group B both mention the SAME graph-grounded entity, plus a
214
- * second shared content token ("predicate"/aspect) that co-occurs with the entity in at least
215
- * one sentence on each side — and one side's co-occurring sentence carries a closed-set
216
- * negation marker while the other side's does not (opposite polarity about the same claim,
217
- * PLAN_COMPLETIONS.md §1.3's own "simpler token-level polarity check"). Returns
218
- * `{ licensingTest, evidence }` or null.
219
- */
148
+ /** CONTRADICTS — groups A/B share a graph-grounded entity plus a second co-occurring token
149
+ * ("aspect"), and one side's matching sentence is negated while the other's isn't. Returns
150
+ * `{ licensingTest, evidence }` or null. */
220
151
  function testContradicts(a, b, graphTerms) {
221
152
  const entitiesA = entitiesOf(a, graphTerms);
222
153
  const entitiesB = entitiesOf(b, graphTerms);
@@ -271,14 +202,9 @@ function isProperSubset(small, big) {
271
202
  return small.size > 0 && small.size < big.size && [...small].every((t) => big.has(t));
272
203
  }
273
204
 
274
- /**
275
- * ELABORATES — one group's graph-grounded entity set is a PROPER SUBSET of the other's: the
276
- * WIDER group (the superset) elaborates the NARROWER one (the subset) — it covers everything
277
- * the narrower group's entities do, plus more. Equal entity sets never count (neither is a
278
- * *proper* subset of the other) — two groups about exactly the same entities are not in an
279
- * elaboration relationship by this test. Returns `{ wider: "a"|"b", licensingTest, evidence }`
280
- * or null.
281
- */
205
+ /** ELABORATES — one group's graph-grounded entity set is a PROPER SUBSET of the other's; the
206
+ * wider group elaborates the narrower one. Equal sets never count. Returns
207
+ * `{ wider: "a"|"b", licensingTest, evidence }` or null. */
282
208
  function testElaborates(a, b, graphTerms) {
283
209
  const entitiesA = new Set(entitiesOf(a, graphTerms));
284
210
  const entitiesB = new Set(entitiesOf(b, graphTerms));
@@ -300,15 +226,10 @@ function testElaborates(a, b, graphTerms) {
300
226
  return null;
301
227
  }
302
228
 
303
- /**
304
- * EXEMPLIFIES — `general` names a class-level term (some taught fact has it as the OBJECT of
305
- * an ISA_PREDICATES edge i.e. something else is taught to BE one of it), and `instance`
306
- * names a graph-grounded entity taught to BE one, directly (ISA_PREDICATES edge: instance ->
307
- * general). Asymmetric and directional by construction: `instance`'s group exemplifies
308
- * `general`'s group, never the reverse in the same call — callers probe both directions by
309
- * calling this twice with the groups swapped (see inferRelations below). Returns
310
- * `{ licensingTest, evidence }` or null.
311
- */
229
+ /** EXEMPLIFIES — `general` names a class-level term (object of an ISA_PREDICATES edge), and
230
+ * `instance` names an entity taught to BE one directly. Asymmetric: callers probe both
231
+ * directions by calling this twice with groups swapped (see inferRelations below). Returns
232
+ * `{ licensingTest, evidence }` or null. */
312
233
  function testExemplifies(general, instance, rows, graphTerms) {
313
234
  const generalEntities = entitiesOf(general, graphTerms);
314
235
  const instanceEntities = entitiesOf(instance, graphTerms);
@@ -332,22 +253,14 @@ function testExemplifies(general, instance, rows, graphTerms) {
332
253
  }
333
254
 
334
255
  /**
335
- * Stage 3 — cross-group inference. For every unordered pair of groups (group.mjs's groupHits()
336
- * output, or any `{ id, members: [{id, text}] }` array), tests each of the four closed
337
- * relations (supports/contradicts/elaborates/exemplifies) via its own concrete, named,
338
- * mechanical licensing test — never prose similarity. A relation only appears in the output
339
- * when its own test function returns a hit; every hit carries `licensingTest` (a human-
340
- * readable description of exactly what fired) and `evidence` (the concrete facts/tokens cited)
341
- * — PLAN_COMPLETIONS.md §2's auditability bar ("every cross-group claim must cite the
342
- * two-or-more groups and the inference kind that licensed it").
256
+ * Stage 3 — cross-group inference. For every unordered pair of groups, tests each of the four
257
+ * closed relations and includes a hit only when its test function fires.
343
258
  *
344
259
  * @param {Array<{id:string, members:Array<{id:string,text:string}>}>} groups
345
260
  * @param {object} memory an already-loaded memory/core.mjs loadMemory() payload
346
261
  * @param {object} [opts] reserved for future tuning; unused today
347
262
  * @returns {Promise<Array<{from:string, to:string, relation:"supports"|"contradicts"|"elaborates"|"exemplifies", licensingTest:string, evidence:object}>>}
348
- * deterministic: groups are processed in id-sorted pairwise order, and the final list is
349
- * additionally stable-sorted by (from, to, relation) so output order never depends on
350
- * incidental iteration order anywhere upstream.
263
+ * deterministic: id-sorted pairwise order, stable-sorted by (from, to, relation).
351
264
  */
352
265
  // eslint-disable-next-line no-unused-vars -- opts reserved, see docblock
353
266
  export async function inferRelations(groups, memory, opts = {}) {
@@ -381,8 +294,7 @@ export async function inferRelations(groups, memory, opts = {}) {
381
294
  out.push({ from, to, relation: "elaborates", licensingTest: ela.licensingTest, evidence: ela.evidence });
382
295
  }
383
296
 
384
- // Both directions probed independently — "A exemplifies B" and "B exemplifies A" are
385
- // genuinely different claims, each licensed (or not) by its own class/instance test.
297
+ // Both directions probed independently — different claims, each its own test.
386
298
  const bExemplifiesA = testExemplifies(A, B, rows, graphTerms);
387
299
  if (bExemplifiesA) out.push({ from: B.id, to: A.id, relation: "exemplifies", licensingTest: bExemplifiesA.licensingTest, evidence: bExemplifiesA.evidence });
388
300
  const aExemplifiesB = testExemplifies(B, A, rows, graphTerms);