@polycode-projects/the-mechanical-code-talker 1.9.2 → 1.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +441 -202
  2. package/bin/tmct.mjs +126 -1
  3. package/package.json +4 -2
  4. package/src/answer-variants.mjs +8 -36
  5. package/src/ask-browser-entry.mjs +5 -23
  6. package/src/ask-browser.bundle.js +1 -2
  7. package/src/ask-nlp.mjs +9 -23
  8. package/src/ask-vocab.mjs +139 -589
  9. package/src/ask.mjs +627 -1729
  10. package/src/chat.mjs +1684 -2872
  11. package/src/cli-args.mjs +14 -28
  12. package/src/codegraph.mjs +236 -644
  13. package/src/completions/complete.mjs +18 -62
  14. package/src/completions/graph-adapter.mjs +14 -60
  15. package/src/completions/group.mjs +12 -68
  16. package/src/completions/infer.mjs +38 -126
  17. package/src/completions/prune.mjs +17 -70
  18. package/src/completions/rank.mjs +16 -69
  19. package/src/completions/search.mjs +8 -31
  20. package/src/concept.mjs +32 -88
  21. package/src/conformance.mjs +11 -15
  22. package/src/corpus/conceptnet.mjs +31 -89
  23. package/src/corpus/templates.mjs +19 -45
  24. package/src/corpus/unknown-ingest.mjs +31 -92
  25. package/src/embed.mjs +10 -22
  26. package/src/extensions.mjs +50 -154
  27. package/src/finish.mjs +35 -91
  28. package/src/grammar/ace.mjs +16 -40
  29. package/src/grammar/assert.mjs +1 -1
  30. package/src/grammar/lexicon-core.json +1 -1
  31. package/src/grammar/lexicon.mjs +9 -27
  32. package/src/graph-merge.mjs +2 -3
  33. package/src/hash.mjs +6 -14
  34. package/src/index.mjs +6 -10
  35. package/src/init.mjs +38 -125
  36. package/src/interpret/fuzzy.mjs +10 -29
  37. package/src/interpret/merge.mjs +9 -27
  38. package/src/interpret/normalize.mjs +137 -585
  39. package/src/interpret/pipeline.mjs +23 -71
  40. package/src/interpret/strategies/ace.mjs +7 -31
  41. package/src/interpret/strategies/constructions.mjs +14 -41
  42. package/src/interpret/strategies/grammar.mjs +21 -60
  43. package/src/interpret/strategies/keywords.mjs +42 -131
  44. package/src/interpret/strategies/noise-strip.mjs +18 -89
  45. package/src/memory/bias.mjs +11 -54
  46. package/src/memory/blocks.mjs +18 -69
  47. package/src/memory/core.mjs +171 -591
  48. package/src/memory/fold.mjs +0 -0
  49. package/src/memory/inspect.mjs +7 -25
  50. package/src/memory/shacl.mjs +10 -39
  51. package/src/memory/trust.mjs +26 -127
  52. package/src/memory-ask-browser-entry.mjs +7 -30
  53. package/src/memory-ask-browser.bundle.js +1 -1
  54. package/src/paraphrase.mjs +20 -53
  55. package/src/planning.mjs +15 -157
  56. package/src/prose-nlp.mjs +4 -17
  57. package/src/prose.mjs +19 -67
  58. package/src/providers/bootstrap.mjs +1 -2
  59. package/src/providers/fixture.mjs +1 -2
  60. package/src/providers/graph-service.mjs +28 -59
  61. package/src/repository-interface.mjs +6 -8
  62. package/src/router/drive.mjs +183 -0
  63. package/src/router/goal-reasoner.mjs +66 -231
  64. package/src/router/guardrail.mjs +20 -58
  65. package/src/router/planner.mjs +15 -46
  66. package/src/router/registry.mjs +13 -43
  67. package/src/router/resolver.mjs +46 -131
  68. package/src/router/results.mjs +231 -0
  69. package/src/schema-docs.mjs +10 -27
  70. package/src/server-http.mjs +10 -19
  71. package/src/server.mjs +22 -28
  72. package/src/sessions.mjs +15 -30
  73. package/src/source-slice.mjs +5 -7
  74. package/src/source.mjs +10 -20
  75. package/src/syllogise.mjs +187 -575
  76. package/src/telemetry.mjs +3 -3
  77. package/src/toml-config.mjs +4 -4
  78. package/src/tui/app.mjs +9 -19
  79. package/src/viz.mjs +66 -123
  80. package/src/wink-model.mjs +10 -24
@@ -1,57 +1,14 @@
1
- // completions/prune.mjs — Stage 5 ("drop non-contributing elements") of PLAN_COMPLETIONS.md's
2
- // six-stage mechanical-text-generation pipeline. §4's staging table (row 3, "Stage 5+6"): "any
3
- // retrieved span that ends up in no surviving group, feeds no asserted inference, and is not
4
- // selected by Stage 4's ranking gets cut, explicitly, with the drop recorded (not silently
5
- // discarded) so the pipeline's own working set is auditable end to end" (§1.5).
6
- //
7
- // Reading the plan's own sentence precisely: the three conditions are joined by AND, so a span
8
- // is dropped only when ALL three hold — equivalently, a span is KEPT when ANY of the three is
9
- // false: it IS in a surviving group, OR it DOES feed an asserted inference, OR it IS selected by
10
- // ranking. This module implements that OR exactly, at SENTENCE granularity (rank.mjs's own
11
- // ranking unit — the thing that actually composes into the assembled prose), with a "hit never
12
- // grouped" defensive check underneath it for the coarser block/hit granularity search.mjs hands
13
- // group.mjs (group.mjs's own contract is that every hit lands in SOME group, so this branch is
14
- // a guard against that invariant breaking silently, not an expected live path today).
15
- //
16
- // Inputs are the pipeline's own already-computed intermediate state — this module computes
17
- // NOTHING new about relevance; it only DECIDES keep/drop from what search.mjs/group.mjs/
18
- // infer.mjs/rank.mjs already produced, and records why. `rankedByGroup` is deliberately an
19
- // INPUT (rank.mjs's own rankSentences() output per group), not recomputed here — pruning is a
20
- // pure decision layer over ranking's output, not a second ranker.
21
- //
22
- // THRESHOLD + REASONING (the plan's own "your call on a sensible cutoff, document your
23
- // reasoning"):
24
- // - top-K per group (default 3, `opts.maxSentencesPerGroup`): rankSentences() is already
25
- // best-first per group; keeping only the top K bounds the assembled completion's size in a
26
- // VISIBLE, documented, override-able way. An unbounded "keep everything with any positive
27
- // score" would make the completion grow without limit as a group's member count grows —
28
- // that is itself a silent, unaudited cap in the opposite direction (nothing stops it from
29
- // ballooning to the size of the whole retrieved corpus for a broad-enough prompt); a small
30
- // explicit K is the "no silent caps" discipline applied honestly — the cap is visible in the
31
- // signature, in this comment, and in every drop log entry it produces.
32
- // - positive score (`score > 0`): rankSentences()'s own scoring is 0 exactly when a sentence
33
- // shares no informative (or, under query-focus, no query-overlapping) token with anything —
34
- // rank.mjs's own "never a guessed match" discipline. A zero-information sentence contributes
35
- // nothing to the completion's content by rankSentences()'s own definition, so it never
36
- // qualifies on ranking grounds alone, however small K is set.
37
- // - relation-anchor salvage: a group that feeds an asserted cross-group inference (infer.mjs)
38
- // but produces ZERO ranking-qualified sentences (every one of its sentences is either
39
- // outside top-K or zero-scored) is not silenced outright — its single top-ranked sentence is
40
- // kept as the group's extractive anchor, so a cross-group claim this pipeline actually
41
- // ASSERTS (with cited licensing evidence) always has at least one real, traceable sentence
42
- // behind it in the assembled text. This is the concrete realisation of the plan's OR: the
43
- // three drop conditions must ALL hold, and "feeds no asserted inference" is one of them.
44
- //
45
- // Determinism: no randomness anywhere. Groups are processed in a fixed (id-sorted) order;
46
- // rankedByGroup's own sentence order (rank.mjs's own deterministic tiebreak) is preserved
47
- // exactly; the output kept/dropped lists are therefore fully deterministic given deterministic
48
- // inputs — see test/completions-prune.test.mjs's own double-run diff.
1
+ // completions/prune.mjs — Stage 5 ("drop non-contributing elements"), at SENTENCE
2
+ // granularity: a sentence is kept if it's in a surviving group AND (feeds an asserted
3
+ // inference OR clears the per-group top-K/positive-score ranking cutoff); otherwise it's
4
+ // dropped with a recorded reason. A group that feeds an inference but has no ranking-
5
+ // qualified sentence still keeps its top-ranked sentence as an extractive anchor, so every
6
+ // asserted relation traces to a real sentence. Pure decision layer — computes nothing new
7
+ // about relevance, only reads what search/group/infer/rank already produced.
49
8
 
50
9
  const DEFAULT_MAX_SENTENCES_PER_GROUP = 3;
51
10
 
52
- /** Every group id referenced as either side of an asserted relation (infer.mjs's inferRelations()
53
- * output) — the "feeds an asserted inference" test, at group granularity (the granularity
54
- * relations are actually asserted at). */
11
+ /** Every group id referenced as either side of an asserted relation. */
55
12
  function relatedGroupIdsOf(relations) {
56
13
  const set = new Set();
57
14
  for (const r of relations) {
@@ -62,30 +19,22 @@ function relatedGroupIdsOf(relations) {
62
19
  }
63
20
 
64
21
  /**
65
- * Stage 5 — pruning. Decides, per sentence, KEEP or DROP, from the pipeline's own already-
66
- * computed intermediate state — never recomputing relevance itself.
22
+ * Stage 5 — pruning. Decides, per sentence, KEEP or DROP.
67
23
  *
68
24
  * @param {object} state
69
- * @param {Array<{id:string, text?:string}>} [state.hits] search.mjs's broadSearch() output (used
70
- * only for the defensive "never grouped" check below — group.mjs's own contract is that this
71
- * branch should never actually fire today).
25
+ * @param {Array<{id:string, text?:string}>} [state.hits] search.mjs's broadSearch() output
72
26
  * @param {Array<{id:string, memberIds:string[], label?:string}>} [state.groups] group.mjs's
73
- * groupHits() output.
27
+ * groupHits() output
74
28
  * @param {Array<{from:string, to:string, relation:string}>} [state.relations] infer.mjs's
75
- * inferRelations() output.
29
+ * inferRelations() output
76
30
  * @param {Object<string, Array<{sentence:string, score:number, sourceBlockId:string}>>}
77
- * [state.rankedByGroup] rank.mjs's rankSentences() output, ONE ENTRY PER GROUP, keyed by
78
- * group id — an INPUT to this module, not recomputed here (see file header).
31
+ * [state.rankedByGroup] rank.mjs's rankSentences() output, keyed by group id
79
32
  * @param {object} [opts]
80
- * @param {number} [opts.maxSentencesPerGroup=3] the top-K-per-group ranking cutoff (see file
81
- * header for the reasoning behind the default).
33
+ * @param {number} [opts.maxSentencesPerGroup=3] the top-K-per-group ranking cutoff
82
34
  * @returns {{
83
35
  * kept: Array<{sentence:string, score:number, sourceBlockId:string, groupId:string, groupLabel:string}>,
84
36
  * dropped: Array<{item:object, reason:string}>
85
- * }} `kept` is sentence-granular, ordered by (group id, rank order within the group) a stable,
86
- * deterministic order the caller can assemble directly. `dropped` is itemized, one entry per
87
- * dropped hit/sentence, each carrying a human-readable `reason` (never a silent discard —
88
- * PLAN_COMPLETIONS.md §1.5/§2's own auditability bar).
37
+ * }} `kept` is ordered by (group id, rank order); `dropped` carries a human-readable `reason`.
89
38
  */
90
39
  export function pruneCompletion(state = {}, { maxSentencesPerGroup = DEFAULT_MAX_SENTENCES_PER_GROUP } = {}) {
91
40
  const hits = Array.isArray(state.hits) ? state.hits : [];
@@ -96,8 +45,7 @@ export function pruneCompletion(state = {}, { maxSentencesPerGroup = DEFAULT_MAX
96
45
  const kept = [];
97
46
  const dropped = [];
98
47
 
99
- // "never grouped" — defensive: every hit should land in SOME group (group.mjs never drops a
100
- // hit), so this guards the invariant explicitly rather than silently assuming it holds.
48
+ // "never grouped" — defensive: group.mjs's contract is every hit lands in some group.
101
49
  const groupedHitIds = new Set();
102
50
  for (const g of groups) for (const id of g.memberIds || []) groupedHitIds.add(id);
103
51
  for (const h of hits) {
@@ -124,8 +72,7 @@ export function pruneCompletion(state = {}, { maxSentencesPerGroup = DEFAULT_MAX
124
72
 
125
73
  let anchor = null;
126
74
  if (!qualifying.length && groupFeedsInference && ranked.length) {
127
- // salvage: the group's own single top-ranked sentence, kept as the extractive anchor for
128
- // its asserted cross-group relation even though it didn't clear the ranking cutoff alone.
75
+ // salvage: keep the group's top-ranked sentence as the extractive anchor.
129
76
  anchor = ranked[0];
130
77
  qualifying.push(anchor);
131
78
  const idx = rest.indexOf(anchor);
@@ -1,40 +1,13 @@
1
- // completions/rank.mjs — Stage 4 ("mechanical summarization") of PLAN_COMPLETIONS.md's
2
- // six-stage mechanical-text-generation pipeline. Stage-2-of-staging scope per the plan's own
3
- // §4 table: extractive SENTENCE ranking within a group.mjs group no cross-group inference
4
- // (Stage 3, §1.3 — separately scoped), pruning (Stage 5), or voice pass (Stage 6) happens
5
- // here.
6
- //
7
- // "Extractive sentence selection over the grouped-and-inferred material, not abstractive
8
- // rewriting... query-focused multi-document summarization (feature-fusion sentence
9
- // selection, graph-ranking approaches in the LexRank/TextRank family, clustering-cum-ranking
10
- // as in CoRank)" — PLAN_COMPLETIONS.md §1.4/§4. The graph-ranking machinery this stage needs
11
- // already exists and ships: memory/blocks.mjs's rankBlocks() (PageRank, d=0.85, 20
12
- // iterations, over the shared-token block-similarity graph) and degreeOf() (hub dampening),
13
- // both generic over any `{ id: tokens[] }` map — NOT block-specific despite the module name.
14
- // This file reuses rankBlocks()/degreeOf() VERBATIM at sentence granularity (no PageRank
15
- // reimplementation), combined with the same idf = log(1 + N/(1+df)) formula group.mjs already
16
- // replicates for group-scoped (not whole-corpus-scoped) weighting, and the same
17
- // idfSum * (1+rank) / sqrt(1+degree) combination retrieveBlocks() uses to fuse relevance,
18
- // centrality, and hub-dampening into one score.
19
- //
20
- // New in this file: splitSentences() (no sentence-splitter existed anywhere in this repo —
21
- // grepped for "sentence" across src/ and found none; a simple regex splitter is intentional
22
- // here per the dispatch's own instruction not to pull in an NLP dependency for this) and the
23
- // sentence-level id/token/scoring wiring that adapts rankBlocks/degreeOf from block to
24
- // sentence granularity.
25
- //
26
- // Determinism: no randomness anywhere. Same group in (same members, same text, same order)
27
- // -> same ranked sentence list out (stable score-descending sort with a deterministic
28
- // sourceBlockId/sentence-text tiebreak) — see test/completions-stage2.test.mjs's double-run
29
- // diff, the exact discipline test/completions-stage0.test.mjs established for search+group.
1
+ // completions/rank.mjs — Stage 4 ("mechanical summarization"): extractive SENTENCE ranking
2
+ // within a group.mjs group. Reuses memory/blocks.mjs's rankBlocks() (PageRank) and degreeOf()
3
+ // (hub dampening) verbatim at sentence granularity, combined with group-scoped IDF the same
4
+ // way retrieveBlocks() fuses relevance/centrality/hub-dampening into one score.
5
+ // splitSentences() is a simple regex splitter — deliberately not an NLP dependency.
30
6
 
31
7
  import { degreeOf, rankBlocks, tokenizeBlock, OVERLAP_MIN } from "../memory/blocks.mjs";
32
8
  import { STOPWORDS } from "../prose.mjs";
33
9
 
34
- // Same content-token filter group.mjs applies to its own adjacency/labeling (not exported
35
- // from group.mjs, so replicated here rather than reached across files — group.mjs's own
36
- // header explains why raw tokenizeBlock output is unsuitable for unweighted overlap: it
37
- // re-admits stopwords/filler that would collapse unrelated sentences into false edges).
10
+ // Same content-token filter group.mjs applies (not exported, so replicated here).
38
11
  const isContentToken = (t) => /^[a-z0-9]+$/.test(t) && !STOPWORDS.has(t);
39
12
 
40
13
  /** tokenizeBlock(text), narrowed to real content tokens — see isContentToken above. */
@@ -43,16 +16,11 @@ function contentTokens(text) {
43
16
  }
44
17
 
45
18
  // Sentence boundary: a run of [.!?] followed by whitespace and an uppercase letter or digit —
46
- // deliberately simple (no abbreviation dictionary, no NLP dependency, per the dispatch's own
47
- // instruction). Applied per-line (a block's own "Q: ...\nA: ..." shape already gives clean
48
- // boundaries at the newline; this regex further splits any line that itself holds more than
49
- // one sentence).
19
+ // deliberately simple (no abbreviation dictionary, no NLP dependency).
50
20
  const SENTENCE_SPLIT_RE = /(?<=[.!?])\s+(?=[A-Z0-9])/;
51
21
 
52
22
  /**
53
- * Split raw block text into trimmed, non-empty sentences (order-preserving, no dedup
54
- * dedup, if ever needed, is the caller's call). Splits on blank lines first (a block's own
55
- * natural line structure), then on sentence-ending punctuation within each line.
23
+ * Split raw block text into trimmed, non-empty sentences (order-preserving, no dedup).
56
24
  *
57
25
  * @param {string} text
58
26
  * @returns {string[]}
@@ -71,26 +39,16 @@ export function splitSentences(text) {
71
39
  }
72
40
 
73
41
  /**
74
- * Stage 4 — extractive sentence ranking within one group.mjs group. Splits every member's
75
- * text into sentences, builds a sentence-level shared-token-overlap similarity graph (the
76
- * exact adjacency buildNeighbours/rankBlocks/degreeOf already establish, just re-keyed to
77
- * sentence ids instead of block ids), runs the same PageRank (rankBlocks) and hub-dampening
78
- * (degreeOf) over it, and combines with group-scoped IDF the same way retrieveBlocks() fuses
79
- * relevance × centrality × hub-dampening — self-weighted (LexRank-style: a sentence's own
80
- * rare/informative tokens) unless `opts.query` is supplied, in which case only tokens
81
- * overlapping the query are IDF-summed (query-focused summarization, PLAN_COMPLETIONS.md's
82
- * own literature framing) — a sentence with zero query overlap honestly scores 0 rather than
83
- * silently falling back to self-weighting, the same "never a guessed match" discipline
84
- * retrieveBlocks() applies when idfSum <= 0.
42
+ * Stage 4 — extractive sentence ranking within one group.mjs group: splits every member's
43
+ * text into sentences, ranks them by PageRank + hub-dampening + IDF (self-weighted, or
44
+ * query-focused when `opts.query` is given a sentence with zero query overlap scores 0
45
+ * rather than falling back to self-weighting).
85
46
  *
86
47
  * @param {{members: Array<{id:string, text:string}>}} group a group.mjs groupHits() entry
87
- * (or any object shaped `{ members: [{id, text}, ...] }`)
88
48
  * @param {object} [opts]
89
49
  * @param {number} [opts.overlapMin=OVERLAP_MIN] shared content-token threshold for a
90
- * sentence-similarity edge — defaults to the same value group.mjs/blocks.mjs use, so
91
- * grouping, block-ranking, and sentence-ranking all agree on what "related" means unless
92
- * the caller deliberately overrides it.
93
- * @param {string|null} [opts.query=null] optional query text to focus ranking on (see above)
50
+ * sentence-similarity edge
51
+ * @param {string|null} [opts.query=null] optional query text to focus ranking on
94
52
  * @returns {Array<{sentence:string, score:number, sourceBlockId:string}>} best-first;
95
53
  * deterministic tiebreak (sourceBlockId, then sentence text) on equal score.
96
54
  */
@@ -98,10 +56,7 @@ export function rankSentences(group, { overlapMin = OVERLAP_MIN, query = null }
98
56
  const members = Array.isArray(group?.members) ? group.members : [];
99
57
  if (!members.length) return [];
100
58
 
101
- // One entry per sentence: a stable id ("<blockId>#<index>") so PageRank/degree/IDF can be
102
- // keyed exactly like rankBlocks/degreeOf already key blocks — deterministic since it's
103
- // derived purely from member order (group.mjs's members are already id-sorted) and each
104
- // block's own sentence order.
59
+ // One entry per sentence, with a stable id for PageRank/degree/IDF keying.
105
60
  const sentences = [];
106
61
  for (const m of members) {
107
62
  const parts = splitSentences(m?.text || "");
@@ -114,13 +69,10 @@ export function rankSentences(group, { overlapMin = OVERLAP_MIN, query = null }
114
69
  const tokensById = {};
115
70
  for (const s of sentences) tokensById[s.id] = contentTokens(s.sentence);
116
71
 
117
- // Reused verbatim — the exact PageRank/hub-dampening machinery rankBlocks()/degreeOf()
118
- // already run for block-level ranking, generic over any id->tokens map.
119
72
  const ranks = rankBlocks(tokensById, { overlapMin });
120
73
  const degrees = degreeOf(tokensById, { overlapMin });
121
74
 
122
- // IDF scoped to THIS group's sentence set (df/N here, not the whole corpus) — the same
123
- // scoping discipline group.mjs applies to its own label tokens.
75
+ // IDF scoped to THIS group's sentence set (df/N), not the whole corpus.
124
76
  const ids = Object.keys(tokensById);
125
77
  const N = ids.length;
126
78
  const df = new Map();
@@ -138,11 +90,6 @@ export function rankSentences(group, { overlapMin = OVERLAP_MIN, query = null }
138
90
  for (const t of idfTokens) idfSum += idf(t);
139
91
  const rank = ranks[s.id] ?? 0;
140
92
  const degree = degrees[s.id] ?? 0;
141
- // idfSum * (1 + rank) / sqrt(1 + degree) — retrieveBlocks()'s own combination formula
142
- // (memory/blocks.mjs), reused at sentence granularity: IDF-weighted informativeness
143
- // decides content, PageRank centrality breaks ties toward well-connected (consensus)
144
- // sentences, and the degree divisor dampens a sentence that merely shares vocabulary
145
- // with disproportionately many others from winning on inflated rank alone.
146
93
  const score = (idfSum * (1 + rank)) / Math.sqrt(1 + degree);
147
94
  return { sentence: s.sentence, score, sourceBlockId: s.sourceBlockId };
148
95
  });
@@ -1,25 +1,6 @@
1
- // completions/search.mjs — Stage 1 ("broad search") of PLAN_COMPLETIONS.md's six-stage
2
- // mechanical-text-generation pipeline. This is Stage-0 scope ONLY per the plan's own §4
3
- // staging table (Stage 1 + Stage 2, no inference/summarization/pruning/voice-pass yet — see
4
- // group.mjs for Stage 2).
5
- //
6
- // Near-total reuse, as the plan's own prior research established: this is a thin composition
7
- // of retrieveBlocks (src/memory/blocks.mjs's best-first block ranker) and the graph
8
- // search()/ask() services (src/providers/graph-service.mjs, contracted by
9
- // src/repository-interface.mjs's SERVICE_GROUPS.search) — a WIDER query shape ("broad
10
- // search": more hits per source, both sources asked) rather than any new retrieval
11
- // machinery. Nothing here re-implements ranking, tokenizing, or graph traversal.
12
- //
13
- // Granularity note (explicit, per the strategy-advisor's Stage-0 correction): retrieveBlocks
14
- // only ever returns whole BLOCK text — one node per ~800-token document/transcript chunk, the
15
- // unit memory/blocks.mjs's index actually stores. PLAN_COMPLETIONS.md's own prose loosely
16
- // says "spans"; this module does NOT invent sub-block span segmentation. Every hit this
17
- // module returns is whole-block (or a whole graph-search/ask result) granularity. Finer-grain
18
- // spans, if the pipeline ever needs them, are a real future increment, out of scope here.
19
- //
20
- // Determinism: retrieveBlocks is already deterministic (stable sort with an id tiebreak);
21
- // the graph service's search()/ask() are pure functions over a fixed graph. Given the same
22
- // dir/query/graphService, broadSearch() always returns the same array in the same order.
1
+ // completions/search.mjs — Stage 1 ("broad search"): a thin composition of retrieveBlocks
2
+ // and the graph search()/ask() services, asking both sources more widely than a single-answer
3
+ // query. Returns whole-block/whole-result hits only no sub-block span segmentation.
23
4
 
24
5
  import { retrieveBlocks } from "../memory/blocks.mjs";
25
6
 
@@ -36,17 +17,13 @@ const DEFAULT_GRAPH_LIMIT = 8;
36
17
  * @param {object} [opts]
37
18
  * @param {number} [opts.blockK=8] retrieveBlocks' k (how many blocks to pull)
38
19
  * @param {object|null} [opts.graphService=null] an optional Repository-Interface service
39
- * (e.g. createGraphService(graph) from src/providers/graph-service.mjs, or any provider
40
- * satisfying SERVICE_GROUPS.search's `search`/`ask`). When supplied, its search() and
41
- * ask() services are queried too. Omitted -> block-only search (still a valid, honest
42
- * broad search; graph access is opt-in, not required — this module never constructs a
43
- * graph service itself, it only calls one it's handed).
20
+ * (e.g. createGraphService(graph)). When supplied, its search() and ask() are queried too;
21
+ * omitted -> block-only search.
44
22
  * @param {number} [opts.graphLimit=8] graph search()'s result limit
45
23
  * @returns {Promise<Array<{source:"block"|"graph-search"|"graph-ask", id:string, text:string, score:number}>>}
46
- * best-first within each source; blocks first, then graph-search, then graph-ask (stable,
47
- * deterministic order — never shuffled/merged by score across sources, since block scores
48
- * and graph relevance are not on a comparable scale; Stage 2's grouping doesn't need them
49
- * to be).
24
+ * best-first within each source; blocks first, then graph-search, then graph-ask (never
25
+ * shuffled/merged by score across sources block scores and graph relevance aren't
26
+ * comparable).
50
27
  */
51
28
  export async function broadSearch(dir, query, {
52
29
  blockK = DEFAULT_BLOCK_K, graphService = null, graphLimit = DEFAULT_GRAPH_LIMIT,
package/src/concept.mjs CHANGED
@@ -1,30 +1,15 @@
1
- // concept.mjs — "the concept force": compose a THREE-BAND answer to a vague
2
- // "what is a X" touch, when tmct KNOWS the concept X (a curated definition) AND
3
- // HAS instances of it (individuals in the code graph and/or remembered isa facts).
4
- //
5
- // 1. THE FACT — the definition of X (lead clause of the corpus/seon entry).
6
- // 2. THE EXAMPLES — real instances of X: code-graph individuals whose class maps
7
- // to X (capped ~3, stable graph order, each a real node), plus any remembered
8
- // "A is a X" facts.
9
- // 3. THE GUIDED FOLLOW-UP — 2-3 concrete, RUNNABLE next questions built from the
10
- // real instances × the query shapes valid for that kind, EACH PRE-CHECKED by
11
- // actually running it through ask() so a suggestion can never miss.
12
- //
13
- // PURE given (graph, term, {definition, factRows}) — the follow-up validator calls
14
- // ask() (deterministic, no model), so the whole composition is reproducible. The
15
- // caller (chat.mjs) owns the async edges: loading corpus/seon/definitions.jsonl and
16
- // the memory fact rows, and rendering through the response template. This module
17
- // never fabricates: every example is a real individual/fact and every follow-up is
18
- // validated against the same graph before it is offered.
1
+ // concept.mjs — "the concept force": compose a three-band answer (fact,
2
+ // examples, guided follow-up) to a vague "what is a X" touch when X is a
3
+ // known, instantiated concept. Pure given (graph, term, {definition,
4
+ // factRows}); never fabricates — every example is a real individual/fact and
5
+ // every follow-up is pre-validated by actually running it through ask().
19
6
 
20
7
  import { ask } from "./ask.mjs";
21
8
  import { relationKind } from "./codegraph.mjs";
22
9
 
23
- /** A vague concept term (normalized, singular — normFactTerm's output) → the graph
24
- * individual `class` it enumerates. The closed set of code-structure concepts the
25
- * seon lexicon + the graph both understand; anything outside it is not a "concept
26
- * force" touch (a general-vocabulary term like "cache" has a definition but no
27
- * enumerable graph class, so it falls back to the ordinary definition surface). */
10
+ /** A vague concept term (normalized, singular) → the graph individual `class`
11
+ * it enumerates. Outside this closed set (e.g. "cache"), the term falls back
12
+ * to the ordinary definition surface instead. */
28
13
  export const CONCEPT_CLASS = Object.freeze({
29
14
  class: "Class",
30
15
  module: "Module",
@@ -52,11 +37,8 @@ const CLASS_NOUN = Object.freeze({
52
37
  * instance, so it never contributes an example. */
53
38
  const ISA_INSTANCE_PREDICATES = new Set(["rdf:type"]);
54
39
 
55
- /** Per graph-class, the candidate follow-up shapes in priority order. Each builder
56
- * takes ONE real instance label and returns a query string; the builder is offered
57
- * only after the query VALIDATES against the live graph, so a shape that can't
58
- * resolve for any instance is silently dropped. Shapes are curated to be exactly
59
- * the ones ask.mjs's grammar answers for that kind. */
40
+ /** Per graph-class, the candidate follow-up shapes in priority order each
41
+ * offered only once its query validates against the live graph. */
60
42
  const FOLLOWUP_SHAPES = Object.freeze({
61
43
  Class: [
62
44
  (x) => `which classes inherit from ${x}`,
@@ -125,11 +107,8 @@ function resolves(graph, query) {
125
107
  }
126
108
  }
127
109
 
128
- /** Build up to MAX_FOLLOWUPS validated follow-ups for a class's instances. For each
129
- * shape in priority order, find the instances whose query resolves and offer it for
130
- * the first such instance NOT already used by an earlier follow-up (so the set
131
- * showcases DIFFERENT real nodes where possible); a shape no instance satisfies is
132
- * dropped entirely. Deterministic (graph order in, first-fit out). */
110
+ /** Up to MAX_FOLLOWUPS validated follow-ups: first-fit over shapes in
111
+ * priority order, preferring an instance not already used. */
133
112
  function buildFollowups(graph, cls, instanceLabels) {
134
113
  const shapes = FOLLOWUP_SHAPES[cls] || [];
135
114
  const used = new Set();
@@ -145,14 +124,8 @@ function buildFollowups(graph, cls, instanceLabels) {
145
124
  return out;
146
125
  }
147
126
 
148
- /** Compose the three bands for a concept term, or null when it is NOT a concept-force
149
- * case — the term is not a known enumerable concept, has no curated definition, or
150
- * has NO instances anywhere (code graph and memory both empty). Returns the pieces as
151
- * strings so the caller can render them through a data template:
152
- * { definition, examples, followups, instances:[{id,label,type,module}] }
153
- * `examples` is always non-empty when non-null (we only fire with real instances);
154
- * `followups` is "" when no validated next-question exists, else a "\nWant to go
155
- * deeper? Try:\n • …" block. */
127
+ /** Compose the three bands for a concept term, or null when it's not a known
128
+ * enumerable concept, has no curated definition, or has no instances anywhere. */
156
129
  export function composeConcept(graph, term, { definition = null, factRows = [] } = {}) {
157
130
  const cls = CONCEPT_CLASS[term];
158
131
  if (!cls || !definition) return null;
@@ -225,20 +198,12 @@ export function composeConcept(graph, term, { definition = null, factRows = [] }
225
198
  };
226
199
  }
227
200
 
228
- // ============================================================================
229
- // THE RELATION CONCEPT FORCE the same three-band shape (definition + real
230
- // example EDGES + validated follow-ups) for a vague touch on a RELATION/edge kind
231
- // ("what about imports", "what are the calls", "tell me about contains"). Where
232
- // composeConcept enumerates INDIVIDUALS of a class, composeRelation enumerates the
233
- // EDGES of a relation kind, and seeds its follow-ups from real edge endpoints.
234
- // PURE given (graph, relTerm, {definition}); every example is a real edge and every
235
- // follow-up is validated via resolves() before it is offered. Never fabricates.
236
- // ============================================================================
237
-
238
- /** A vague relation term (lower-cased) → the internal concept key it enumerates.
239
- * The closed set of edge-kind concepts the seon relation table + the graph both
240
- * understand; a term outside it is not a relation-force touch. Nouns, gerunds and
241
- * a couple of synonyms all collapse to one key. */
201
+ // ---- THE RELATION CONCEPT FORCE — same three bands, but for a vague touch on
202
+ // a RELATION/edge kind ("what about imports"): composeConcept enumerates
203
+ // INDIVIDUALS, composeRelation enumerates EDGES. ----
204
+
205
+ /** A vague relation term (lower-cased) the internal concept key it
206
+ * enumerates. Nouns, gerunds and synonyms all collapse to one key. */
242
207
  export const RELATION_TERM = Object.freeze({
243
208
  import: "imports", imports: "imports", importing: "imports", imported: "imports",
244
209
  call: "calls", calls: "calls", calling: "calls", called: "calls", invoke: "calls", invokes: "calls", invoking: "calls",
@@ -249,11 +214,8 @@ export const RELATION_TERM = Object.freeze({
249
214
  define: "defines", defines: "defines", defining: "defines", defined: "defines", definition: "defines", definitions: "defines", declaration: "defines",
250
215
  touch: "touches", touches: "touches", touching: "touches", touched: "touches",
251
216
  cochange: "cochange", "co-change": "cochange", "change-coupling": "cochange", coupled: "cochange",
252
- // "export"/"exports" is ALSO a curated seon lexicon noun (corpus/seon/definitions.jsonl
253
- // "export"), same shape as "imports"but that meta reading only owns the "what
254
- // does export mean"/"what is an export" shape; vagueTouchTermOf/relationTermOf are
255
- // deliberately scoped to the NON-meta "what about X"/"tell me about X" touch (see
256
- // relationTermOf's own docblock, frozen case am-meta-imports), so no conflict here.
217
+ // "export"/"exports" is also a curated seon lexicon noun, but that meta reading
218
+ // only owns "what does export mean" — no conflict with this vague-touch table.
257
219
  export: "reexports", exports: "reexports", exporting: "reexports", exported: "reexports",
258
220
  reexport: "reexports", reexports: "reexports", reexporting: "reexports",
259
221
  "re-export": "reexports", "re-exports": "reexports", "re-exporting": "reexports",
@@ -288,11 +250,9 @@ const RELATION_RENDER = Object.freeze({
288
250
  reexports: { verb: "re-exports", edgeNoun: "re-export" },
289
251
  });
290
252
 
291
- /** Per concept key, the candidate follow-up shapes in priority order. Each shape
292
- * draws a real endpoint from one SIDE of the edges (subject or object) and builds a
293
- * query; a shape is offered only once the query VALIDATES against the live graph
294
- * (resolves()), so a shape no endpoint satisfies is silently dropped. Curated to be
295
- * exactly the shapes ask.mjs answers for that kind. */
253
+ /** Per concept key, the candidate follow-up shapes in priority order each
254
+ * draws an endpoint from one side of the edges and is offered only once its
255
+ * query validates against the live graph. */
296
256
  const RELATION_FOLLOWUP_SHAPES = Object.freeze({
297
257
  imports: [
298
258
  { side: "obj", make: (x) => `which modules import ${x}` },
@@ -340,10 +300,8 @@ const MAX_EDGE_EXAMPLES = 3;
340
300
  const edgeSubjectLabel = (e) => String(e.subjectLabel || e.subject);
341
301
  const edgeObjectLabel = (e) => String(e.objectLabel || e.object);
342
302
 
343
- /** Build up to MAX_FOLLOWUPS validated follow-ups for a relation's edges. Same
344
- * first-fit discipline as buildFollowups: for each shape in priority order, find the
345
- * endpoints (of that shape's side) whose query resolves and offer it for the first
346
- * such endpoint not already used, so the set showcases DIFFERENT real nodes. */
303
+ /** Up to MAX_FOLLOWUPS validated follow-ups for a relation's edges — same
304
+ * first-fit discipline as buildFollowups. */
347
305
  function buildRelationFollowups(graph, key, subjLabels, objLabels) {
348
306
  const shapes = RELATION_FOLLOWUP_SHAPES[key] || [];
349
307
  const used = new Set();
@@ -360,25 +318,11 @@ function buildRelationFollowups(graph, key, subjLabels, objLabels) {
360
318
  return out;
361
319
  }
362
320
 
363
- /** Compose the three bands for a RELATION concept term, or null when it is NOT a
364
- * relation-force case at all the term is not a known enumerable relation, or has no
365
- * curated definition. Returns the same string-band shape composeConcept does:
366
- * { definition, examples, followups, followupQueries, remainder, noun }
367
- * `examples` is always non-empty when non-null; `followups` is "" when no validated
368
- * next-question exists.
369
- *
370
- * A known relation whose graph has ZERO edges of that kind is NOT null — it degrades
371
- * to a two-band answer (the definition + an explicit "this codebase has no X edges"
372
- * line, `examples`-shaped so the caller renders it identically). Found live (an
373
- * advisor tick on the 0.9.14 Tier-2 playtest cycle): returning null here for the
374
- * zero-edge case let the caller's OWN raw grammar attempt at the vague-touch text
375
- * ("what about exports", "tell me about reexports") stand instead — but that text was
376
- * never meant to be parsed as an object search, so on a graph with no reexports edges
377
- * (the realistic case for most repos, e.g. examples/mini-webapp) it fell through to a
378
- * garbled `no module matching "about"/"exports" found`, not an honest miss. A relation
379
- * kind the graph has NEVER SEEN AT ALL (relationKind returns nothing in RELATION_KINDS
380
- * for this graph's shape) still degrades the same way — the definition is always
381
- * worth stating; only the fabricated edge is refused. */
321
+ /** Compose the three bands for a RELATION concept term, or null when it's not
322
+ * a known enumerable relation or has no curated definition. A known relation
323
+ * with ZERO edges is NOT null — it degrades to a two-band "no X edges"
324
+ * answer, since returning null here would fall through to a garbled raw
325
+ * grammar miss instead of an honest one. */
382
326
  export function composeRelation(graph, relTerm, { definition = null } = {}) {
383
327
  const key = RELATION_TERM[String(relTerm || "").toLowerCase()];
384
328
  if (!key || !definition) return null;
@@ -1,15 +1,11 @@
1
1
  // conformance.mjs — the Repository-Interface CONTRACT TEST SUITE as a reusable kit.
2
2
  //
3
- // archive/PLAN_REPOSITORY_INTERFACE.md deliverable 3: an implementation is CONFORMANT iff it
4
- // passes `runConformance(name, makeProvider)`. tmct's own fixture + bootstrap providers
5
- // pass it in `npm test`; an EXTERNAL producer (seonix) imports this kit from the
6
- // published package and runs the SAME suite against its native provider to claim
7
- // conformance — conformance is the suite, not prose. It is provider-agnostic: it
8
- // asserts the SHAPE + the error contract; data-bearing truth is asserted by the caller
9
- // where its provider carries data.
3
+ // An implementation is CONFORMANT iff it passes `runConformance(name, makeProvider)`.
4
+ // Provider-agnostic: it asserts the shape + error contract; data-bearing truth is
5
+ // asserted by the caller where its provider carries data.
10
6
  //
11
- // Public surface (exported here + via the package "./conformance" subpath):
12
- // runConformance(name, makeProvider), assertResult, assertIndividual, assertEdge.
7
+ // Public surface: runConformance(name, makeProvider), assertResult, assertIndividual,
8
+ // assertEdge.
13
9
  import { test } from "node:test";
14
10
  import assert from "node:assert/strict";
15
11
  import {
@@ -114,8 +110,8 @@ export function runConformance(name, makeProvider) {
114
110
  assert.throws(() => svc.edges("no:such:id", "not-a-real-kind"), TypeError);
115
111
  });
116
112
 
117
- // snippet: UNCHANGED by INTERFACE_VERSION 1.1.0 — it has nothing useful without fs, so it
118
- // still honestly misses NO_SOURCE (or UNRESOLVED_TERM on an empty graph) with no working tree.
113
+ // snippet has nothing useful without fs, so it honestly misses NO_SOURCE (or
114
+ // UNRESOLVED_TERM on an empty graph) with no working tree.
119
115
  test(`[${name}] snippet answers NO_SOURCE (not a throw) when no working tree`, async () => {
120
116
  const svc = makeProvider();
121
117
  if (svc.sourceAccess) return; // covered by the source-capable branch below instead
@@ -129,10 +125,10 @@ export function runConformance(name, makeProvider) {
129
125
  );
130
126
  });
131
127
 
132
- // context: NARROWED by INTERFACE_VERSION 1.1.0 contextPlan/sizeBundle/renderGraphOnlyBundle
133
- // are pure graph queries, so a graph-only provider (no working tree) now returns a REAL HIT
134
- // for any resolvable symbol; only an unresolvable symbol still misses (UNRESOLVED_TERM). See
135
- // repository-interface.mjs's context service entry for the full rationale.
128
+ // context: contextPlan/sizeBundle/renderGraphOnlyBundle are pure graph queries, so a
129
+ // graph-only provider (no working tree) returns a REAL HIT for any resolvable symbol;
130
+ // only an unresolvable symbol still misses (UNRESOLVED_TERM). See repository-interface.mjs's
131
+ // context service entry for the full rationale.
136
132
  test(`[${name}] context returns a graph-only HIT for a resolvable symbol, even with no working tree`, async () => {
137
133
  const svc = makeProvider();
138
134
  if (svc.sourceAccess) return; // covered by the source-capable branch below instead