@polycode-projects/the-mechanical-code-talker 1.9.2 → 1.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +441 -202
- package/bin/tmct.mjs +126 -1
- package/package.json +4 -2
- package/src/answer-variants.mjs +8 -36
- package/src/ask-browser-entry.mjs +5 -23
- package/src/ask-browser.bundle.js +1 -2
- package/src/ask-nlp.mjs +9 -23
- package/src/ask-vocab.mjs +139 -589
- package/src/ask.mjs +627 -1729
- package/src/chat.mjs +1684 -2872
- package/src/cli-args.mjs +14 -28
- package/src/codegraph.mjs +236 -644
- package/src/completions/complete.mjs +18 -62
- package/src/completions/graph-adapter.mjs +14 -60
- package/src/completions/group.mjs +12 -68
- package/src/completions/infer.mjs +38 -126
- package/src/completions/prune.mjs +17 -70
- package/src/completions/rank.mjs +16 -69
- package/src/completions/search.mjs +8 -31
- package/src/concept.mjs +32 -88
- package/src/conformance.mjs +11 -15
- package/src/corpus/conceptnet.mjs +31 -89
- package/src/corpus/templates.mjs +19 -45
- package/src/corpus/unknown-ingest.mjs +31 -92
- package/src/embed.mjs +10 -22
- package/src/extensions.mjs +50 -154
- package/src/finish.mjs +35 -91
- package/src/grammar/ace.mjs +16 -40
- package/src/grammar/assert.mjs +1 -1
- package/src/grammar/lexicon-core.json +1 -1
- package/src/grammar/lexicon.mjs +9 -27
- package/src/graph-merge.mjs +2 -3
- package/src/hash.mjs +6 -14
- package/src/index.mjs +6 -10
- package/src/init.mjs +38 -125
- package/src/interpret/fuzzy.mjs +10 -29
- package/src/interpret/merge.mjs +9 -27
- package/src/interpret/normalize.mjs +137 -585
- package/src/interpret/pipeline.mjs +23 -71
- package/src/interpret/strategies/ace.mjs +7 -31
- package/src/interpret/strategies/constructions.mjs +14 -41
- package/src/interpret/strategies/grammar.mjs +21 -60
- package/src/interpret/strategies/keywords.mjs +42 -131
- package/src/interpret/strategies/noise-strip.mjs +18 -89
- package/src/memory/bias.mjs +11 -54
- package/src/memory/blocks.mjs +18 -69
- package/src/memory/core.mjs +171 -591
- package/src/memory/fold.mjs +0 -0
- package/src/memory/inspect.mjs +7 -25
- package/src/memory/shacl.mjs +10 -39
- package/src/memory/trust.mjs +26 -127
- package/src/memory-ask-browser-entry.mjs +7 -30
- package/src/memory-ask-browser.bundle.js +1 -1
- package/src/paraphrase.mjs +20 -53
- package/src/planning.mjs +15 -157
- package/src/prose-nlp.mjs +4 -17
- package/src/prose.mjs +19 -67
- package/src/providers/bootstrap.mjs +1 -2
- package/src/providers/fixture.mjs +1 -2
- package/src/providers/graph-service.mjs +28 -59
- package/src/repository-interface.mjs +6 -8
- package/src/router/drive.mjs +183 -0
- package/src/router/goal-reasoner.mjs +66 -231
- package/src/router/guardrail.mjs +20 -58
- package/src/router/planner.mjs +15 -46
- package/src/router/registry.mjs +13 -43
- package/src/router/resolver.mjs +46 -131
- package/src/router/results.mjs +231 -0
- package/src/schema-docs.mjs +10 -27
- package/src/server-http.mjs +10 -19
- package/src/server.mjs +22 -28
- package/src/sessions.mjs +15 -30
- package/src/source-slice.mjs +5 -7
- package/src/source.mjs +10 -20
- package/src/syllogise.mjs +187 -575
- package/src/telemetry.mjs +3 -3
- package/src/toml-config.mjs +4 -4
- package/src/tui/app.mjs +9 -19
- package/src/viz.mjs +66 -123
- package/src/wink-model.mjs +10 -24
|
@@ -1,57 +1,14 @@
|
|
|
1
|
-
// completions/prune.mjs — Stage 5 ("drop non-contributing elements")
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
// is dropped only when ALL three hold — equivalently, a span is KEPT when ANY of the three is
|
|
9
|
-
// false: it IS in a surviving group, OR it DOES feed an asserted inference, OR it IS selected by
|
|
10
|
-
// ranking. This module implements that OR exactly, at SENTENCE granularity (rank.mjs's own
|
|
11
|
-
// ranking unit — the thing that actually composes into the assembled prose), with a "hit never
|
|
12
|
-
// grouped" defensive check underneath it for the coarser block/hit granularity search.mjs hands
|
|
13
|
-
// group.mjs (group.mjs's own contract is that every hit lands in SOME group, so this branch is
|
|
14
|
-
// a guard against that invariant breaking silently, not an expected live path today).
|
|
15
|
-
//
|
|
16
|
-
// Inputs are the pipeline's own already-computed intermediate state — this module computes
|
|
17
|
-
// NOTHING new about relevance; it only DECIDES keep/drop from what search.mjs/group.mjs/
|
|
18
|
-
// infer.mjs/rank.mjs already produced, and records why. `rankedByGroup` is deliberately an
|
|
19
|
-
// INPUT (rank.mjs's own rankSentences() output per group), not recomputed here — pruning is a
|
|
20
|
-
// pure decision layer over ranking's output, not a second ranker.
|
|
21
|
-
//
|
|
22
|
-
// THRESHOLD + REASONING (the plan's own "your call on a sensible cutoff, document your
|
|
23
|
-
// reasoning"):
|
|
24
|
-
// - top-K per group (default 3, `opts.maxSentencesPerGroup`): rankSentences() is already
|
|
25
|
-
// best-first per group; keeping only the top K bounds the assembled completion's size in a
|
|
26
|
-
// VISIBLE, documented, override-able way. An unbounded "keep everything with any positive
|
|
27
|
-
// score" would make the completion grow without limit as a group's member count grows —
|
|
28
|
-
// that is itself a silent, unaudited cap in the opposite direction (nothing stops it from
|
|
29
|
-
// ballooning to the size of the whole retrieved corpus for a broad-enough prompt); a small
|
|
30
|
-
// explicit K is the "no silent caps" discipline applied honestly — the cap is visible in the
|
|
31
|
-
// signature, in this comment, and in every drop log entry it produces.
|
|
32
|
-
// - positive score (`score > 0`): rankSentences()'s own scoring is 0 exactly when a sentence
|
|
33
|
-
// shares no informative (or, under query-focus, no query-overlapping) token with anything —
|
|
34
|
-
// rank.mjs's own "never a guessed match" discipline. A zero-information sentence contributes
|
|
35
|
-
// nothing to the completion's content by rankSentences()'s own definition, so it never
|
|
36
|
-
// qualifies on ranking grounds alone, however small K is set.
|
|
37
|
-
// - relation-anchor salvage: a group that feeds an asserted cross-group inference (infer.mjs)
|
|
38
|
-
// but produces ZERO ranking-qualified sentences (every one of its sentences is either
|
|
39
|
-
// outside top-K or zero-scored) is not silenced outright — its single top-ranked sentence is
|
|
40
|
-
// kept as the group's extractive anchor, so a cross-group claim this pipeline actually
|
|
41
|
-
// ASSERTS (with cited licensing evidence) always has at least one real, traceable sentence
|
|
42
|
-
// behind it in the assembled text. This is the concrete realisation of the plan's OR: the
|
|
43
|
-
// three drop conditions must ALL hold, and "feeds no asserted inference" is one of them.
|
|
44
|
-
//
|
|
45
|
-
// Determinism: no randomness anywhere. Groups are processed in a fixed (id-sorted) order;
|
|
46
|
-
// rankedByGroup's own sentence order (rank.mjs's own deterministic tiebreak) is preserved
|
|
47
|
-
// exactly; the output kept/dropped lists are therefore fully deterministic given deterministic
|
|
48
|
-
// inputs — see test/completions-prune.test.mjs's own double-run diff.
|
|
1
|
+
// completions/prune.mjs — Stage 5 ("drop non-contributing elements"), at SENTENCE
|
|
2
|
+
// granularity: a sentence is kept if it's in a surviving group AND (feeds an asserted
|
|
3
|
+
// inference OR clears the per-group top-K/positive-score ranking cutoff); otherwise it's
|
|
4
|
+
// dropped with a recorded reason. A group that feeds an inference but has no ranking-
|
|
5
|
+
// qualified sentence still keeps its top-ranked sentence as an extractive anchor, so every
|
|
6
|
+
// asserted relation traces to a real sentence. Pure decision layer — computes nothing new
|
|
7
|
+
// about relevance, only reads what search/group/infer/rank already produced.
|
|
49
8
|
|
|
50
9
|
const DEFAULT_MAX_SENTENCES_PER_GROUP = 3;
|
|
51
10
|
|
|
52
|
-
/** Every group id referenced as either side of an asserted relation
|
|
53
|
-
* output) — the "feeds an asserted inference" test, at group granularity (the granularity
|
|
54
|
-
* relations are actually asserted at). */
|
|
11
|
+
/** Every group id referenced as either side of an asserted relation. */
|
|
55
12
|
function relatedGroupIdsOf(relations) {
|
|
56
13
|
const set = new Set();
|
|
57
14
|
for (const r of relations) {
|
|
@@ -62,30 +19,22 @@ function relatedGroupIdsOf(relations) {
|
|
|
62
19
|
}
|
|
63
20
|
|
|
64
21
|
/**
|
|
65
|
-
* Stage 5 — pruning. Decides, per sentence, KEEP or DROP
|
|
66
|
-
* computed intermediate state — never recomputing relevance itself.
|
|
22
|
+
* Stage 5 — pruning. Decides, per sentence, KEEP or DROP.
|
|
67
23
|
*
|
|
68
24
|
* @param {object} state
|
|
69
|
-
* @param {Array<{id:string, text?:string}>} [state.hits] search.mjs's broadSearch() output
|
|
70
|
-
* only for the defensive "never grouped" check below — group.mjs's own contract is that this
|
|
71
|
-
* branch should never actually fire today).
|
|
25
|
+
* @param {Array<{id:string, text?:string}>} [state.hits] search.mjs's broadSearch() output
|
|
72
26
|
* @param {Array<{id:string, memberIds:string[], label?:string}>} [state.groups] group.mjs's
|
|
73
|
-
* groupHits() output
|
|
27
|
+
* groupHits() output
|
|
74
28
|
* @param {Array<{from:string, to:string, relation:string}>} [state.relations] infer.mjs's
|
|
75
|
-
* inferRelations() output
|
|
29
|
+
* inferRelations() output
|
|
76
30
|
* @param {Object<string, Array<{sentence:string, score:number, sourceBlockId:string}>>}
|
|
77
|
-
* [state.rankedByGroup] rank.mjs's rankSentences() output,
|
|
78
|
-
* group id — an INPUT to this module, not recomputed here (see file header).
|
|
31
|
+
* [state.rankedByGroup] rank.mjs's rankSentences() output, keyed by group id
|
|
79
32
|
* @param {object} [opts]
|
|
80
|
-
* @param {number} [opts.maxSentencesPerGroup=3] the top-K-per-group ranking cutoff
|
|
81
|
-
* header for the reasoning behind the default).
|
|
33
|
+
* @param {number} [opts.maxSentencesPerGroup=3] the top-K-per-group ranking cutoff
|
|
82
34
|
* @returns {{
|
|
83
35
|
* kept: Array<{sentence:string, score:number, sourceBlockId:string, groupId:string, groupLabel:string}>,
|
|
84
36
|
* dropped: Array<{item:object, reason:string}>
|
|
85
|
-
* }} `kept` is
|
|
86
|
-
* deterministic order the caller can assemble directly. `dropped` is itemized, one entry per
|
|
87
|
-
* dropped hit/sentence, each carrying a human-readable `reason` (never a silent discard —
|
|
88
|
-
* PLAN_COMPLETIONS.md §1.5/§2's own auditability bar).
|
|
37
|
+
* }} `kept` is ordered by (group id, rank order); `dropped` carries a human-readable `reason`.
|
|
89
38
|
*/
|
|
90
39
|
export function pruneCompletion(state = {}, { maxSentencesPerGroup = DEFAULT_MAX_SENTENCES_PER_GROUP } = {}) {
|
|
91
40
|
const hits = Array.isArray(state.hits) ? state.hits : [];
|
|
@@ -96,8 +45,7 @@ export function pruneCompletion(state = {}, { maxSentencesPerGroup = DEFAULT_MAX
|
|
|
96
45
|
const kept = [];
|
|
97
46
|
const dropped = [];
|
|
98
47
|
|
|
99
|
-
// "never grouped" — defensive: every hit
|
|
100
|
-
// hit), so this guards the invariant explicitly rather than silently assuming it holds.
|
|
48
|
+
// "never grouped" — defensive: group.mjs's contract is every hit lands in some group.
|
|
101
49
|
const groupedHitIds = new Set();
|
|
102
50
|
for (const g of groups) for (const id of g.memberIds || []) groupedHitIds.add(id);
|
|
103
51
|
for (const h of hits) {
|
|
@@ -124,8 +72,7 @@ export function pruneCompletion(state = {}, { maxSentencesPerGroup = DEFAULT_MAX
|
|
|
124
72
|
|
|
125
73
|
let anchor = null;
|
|
126
74
|
if (!qualifying.length && groupFeedsInference && ranked.length) {
|
|
127
|
-
// salvage: the group's
|
|
128
|
-
// its asserted cross-group relation even though it didn't clear the ranking cutoff alone.
|
|
75
|
+
// salvage: keep the group's top-ranked sentence as the extractive anchor.
|
|
129
76
|
anchor = ranked[0];
|
|
130
77
|
qualifying.push(anchor);
|
|
131
78
|
const idx = rest.indexOf(anchor);
|
package/src/completions/rank.mjs
CHANGED
|
@@ -1,40 +1,13 @@
|
|
|
1
|
-
// completions/rank.mjs — Stage 4 ("mechanical summarization")
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// "Extractive sentence selection over the grouped-and-inferred material, not abstractive
|
|
8
|
-
// rewriting... query-focused multi-document summarization (feature-fusion sentence
|
|
9
|
-
// selection, graph-ranking approaches in the LexRank/TextRank family, clustering-cum-ranking
|
|
10
|
-
// as in CoRank)" — PLAN_COMPLETIONS.md §1.4/§4. The graph-ranking machinery this stage needs
|
|
11
|
-
// already exists and ships: memory/blocks.mjs's rankBlocks() (PageRank, d=0.85, 20
|
|
12
|
-
// iterations, over the shared-token block-similarity graph) and degreeOf() (hub dampening),
|
|
13
|
-
// both generic over any `{ id: tokens[] }` map — NOT block-specific despite the module name.
|
|
14
|
-
// This file reuses rankBlocks()/degreeOf() VERBATIM at sentence granularity (no PageRank
|
|
15
|
-
// reimplementation), combined with the same idf = log(1 + N/(1+df)) formula group.mjs already
|
|
16
|
-
// replicates for group-scoped (not whole-corpus-scoped) weighting, and the same
|
|
17
|
-
// idfSum * (1+rank) / sqrt(1+degree) combination retrieveBlocks() uses to fuse relevance,
|
|
18
|
-
// centrality, and hub-dampening into one score.
|
|
19
|
-
//
|
|
20
|
-
// New in this file: splitSentences() (no sentence-splitter existed anywhere in this repo —
|
|
21
|
-
// grepped for "sentence" across src/ and found none; a simple regex splitter is intentional
|
|
22
|
-
// here per the dispatch's own instruction not to pull in an NLP dependency for this) and the
|
|
23
|
-
// sentence-level id/token/scoring wiring that adapts rankBlocks/degreeOf from block to
|
|
24
|
-
// sentence granularity.
|
|
25
|
-
//
|
|
26
|
-
// Determinism: no randomness anywhere. Same group in (same members, same text, same order)
|
|
27
|
-
// -> same ranked sentence list out (stable score-descending sort with a deterministic
|
|
28
|
-
// sourceBlockId/sentence-text tiebreak) — see test/completions-stage2.test.mjs's double-run
|
|
29
|
-
// diff, the exact discipline test/completions-stage0.test.mjs established for search+group.
|
|
1
|
+
// completions/rank.mjs — Stage 4 ("mechanical summarization"): extractive SENTENCE ranking
|
|
2
|
+
// within a group.mjs group. Reuses memory/blocks.mjs's rankBlocks() (PageRank) and degreeOf()
|
|
3
|
+
// (hub dampening) verbatim at sentence granularity, combined with group-scoped IDF the same
|
|
4
|
+
// way retrieveBlocks() fuses relevance/centrality/hub-dampening into one score.
|
|
5
|
+
// splitSentences() is a simple regex splitter — deliberately not an NLP dependency.
|
|
30
6
|
|
|
31
7
|
import { degreeOf, rankBlocks, tokenizeBlock, OVERLAP_MIN } from "../memory/blocks.mjs";
|
|
32
8
|
import { STOPWORDS } from "../prose.mjs";
|
|
33
9
|
|
|
34
|
-
// Same content-token filter group.mjs applies
|
|
35
|
-
// from group.mjs, so replicated here rather than reached across files — group.mjs's own
|
|
36
|
-
// header explains why raw tokenizeBlock output is unsuitable for unweighted overlap: it
|
|
37
|
-
// re-admits stopwords/filler that would collapse unrelated sentences into false edges).
|
|
10
|
+
// Same content-token filter group.mjs applies (not exported, so replicated here).
|
|
38
11
|
const isContentToken = (t) => /^[a-z0-9]+$/.test(t) && !STOPWORDS.has(t);
|
|
39
12
|
|
|
40
13
|
/** tokenizeBlock(text), narrowed to real content tokens — see isContentToken above. */
|
|
@@ -43,16 +16,11 @@ function contentTokens(text) {
|
|
|
43
16
|
}
|
|
44
17
|
|
|
45
18
|
// Sentence boundary: a run of [.!?] followed by whitespace and an uppercase letter or digit —
|
|
46
|
-
// deliberately simple (no abbreviation dictionary, no NLP dependency
|
|
47
|
-
// instruction). Applied per-line (a block's own "Q: ...\nA: ..." shape already gives clean
|
|
48
|
-
// boundaries at the newline; this regex further splits any line that itself holds more than
|
|
49
|
-
// one sentence).
|
|
19
|
+
// deliberately simple (no abbreviation dictionary, no NLP dependency).
|
|
50
20
|
const SENTENCE_SPLIT_RE = /(?<=[.!?])\s+(?=[A-Z0-9])/;
|
|
51
21
|
|
|
52
22
|
/**
|
|
53
|
-
* Split raw block text into trimmed, non-empty sentences (order-preserving, no dedup
|
|
54
|
-
* dedup, if ever needed, is the caller's call). Splits on blank lines first (a block's own
|
|
55
|
-
* natural line structure), then on sentence-ending punctuation within each line.
|
|
23
|
+
* Split raw block text into trimmed, non-empty sentences (order-preserving, no dedup).
|
|
56
24
|
*
|
|
57
25
|
* @param {string} text
|
|
58
26
|
* @returns {string[]}
|
|
@@ -71,26 +39,16 @@ export function splitSentences(text) {
|
|
|
71
39
|
}
|
|
72
40
|
|
|
73
41
|
/**
|
|
74
|
-
* Stage 4 — extractive sentence ranking within one group.mjs group
|
|
75
|
-
* text into sentences,
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
* (degreeOf) over it, and combines with group-scoped IDF the same way retrieveBlocks() fuses
|
|
79
|
-
* relevance × centrality × hub-dampening — self-weighted (LexRank-style: a sentence's own
|
|
80
|
-
* rare/informative tokens) unless `opts.query` is supplied, in which case only tokens
|
|
81
|
-
* overlapping the query are IDF-summed (query-focused summarization, PLAN_COMPLETIONS.md's
|
|
82
|
-
* own literature framing) — a sentence with zero query overlap honestly scores 0 rather than
|
|
83
|
-
* silently falling back to self-weighting, the same "never a guessed match" discipline
|
|
84
|
-
* retrieveBlocks() applies when idfSum <= 0.
|
|
42
|
+
* Stage 4 — extractive sentence ranking within one group.mjs group: splits every member's
|
|
43
|
+
* text into sentences, ranks them by PageRank + hub-dampening + IDF (self-weighted, or
|
|
44
|
+
* query-focused when `opts.query` is given — a sentence with zero query overlap scores 0
|
|
45
|
+
* rather than falling back to self-weighting).
|
|
85
46
|
*
|
|
86
47
|
* @param {{members: Array<{id:string, text:string}>}} group a group.mjs groupHits() entry
|
|
87
|
-
* (or any object shaped `{ members: [{id, text}, ...] }`)
|
|
88
48
|
* @param {object} [opts]
|
|
89
49
|
* @param {number} [opts.overlapMin=OVERLAP_MIN] shared content-token threshold for a
|
|
90
|
-
* sentence-similarity edge
|
|
91
|
-
*
|
|
92
|
-
* the caller deliberately overrides it.
|
|
93
|
-
* @param {string|null} [opts.query=null] optional query text to focus ranking on (see above)
|
|
50
|
+
* sentence-similarity edge
|
|
51
|
+
* @param {string|null} [opts.query=null] optional query text to focus ranking on
|
|
94
52
|
* @returns {Array<{sentence:string, score:number, sourceBlockId:string}>} best-first;
|
|
95
53
|
* deterministic tiebreak (sourceBlockId, then sentence text) on equal score.
|
|
96
54
|
*/
|
|
@@ -98,10 +56,7 @@ export function rankSentences(group, { overlapMin = OVERLAP_MIN, query = null }
|
|
|
98
56
|
const members = Array.isArray(group?.members) ? group.members : [];
|
|
99
57
|
if (!members.length) return [];
|
|
100
58
|
|
|
101
|
-
// One entry per sentence
|
|
102
|
-
// keyed exactly like rankBlocks/degreeOf already key blocks — deterministic since it's
|
|
103
|
-
// derived purely from member order (group.mjs's members are already id-sorted) and each
|
|
104
|
-
// block's own sentence order.
|
|
59
|
+
// One entry per sentence, with a stable id for PageRank/degree/IDF keying.
|
|
105
60
|
const sentences = [];
|
|
106
61
|
for (const m of members) {
|
|
107
62
|
const parts = splitSentences(m?.text || "");
|
|
@@ -114,13 +69,10 @@ export function rankSentences(group, { overlapMin = OVERLAP_MIN, query = null }
|
|
|
114
69
|
const tokensById = {};
|
|
115
70
|
for (const s of sentences) tokensById[s.id] = contentTokens(s.sentence);
|
|
116
71
|
|
|
117
|
-
// Reused verbatim — the exact PageRank/hub-dampening machinery rankBlocks()/degreeOf()
|
|
118
|
-
// already run for block-level ranking, generic over any id->tokens map.
|
|
119
72
|
const ranks = rankBlocks(tokensById, { overlapMin });
|
|
120
73
|
const degrees = degreeOf(tokensById, { overlapMin });
|
|
121
74
|
|
|
122
|
-
// IDF scoped to THIS group's sentence set (df/N
|
|
123
|
-
// scoping discipline group.mjs applies to its own label tokens.
|
|
75
|
+
// IDF scoped to THIS group's sentence set (df/N), not the whole corpus.
|
|
124
76
|
const ids = Object.keys(tokensById);
|
|
125
77
|
const N = ids.length;
|
|
126
78
|
const df = new Map();
|
|
@@ -138,11 +90,6 @@ export function rankSentences(group, { overlapMin = OVERLAP_MIN, query = null }
|
|
|
138
90
|
for (const t of idfTokens) idfSum += idf(t);
|
|
139
91
|
const rank = ranks[s.id] ?? 0;
|
|
140
92
|
const degree = degrees[s.id] ?? 0;
|
|
141
|
-
// idfSum * (1 + rank) / sqrt(1 + degree) — retrieveBlocks()'s own combination formula
|
|
142
|
-
// (memory/blocks.mjs), reused at sentence granularity: IDF-weighted informativeness
|
|
143
|
-
// decides content, PageRank centrality breaks ties toward well-connected (consensus)
|
|
144
|
-
// sentences, and the degree divisor dampens a sentence that merely shares vocabulary
|
|
145
|
-
// with disproportionately many others from winning on inflated rank alone.
|
|
146
93
|
const score = (idfSum * (1 + rank)) / Math.sqrt(1 + degree);
|
|
147
94
|
return { sentence: s.sentence, score, sourceBlockId: s.sourceBlockId };
|
|
148
95
|
});
|
|
@@ -1,25 +1,6 @@
|
|
|
1
|
-
// completions/search.mjs — Stage 1 ("broad search") of
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// group.mjs for Stage 2).
|
|
5
|
-
//
|
|
6
|
-
// Near-total reuse, as the plan's own prior research established: this is a thin composition
|
|
7
|
-
// of retrieveBlocks (src/memory/blocks.mjs's best-first block ranker) and the graph
|
|
8
|
-
// search()/ask() services (src/providers/graph-service.mjs, contracted by
|
|
9
|
-
// src/repository-interface.mjs's SERVICE_GROUPS.search) — a WIDER query shape ("broad
|
|
10
|
-
// search": more hits per source, both sources asked) rather than any new retrieval
|
|
11
|
-
// machinery. Nothing here re-implements ranking, tokenizing, or graph traversal.
|
|
12
|
-
//
|
|
13
|
-
// Granularity note (explicit, per the strategy-advisor's Stage-0 correction): retrieveBlocks
|
|
14
|
-
// only ever returns whole BLOCK text — one node per ~800-token document/transcript chunk, the
|
|
15
|
-
// unit memory/blocks.mjs's index actually stores. PLAN_COMPLETIONS.md's own prose loosely
|
|
16
|
-
// says "spans"; this module does NOT invent sub-block span segmentation. Every hit this
|
|
17
|
-
// module returns is whole-block (or a whole graph-search/ask result) granularity. Finer-grain
|
|
18
|
-
// spans, if the pipeline ever needs them, are a real future increment, out of scope here.
|
|
19
|
-
//
|
|
20
|
-
// Determinism: retrieveBlocks is already deterministic (stable sort with an id tiebreak);
|
|
21
|
-
// the graph service's search()/ask() are pure functions over a fixed graph. Given the same
|
|
22
|
-
// dir/query/graphService, broadSearch() always returns the same array in the same order.
|
|
1
|
+
// completions/search.mjs — Stage 1 ("broad search"): a thin composition of retrieveBlocks
|
|
2
|
+
// and the graph search()/ask() services, asking both sources more widely than a single-answer
|
|
3
|
+
// query. Returns whole-block/whole-result hits only — no sub-block span segmentation.
|
|
23
4
|
|
|
24
5
|
import { retrieveBlocks } from "../memory/blocks.mjs";
|
|
25
6
|
|
|
@@ -36,17 +17,13 @@ const DEFAULT_GRAPH_LIMIT = 8;
|
|
|
36
17
|
* @param {object} [opts]
|
|
37
18
|
* @param {number} [opts.blockK=8] retrieveBlocks' k (how many blocks to pull)
|
|
38
19
|
* @param {object|null} [opts.graphService=null] an optional Repository-Interface service
|
|
39
|
-
* (e.g. createGraphService(graph)
|
|
40
|
-
*
|
|
41
|
-
* ask() services are queried too. Omitted -> block-only search (still a valid, honest
|
|
42
|
-
* broad search; graph access is opt-in, not required — this module never constructs a
|
|
43
|
-
* graph service itself, it only calls one it's handed).
|
|
20
|
+
* (e.g. createGraphService(graph)). When supplied, its search() and ask() are queried too;
|
|
21
|
+
* omitted -> block-only search.
|
|
44
22
|
* @param {number} [opts.graphLimit=8] graph search()'s result limit
|
|
45
23
|
* @returns {Promise<Array<{source:"block"|"graph-search"|"graph-ask", id:string, text:string, score:number}>>}
|
|
46
|
-
* best-first within each source; blocks first, then graph-search, then graph-ask (
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
* to be).
|
|
24
|
+
* best-first within each source; blocks first, then graph-search, then graph-ask (never
|
|
25
|
+
* shuffled/merged by score across sources — block scores and graph relevance aren't
|
|
26
|
+
* comparable).
|
|
50
27
|
*/
|
|
51
28
|
export async function broadSearch(dir, query, {
|
|
52
29
|
blockK = DEFAULT_BLOCK_K, graphService = null, graphLimit = DEFAULT_GRAPH_LIMIT,
|
package/src/concept.mjs
CHANGED
|
@@ -1,30 +1,15 @@
|
|
|
1
|
-
// concept.mjs — "the concept force": compose a
|
|
2
|
-
// "what is a X" touch
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// 2. THE EXAMPLES — real instances of X: code-graph individuals whose class maps
|
|
7
|
-
// to X (capped ~3, stable graph order, each a real node), plus any remembered
|
|
8
|
-
// "A is a X" facts.
|
|
9
|
-
// 3. THE GUIDED FOLLOW-UP — 2-3 concrete, RUNNABLE next questions built from the
|
|
10
|
-
// real instances × the query shapes valid for that kind, EACH PRE-CHECKED by
|
|
11
|
-
// actually running it through ask() so a suggestion can never miss.
|
|
12
|
-
//
|
|
13
|
-
// PURE given (graph, term, {definition, factRows}) — the follow-up validator calls
|
|
14
|
-
// ask() (deterministic, no model), so the whole composition is reproducible. The
|
|
15
|
-
// caller (chat.mjs) owns the async edges: loading corpus/seon/definitions.jsonl and
|
|
16
|
-
// the memory fact rows, and rendering through the response template. This module
|
|
17
|
-
// never fabricates: every example is a real individual/fact and every follow-up is
|
|
18
|
-
// validated against the same graph before it is offered.
|
|
1
|
+
// concept.mjs — "the concept force": compose a three-band answer (fact,
|
|
2
|
+
// examples, guided follow-up) to a vague "what is a X" touch when X is a
|
|
3
|
+
// known, instantiated concept. Pure given (graph, term, {definition,
|
|
4
|
+
// factRows}); never fabricates — every example is a real individual/fact and
|
|
5
|
+
// every follow-up is pre-validated by actually running it through ask().
|
|
19
6
|
|
|
20
7
|
import { ask } from "./ask.mjs";
|
|
21
8
|
import { relationKind } from "./codegraph.mjs";
|
|
22
9
|
|
|
23
|
-
/** A vague concept term (normalized, singular
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* force" touch (a general-vocabulary term like "cache" has a definition but no
|
|
27
|
-
* enumerable graph class, so it falls back to the ordinary definition surface). */
|
|
10
|
+
/** A vague concept term (normalized, singular) → the graph individual `class`
|
|
11
|
+
* it enumerates. Outside this closed set (e.g. "cache"), the term falls back
|
|
12
|
+
* to the ordinary definition surface instead. */
|
|
28
13
|
export const CONCEPT_CLASS = Object.freeze({
|
|
29
14
|
class: "Class",
|
|
30
15
|
module: "Module",
|
|
@@ -52,11 +37,8 @@ const CLASS_NOUN = Object.freeze({
|
|
|
52
37
|
* instance, so it never contributes an example. */
|
|
53
38
|
const ISA_INSTANCE_PREDICATES = new Set(["rdf:type"]);
|
|
54
39
|
|
|
55
|
-
/** Per graph-class, the candidate follow-up shapes in priority order
|
|
56
|
-
*
|
|
57
|
-
* only after the query VALIDATES against the live graph, so a shape that can't
|
|
58
|
-
* resolve for any instance is silently dropped. Shapes are curated to be exactly
|
|
59
|
-
* the ones ask.mjs's grammar answers for that kind. */
|
|
40
|
+
/** Per graph-class, the candidate follow-up shapes in priority order — each
|
|
41
|
+
* offered only once its query validates against the live graph. */
|
|
60
42
|
const FOLLOWUP_SHAPES = Object.freeze({
|
|
61
43
|
Class: [
|
|
62
44
|
(x) => `which classes inherit from ${x}`,
|
|
@@ -125,11 +107,8 @@ function resolves(graph, query) {
|
|
|
125
107
|
}
|
|
126
108
|
}
|
|
127
109
|
|
|
128
|
-
/**
|
|
129
|
-
*
|
|
130
|
-
* the first such instance NOT already used by an earlier follow-up (so the set
|
|
131
|
-
* showcases DIFFERENT real nodes where possible); a shape no instance satisfies is
|
|
132
|
-
* dropped entirely. Deterministic (graph order in, first-fit out). */
|
|
110
|
+
/** Up to MAX_FOLLOWUPS validated follow-ups: first-fit over shapes in
|
|
111
|
+
* priority order, preferring an instance not already used. */
|
|
133
112
|
function buildFollowups(graph, cls, instanceLabels) {
|
|
134
113
|
const shapes = FOLLOWUP_SHAPES[cls] || [];
|
|
135
114
|
const used = new Set();
|
|
@@ -145,14 +124,8 @@ function buildFollowups(graph, cls, instanceLabels) {
|
|
|
145
124
|
return out;
|
|
146
125
|
}
|
|
147
126
|
|
|
148
|
-
/** Compose the three bands for a concept term, or null when it
|
|
149
|
-
*
|
|
150
|
-
* has NO instances anywhere (code graph and memory both empty). Returns the pieces as
|
|
151
|
-
* strings so the caller can render them through a data template:
|
|
152
|
-
* { definition, examples, followups, instances:[{id,label,type,module}] }
|
|
153
|
-
* `examples` is always non-empty when non-null (we only fire with real instances);
|
|
154
|
-
* `followups` is "" when no validated next-question exists, else a "\nWant to go
|
|
155
|
-
* deeper? Try:\n • …" block. */
|
|
127
|
+
/** Compose the three bands for a concept term, or null when it's not a known
|
|
128
|
+
* enumerable concept, has no curated definition, or has no instances anywhere. */
|
|
156
129
|
export function composeConcept(graph, term, { definition = null, factRows = [] } = {}) {
|
|
157
130
|
const cls = CONCEPT_CLASS[term];
|
|
158
131
|
if (!cls || !definition) return null;
|
|
@@ -225,20 +198,12 @@ export function composeConcept(graph, term, { definition = null, factRows = [] }
|
|
|
225
198
|
};
|
|
226
199
|
}
|
|
227
200
|
|
|
228
|
-
//
|
|
229
|
-
//
|
|
230
|
-
//
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
// PURE given (graph, relTerm, {definition}); every example is a real edge and every
|
|
235
|
-
// follow-up is validated via resolves() before it is offered. Never fabricates.
|
|
236
|
-
// ============================================================================
|
|
237
|
-
|
|
238
|
-
/** A vague relation term (lower-cased) → the internal concept key it enumerates.
|
|
239
|
-
* The closed set of edge-kind concepts the seon relation table + the graph both
|
|
240
|
-
* understand; a term outside it is not a relation-force touch. Nouns, gerunds and
|
|
241
|
-
* a couple of synonyms all collapse to one key. */
|
|
201
|
+
// ---- THE RELATION CONCEPT FORCE — same three bands, but for a vague touch on
|
|
202
|
+
// a RELATION/edge kind ("what about imports"): composeConcept enumerates
|
|
203
|
+
// INDIVIDUALS, composeRelation enumerates EDGES. ----
|
|
204
|
+
|
|
205
|
+
/** A vague relation term (lower-cased) → the internal concept key it
|
|
206
|
+
* enumerates. Nouns, gerunds and synonyms all collapse to one key. */
|
|
242
207
|
export const RELATION_TERM = Object.freeze({
|
|
243
208
|
import: "imports", imports: "imports", importing: "imports", imported: "imports",
|
|
244
209
|
call: "calls", calls: "calls", calling: "calls", called: "calls", invoke: "calls", invokes: "calls", invoking: "calls",
|
|
@@ -249,11 +214,8 @@ export const RELATION_TERM = Object.freeze({
|
|
|
249
214
|
define: "defines", defines: "defines", defining: "defines", defined: "defines", definition: "defines", definitions: "defines", declaration: "defines",
|
|
250
215
|
touch: "touches", touches: "touches", touching: "touches", touched: "touches",
|
|
251
216
|
cochange: "cochange", "co-change": "cochange", "change-coupling": "cochange", coupled: "cochange",
|
|
252
|
-
// "export"/"exports" is
|
|
253
|
-
// "
|
|
254
|
-
// does export mean"/"what is an export" shape; vagueTouchTermOf/relationTermOf are
|
|
255
|
-
// deliberately scoped to the NON-meta "what about X"/"tell me about X" touch (see
|
|
256
|
-
// relationTermOf's own docblock, frozen case am-meta-imports), so no conflict here.
|
|
217
|
+
// "export"/"exports" is also a curated seon lexicon noun, but that meta reading
|
|
218
|
+
// only owns "what does export mean" — no conflict with this vague-touch table.
|
|
257
219
|
export: "reexports", exports: "reexports", exporting: "reexports", exported: "reexports",
|
|
258
220
|
reexport: "reexports", reexports: "reexports", reexporting: "reexports",
|
|
259
221
|
"re-export": "reexports", "re-exports": "reexports", "re-exporting": "reexports",
|
|
@@ -288,11 +250,9 @@ const RELATION_RENDER = Object.freeze({
|
|
|
288
250
|
reexports: { verb: "re-exports", edgeNoun: "re-export" },
|
|
289
251
|
});
|
|
290
252
|
|
|
291
|
-
/** Per concept key, the candidate follow-up shapes in priority order
|
|
292
|
-
* draws
|
|
293
|
-
* query
|
|
294
|
-
* (resolves()), so a shape no endpoint satisfies is silently dropped. Curated to be
|
|
295
|
-
* exactly the shapes ask.mjs answers for that kind. */
|
|
253
|
+
/** Per concept key, the candidate follow-up shapes in priority order — each
|
|
254
|
+
* draws an endpoint from one side of the edges and is offered only once its
|
|
255
|
+
* query validates against the live graph. */
|
|
296
256
|
const RELATION_FOLLOWUP_SHAPES = Object.freeze({
|
|
297
257
|
imports: [
|
|
298
258
|
{ side: "obj", make: (x) => `which modules import ${x}` },
|
|
@@ -340,10 +300,8 @@ const MAX_EDGE_EXAMPLES = 3;
|
|
|
340
300
|
const edgeSubjectLabel = (e) => String(e.subjectLabel || e.subject);
|
|
341
301
|
const edgeObjectLabel = (e) => String(e.objectLabel || e.object);
|
|
342
302
|
|
|
343
|
-
/**
|
|
344
|
-
* first-fit discipline as buildFollowups
|
|
345
|
-
* endpoints (of that shape's side) whose query resolves and offer it for the first
|
|
346
|
-
* such endpoint not already used, so the set showcases DIFFERENT real nodes. */
|
|
303
|
+
/** Up to MAX_FOLLOWUPS validated follow-ups for a relation's edges — same
|
|
304
|
+
* first-fit discipline as buildFollowups. */
|
|
347
305
|
function buildRelationFollowups(graph, key, subjLabels, objLabels) {
|
|
348
306
|
const shapes = RELATION_FOLLOWUP_SHAPES[key] || [];
|
|
349
307
|
const used = new Set();
|
|
@@ -360,25 +318,11 @@ function buildRelationFollowups(graph, key, subjLabels, objLabels) {
|
|
|
360
318
|
return out;
|
|
361
319
|
}
|
|
362
320
|
|
|
363
|
-
/** Compose the three bands for a RELATION concept term, or null when it
|
|
364
|
-
*
|
|
365
|
-
*
|
|
366
|
-
*
|
|
367
|
-
*
|
|
368
|
-
* next-question exists.
|
|
369
|
-
*
|
|
370
|
-
* A known relation whose graph has ZERO edges of that kind is NOT null — it degrades
|
|
371
|
-
* to a two-band answer (the definition + an explicit "this codebase has no X edges"
|
|
372
|
-
* line, `examples`-shaped so the caller renders it identically). Found live (an
|
|
373
|
-
* advisor tick on the 0.9.14 Tier-2 playtest cycle): returning null here for the
|
|
374
|
-
* zero-edge case let the caller's OWN raw grammar attempt at the vague-touch text
|
|
375
|
-
* ("what about exports", "tell me about reexports") stand instead — but that text was
|
|
376
|
-
* never meant to be parsed as an object search, so on a graph with no reexports edges
|
|
377
|
-
* (the realistic case for most repos, e.g. examples/mini-webapp) it fell through to a
|
|
378
|
-
* garbled `no module matching "about"/"exports" found`, not an honest miss. A relation
|
|
379
|
-
* kind the graph has NEVER SEEN AT ALL (relationKind returns nothing in RELATION_KINDS
|
|
380
|
-
* for this graph's shape) still degrades the same way — the definition is always
|
|
381
|
-
* worth stating; only the fabricated edge is refused. */
|
|
321
|
+
/** Compose the three bands for a RELATION concept term, or null when it's not
|
|
322
|
+
* a known enumerable relation or has no curated definition. A known relation
|
|
323
|
+
* with ZERO edges is NOT null — it degrades to a two-band "no X edges"
|
|
324
|
+
* answer, since returning null here would fall through to a garbled raw
|
|
325
|
+
* grammar miss instead of an honest one. */
|
|
382
326
|
export function composeRelation(graph, relTerm, { definition = null } = {}) {
|
|
383
327
|
const key = RELATION_TERM[String(relTerm || "").toLowerCase()];
|
|
384
328
|
if (!key || !definition) return null;
|
package/src/conformance.mjs
CHANGED
|
@@ -1,15 +1,11 @@
|
|
|
1
1
|
// conformance.mjs — the Repository-Interface CONTRACT TEST SUITE as a reusable kit.
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// published package and runs the SAME suite against its native provider to claim
|
|
7
|
-
// conformance — conformance is the suite, not prose. It is provider-agnostic: it
|
|
8
|
-
// asserts the SHAPE + the error contract; data-bearing truth is asserted by the caller
|
|
9
|
-
// where its provider carries data.
|
|
3
|
+
// An implementation is CONFORMANT iff it passes `runConformance(name, makeProvider)`.
|
|
4
|
+
// Provider-agnostic: it asserts the shape + error contract; data-bearing truth is
|
|
5
|
+
// asserted by the caller where its provider carries data.
|
|
10
6
|
//
|
|
11
|
-
// Public surface (
|
|
12
|
-
//
|
|
7
|
+
// Public surface: runConformance(name, makeProvider), assertResult, assertIndividual,
|
|
8
|
+
// assertEdge.
|
|
13
9
|
import { test } from "node:test";
|
|
14
10
|
import assert from "node:assert/strict";
|
|
15
11
|
import {
|
|
@@ -114,8 +110,8 @@ export function runConformance(name, makeProvider) {
|
|
|
114
110
|
assert.throws(() => svc.edges("no:such:id", "not-a-real-kind"), TypeError);
|
|
115
111
|
});
|
|
116
112
|
|
|
117
|
-
// snippet
|
|
118
|
-
//
|
|
113
|
+
// snippet has nothing useful without fs, so it honestly misses NO_SOURCE (or
|
|
114
|
+
// UNRESOLVED_TERM on an empty graph) with no working tree.
|
|
119
115
|
test(`[${name}] snippet answers NO_SOURCE (not a throw) when no working tree`, async () => {
|
|
120
116
|
const svc = makeProvider();
|
|
121
117
|
if (svc.sourceAccess) return; // covered by the source-capable branch below instead
|
|
@@ -129,10 +125,10 @@ export function runConformance(name, makeProvider) {
|
|
|
129
125
|
);
|
|
130
126
|
});
|
|
131
127
|
|
|
132
|
-
// context:
|
|
133
|
-
//
|
|
134
|
-
//
|
|
135
|
-
//
|
|
128
|
+
// context: contextPlan/sizeBundle/renderGraphOnlyBundle are pure graph queries, so a
|
|
129
|
+
// graph-only provider (no working tree) returns a REAL HIT for any resolvable symbol;
|
|
130
|
+
// only an unresolvable symbol still misses (UNRESOLVED_TERM). See repository-interface.mjs's
|
|
131
|
+
// context service entry for the full rationale.
|
|
136
132
|
test(`[${name}] context returns a graph-only HIT for a resolvable symbol, even with no working tree`, async () => {
|
|
137
133
|
const svc = makeProvider();
|
|
138
134
|
if (svc.sourceAccess) return; // covered by the source-capable branch below instead
|