@polycode-projects/the-mechanical-code-talker 1.9.1 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +441 -217
- package/bin/tmct.mjs +126 -1
- package/corpus/seon/README.md +1 -2
- package/package.json +4 -2
- package/src/answer-variants.mjs +8 -36
- package/src/ask-browser-entry.mjs +5 -23
- package/src/ask-browser.bundle.js +1 -2
- package/src/ask-nlp.mjs +9 -23
- package/src/ask-vocab.mjs +139 -589
- package/src/ask.mjs +627 -1729
- package/src/chat.mjs +1684 -2874
- package/src/cli-args.mjs +14 -28
- package/src/codegraph.mjs +236 -644
- package/src/completions/complete.mjs +18 -62
- package/src/completions/graph-adapter.mjs +14 -60
- package/src/completions/group.mjs +12 -68
- package/src/completions/infer.mjs +38 -126
- package/src/completions/prune.mjs +17 -70
- package/src/completions/rank.mjs +16 -69
- package/src/completions/search.mjs +8 -31
- package/src/concept.mjs +32 -88
- package/src/conformance.mjs +11 -15
- package/src/corpus/conceptnet.mjs +31 -89
- package/src/corpus/templates.mjs +19 -45
- package/src/corpus/unknown-ingest.mjs +31 -92
- package/src/embed.mjs +10 -22
- package/src/extensions.mjs +50 -154
- package/src/finish.mjs +35 -91
- package/src/grammar/ace.mjs +16 -40
- package/src/grammar/assert.mjs +1 -1
- package/src/grammar/lexicon-core.json +1 -1
- package/src/grammar/lexicon.mjs +9 -27
- package/src/graph-merge.mjs +2 -3
- package/src/hash.mjs +6 -14
- package/src/index.mjs +6 -10
- package/src/init.mjs +38 -125
- package/src/interpret/fuzzy.mjs +10 -29
- package/src/interpret/merge.mjs +9 -27
- package/src/interpret/normalize.mjs +137 -585
- package/src/interpret/pipeline.mjs +23 -71
- package/src/interpret/strategies/ace.mjs +7 -31
- package/src/interpret/strategies/constructions.mjs +14 -41
- package/src/interpret/strategies/grammar.mjs +21 -60
- package/src/interpret/strategies/keywords.mjs +42 -131
- package/src/interpret/strategies/noise-strip.mjs +18 -89
- package/src/memory/bias.mjs +11 -54
- package/src/memory/blocks.mjs +18 -69
- package/src/memory/core.mjs +171 -591
- package/src/memory/fold.mjs +0 -0
- package/src/memory/inspect.mjs +7 -25
- package/src/memory/shacl.mjs +10 -39
- package/src/memory/trust.mjs +26 -127
- package/src/memory-ask-browser-entry.mjs +7 -30
- package/src/memory-ask-browser.bundle.js +1 -1
- package/src/paraphrase.mjs +20 -53
- package/src/planning.mjs +15 -157
- package/src/prose-nlp.mjs +4 -17
- package/src/prose.mjs +19 -67
- package/src/providers/bootstrap.mjs +1 -2
- package/src/providers/fixture.mjs +1 -2
- package/src/providers/graph-service.mjs +28 -59
- package/src/repository-interface.mjs +6 -8
- package/src/router/drive.mjs +183 -0
- package/src/router/goal-reasoner.mjs +66 -231
- package/src/router/guardrail.mjs +20 -58
- package/src/router/planner.mjs +15 -46
- package/src/router/registry.mjs +13 -43
- package/src/router/resolver.mjs +46 -131
- package/src/router/results.mjs +231 -0
- package/src/schema-docs.mjs +10 -27
- package/src/server-http.mjs +10 -19
- package/src/server.mjs +22 -28
- package/src/sessions.mjs +15 -30
- package/src/source-slice.mjs +5 -7
- package/src/source.mjs +10 -20
- package/src/syllogise.mjs +187 -575
- package/src/telemetry.mjs +3 -3
- package/src/toml-config.mjs +4 -4
- package/src/tui/app.mjs +9 -19
- package/src/viz.mjs +66 -123
- package/src/wink-model.mjs +10 -24
package/src/planning.mjs
CHANGED
|
@@ -1,35 +1,6 @@
|
|
|
1
|
-
// planning.mjs — a domain-agnostic bounded state-space search primitive
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// "generalizing findIsaChain from 'walk pre-loaded class edges' to 'walk
|
|
5
|
-
// on-demand successor states' is a moderate, in-house-idiom-consistent
|
|
6
|
-
// extension, not a foreign paradigm").
|
|
7
|
-
//
|
|
8
|
-
// `src/syllogise.mjs`'s `findIsaChain` is, in shape, already a bounded rooted
|
|
9
|
-
// BFS path search: it walks a FIXED, pre-loaded edge list (`typeEdges`/
|
|
10
|
-
// `subClassEdges`) from a start node to a target set, frontier-expansion
|
|
11
|
-
// style, checking the frontier for a hit BEFORE extending it one hop further,
|
|
12
|
-
// stopping the instant a target is reached or the hop budget is exhausted.
|
|
13
|
-
//
|
|
14
|
-
// Real planning (Hanoi, or anything with actions) needs the same shape over a
|
|
15
|
-
// state space where successors are NOT pre-loaded — they are generated ON
|
|
16
|
-
// DEMAND by applying an action to the CURRENT state. `findActionPath` below
|
|
17
|
-
// is that generalization: same frontier/seen-set/check-then-extend/shortest-
|
|
18
|
-
// path discipline as `findIsaChain`, but the "edges" come from calling the
|
|
19
|
-
// caller-supplied `applyActions(state)` fresh at every expansion, instead of
|
|
20
|
-
// looking them up in a fixed array.
|
|
21
|
-
//
|
|
22
|
-
// Deliberately NOT sharing code with `findIsaChain` itself: that function's
|
|
23
|
-
// edge lists are pre-built ONCE into a `Map` before the search loop even
|
|
24
|
-
// starts (`subSucc`, `syllogise.mjs:291-296`) — a real, load-bearing
|
|
25
|
-
// optimization for its domain (static edges, looked up many times) that does
|
|
26
|
-
// not apply here (successors are computed fresh, never looked up twice for
|
|
27
|
-
// the same state). Extracting a "shared" BFS core would either lose that
|
|
28
|
-
// optimization or force `findActionPath` to fake a static edge list, so this
|
|
29
|
-
// lands as an independent sibling, following the same DISCIPLINE, not the
|
|
30
|
-
// same code path. `findIsaChain` itself is untouched by this file.
|
|
31
|
-
//
|
|
32
|
-
// Pure, no I/O, deterministic given a deterministic `applyActions`.
|
|
1
|
+
// planning.mjs — a domain-agnostic bounded state-space search primitive,
|
|
2
|
+
// generalizing syllogise.mjs's findIsaChain (fixed pre-loaded edges) to
|
|
3
|
+
// on-demand successor generation via applyActions(state). Pure, no I/O.
|
|
33
4
|
|
|
34
5
|
/** Default state-identity key: plain values compare by `String()`, plain
|
|
35
6
|
* objects by a stable-ish `JSON.stringify` (good enough for a toy/plain-
|
|
@@ -40,15 +11,8 @@ function defaultStateKey(state) {
|
|
|
40
11
|
return String(state);
|
|
41
12
|
}
|
|
42
13
|
|
|
43
|
-
/**
|
|
44
|
-
*
|
|
45
|
-
* `findActionPath` and `findReachableSet` below (seeding a frontier from a
|
|
46
|
-
* start state has no goal/accumulation semantics to differ on: it is pure
|
|
47
|
-
* "call `applyActions` once, wrap each result"), so this one small step is
|
|
48
|
-
* genuinely, safely shared rather than duplicated verbatim in both
|
|
49
|
-
* functions. See the file-header note above `findActionPath` for why the
|
|
50
|
-
* REST of the two functions' bodies are deliberately NOT merged the same
|
|
51
|
-
* way. */
|
|
14
|
+
/** One-hop expansion of `startState` into an initial frontier — shared by
|
|
15
|
+
* findActionPath and findReachableSet below. */
|
|
52
16
|
function seedFrontier(startState, applyActions) {
|
|
53
17
|
const frontier = [];
|
|
54
18
|
for (const { action, nextState } of applyActions(startState) || []) {
|
|
@@ -58,50 +22,17 @@ function seedFrontier(startState, applyActions) {
|
|
|
58
22
|
}
|
|
59
23
|
|
|
60
24
|
/**
|
|
61
|
-
* Bounded, cycle-safe, shortest-path-first
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
* `stateKey` (default: `String()`/`JSON.stringify()`).
|
|
66
|
-
* - `isGoal(state) -> boolean` — goal predicate, checked BEFORE a state is
|
|
67
|
-
* expanded (never after — see the hop-counting discipline below).
|
|
68
|
-
* - `applyActions(state) -> Array<{ action, nextState }>` — the caller's
|
|
69
|
-
* domain logic: given the CURRENT state, the legal (action, resulting-
|
|
70
|
-
* state) pairs reachable in exactly one step. Called fresh every time a
|
|
71
|
-
* state is expanded; nothing is precomputed or cached across calls.
|
|
72
|
-
* - `opts.maxDepth` (default 50) — hop budget, mirrors `findIsaChain`'s
|
|
73
|
-
* `maxHops`: the frontier is checked for the goal AT every depth up to
|
|
74
|
-
* and including `maxDepth`, but never extended past it (check-then-
|
|
75
|
-
* extend — `findIsaChain`'s own comment on this exact off-by-one:
|
|
76
|
-
* "the frontier is checked AT every length up to and including maxHops,
|
|
77
|
-
* never one hop beyond it").
|
|
78
|
-
* - `opts.stateKey(state) -> string` — override the default identity key
|
|
79
|
-
* when `startState`/successor states are richer than a plain
|
|
80
|
-
* string/number/JSON-able object.
|
|
81
|
-
*
|
|
82
|
-
* Returns `{ actions: [...], states: [startState, ...,goalState] }` on
|
|
83
|
-
* success (the full action sequence AND the resulting state at each step, so
|
|
84
|
-
* a caller can actually execute the plan, not just know one exists), or
|
|
85
|
-
* `null` when no path reaches a goal state within `maxDepth` — an honest
|
|
86
|
-
* miss, never a guessed/truncated path.
|
|
87
|
-
*
|
|
88
|
-
* Cycle-safe via a `seen` state-key set (this function's direct precedent:
|
|
89
|
-
* `findIsaChain`'s own `seen` set, `syllogise.mjs:311`) — a state is only
|
|
90
|
-
* ever expanded once, the first (shortest) path to reach it, so a domain
|
|
91
|
-
* with cycles (two states that can reach each other) still terminates and
|
|
92
|
-
* still returns the correct shortest path, never loops.
|
|
25
|
+
* Bounded, cycle-safe, shortest-path-first BFS over a state space whose
|
|
26
|
+
* successors are generated on demand via `applyActions(state)`, checked
|
|
27
|
+
* against `isGoal` before each expansion up to `opts.maxDepth` hops.
|
|
28
|
+
* Returns `{ actions, states }` (the full plan) or `null` on an honest miss.
|
|
93
29
|
*/
|
|
94
30
|
export function findActionPath(startState, isGoal, applyActions, { maxDepth = 50, stateKey = defaultStateKey } = {}) {
|
|
95
31
|
if (isGoal(startState)) return { actions: [], states: [startState] };
|
|
96
32
|
|
|
97
33
|
let frontier = seedFrontier(startState, applyActions);
|
|
98
34
|
|
|
99
|
-
//
|
|
100
|
-
// first check) — exactly `findIsaChain`'s own "hop counts the LENGTH of the
|
|
101
|
-
// paths currently in frontier" discipline. Check-then-extend, and never
|
|
102
|
-
// extend past maxDepth: the frontier is checked at every depth up to and
|
|
103
|
-
// including maxDepth, never one hop beyond it (the off-by-one findIsaChain
|
|
104
|
-
// itself once had and fixed — not reintroduced here).
|
|
35
|
+
// Check-then-extend: never extend past maxDepth.
|
|
105
36
|
const seen = new Set([stateKey(startState)]);
|
|
106
37
|
for (let depth = 1; depth <= maxDepth && frontier.length; depth += 1) {
|
|
107
38
|
for (const entry of frontier) if (isGoal(entry.state)) return { actions: entry.actions, states: entry.states };
|
|
@@ -122,91 +53,18 @@ export function findActionPath(startState, isGoal, applyActions, { maxDepth = 50
|
|
|
122
53
|
return null;
|
|
123
54
|
}
|
|
124
55
|
|
|
125
|
-
// ---------------------------------------------------------------------------
|
|
126
|
-
// findReachableSet — PLAN_TAUGHT_RELATIONS.md, Item 6 (recursive/reachability
|
|
127
|
-
// rules) kernel half. Query-side need: "list every X reachable from Y" (e.g.
|
|
128
|
-
// "list the descendants of ahab") is REACHABILITY-SET ENUMERATION, not
|
|
129
|
-
// single-goal search — checked against `findActionPath` above's own body:
|
|
130
|
-
// it returns the INSTANT the first goal-satisfying state is found (the
|
|
131
|
-
// `for (const entry of frontier) if (isGoal(entry.state)) return …` line),
|
|
132
|
-
// so there is no way to keep it running to collect every reachable node
|
|
133
|
-
// without changing both its halting condition (early-return vs. never-
|
|
134
|
-
// return-early) AND its return shape (one path vs. every path) — a
|
|
135
|
-
// genuinely new function, not a parameter tweak, per the plan's own
|
|
136
|
-
// analysis.
|
|
137
|
-
//
|
|
138
|
-
// Code-sharing decision (asked for explicitly, decided fresh here rather
|
|
139
|
-
// than copying the plan doc's framing verbatim): the file-header reasoning
|
|
140
|
-
// for why `findActionPath` is an independent SIBLING of `findIsaChain`
|
|
141
|
-
// (lines 1-32) is "pre-built static edge maps vs. on-demand successor
|
|
142
|
-
// generation don't share an implementation, only a discipline." That
|
|
143
|
-
// reasoning does NOT distinguish `findActionPath` from `findReachableSet`
|
|
144
|
-
// — both call the caller's `applyActions(state)` fresh at every expansion;
|
|
145
|
-
// neither pre-builds anything. So on the file's own stated logic, these two
|
|
146
|
-
// are legitimately closer to each other than either is to `findIsaChain`,
|
|
147
|
-
// and it's worth asking whether MORE sharing is warranted here specifically
|
|
148
|
-
// — not just repeating the same verdict by default.
|
|
149
|
-
//
|
|
150
|
-
// Having written both bodies out, the answer is: share the one step that is
|
|
151
|
-
// truly identical (`seedFrontier` above — a single `applyActions(startState)`
|
|
152
|
-
// call with no goal/accumulation semantics to differ on), but keep the main
|
|
153
|
-
// expand-loop bodies independent. The reason isn't "different edge
|
|
154
|
-
// generation" this time — it's that the two loops' HALTING and RESULT-
|
|
155
|
-
// COLLECTION semantics are irreducibly different: `findActionPath` returns
|
|
156
|
-
// the instant ANY frontier entry satisfies `isGoal`, discarding the rest of
|
|
157
|
-
// the frontier and every state it hasn't reached yet; `findReachableSet`
|
|
158
|
-
// never returns early, has no predicate at all, and must keep every
|
|
159
|
-
// newly-seen state (not just one) across the entire bounded search. Forcing
|
|
160
|
-
// both through one shared "expand a frontier" core would mean threading an
|
|
161
|
-
// optional `isGoal` (or a sentinel "never" predicate) AND an accumulator
|
|
162
|
-
// mode through a single function — that parameter surface would itself
|
|
163
|
-
// recreate the complexity the merge was meant to remove, for a savings of
|
|
164
|
-
// roughly the ~10-line inner loop. Given the file's own established
|
|
165
|
-
// precedent of favoring readable independent siblings over cleverly
|
|
166
|
-
// parameterized cores, and that the one truly shared step already isn't
|
|
167
|
-
// duplicated (`seedFrontier`), landing this as an independent sibling
|
|
168
|
-
// remains the right call — just not for the identical reason `findIsaChain`
|
|
169
|
-
// vs. `findActionPath` had.
|
|
170
56
|
/**
|
|
171
|
-
* Bounded, cycle-safe
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
*
|
|
176
|
-
* meaning and defaults as `findActionPath` (see above); successors are
|
|
177
|
-
* generated ON DEMAND by calling `applyActions(state)` fresh at every
|
|
178
|
-
* expansion, nothing precomputed or cached.
|
|
179
|
-
* - Deliberately NO `isGoal` parameter: every state reachable from
|
|
180
|
-
* `startState` (EXCLUDING `startState` itself — the start is where you
|
|
181
|
-
* already are, not a reachable result) within the hop budget is a
|
|
182
|
-
* result, not just one goal-satisfying state.
|
|
183
|
-
*
|
|
184
|
-
* Returns an array of `{ node, path: { actions, states } }` — one entry per
|
|
185
|
-
* distinct reachable state, `path` mirroring `findActionPath`'s own
|
|
186
|
-
* `{ actions, states }` return shape (the action sequence AND intermediate
|
|
187
|
-
* states from `startState` to that node), so a caller gets "how did we get
|
|
188
|
-
* here" for every reachable node, not just one. Returns `[]` (never
|
|
189
|
-
* `null`/`undefined`) when nothing is reachable within budget — reachability
|
|
190
|
-
* enumeration has no "miss" case the way single-goal search does; an empty
|
|
191
|
-
* result set is itself the honest, complete answer.
|
|
192
|
-
*
|
|
193
|
-
* Cycle-safe via the same `seen` state-key convention as `findActionPath`: a
|
|
194
|
-
* state is recorded (and expanded) only the FIRST time it is reached, so the
|
|
195
|
-
* shortest path to it is what gets stored, and a state reachable by two
|
|
196
|
-
* different routes (or sitting inside a genuine cycle) is reported exactly
|
|
197
|
-
* once, never duplicated, and never causes an infinite loop.
|
|
57
|
+
* Bounded, cycle-safe BFS enumeration of every state reachable from
|
|
58
|
+
* `startState` within `maxDepth` hops (no `isGoal` — excludes `startState`
|
|
59
|
+
* itself). Shares seedFrontier with findActionPath but keeps its own
|
|
60
|
+
* expand-loop (collect-everything vs. early-return-on-first-hit). Returns
|
|
61
|
+
* `[{ node, path: { actions, states } }]`, `[]` when nothing is reachable.
|
|
198
62
|
*/
|
|
199
63
|
export function findReachableSet(startState, applyActions, { maxDepth = 50, stateKey = defaultStateKey } = {}) {
|
|
200
64
|
let frontier = seedFrontier(startState, applyActions);
|
|
201
65
|
|
|
202
66
|
const seen = new Set([stateKey(startState)]);
|
|
203
67
|
const results = [];
|
|
204
|
-
// Single combined loop (not findActionPath's check-then-separate-extend):
|
|
205
|
-
// there is no per-iteration early return to protect here, so recording a
|
|
206
|
-
// newly-seen state and deciding whether to expand it past it can live in
|
|
207
|
-
// the same pass without losing any of findActionPath's check-then-extend
|
|
208
|
-
// discipline — a state discovered exactly at maxDepth is still recorded
|
|
209
|
-
// (it IS reachable within budget) but is never expanded past it.
|
|
210
68
|
for (let depth = 1; depth <= maxDepth && frontier.length; depth += 1) {
|
|
211
69
|
const next = [];
|
|
212
70
|
for (const entry of frontier) {
|
package/src/prose-nlp.mjs
CHANGED
|
@@ -1,21 +1,8 @@
|
|
|
1
1
|
// prose-nlp.mjs — the OPTIONAL wink-nlp lemma loader behind prose.mjs's LEMMA layer.
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// duplicated createRequire lines are gone; the coupling this file avoids is to
|
|
7
|
-
// ask-nlp.mjs's export shape, not to a leaf model loader.
|
|
8
|
-
//
|
|
9
|
-
// BOUNDARY (same as ask-nlp.mjs, hard): Node-only path, never inlined into the
|
|
10
|
-
// viewer bundle. prose.mjs is itself never inlined by viz.mjs's askSource(), so
|
|
11
|
-
// nothing browser-side can reach this module. The wink pair is loaded lazily (Node
|
|
12
|
-
// createRequire fallback, or the browser registration seam), failure cached as null:
|
|
13
|
-
// a checkout without the optional deps simply builds no lemma layer (honestly
|
|
14
|
-
// absent), it never throws.
|
|
15
|
-
//
|
|
16
|
-
// Determinism: wink's lemmatiser is a fixed trained model with no sampling — the
|
|
17
|
-
// same token always yields the same lemma across runs and processes, which is what
|
|
18
|
-
// lets the lemma layer meet the "byte-identical proseIndex across builds" contract.
|
|
2
|
+
// Node-only, never inlined into the viewer bundle; loaded lazily and cached, so a
|
|
3
|
+
// checkout without the optional deps builds no lemma layer (honestly absent),
|
|
4
|
+
// never throws. Wink's lemmatiser is deterministic (no sampling), which is what
|
|
5
|
+
// keeps the built proseIndex byte-identical across builds.
|
|
19
6
|
|
|
20
7
|
import { winkInstance } from "./wink-model.mjs";
|
|
21
8
|
|
package/src/prose.mjs
CHANGED
|
@@ -1,35 +1,11 @@
|
|
|
1
|
-
// prose.mjs — the second-pass prose extraction + cross-reference index
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// 1. identifier decomposition — names are often literal sentence fragments
|
|
7
|
-
// ("calculateTotalPriceIncludingTax" -> calculate/total/price/including/tax); splitting
|
|
8
|
-
// them turns every symbol/module name into free-text search surface for zero extra cost.
|
|
9
|
-
// 2. prose literals — docstrings/doc-comments already captured as the `doc` attribute.
|
|
10
|
-
// Python (extract_ast.py) and JS/TS (jsts_tsc.mjs's firstDocLine) already populate this;
|
|
11
|
-
// C#/Java's PRIMARY extractors (cs_roslyn.mjs, java_javaparser.mjs) shell out to compiled
|
|
12
|
-
// Roslyn/JavaParser binaries — adding doc capture there means modifying and rebuilding
|
|
13
|
-
// external .NET/JVM tooling, not a JS-side change. Deliberately deferred (PLAN_PROSE_INDEX.md
|
|
14
|
-
// backlog); this pass consumes whatever `doc` is already present, regardless of source
|
|
15
|
-
// language, so C#/Java modules still get identifier-decomposition tokens today.
|
|
16
|
-
//
|
|
17
|
-
// Tokenizer pattern (stopwords, length bounds, per-doc token cap) mirrors marginalia's
|
|
18
|
-
// app/lib/text-index.mjs — a proven lexical-inverted-index tokenizer for the same "search by
|
|
19
|
-
// keyword, no embeddings, no stemmer dependency" problem, adapted for code identifiers.
|
|
20
|
-
//
|
|
21
|
-
// Storage: (a) a `prose_tokens` attribute (space-joined, deduped, sorted) on each individual —
|
|
22
|
-
// the graph stays self-describing, works if a consumer only has one individual in hand; AND
|
|
23
|
-
// (b) a real inverted index (word -> [individual ids]) built from those same tokens and
|
|
24
|
-
// attached as `entities.proseIndex` — an O(1) word lookup for consumers (resolveObject-style
|
|
25
|
-
// fuzzy object-term resolution, scoreModules-style lexical boosting) instead of scanning every
|
|
26
|
-
// individual's attributes. Both are derived from the identical token set, so they can never
|
|
27
|
-
// disagree; (b) is just (a) inverted once, cheaply, at build time.
|
|
1
|
+
// prose.mjs — the second-pass prose extraction + cross-reference index.
|
|
2
|
+
// Deterministic, no model calls: tokenizes each individual's decomposed name
|
|
3
|
+
// plus any captured `doc` text into a combined set, stored both as a
|
|
4
|
+
// `prose_tokens` attribute (self-describing) and inverted into
|
|
5
|
+
// `entities.proseIndex` (word -> ids) for O(1) lookup.
|
|
28
6
|
|
|
29
|
-
// Exported
|
|
30
|
-
//
|
|
31
|
-
// built for code identifiers, where stopword-shaped fragments are rare) down to real content
|
|
32
|
-
// words before using it as a text-clustering similarity signal.
|
|
7
|
+
// Exported so src/completions/group.mjs can filter splitIdentifierWords'
|
|
8
|
+
// output (which doesn't apply this list itself) down to real content words.
|
|
33
9
|
export const STOPWORDS = new Set(
|
|
34
10
|
("a an and or but the of to in on at for with from by as is are was were be been being " +
|
|
35
11
|
"it its this that these those i you he she they we me my your our do does did not no " +
|
|
@@ -38,7 +14,7 @@ export const STOPWORDS = new Set(
|
|
|
38
14
|
"where why how").split(/\s+/),
|
|
39
15
|
);
|
|
40
16
|
|
|
41
|
-
const MAX_TOKEN_LEN = 40; // drops hash-like/garbage tokens
|
|
17
|
+
const MAX_TOKEN_LEN = 40; // drops hash-like/garbage tokens
|
|
42
18
|
const MAX_TOKENS_PER_DOC = 120; // bounds cost on a pathologically long docstring/name
|
|
43
19
|
|
|
44
20
|
/** Split an identifier or a path-like name into lowercase word tokens.
|
|
@@ -81,17 +57,10 @@ export function proseTokensFor({ name, doc } = {}) {
|
|
|
81
57
|
return [...set].sort();
|
|
82
58
|
}
|
|
83
59
|
|
|
84
|
-
/**
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
* attribute if present (docstring/doc-comment). Commit is different: its `label` is a
|
|
89
|
-
* truncated SHA (hex noise if decomposed, e.g. "e6a9419567f7" -> "9419567" garbage) — skip
|
|
90
|
-
* decomposing it and tokenize its `message` attribute instead, which is the real prose
|
|
91
|
-
* (commit messages are often the richest free text in the whole graph). Mutates and
|
|
92
|
-
* returns the same array — safe to call once after the typed individuals are built.
|
|
93
|
-
* `enabled=false` is a no-op (the disable path `TMCT_PROSE_INDEX=0` in a graph writer), so
|
|
94
|
-
* the core typed graph is never affected by turning this pass off. */
|
|
60
|
+
/** Attach a `prose_tokens` attribute to every individual, from its (decomposed)
|
|
61
|
+
* name and captured doc text — except Commit, whose `label` is a truncated
|
|
62
|
+
* SHA, not a decomposable identifier: it tokenizes `message` instead. Mutates
|
|
63
|
+
* and returns the same array; `enabled=false` is a no-op. */
|
|
95
64
|
export function attachProseTokens(individuals, { enabled = true } = {}) {
|
|
96
65
|
if (!enabled) return individuals;
|
|
97
66
|
for (const ind of individuals) {
|
|
@@ -130,8 +99,7 @@ export function buildProseIndex(individuals) {
|
|
|
130
99
|
* tokenized the same way as a docstring), ranked by overlap count (most shared words
|
|
131
100
|
* first). This is the integration point for ask.mjs's resolveObject (fuzzy object-term
|
|
132
101
|
* resolution beyond exact/substring match) and codegraph.mjs's scoreModules (a lexical
|
|
133
|
-
* boost source)
|
|
134
|
-
* wired into either file here to avoid colliding with concurrent work on them.
|
|
102
|
+
* boost source); not wired into either file here.
|
|
135
103
|
* `proseIndex` is `entities.proseIndex` (buildProseIndex's output). */
|
|
136
104
|
export function lookupByProseTokens(proseIndex, query, { limit = 10 } = {}) {
|
|
137
105
|
const queryTokens = [...new Set([...splitIdentifierWords(query), ...tokenizeProse(query)])];
|
|
@@ -148,28 +116,12 @@ export function lookupByProseTokens(proseIndex, query, { limit = 10 } = {}) {
|
|
|
148
116
|
.map(([id, score]) => ({ id, score }));
|
|
149
117
|
}
|
|
150
118
|
|
|
151
|
-
/**
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
157
|
-
* normalised form still resolves.
|
|
158
|
-
*
|
|
159
|
-
* Layer shape consumed (an inverted index keyed by the NORMALISED token, mirroring the verbatim
|
|
160
|
-
* top level, just normalised):
|
|
161
|
-
* proseIndex["tmct:layers"] = { <layerName>: { <normalisedToken>: [id, …] }, … }
|
|
162
|
-
* A posting may be a plain id array or `{ ids: [...] }` — both are tolerated. The raw query
|
|
163
|
-
* token is looked up directly against every layer's keys, so a token whose surface form is
|
|
164
|
-
* already a canonical/stem/lemma/spell-corrected key hits; the accessor never itself normalises
|
|
165
|
-
* the query (it owns no normaliser — those live in the concurrent ask/prose-nlp surface), so it
|
|
166
|
-
* can never disagree with the build's normalisation, only under-fire safely.
|
|
167
|
-
*
|
|
168
|
-
* Returns { ids, via }: `ids` a deduped, sorted (stable/deterministic) id list; `via` the
|
|
169
|
-
* sorted layer names that produced them, joined with "+", for a scorer's provenance — or null
|
|
170
|
-
* when nothing hit. Absent / malformed / pre-layers `proseIndex` → { ids: [], via: null }: a
|
|
171
|
-
* safe no-op, so the opt-in flag degrades to nothing on a graph indexed before layers existed.
|
|
172
|
-
* Accepts either a `proseIndex` object or a parsed graph (reads its `.proseIndex`). */
|
|
119
|
+
/** Read accessor for the NORMALISED prose layers (spell-corrected /
|
|
120
|
+
* canonical-schema-term / stem / lemma) under `proseIndex["tmct:layers"] =
|
|
121
|
+
* { layerName: { normalisedToken: [id, …] } }`, so a query word that only
|
|
122
|
+
* overlaps a module via a normalised form still resolves. Returns
|
|
123
|
+
* { ids, via } (deduped/sorted ids, the layer names that matched); a safe
|
|
124
|
+
* no-op `{ ids: [], via: null }` on an absent/pre-layers proseIndex. */
|
|
173
125
|
export function proseLayerHits(proseIndex, token) {
|
|
174
126
|
const src = proseIndex && (proseIndex["tmct:layers"] ? proseIndex : proseIndex.proseIndex);
|
|
175
127
|
const layers = src && src["tmct:layers"];
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
// The BOOTSTRAP reference provider — the empty/degenerate graph a fresh repo
|
|
2
|
-
// "contains" before anything is indexed.
|
|
3
|
-
// 2: "bootstrap returns honest empties".
|
|
2
|
+
// "contains" before anything is indexed.
|
|
4
3
|
//
|
|
5
4
|
// It implements every Repository-Interface service over the empty bootstrap
|
|
6
5
|
// payload (src/source.mjs emptyEntities): every id-taking service returns
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
// The FIXTURE reference provider — a small, real, self-contained code graph that
|
|
2
|
-
// implements every Repository-Interface service.
|
|
3
|
-
// deliverable 2: "the executable specification an external producer reads first".
|
|
2
|
+
// implements every Repository-Interface service.
|
|
4
3
|
//
|
|
5
4
|
// It is a degenerate provider in the sense that its graph is tiny and its source
|
|
6
5
|
// bodies are absent (snippet/context answer NO_SOURCE) — but every OTHER service
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
// The reference Repository-Interface service over a parsed code graph.
|
|
2
|
-
// archive/PLAN_REPOSITORY_INTERFACE.md — "the executable specification".
|
|
3
2
|
//
|
|
4
3
|
// createGraphService(graph) returns a typed service object implementing EVERY
|
|
5
4
|
// service in src/repository-interface.mjs over the `{ individuals, byId,
|
|
@@ -67,14 +66,10 @@ function groupMetaForKind(graph, kind) {
|
|
|
67
66
|
const CONTEXT_BODY_MAX_LINES = 200; // mirrors server.mjs's SNIPPET_MAX_LINES for the source-capable body sections
|
|
68
67
|
const CONTEXT_INLINE_CALLEE_LOC = 120; // mirrors server.mjs's INLINE_CALLEE_LOC budget
|
|
69
68
|
|
|
70
|
-
/** The fs-dependent half of context()'s bundle — anchor
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
* which codegraph.mjs deliberately never touches. Read failures degrade silently (an
|
|
75
|
-
* omitted section), matching buildContextBundle's own graceful-degradation behavior —
|
|
76
|
-
* this is a best-effort enrichment on top of an already-real graph-only hit, not a new
|
|
77
|
-
* failure mode. Returns "" when nothing could be rendered. */
|
|
69
|
+
/** The fs-dependent half of context()'s bundle — anchor/exemplar/inlined-callee body TEXT —
|
|
70
|
+
* layered on top of renderGraphOnlyBundle's pure sections when source-capable. Read
|
|
71
|
+
* failures degrade silently (an omitted section, never a throw). Returns "" when nothing
|
|
72
|
+
* could be rendered. */
|
|
78
73
|
async function renderSourceBodies(plan, mask, { readFile, repoRoot }) {
|
|
79
74
|
if (!plan.moduleLabel) return "";
|
|
80
75
|
let lines = null;
|
|
@@ -123,17 +118,14 @@ async function renderSourceBodies(plan, mask, { readFile, repoRoot }) {
|
|
|
123
118
|
* @param {object} graph a parseEntities() result
|
|
124
119
|
* @param {object} [opts]
|
|
125
120
|
* @param {boolean} [opts.sourceAccess=false] whether source services (snippet, context) read
|
|
126
|
-
* real fs bodies. When true, `repoRoot` + `readFile` are
|
|
127
|
-
*
|
|
128
|
-
* fs is an explicit INJECTED capability, never an ambient import).
|
|
121
|
+
* real fs bodies. When true, `repoRoot` + `readFile` are required (fs is an injected
|
|
122
|
+
* capability, never an ambient import).
|
|
129
123
|
* @param {string} [opts.repoRoot] absolute repo root; required when sourceAccess is true.
|
|
130
|
-
* @param {Function} [opts.readFile] async (path, encoding) => string
|
|
131
|
-
*
|
|
132
|
-
* @param {object|null} [opts.tel] an optional telemetry sink ({ record(fields) }
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
* only, never raw text/body. Null (the default) skips the wrapping loop entirely — zero
|
|
136
|
-
* overhead, and fixtureProvider()/bootstrapProvider() (which pass no tel) are unaffected.
|
|
124
|
+
* @param {Function} [opts.readFile] async (path, encoding) => string; required when
|
|
125
|
+
* sourceAccess is true.
|
|
126
|
+
* @param {object|null} [opts.tel] an optional telemetry sink ({ record(fields) }). When
|
|
127
|
+
* present, every service is wrapped once to time it and record counts only, never raw
|
|
128
|
+
* text/body.
|
|
137
129
|
* @returns the typed service object
|
|
138
130
|
*/
|
|
139
131
|
export function createGraphService(graph, { sourceAccess = false, repoRoot = null, readFile = null, tel = null } = {}) {
|
|
@@ -192,7 +184,7 @@ export function createGraphService(graph, { sourceAccess = false, repoRoot = nul
|
|
|
192
184
|
const bases = inherits
|
|
193
185
|
.filter((e) => e.subject === classId)
|
|
194
186
|
.map((e) => byId.get(e.object) ? toIndividual(byId.get(e.object)) : { id: e.object, label: e.objectLabel || e.object, class: "Class", attributes: [] });
|
|
195
|
-
// transitive reverse inheritance closure
|
|
187
|
+
// transitive reverse inheritance closure
|
|
196
188
|
const childrenOf = new Map();
|
|
197
189
|
for (const e of inherits) {
|
|
198
190
|
if (!childrenOf.has(e.object)) childrenOf.set(e.object, []);
|
|
@@ -256,9 +248,7 @@ export function createGraphService(graph, { sourceAccess = false, repoRoot = nul
|
|
|
256
248
|
const ind = resolveId(id);
|
|
257
249
|
if (!ind) return miss(MISS_REASONS.UNRESOLVED_TERM, { term: id });
|
|
258
250
|
const meta = groupMetaForKind(graph, kind);
|
|
259
|
-
// edge order is stable/memoized
|
|
260
|
-
// slice after filter/map is a safe, backward-compatible pagination: an omitted `limit`
|
|
261
|
-
// leaves the full list untouched (limit=undefined → slice(offset) → everything from offset).
|
|
251
|
+
// edge order is stable/memoized; an omitted `limit` leaves the full list untouched.
|
|
262
252
|
let edges = edgesOfKind(graph, kind)
|
|
263
253
|
.filter((e) => e.subject === id)
|
|
264
254
|
.map((e) => toEdge(e, meta));
|
|
@@ -274,11 +264,8 @@ export function createGraphService(graph, { sourceAccess = false, repoRoot = nul
|
|
|
274
264
|
return hit({ total, levels });
|
|
275
265
|
},
|
|
276
266
|
|
|
277
|
-
//
|
|
278
|
-
//
|
|
279
|
-
// source-reaching service should always `await` it regardless of whether THIS
|
|
280
|
-
// particular provider happens to be source-capable (awaiting a non-Promise value is a
|
|
281
|
-
// safe no-op, so this is backward-compatible for a caller that already awaits).
|
|
267
|
+
// Async (returns Promise<Result>): callers should always await a source-reaching
|
|
268
|
+
// service regardless of whether this provider happens to be source-capable.
|
|
282
269
|
async snippet(id) {
|
|
283
270
|
const ind = resolveId(id);
|
|
284
271
|
if (!ind) return miss(MISS_REASONS.UNRESOLVED_TERM, { term: id });
|
|
@@ -296,18 +283,13 @@ export function createGraphService(graph, { sourceAccess = false, repoRoot = nul
|
|
|
296
283
|
});
|
|
297
284
|
return hit({ path: site.path, span: { start: site.start, end: site.end }, body: sliced.text });
|
|
298
285
|
} catch (e) {
|
|
299
|
-
// A path-traversal ToolError or any other read failure both land here — honestly,
|
|
300
|
-
// never a throw (the interface's error contract: a clean miss is a value).
|
|
301
286
|
return miss(MISS_REASONS.NO_SOURCE, { term: id, detail: `could not read ${site.path}: ${e?.message || e}` });
|
|
302
287
|
}
|
|
303
288
|
},
|
|
304
289
|
|
|
305
|
-
//
|
|
306
|
-
//
|
|
307
|
-
//
|
|
308
|
-
// globals/tests/exports/insertion-region, everything EXCEPT anchor/exemplar/inlined-callee
|
|
309
|
-
// body TEXT. Only an unresolvable symbol still misses (UNRESOLVED_TERM). A source-capable
|
|
310
|
-
// provider layers the body sections on top via renderSourceBodies (below).
|
|
290
|
+
// A graph-only HIT for any resolvable symbol (siblings/registration/globals/tests/
|
|
291
|
+
// exports/insertion-region), everything except anchor/exemplar/inlined-callee body TEXT.
|
|
292
|
+
// A source-capable provider layers the body sections on top via renderSourceBodies.
|
|
311
293
|
async context(symbol, { depth = "auto" } = {}) {
|
|
312
294
|
const { match } = resolveSymbol(graph, String(symbol ?? ""));
|
|
313
295
|
if (!match) return miss(MISS_REASONS.UNRESOLVED_TERM, { term: String(symbol ?? "") });
|
|
@@ -389,14 +371,9 @@ export function createGraphService(graph, { sourceAccess = false, repoRoot = nul
|
|
|
389
371
|
return hit({ commits });
|
|
390
372
|
},
|
|
391
373
|
|
|
392
|
-
// Ranked lexical search,
|
|
393
|
-
//
|
|
394
|
-
//
|
|
395
|
-
// symbol-mode (kind names a symbol kind) ranks via scoreSymbolsRanked. name/decorator
|
|
396
|
-
// filters apply in SYMBOL mode only — module mode never supported them in codegraph.mjs
|
|
397
|
-
// either (renderSearch's module branch ignores both beyond the "was anything specified"
|
|
398
|
-
// check), so this does not invent a new filter semantic. Results are capped at
|
|
399
|
-
// `limit` (default SEARCH_LIMIT), sliced after the full ranked array is computed.
|
|
374
|
+
// Ranked lexical search: module-mode (no kind, or kind="module") ranks via
|
|
375
|
+
// searchModulesRanked; symbol-mode (kind names a symbol kind) ranks via
|
|
376
|
+
// scoreSymbolsRanked, with name/decorator filters. Results capped at `limit`.
|
|
400
377
|
search(query, { kind = "", name = "", decorator = "", limit = SEARCH_LIMIT, offset = 0 } = {}) {
|
|
401
378
|
const rawQuery = String(query || "");
|
|
402
379
|
const k = String(kind || "").trim().toLowerCase();
|
|
@@ -411,8 +388,8 @@ export function createGraphService(graph, { sourceAccess = false, repoRoot = nul
|
|
|
411
388
|
const tokens = rawQuery.toLowerCase().split(/[^a-z0-9_]+/).filter(Boolean);
|
|
412
389
|
rankedInds = scoreSymbolsRanked(graph, tokens, { kind: k, decFilter: dec, nameRe }).map((s) => s.ind);
|
|
413
390
|
} else {
|
|
414
|
-
// label
|
|
415
|
-
//
|
|
391
|
+
// label -> individual, scoped to this call: maps searchModulesRanked's path labels
|
|
392
|
+
// back to real Individuals.
|
|
416
393
|
const byLabel = new Map();
|
|
417
394
|
for (const i of graph.individuals) if ((i.class || "") === "Module") byLabel.set(i.label, i);
|
|
418
395
|
rankedInds = searchModulesRanked(graph, rawQuery)
|
|
@@ -429,11 +406,8 @@ export function createGraphService(graph, { sourceAccess = false, repoRoot = nul
|
|
|
429
406
|
},
|
|
430
407
|
};
|
|
431
408
|
|
|
432
|
-
// Optional telemetry
|
|
433
|
-
//
|
|
434
|
-
// text/body; see responseCounts below and telemetry.mjs's redact() as a second net). When
|
|
435
|
-
// `tel` is null (the default), this loop does not run at all: zero overhead, and
|
|
436
|
-
// fixtureProvider()/bootstrapProvider() (which pass no `tel`) are completely unaffected.
|
|
409
|
+
// Optional telemetry: wrap every RI service once here to time it and record counts only,
|
|
410
|
+
// never raw text/body. `tel` null (the default) skips the wrapping loop entirely.
|
|
437
411
|
if (tel) {
|
|
438
412
|
for (const name of SERVICES) {
|
|
439
413
|
const orig = svc[name];
|
|
@@ -441,10 +415,7 @@ export function createGraphService(graph, { sourceAccess = false, repoRoot = nul
|
|
|
441
415
|
svc[name] = (...args) => {
|
|
442
416
|
const t0 = performance.now();
|
|
443
417
|
const result = orig.apply(svc, args);
|
|
444
|
-
// snippet/context are
|
|
445
|
-
// promise; every other service is synchronous — record immediately, return the value
|
|
446
|
-
// as-is. Detected at the ACTUAL call (not a static per-service list) so this is correct
|
|
447
|
-
// even if a future service's sync/async-ness varies by branch.
|
|
418
|
+
// snippet/context are async; record after settling but still return a promise.
|
|
448
419
|
if (result && typeof result.then === "function") {
|
|
449
420
|
return result.then((r) => {
|
|
450
421
|
recordTelemetry(tel, name, performance.now() - t0, r);
|
|
@@ -468,10 +439,8 @@ function recordTelemetry(tel, name, ms, result) {
|
|
|
468
439
|
} catch { /* telemetry must never break the caller's turn */ }
|
|
469
440
|
}
|
|
470
441
|
|
|
471
|
-
/** Counts only, never raw text/body: { ok, count } — count
|
|
472
|
-
*
|
|
473
|
-
* single generic aggregate rather than guessing each service's own field names. A miss
|
|
474
|
-
* records its reason (a closed-set token, not free text) instead of a count. */
|
|
442
|
+
/** Counts only, never raw text/body: { ok, count } — count sums every array-valued field's
|
|
443
|
+
* length under result.value. A miss records its reason instead of a count. */
|
|
475
444
|
function responseCounts(result) {
|
|
476
445
|
if (!result || typeof result !== "object" || result.ok !== true) {
|
|
477
446
|
return { ok: false, reason: result?.miss?.reason || null };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// The Repository Interface — tmct's OWNED, versioned contract between "interpret
|
|
2
2
|
// the query" (tmct, the brittle side) and "ask the graph for truth" (a provider,
|
|
3
|
-
// the stable side).
|
|
3
|
+
// the stable side).
|
|
4
4
|
//
|
|
5
5
|
// tmct defines and versions this shape; a provider (seonix, a fixture, a browser
|
|
6
6
|
// page) IMPLEMENTS it over its native graph. Both sides already agree on the
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
/** SemVer of the interface. Additive-by-default: new services / optional args are
|
|
19
19
|
* minor bumps; the suite for version N stays green under N+1. A breaking change
|
|
20
|
-
* is a new MAJOR with its own suite
|
|
20
|
+
* is a new MAJOR with its own suite. */
|
|
21
21
|
export const INTERFACE_VERSION = "1.1.0";
|
|
22
22
|
|
|
23
23
|
/** The OWL vocabulary the types are grounded in. */
|
|
@@ -275,15 +275,13 @@ export const REPOSITORY_INTERFACE = Object.freeze({
|
|
|
275
275
|
context: {
|
|
276
276
|
group: "source", args: { symbol: "string", depth: "min|auto|full?" },
|
|
277
277
|
result: "{ text: string, tier: string } (a sized edit bundle) — Promise<Result>",
|
|
278
|
-
// INTERFACE_VERSION 1.1.0 (2026-07): NARROWED miss contract — context() used to
|
|
279
|
-
// unconditionally miss(NO_SOURCE) (its whole edit bundle was implemented as fs-only).
|
|
280
278
|
// contextPlan/sizeBundle/renderGraphOnlyBundle are pure graph queries, so a graph-only
|
|
281
|
-
// provider (sourceAccess:false)
|
|
279
|
+
// provider (sourceAccess:false) returns a REAL HIT for any resolvable symbol —
|
|
282
280
|
// siblings, registration, class members, __all__/re-exports, insertion region, covering
|
|
283
281
|
// tests, co-change — everything except anchor/exemplar/inlined-callee BODY TEXT, which
|
|
284
|
-
// still needs a source-capable provider. NO_SOURCE is
|
|
285
|
-
//
|
|
286
|
-
//
|
|
282
|
+
// still needs a source-capable provider. NO_SOURCE is therefore not a miss reason
|
|
283
|
+
// context() can return (not in the list below) — the only remaining miss is an
|
|
284
|
+
// unresolvable symbol.
|
|
287
285
|
misses: ["UNRESOLVED_TERM"], concurrency: CONCURRENT_SAFE,
|
|
288
286
|
purpose: "The composed edit bundle (exemplar, siblings, registration, insertion region). A graph-only provider returns the graph-only sections as a real hit; a source-capable provider (sourceAccess:true, repoRoot, readFile) additionally includes the anchor/exemplar/inlined-callee body text.",
|
|
289
287
|
note: "ASYNC (returns Promise<Result>) — see snippet's note; source-capable rendering is fs I/O.",
|