@polycode-projects/the-mechanical-code-talker 1.5.4 → 1.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +123 -14
- package/ROADMAP.md +233 -1392
- package/bin/tmct.mjs +479 -98
- package/corpus/README.md +3 -0
- package/corpus/generated/README.md +43 -0
- package/corpus/generated/ace-surface-variants.jsonl +17 -0
- package/corpus/generated/manifest.json +9 -0
- package/corpus/tier2/generate.mjs +14668 -0
- package/corpus/tier2/human-examples-large.jsonl +1928 -0
- package/corpus/tier2/human-examples-medium.jsonl +356 -0
- package/corpus/tier2/human-examples.jsonl +120 -0
- package/corpus/tier2/human-large.jsonl +12001 -0
- package/corpus/tier2/human-medium.jsonl +944 -0
- package/corpus/tier2/human.jsonl +664 -0
- package/corpus/tier2/manifest.json +42 -0
- package/package.json +14 -8
- package/src/answer-variants.json +47 -0
- package/src/answer-variants.mjs +67 -0
- package/src/ask-browser-entry.mjs +34 -0
- package/src/ask-browser.bundle.js +5095 -0
- package/src/ask-vocab.mjs +93 -8
- package/src/ask.mjs +451 -49
- package/src/chat.mjs +1391 -141
- package/src/cli-args.mjs +164 -0
- package/src/codegraph.mjs +170 -32
- package/src/completions/graph-adapter.mjs +118 -0
- package/src/extensions.mjs +100 -19
- package/src/grammar/ace.mjs +85 -3
- package/src/grammar/lexicon-core.json +9531 -63
- package/src/grammar/lexicon.mjs +58 -8
- package/src/graph-merge.mjs +114 -0
- package/src/index.mjs +14 -0
- package/src/init.mjs +40 -14
- package/src/interpret/normalize.mjs +88 -3
- package/src/interpret/strategies/grammar.mjs +10 -0
- package/src/interpret/strategies/keywords.mjs +20 -0
- package/src/interpret/strategies/noise-strip.mjs +73 -4
- package/src/memory/core.mjs +466 -8
- package/src/router/goal-reasoner.mjs +41 -7
- package/src/router/guardrail.mjs +37 -7
- package/src/router/resolver.mjs +50 -4
- package/src/sessions.mjs +5 -1
- package/src/source.mjs +54 -1
- package/src/syllogise.mjs +398 -27
- package/src/toml-config.mjs +13 -4
- package/src/viz.mjs +541 -0
package/src/grammar/lexicon.mjs
CHANGED
|
@@ -154,26 +154,76 @@ export function loadLexicon(extra, ns = DEFAULT_NS) {
|
|
|
154
154
|
return lex;
|
|
155
155
|
}
|
|
156
156
|
|
|
157
|
-
/** Noun lookup with plural folding; returns the entry ({lemma, property?}) or null.
|
|
158
|
-
|
|
157
|
+
/** Noun lookup with plural folding; returns the entry ({lemma, property?}) or null.
|
|
158
|
+
* `opts.singularOnly` (set by ace.mjs's resolveNP for an "a"/"an" determiner —
|
|
159
|
+
* the only ACE determiners that are grammatically singular-ONLY) prunes the
|
|
160
|
+
* irregular-plural fold when the SAME surface word is ALSO declared as its own
|
|
161
|
+
* standalone noun (die/dice, person/people, tooth/teeth): a singular-only
|
|
162
|
+
* determiner is incompatible with the plural-fold reading but fully compatible
|
|
163
|
+
* with the standalone-singular reading, so that's the one grammar agreement
|
|
164
|
+
* allows. A general rule keyed on determiner agreement, not a per-word carve-out
|
|
165
|
+
* — see lookupNounCandidates below for the multi-candidate form this wraps. */
|
|
166
|
+
export function lookupNoun(lexicon, word, opts = {}) {
|
|
167
|
+
return lookupNounCandidates(lexicon, word, opts)[0] ?? null;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Every lexicon entry `word` could plausibly resolve to, ranked with the SAME
|
|
171
|
+
* top choice lookupNoun would return (opts.singularOnly applies identically)
|
|
172
|
+
* but without discarding a genuine alternate — e.g. the die/dice collision
|
|
173
|
+
* returns BOTH the `dice` and `die` entries (order depends on
|
|
174
|
+
* opts.singularOnly), a regular -s fold with both forms independently
|
|
175
|
+
* declared returns both. Additive: existing callers that only want the single
|
|
176
|
+
* best answer keep using lookupNoun untouched. */
|
|
177
|
+
export function lookupNounCandidates(lexicon, word, opts = {}) {
|
|
159
178
|
const w = String(word ?? "").toLowerCase();
|
|
179
|
+
const standalone = lexicon.nouns.get(w);
|
|
160
180
|
const irregular = lexicon.nounPlurals.get(w);
|
|
161
|
-
|
|
181
|
+
const out = [];
|
|
182
|
+
const seen = new Set();
|
|
183
|
+
const push = (entry) => {
|
|
184
|
+
if (entry && !seen.has(entry.lemma)) { seen.add(entry.lemma); out.push(entry); }
|
|
185
|
+
};
|
|
186
|
+
if (irregular) {
|
|
187
|
+
const irregularEntry = lexicon.nouns.get(irregular) ?? null;
|
|
188
|
+
if (opts.singularOnly && standalone) {
|
|
189
|
+
// grammatical-agreement pruning: "a"/"an" rules out the plural-fold
|
|
190
|
+
// reading, so the standalone singular entry is ranked FIRST here.
|
|
191
|
+
push(standalone);
|
|
192
|
+
push(irregularEntry);
|
|
193
|
+
} else {
|
|
194
|
+
push(irregularEntry);
|
|
195
|
+
push(standalone);
|
|
196
|
+
}
|
|
197
|
+
return out;
|
|
198
|
+
}
|
|
162
199
|
for (const cand of foldCandidates(w)) {
|
|
163
|
-
|
|
164
|
-
if (hit) return hit;
|
|
200
|
+
push(lexicon.nouns.get(cand));
|
|
165
201
|
}
|
|
166
|
-
return
|
|
202
|
+
return out;
|
|
167
203
|
}
|
|
168
204
|
|
|
169
205
|
/** Verb lookup with 3sg folding; returns the entry ({lemma, prep?, predicate?}) or null. */
|
|
170
206
|
export function lookupVerb(lexicon, word) {
|
|
207
|
+
return lookupVerbCandidates(lexicon, word)[0] ?? null;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** Every verb entry `word` could plausibly resolve to via foldCandidates,
|
|
211
|
+
* most-specific-fold-first (same order lookupVerb's single answer already
|
|
212
|
+
* used) — additive sibling of lookupNounCandidates, for a caller that wants
|
|
213
|
+
* to know about a genuine fold collision instead of only the first hit. In
|
|
214
|
+
* practice a verb fold rarely collides (unlike nouns' irregular-plural
|
|
215
|
+
* table), but the shape is symmetric with lookupNounCandidates on purpose —
|
|
216
|
+
* ace.mjs's multi-candidate relation search (parseRelationHits) reads
|
|
217
|
+
* whichever of the two a token's part of speech calls for. */
|
|
218
|
+
export function lookupVerbCandidates(lexicon, word) {
|
|
171
219
|
const w = String(word ?? "").toLowerCase();
|
|
220
|
+
const out = [];
|
|
221
|
+
const seen = new Set();
|
|
172
222
|
for (const cand of foldCandidates(w)) {
|
|
173
223
|
const hit = lexicon.verbs.get(cand);
|
|
174
|
-
if (hit)
|
|
224
|
+
if (hit && !seen.has(hit.lemma)) { seen.add(hit.lemma); out.push(hit); }
|
|
175
225
|
}
|
|
176
|
-
return
|
|
226
|
+
return out;
|
|
177
227
|
}
|
|
178
228
|
|
|
179
229
|
/** Adjective lookup (exact lemma); returns {lemma, type, property?, value?} or null. */
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// graph-merge.mjs — multi-graph payload merging (the CLI/config unification
|
|
2
|
+
// batch's multi-graph support). Used ONLY by source.mjs's fetchEntities when a
|
|
3
|
+
// config carries more than one graph file (`config.graphFiles.length > 1`);
|
|
4
|
+
// the single-graph path never calls this — that byte-identical guarantee
|
|
5
|
+
// lives in source.mjs, not here.
|
|
6
|
+
//
|
|
7
|
+
// Individual ids are NOT collision-safe across repos — codegraph.mjs builds
|
|
8
|
+
// ids as `mod:${relativePath}` (repo-relative), so two graphs describing
|
|
9
|
+
// similarly-structured repos can collide. mergeEntityPayloads concatenates the
|
|
10
|
+
// straightforward arrays (classes/vocabulary/objectProperties/individuals),
|
|
11
|
+
// unions proseIndex (merging the id-array per word key), and — Option A, only
|
|
12
|
+
// on an ACTUAL collision — prefixes the specific colliding ids (and every
|
|
13
|
+
// in-payload reference to them: derived_from entries, mentions, edge subject/
|
|
14
|
+
// object, proseIndex entries) with `<graphName>/`. Ids that never collide pass
|
|
15
|
+
// through untouched, so the common (no-collision) case stays fully readable.
|
|
16
|
+
|
|
17
|
+
/** Every individual id a payload declares, as a Set (cheap membership tests). */
|
|
18
|
+
function idsOf(payload) {
|
|
19
|
+
const s = new Set();
|
|
20
|
+
for (const ind of Array.isArray(payload?.individuals) ? payload.individuals : []) {
|
|
21
|
+
if (ind && ind.id) s.add(ind.id);
|
|
22
|
+
}
|
|
23
|
+
return s;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Merge N single-graph entities payloads into one.
|
|
28
|
+
*
|
|
29
|
+
* @param {Array<{file?: string, payload: object, name?: string}>} entries
|
|
30
|
+
* one entry per graph file already read+parsed; `name` is an explicit
|
|
31
|
+
* tmct.toml `[[graphs]]` name, defaulting to the entry's array index
|
|
32
|
+
* (stringified) when absent.
|
|
33
|
+
* @returns {object} a payload of the exact shape parseEntities/fetchEntities
|
|
34
|
+
* already produce for a single graph: {generated_at, classes, vocabulary,
|
|
35
|
+
* objectProperties, individuals, proseIndex, bootstrap?}.
|
|
36
|
+
*/
|
|
37
|
+
export function mergeEntityPayloads(entries) {
|
|
38
|
+
const list = (Array.isArray(entries) ? entries : []).map((e, i) => ({
|
|
39
|
+
file: e?.file,
|
|
40
|
+
payload: e?.payload || {},
|
|
41
|
+
name: e?.name != null && String(e.name).length ? String(e.name) : String(i),
|
|
42
|
+
}));
|
|
43
|
+
|
|
44
|
+
// Set-based collision detection, O(n): an id "collides" when it appears in
|
|
45
|
+
// more than one payload's own individuals list.
|
|
46
|
+
const idSets = list.map(({ payload }) => idsOf(payload));
|
|
47
|
+
const seenInCount = new Map();
|
|
48
|
+
for (const s of idSets) for (const id of s) seenInCount.set(id, (seenInCount.get(id) || 0) + 1);
|
|
49
|
+
const colliding = new Set([...seenInCount.entries()].filter(([, n]) => n > 1).map(([id]) => id));
|
|
50
|
+
|
|
51
|
+
const merged = {
|
|
52
|
+
generated_at: "",
|
|
53
|
+
classes: [],
|
|
54
|
+
vocabulary: [],
|
|
55
|
+
objectProperties: [],
|
|
56
|
+
individuals: [],
|
|
57
|
+
proseIndex: {},
|
|
58
|
+
};
|
|
59
|
+
let latestGeneratedAt = "";
|
|
60
|
+
let everyPayloadIsBootstrap = list.length > 0;
|
|
61
|
+
|
|
62
|
+
for (const { payload, name } of list) {
|
|
63
|
+
everyPayloadIsBootstrap = everyPayloadIsBootstrap && Boolean(payload.bootstrap);
|
|
64
|
+
if (typeof payload.generated_at === "string" && payload.generated_at > latestGeneratedAt) {
|
|
65
|
+
latestGeneratedAt = payload.generated_at;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const rewriteId = (id) => (colliding.has(id) ? `${name}/${id}` : id);
|
|
69
|
+
|
|
70
|
+
if (Array.isArray(payload.classes)) merged.classes.push(...payload.classes);
|
|
71
|
+
if (Array.isArray(payload.vocabulary)) merged.vocabulary.push(...payload.vocabulary);
|
|
72
|
+
|
|
73
|
+
for (const ind of Array.isArray(payload.individuals) ? payload.individuals : []) {
|
|
74
|
+
if (!ind) continue;
|
|
75
|
+
const out = { ...ind };
|
|
76
|
+
if (out.id) out.id = rewriteId(out.id);
|
|
77
|
+
if (Array.isArray(out.derived_from)) {
|
|
78
|
+
out.derived_from = out.derived_from.map((r) => rewriteId(r));
|
|
79
|
+
}
|
|
80
|
+
if (Array.isArray(out.mentions)) {
|
|
81
|
+
out.mentions = out.mentions.map((m) => (m && m.id ? { ...m, id: rewriteId(m.id) } : m));
|
|
82
|
+
}
|
|
83
|
+
merged.individuals.push(out);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
for (const grp of Array.isArray(payload.objectProperties) ? payload.objectProperties : []) {
|
|
87
|
+
if (!grp) continue;
|
|
88
|
+
const out = { ...grp };
|
|
89
|
+
if (Array.isArray(out.examples)) {
|
|
90
|
+
out.examples = out.examples.map((e) => {
|
|
91
|
+
if (!e) return e;
|
|
92
|
+
const ne = { ...e };
|
|
93
|
+
if (ne.subject) ne.subject = rewriteId(ne.subject);
|
|
94
|
+
if (ne.object) ne.object = rewriteId(ne.object);
|
|
95
|
+
return ne;
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
merged.objectProperties.push(out);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const proseIndex = payload.proseIndex && typeof payload.proseIndex === "object" ? payload.proseIndex : {};
|
|
102
|
+
for (const [word, ids] of Object.entries(proseIndex)) {
|
|
103
|
+
const bucket = merged.proseIndex[word] || (merged.proseIndex[word] = []);
|
|
104
|
+
for (const id of Array.isArray(ids) ? ids : []) {
|
|
105
|
+
const rewritten = rewriteId(id);
|
|
106
|
+
if (!bucket.includes(rewritten)) bucket.push(rewritten);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
merged.generated_at = latestGeneratedAt;
|
|
112
|
+
if (everyPayloadIsBootstrap) merged.bootstrap = true;
|
|
113
|
+
return merged;
|
|
114
|
+
}
|
package/src/index.mjs
CHANGED
|
@@ -39,3 +39,17 @@ export { foldSessionLogs } from "./memory/fold.mjs";
|
|
|
39
39
|
// (docs/adapter-contract.md): registerProvider() plugs a producer in;
|
|
40
40
|
// fetchEntities() is the one read path.
|
|
41
41
|
export { fetchEntities, registerProvider } from "./source.mjs";
|
|
42
|
+
|
|
43
|
+
// `tmct init` onboarding (also reachable as the `./init` subpath export).
|
|
44
|
+
// init.mjs and toml-config.mjs each export a same-named `CONFIG_FILE`
|
|
45
|
+
// constant ("tmct.toml") — aliased here so both can ride the one `.` entry
|
|
46
|
+
// point without colliding.
|
|
47
|
+
export { initRepo, defaultConfig, renderTomlConfig, PERSONA_PRESETS, CONFIG_FILE as INIT_CONFIG_FILE } from "./init.mjs";
|
|
48
|
+
|
|
49
|
+
// tmct.toml loading (also reachable as the `./toml-config` subpath export).
|
|
50
|
+
export { CONFIG_FILE as TOML_CONFIG_FILE } from "./toml-config.mjs";
|
|
51
|
+
|
|
52
|
+
// The "detailed answer" completions pipeline (also reachable as the
|
|
53
|
+
// `./generateCompletion` and `./createCompletionsGraphAdapter` subpath exports).
|
|
54
|
+
export { generateCompletion } from "./completions/complete.mjs";
|
|
55
|
+
export { createCompletionsGraphAdapter } from "./completions/graph-adapter.mjs";
|
package/src/init.mjs
CHANGED
|
@@ -57,16 +57,34 @@ export function defaultConfig() {
|
|
|
57
57
|
}
|
|
58
58
|
|
|
59
59
|
/** `tmct init --with-persona <name>` presets (Part 7 of the extension-pack
|
|
60
|
-
* batch
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
60
|
+
* batch; PLAN_SEED.md §2 flips which preset is the SHIPPED implicit
|
|
61
|
+
* default): a named bundle of `extensions`/`bias` overrides, written into
|
|
62
|
+
* tmct.toml alongside the plain defaults.
|
|
63
|
+
*
|
|
64
|
+
* - `human` — the NEW implicit default (src/extensions.mjs's
|
|
65
|
+
* BUILTIN_EXTENSIONS already ships `human` active and `seon`/`conceptnet`
|
|
66
|
+
* inactive) made EXPLICIT: an empty `extensions` override (nothing to
|
|
67
|
+
* add — the builtin defaults already are this persona) plus an explicit
|
|
68
|
+
* `[bias] human = 1.0`, so a repo that asks for `--with-persona human`
|
|
69
|
+
* has a self-documenting tmct.toml rather than relying on an unstated
|
|
70
|
+
* implicit default — the exact same discipline `code` below already used
|
|
71
|
+
* for the OLD default.
|
|
72
|
+
* - `code` — TODAY'S OLD IMPLICIT DEFAULT, now something a repo must
|
|
73
|
+
* request explicitly: re-activates `seon`+`conceptnet` (both now
|
|
74
|
+
* shipped inactive) and sets their bias, for a caller (e.g. seonix's own
|
|
75
|
+
* code-domain chat surface) that still wants the software-domain seed.
|
|
76
|
+
* - `empty` — the advanced escape hatch (PLAN_SEED.md §7): deactivates the
|
|
77
|
+
* one bundle now active by default (`human`), leaving a repo genuinely
|
|
78
|
+
* empty of corpus facts for a consumer bringing its own ontology/lexicon/
|
|
79
|
+
* corpus. Does NOT reduce npm package size (§7's own documented caveat —
|
|
80
|
+
* `corpus/` ships unconditionally either way).
|
|
81
|
+
*
|
|
82
|
+
* Kept minimal on purpose — this batch's job is the persona SEAM, not a
|
|
83
|
+
* curated library of presets. */
|
|
68
84
|
export const PERSONA_PRESETS = Object.freeze({
|
|
69
|
-
|
|
85
|
+
human: { extensions: {}, bias: { human: 1.0 } },
|
|
86
|
+
code: { extensions: { seon: { active: true }, conceptnet: { active: true } }, bias: { seon: 1.0, conceptnet: 1.0 } },
|
|
87
|
+
empty: { extensions: { human: { active: false } }, bias: {} },
|
|
70
88
|
});
|
|
71
89
|
|
|
72
90
|
/** Read this package's version (best-effort, for provenance). */
|
|
@@ -105,7 +123,14 @@ export function renderTomlConfig(config = defaultConfig()) {
|
|
|
105
123
|
# Where the code-graph JSON artifact lives, relative to this file. The
|
|
106
124
|
# TMCT_GRAPH_FILE environment variable overrides it at runtime.
|
|
107
125
|
graph_file = ${JSON.stringify(c.graphFile)}
|
|
108
|
-
|
|
126
|
+
${Array.isArray(c.graphFiles) && c.graphFiles.length ? `
|
|
127
|
+
# graph_files — additional graph artifacts (multi-graph). Merged with
|
|
128
|
+
# graph_file at read time (src/graph-merge.mjs); an id that collides
|
|
129
|
+
# across graphs is auto-prefixed with its graph's name, everything else
|
|
130
|
+
# passes through unchanged. Written by \`tmct init --graph\`/\`tmct import
|
|
131
|
+
# --graph\`.
|
|
132
|
+
graph_files = ${JSON.stringify(c.graphFiles)}
|
|
133
|
+
` : ""}
|
|
109
134
|
[corpus]
|
|
110
135
|
# Corpus-tiering policy (ROADMAP Phase 4). The $0-offline default is inviolable;
|
|
111
136
|
# higher tiers are ADDITIVE and never required to answer.
|
|
@@ -135,10 +160,11 @@ ${seed.limit != null ? `limit = ${Number(seed.limit)}` : "# limit = 500"}
|
|
|
135
160
|
if (!Object.keys(extras).length) return base;
|
|
136
161
|
return `${base}
|
|
137
162
|
# Extension packs + bias (src/extensions.mjs) — written by \`tmct init --with-persona\`
|
|
138
|
-
# or a manual edit. Recognized names (seon, conceptnet, tier2-aws,
|
|
139
|
-
# tier2-java) override the shipped defaults; any
|
|
140
|
-
# host-supplied bundle (needs its own "kind").
|
|
141
|
-
# weight table consumed by
|
|
163
|
+
# or a manual edit. Recognized names (human, seon, conceptnet, tier2-aws,
|
|
164
|
+
# tier2-python, tier2-java, tier2-general) override the shipped defaults; any
|
|
165
|
+
# other name declares a new host-supplied bundle (needs its own "kind").
|
|
166
|
+
# [bias] is a flat bundle-name -> weight table consumed by
|
|
167
|
+
# src/memory/bias.mjs's ranking.
|
|
142
168
|
${stringifyToml(extras)}`;
|
|
143
169
|
}
|
|
144
170
|
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
|
|
14
14
|
import {
|
|
15
15
|
CONTRACTIONS, MISSPELLINGS, WRONG_WORDS, G_DROP, FILLER_WORDS,
|
|
16
|
-
NEGATION_FRAMES, COMMIT_CONTENT_FRAMES, VERB_TO_KIND,
|
|
16
|
+
NEGATION_FRAMES, COMMIT_CONTENT_FRAMES, VERB_TO_KIND, ENTITY_TO_TYPE,
|
|
17
17
|
} from "../ask-vocab.mjs";
|
|
18
18
|
|
|
19
19
|
export function escapeRegex(s) {
|
|
@@ -44,6 +44,55 @@ const correctionRe = (table) => new RegExp(
|
|
|
44
44
|
const MISSPELLING_RE = correctionRe(MISSPELLINGS);
|
|
45
45
|
const WRONG_WORD_RE = correctionRe(WRONG_WORDS);
|
|
46
46
|
|
|
47
|
+
// ---- two deferred typo sub-cases (HANDOVER.md 2026-07-12, routed from
|
|
48
|
+
// BENCHMARK_CONVERSATION_1.7.0.md's "Routed backlog": "cochange w/ wat",
|
|
49
|
+
// "tests 4 it") that structurally CANNOT live in the MISSPELLINGS/WRONG_WORDS
|
|
50
|
+
// tables above, for two independent reasons:
|
|
51
|
+
// - correctionRe() builds `\b(...)\b` from a table's keys. A key of "w/"
|
|
52
|
+
// can never satisfy the trailing `\b`: both "/" and the whitespace that
|
|
53
|
+
// follows it are non-word characters, so no word-boundary transition
|
|
54
|
+
// ever happens there — no table entry, however spelled, can match.
|
|
55
|
+
// - "with" and "for" are not canonical grammar-owned words (not in
|
|
56
|
+
// VERB_TO_KIND/ENTITY_TO_TYPE/MODIFIER_TO_KIND/TRIGGER_WORDS), and
|
|
57
|
+
// test/ask-vocab.test.mjs enforces that every correction TABLE value is
|
|
58
|
+
// one of those — by design, so a table entry couldn't rewrite INTO a
|
|
59
|
+
// non-grammar word either.
|
|
60
|
+
// Same species as chat.mjs's VAGUE_TOUCH_TEL_RE/VAGUE_TOUCH_ABUT_RE (a
|
|
61
|
+
// dedicated, narrowly-scoped regex pass instead of a table entry), wired
|
|
62
|
+
// here rather than in chat.mjs because "w/" and leetspeak "4" are GENERAL
|
|
63
|
+
// shorthand a developer can use in any question, not one lane's own anchor
|
|
64
|
+
// word. normalize.mjs's normalizeQuery is the single point both ask.mjs's
|
|
65
|
+
// parseQuery and interpret/pipeline.mjs's normalizeInput funnel every query
|
|
66
|
+
// through, so wiring it here reaches every caller once. ----
|
|
67
|
+
|
|
68
|
+
/** "w/" -> "with". Lookbehind/lookahead require whitespace (or start/end of
|
|
69
|
+
* string) on BOTH sides, so a real path fragment ("src/w/foo.mjs" — the "/"
|
|
70
|
+
* right before "w" is not whitespace) and the DIFFERENT shorthand "w/o"
|
|
71
|
+
* ("without" — a non-whitespace "o" right after the slash) never match. */
|
|
72
|
+
const W_SLASH_RE = /(?<=^|\s)w\/(?=\s|$)/gi;
|
|
73
|
+
|
|
74
|
+
/** "4" meaning the word "for" — deliberately NOT a context-free standalone-
|
|
75
|
+
* digit substitution. fuzzy.mjs's eligibleForCanon() and several independent
|
|
76
|
+
* `/^[a-z]+$/` guards in ask.mjs protect digit tokens everywhere in this
|
|
77
|
+
* codebase (shas, line numbers, counts — "top 4 results", "line 4", "commit
|
|
78
|
+
* 4a2b…") on purpose; a blind "4" -> "for" token rule would corrupt every
|
|
79
|
+
* one of those. Narrowed to two closed, well-justified trigger shapes —
|
|
80
|
+
* chosen SMALLER than the plausible "wait 4 X" / "used 4 X" leetspeak
|
|
81
|
+
* because both of those anchors collide with genuine counts in ordinary
|
|
82
|
+
* English ("wait 4 minutes", "used 4 times"), which this rule must never
|
|
83
|
+
* touch:
|
|
84
|
+
* - a GRATITUDE interjection immediately before "4" (the same closed word
|
|
85
|
+
* list THANKS_PREAMBLE_RE above already trusts as pure gratitude, never
|
|
86
|
+
* a count-report opener) — "thx 4 the help", "cheers 4 that".
|
|
87
|
+
* - the "4 example"/"4 instance" idiom, guarded to fire ONLY when nothing
|
|
88
|
+
* else follows on the same clause (end of string or punctuation next) —
|
|
89
|
+
* "…, 4 example" / "4 example?" is the parenthetical "for example" idiom,
|
|
90
|
+
* but "the 4 example modules" names a genuine COUNT of four example
|
|
91
|
+
* modules, and the trailing-word lookahead refuses to rewrite that.
|
|
92
|
+
*/
|
|
93
|
+
const FOR_DIGIT_THANKS_RE = /\b(thx|thanks|thank\s+you|many\s+thanks|ty|cheers)\s+4\b/gi;
|
|
94
|
+
const FOR_DIGIT_EXAMPLE_RE = /\b4\s+(example|instance)\b(?!\s*[a-z])/gi;
|
|
95
|
+
|
|
47
96
|
/** "that class"/"this module"/"that function" (a context pronoun immediately
|
|
48
97
|
* followed by the SINGULAR kind noun it's already standing in for) -> the bare
|
|
49
98
|
* pronoun alone (0.9.15 Tier-1 single-touch playtest). "which class contains
|
|
@@ -64,6 +113,25 @@ const WRONG_WORD_RE = correctionRe(WRONG_WORDS);
|
|
|
64
113
|
* CONTEXT_PRONOUNS entry). */
|
|
65
114
|
const KIND_NOUN_ANAPHORA_RE = /\b(this|that)\s+(class|module|function|method|attribute|variable|file|commit)\b/gi;
|
|
66
115
|
|
|
116
|
+
/** Read-only PROBE (BENCHMARK_CONVERSATION_1.7.0.md routed backlog C2): does
|
|
117
|
+
* `text` contain a KIND_NOUN_ANAPHORA_RE match, and if so, what ENTITY CLASS
|
|
118
|
+
* does its kind noun name (ENTITY_TO_TYPE, ask-vocab.mjs — the same
|
|
119
|
+
* "file"->"Module" convention every other lane in this grammar already
|
|
120
|
+
* uses)? Returns that class, or null when no such anaphora is present.
|
|
121
|
+
* Deliberately SEPARATE from normalizeQuery's own KIND_NOUN_ANAPHORA_RE
|
|
122
|
+
* replace just above (which permanently collapses "this file" to bare
|
|
123
|
+
* "this", discarding the kind-noun signal for good, by design — see that
|
|
124
|
+
* replace's own docblock): this never mutates its input and has no effect
|
|
125
|
+
* on normalizeQuery's behavior, signature, or any of its many call sites.
|
|
126
|
+
* A caller that needs BOTH the collapsed pronoun AND the kind it stood for
|
|
127
|
+
* (chat.mjs's runAsk, at its pronoun-reuse site) calls this side-channel on
|
|
128
|
+
* the same raw text handed to normalizeQuery — order between the two calls
|
|
129
|
+
* doesn't matter, since this one only ever reads. */
|
|
130
|
+
export function kindNounAnaphoraHint(text) {
|
|
131
|
+
const m = new RegExp(KIND_NOUN_ANAPHORA_RE.source, "i").exec(String(text || ""));
|
|
132
|
+
return m ? (ENTITY_TO_TYPE[m[2].toLowerCase()] || null) : null;
|
|
133
|
+
}
|
|
134
|
+
|
|
67
135
|
// every relation verb phrase VERB_TO_KIND knows, as one alternation (longest-first
|
|
68
136
|
// so a multi-word verb like "inherit from" wins over its own leading word "inherit"
|
|
69
137
|
// appearing elsewhere) — feeds the DOES-X-VERB-ANYTHING-ELSE frame below, which needs
|
|
@@ -191,8 +259,19 @@ const ACK_PREAMBLE_RE = /^(?:(?:ok(?:ay)?|aight|cool|alright|sure|right|fine|gre
|
|
|
191
259
|
* §3: the vague-opener family a genuine first-time stranger types). Same
|
|
192
260
|
* delimiter-required discipline as GREETING/THANKS/ACK_PREAMBLE_RE above —
|
|
193
261
|
* a bare "just poking around" with no question stays small-talk (this file
|
|
194
|
-
* never claims a turn that has no remainder to hand back).
|
|
195
|
-
|
|
262
|
+
* never claims a turn that has no remainder to hand back).
|
|
263
|
+
* HANDOVER.md 2026-07-10 item 3: "first time trying this out"/"first time
|
|
264
|
+
* using this"/"first time here" is the SAME self-orientation species — a
|
|
265
|
+
* genuine stranger's opener, just phrased around their own inexperience
|
|
266
|
+
* rather than what they're doing right now — found live as "hey, first
|
|
267
|
+
* time trying this out - what is in here?" falling straight to the raw
|
|
268
|
+
* grammar wall (GREETING_PREAMBLE_RE peels "hey,", but nothing recognized
|
|
269
|
+
* the remainder as a preamble at all). Added as a sibling alternative in
|
|
270
|
+
* the SAME regex/capture group, so it strips into the identical downstream
|
|
271
|
+
* shape ("just poking around, X" and "first time trying this out, X" both
|
|
272
|
+
* hand back the bare "X" for the ordinary pipeline to answer) rather than a
|
|
273
|
+
* new frame with its own behavior. */
|
|
274
|
+
const BROWSING_PREAMBLE_RE = /^(?:just\s+(?:poking\s+around|looking\s+around|browsing|exploring|checking\s+(?:this|it)\s+out)|first\s+time\s+(?:trying\s+this\s+out|using\s+this|here))\s*[,.—–-]\s*(.+)$/i;
|
|
196
275
|
/** A repeated leading HEDGE ADVERB ("maybe", "possibly", "perhaps") ahead of a
|
|
197
276
|
* polite request verb — the sibling of ACK_PREAMBLE_RE for HEDGING rather than
|
|
198
277
|
* acknowledging (Tier 6 playtest §3's own stacked-politeness example: "could
|
|
@@ -528,6 +607,12 @@ export function normalizeQuery(text) {
|
|
|
528
607
|
q = q.replace(CONTRACTION_RE, (m) => CONTRACTIONS[m.toLowerCase()]);
|
|
529
608
|
q = q.replace(MISSPELLING_RE, (m) => MISSPELLINGS[m.toLowerCase()]);
|
|
530
609
|
q = q.replace(WRONG_WORD_RE, (m) => WRONG_WORDS[m.toLowerCase()]);
|
|
610
|
+
// "w/" -> "with", leetspeak "4" -> "for" (narrow trigger shapes only) — see
|
|
611
|
+
// the two tables' own docblocks just above for why neither can live in the
|
|
612
|
+
// MISSPELLINGS/WRONG_WORDS tables above this function.
|
|
613
|
+
q = q.replace(W_SLASH_RE, "with");
|
|
614
|
+
q = q.replace(FOR_DIGIT_THANKS_RE, (_, w) => `${w} for`);
|
|
615
|
+
q = q.replace(FOR_DIGIT_EXAMPLE_RE, (_, w) => `for ${w}`);
|
|
531
616
|
q = q.replace(KIND_NOUN_ANAPHORA_RE, (_, pron) => pron);
|
|
532
617
|
q = q.replace(G_DROP, "$1ing");
|
|
533
618
|
// closed preamble frames (greeting lead-in, modal wrapper, show/give-me
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
VERB_TO_KIND, ENTITY_TO_TYPE, MODIFIER_TO_KIND,
|
|
9
9
|
META_MEANING_VERBS, WHERE_MARKERS, MENTION_MARKERS,
|
|
10
10
|
INHERITS_REVERSE_VERBS, stripTrailingScopeFiller, stripTrailingDiscourseTag,
|
|
11
|
+
ARTICLE_RELATION_CONTINUATIONS,
|
|
11
12
|
} from "../../ask-vocab.mjs";
|
|
12
13
|
import { escapeRegex } from "../normalize.mjs";
|
|
13
14
|
|
|
@@ -106,6 +107,15 @@ const TEMPLATES = [
|
|
|
106
107
|
build: (m) => {
|
|
107
108
|
const object = stripTrailingDiscourseTag(m[2].trim());
|
|
108
109
|
if (!m[1] && !ENTITY_TO_TYPE[object.toLowerCase()]) return null; // bare form: closed-set only
|
|
110
|
+
// "what is a kind of X" / "what is a subclass of X" (ARTICLE_RELATION_CONTINUATIONS,
|
|
111
|
+
// ask-vocab.mjs): the captured object is itself the tail of a registered inherits
|
|
112
|
+
// verb's own "is a .../are a ..." phrasing, not a term to define — reject so this
|
|
113
|
+
// template yields no candidate at all and only keyword-spot's (unambiguous) reverse-
|
|
114
|
+
// inherits reading survives, instead of a spurious meta/inherits {ambiguousParse} tie.
|
|
115
|
+
const objLower = object.toLowerCase();
|
|
116
|
+
if (m[1] && ARTICLE_RELATION_CONTINUATIONS.some(
|
|
117
|
+
(c) => objLower === c || objLower.startsWith(`${c} `),
|
|
118
|
+
)) return null;
|
|
109
119
|
return { shape: "meta", entityType: null, modifier: "direct", kind: "meta", object: stripTrailingScopeFiller(object) };
|
|
110
120
|
},
|
|
111
121
|
},
|
|
@@ -277,6 +277,26 @@ export function parseKeywordSpot(text, nlp = null) {
|
|
|
277
277
|
return { shape: "ask", entityType: null, modifier: "direct", kind, subject, object };
|
|
278
278
|
}
|
|
279
279
|
if (afterText) return { shape: "reverse", entityType, modifier, kind, object: afterText };
|
|
280
|
+
// "what is a kind of class" / "what inherits from function" (live-caught 2026-07-11
|
|
281
|
+
// follow-up to the ambiguousParse fix, commit 5c858bf): entityHit above is found
|
|
282
|
+
// ANYWHERE in the sentence and marked consumed before afterText is computed, so
|
|
283
|
+
// when the "kind of X"/"inherits X" idiom's object IS ITSELF one of the four
|
|
284
|
+
// code-graph entity-type nouns (class/function/method/module — ENTITY_TO_TYPE's
|
|
285
|
+
// own keys), the entity match swallows the ENTIRE post-verb span as a (wrong, in
|
|
286
|
+
// this shape) grain qualifier, leaving afterText empty and no candidate at all —
|
|
287
|
+
// the query silently fails to parse rather than answering or declining honestly.
|
|
288
|
+
// Scoped narrowly to kind==="inherits" (the one relation whose object is routinely
|
|
289
|
+
// a bare vocabulary noun with no further qualifier) and only when the ENTIRE
|
|
290
|
+
// post-verb span was consumed by the entity match (nothing else remains to be an
|
|
291
|
+
// object): re-read that span as the literal OBJECT text instead, entityType null
|
|
292
|
+
// (it names the thing being asked about here, not a grain filter on some other
|
|
293
|
+
// object). Every other kind/shape is unaffected — this never fires unless
|
|
294
|
+
// afterText is otherwise empty AND the sole cause is an entity-consumed span
|
|
295
|
+
// immediately after the verb.
|
|
296
|
+
if (kind === "inherits" && !beforeText && entityHit && entityHit.start === verbHit.end) {
|
|
297
|
+
const entityText = canonWords.slice(entityHit.start, entityHit.end).join(" ");
|
|
298
|
+
if (entityText) return { shape: "reverse", entityType: null, modifier, kind, object: entityText };
|
|
299
|
+
}
|
|
280
300
|
// forward keeps the spotted entityType ("which modules did commit <sha> touch" is a
|
|
281
301
|
// forward decomposition — subject before the verb — whose asked grain would otherwise
|
|
282
302
|
// be lost); traverse() only consults it for the commit-as-subject grain selection,
|
|
@@ -73,11 +73,62 @@ const KEEP = new Set([
|
|
|
73
73
|
* carried here too so the strategy stands alone) + the cascade's noise list. */
|
|
74
74
|
const CURATED_NOISE = new Set([...wordsOf(FILLER_WORDS), ...wordsOf(CASCADE_NOISE)]);
|
|
75
75
|
|
|
76
|
-
/**
|
|
76
|
+
/** Words this pass keeps but flags as UNCERTAIN (PLAN_CONVERSATION.md Finding 2
|
|
77
|
+
* — the "store"/"keep" gap): wink's `isStopWord` is a generic English
|
|
78
|
+
* dictionary, not purpose-built for this codebase — it happens to flag
|
|
79
|
+
* "keep"/"put"/"get" but not their close synonyms "store"/"hold"/"place"/
|
|
80
|
+
* "save" sitting in the exact same no-relation-verb slot ("where would i
|
|
81
|
+
* keep/store a router"), so a KEPT word surviving the pass above is not
|
|
82
|
+
* necessarily real content. A curated synonym list was tried and rejected
|
|
83
|
+
* (see the file doc / PLAN_CONVERSATION.md): "store"/"hold"/"save" are
|
|
84
|
+
* exactly the words most likely to ALSO be a real identifier ("where does
|
|
85
|
+
* the store live" must not lose its subject). The general, non-curated
|
|
86
|
+
* signal that discriminates the two: wink's POS tagger, reading the WHOLE
|
|
87
|
+
* ORIGINAL sentence for real grammatical context (an isolated 2-word
|
|
88
|
+
* fragment like "store router" tags BOTH words NOUN — confirmed live; the
|
|
89
|
+
* same "store" in "where would i store a router" tags VERB, and in "where
|
|
90
|
+
* does the store live" tags NOUN — also confirmed live, so the isolated
|
|
91
|
+
* object phrase alone can never carry this signal; it must be read here,
|
|
92
|
+
* off the whole sentence, before the phrase is extracted).
|
|
93
|
+
*
|
|
94
|
+
* A KEPT word wink tags VERB here is returned as `maybeNoise`, NOT stripped
|
|
95
|
+
* outright — this function has no graph to check a resolution against
|
|
96
|
+
* (interpret/pipeline.mjs's own documented boundary: "no graph access
|
|
97
|
+
* here"). The caller below turns this into a second candidate reading;
|
|
98
|
+
* ask.mjs's traverse() (where the graph lives) tries both and prunes the
|
|
99
|
+
* one that misses/ties in favor of the one that resolves cleanly —
|
|
100
|
+
* mirroring resolveObject's own grain-word retry (try a variant, keep it
|
|
101
|
+
* only on an unambiguous hit) and grammar/ace.mjs's parseAceAmbiguous
|
|
102
|
+
* ("keep only complete, valid parses"). Never guessed here; always pruned
|
|
103
|
+
* where the evidence (the graph) actually is. */
|
|
104
|
+
function maybeVerbNoiseWords(words, kept, nlp) {
|
|
105
|
+
if (!nlp || typeof nlp.posTags !== "function" || !kept.length) return [];
|
|
106
|
+
const keptSet = new Set(kept.map((w) => w.toLowerCase()));
|
|
107
|
+
let tags;
|
|
108
|
+
try {
|
|
109
|
+
tags = nlp.posTags(words);
|
|
110
|
+
} catch {
|
|
111
|
+
return [];
|
|
112
|
+
}
|
|
113
|
+
const out = [];
|
|
114
|
+
for (let i = 0; i < words.length; i += 1) {
|
|
115
|
+
const w = words[i];
|
|
116
|
+
const lc = w.toLowerCase();
|
|
117
|
+
if (!/^[a-z]+$/.test(w) || KEEP.has(lc) || !keptSet.has(lc)) continue;
|
|
118
|
+
if (tags[i] === "VERB") out.push(lc);
|
|
119
|
+
}
|
|
120
|
+
return out;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Strip the strippable tokens (see the file doc). Returns {text, dropped,
|
|
124
|
+
* maybeNoise} — maybeNoise is the POS-flagged uncertain-word list above,
|
|
125
|
+
* `[]` whenever no `nlp` adapter is available (same graceful-degradation
|
|
126
|
+
* discipline as the rest of this file: never a curated guess, never a throw). */
|
|
77
127
|
export function stripNoise(text, nlp = null) {
|
|
128
|
+
const words = splitWords(text);
|
|
78
129
|
const kept = [];
|
|
79
130
|
const dropped = [];
|
|
80
|
-
for (const w of
|
|
131
|
+
for (const w of words) {
|
|
81
132
|
const lc = w.toLowerCase();
|
|
82
133
|
const strippable = /^[a-z]+$/.test(w) && !KEEP.has(lc)
|
|
83
134
|
&& (CURATED_NOISE.has(lc)
|
|
@@ -85,7 +136,8 @@ export function stripNoise(text, nlp = null) {
|
|
|
85
136
|
if (strippable) dropped.push(w);
|
|
86
137
|
else kept.push(w);
|
|
87
138
|
}
|
|
88
|
-
|
|
139
|
+
const maybeNoise = maybeVerbNoiseWords(words, kept, nlp);
|
|
140
|
+
return { text: kept.join(" "), dropped, maybeNoise };
|
|
89
141
|
}
|
|
90
142
|
|
|
91
143
|
/** Pipeline registration (interpret/pipeline.mjs). */
|
|
@@ -94,7 +146,7 @@ export const noiseStripStrategy = {
|
|
|
94
146
|
class: "noise-stripped",
|
|
95
147
|
run(text, ctx = {}) {
|
|
96
148
|
if (parseAnchored(text)) return null; // the grammar owns the text as-given
|
|
97
|
-
const { text: stripped, dropped } = stripNoise(text, ctx.nlp || null);
|
|
149
|
+
const { text: stripped, dropped, maybeNoise } = stripNoise(text, ctx.nlp || null);
|
|
98
150
|
if (!dropped.length || !stripped) return null;
|
|
99
151
|
// tier 1: the anchored templates over the stripped text — the strictest
|
|
100
152
|
// re-parse, tried first so a template shape is never displaced by a looser
|
|
@@ -105,6 +157,23 @@ export const noiseStripStrategy = {
|
|
|
105
157
|
// (see the file doc for the discipline/cost argument).
|
|
106
158
|
const parsed = parseAnchored(stripped) || parseKeywordSpot(stripped, ctx.nlp || null);
|
|
107
159
|
if (!parsed) return null;
|
|
160
|
+
// Finding 2 extension (PLAN_CONVERSATION.md): a bare "where"/"mentions"
|
|
161
|
+
// question is the ONE shape with no explicit relation verb gating its
|
|
162
|
+
// object (every other decomposition in keywords.mjs requires a real
|
|
163
|
+
// VERB_TO_KIND match before it ever runs), so it's the only place an
|
|
164
|
+
// unlisted light verb ("store", "hold", …) can leak into the object
|
|
165
|
+
// phrase untouched. Scoped to exactly that shape — deliberately not a
|
|
166
|
+
// blanket change to stripNoise's shared criteria, per the file's own
|
|
167
|
+
// construction-scoping caveat. `altObject` rides along on the SAME
|
|
168
|
+
// single candidate (not a second strategy candidate — that would force
|
|
169
|
+
// mergeStrategyResults' pre-resolution ambiguousParse surface on every
|
|
170
|
+
// hit, before anyone has checked whether it's even real ambiguity);
|
|
171
|
+
// ask.mjs's traverse() is the one place both the alternate reading and
|
|
172
|
+
// the graph are available together to actually prune it.
|
|
173
|
+
if (maybeNoise.length && (parsed.shape === "where" || parsed.shape === "mentions") && parsed.object) {
|
|
174
|
+
const altObject = parsed.object.split(/\s+/).filter((w) => !maybeNoise.includes(w.toLowerCase())).join(" ").trim();
|
|
175
|
+
if (altObject && altObject !== parsed.object) parsed.altObject = altObject;
|
|
176
|
+
}
|
|
108
177
|
return {
|
|
109
178
|
strategyId: "noise-strip",
|
|
110
179
|
class: "noise-stripped",
|