@polycode-projects/the-mechanical-code-talker 2.10.5 → 2.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/corpus/sprites/src/sprite-facts.jsonl +18 -0
- package/corpus/worlds/manifest.json +5 -5
- package/corpus/worlds/shards/ashcombe-hall.jsonl.gz +0 -0
- package/corpus/worlds/src/ashcombe-hall.jsonl +27 -0
- package/data/sprites/book-icon.toml +12 -0
- package/data/sprites/cellar-icon.toml +12 -0
- package/data/sprites/drawing-room-icon.toml +13 -0
- package/data/sprites/garden-icon.toml +12 -0
- package/data/sprites/kitchen-icon.toml +13 -0
- package/data/sprites/library-icon.toml +12 -0
- package/data/sprites/pan-icon.toml +11 -0
- package/data/sprites/study-icon.toml +12 -0
- package/data/templates/responses.jsonl +1 -0
- package/package.json +5 -2
- package/src/adapters/corpus/wikipedia-live.mjs +182 -26
- package/src/adapters/corpus/worlds-pack.mjs +8 -2
- package/src/adapters/toml-config.mjs +6 -0
- package/src/domain/ask-vocab.mjs +17 -0
- package/src/domain/ask.mjs +51 -1
- package/src/domain/grammar/ace.mjs +43 -3
- package/src/domain/interpret/normalize.mjs +6 -2
- package/src/domain/interpret/strategies/grammar.mjs +47 -17
- package/src/domain/interpret/strategies/keywords.mjs +53 -4
- package/src/domain/memory/trust.mjs +11 -0
- package/src/domain/router/registry.mjs +8 -1
- package/src/domain/worlds-pack.mjs +50 -0
- package/src/services/adventure-autoplay.mjs +5 -2
- package/src/services/adventure-viz.mjs +301 -33
- package/src/services/adventure.mjs +162 -14
- package/src/services/chat-page-viz.mjs +265 -189
- package/src/services/chat-session.mjs +15 -5
- package/src/services/chat.mjs +471 -79
- package/src/services/code-explorer-viz.mjs +183 -75
- package/src/services/extract-facts.mjs +118 -28
- package/src/services/ingest-viz.mjs +328 -79
- package/src/services/ledger-viz.mjs +99 -0
- package/src/services/memory-panel-viz.mjs +159 -0
- package/src/services/research.mjs +266 -0
- package/src/services/sentences.mjs +19 -0
- package/src/services/spider-fly-viz.mjs +21 -5
- package/src/surfaces/web/adventure-browser-entry.mjs +9 -5
- package/src/surfaces/web/chat-browser-entry.mjs +28 -11
- package/src/surfaces/web/code-explorer-browser-entry.mjs +27 -11
- package/src/surfaces/web/ingest-browser-entry.mjs +123 -41
- package/src/surfaces/web/ledger-browser-entry.mjs +10 -4
- package/src/surfaces/web/memory-ask-browser.bundle.js +116 -116
- package/src/surfaces/web/memory-stats.mjs +53 -0
|
@@ -13,24 +13,55 @@
|
|
|
13
13
|
// The provider seam mirrors reference-pack.mjs's: registerLiveReferenceProvider
|
|
14
14
|
// swaps the whole lookup behind one async { lookup(normTerm) } contract (tests
|
|
15
15
|
// and the demo page stub it); null restores the default provider below.
|
|
16
|
+
//
|
|
17
|
+
// The research lane rides the same provider factory against
|
|
18
|
+
// simple.wikipedia.org (registerResearchProvider/getResearchProvider below),
|
|
19
|
+
// adding two fan-out reads: pageByTitle (an exact linked title costs ONE
|
|
20
|
+
// round trip, no opensearch) and linkedTitles (the lead section's
|
|
21
|
+
// namespace-0 links, document-ordered). Requests identify themselves per
|
|
22
|
+
// Wikimedia's robot etiquette (WIKIMEDIA_USER_AGENT) and carry maxlag so an
|
|
23
|
+
// overloaded replica set is backed off from, not hammered.
|
|
16
24
|
|
|
17
25
|
import { normFactTerm } from "../../domain/hash.mjs";
|
|
18
26
|
import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
|
|
19
27
|
import { isReferenceArticleRow, sentencesUpTo, isaOf, SUMMARY_CHAR_CAP } from "../../domain/reference-pack.mjs";
|
|
20
28
|
|
|
21
29
|
export const WIKIPEDIA_LIVE_ORIGIN = "https://en.wikipedia.org";
|
|
30
|
+
export const SIMPLE_WIKIPEDIA_ORIGIN = "https://simple.wikipedia.org";
|
|
31
|
+
|
|
32
|
+
/** The identification string Wikimedia's robot policy asks API clients to
|
|
33
|
+
* carry, pointing at this project's public site as the contact. Browsers
|
|
34
|
+
* refuse to override the User-Agent request header, so the API-recognised
|
|
35
|
+
* Api-User-Agent header carries the same string there; under Node both are
|
|
36
|
+
* sent. */
|
|
37
|
+
export const WIKIMEDIA_USER_AGENT = "the-mechanical-code-talker (+https://polycode-projects.gitlab.io/the-mechanical-code-talker/)";
|
|
22
38
|
|
|
23
39
|
const DEFAULT_TIMEOUT_MS = 4000;
|
|
24
40
|
const DEFAULT_MIN_INTERVAL_MS = 2000;
|
|
25
41
|
const RETRY_AFTER_FLOOR_MS = 5000;
|
|
42
|
+
// Action-API requests carry maxlag so an overloaded replica set answers with
|
|
43
|
+
// an error we back off from instead of adding to its load (Wikimedia's own
|
|
44
|
+
// recommended default for non-interactive clients).
|
|
45
|
+
const MAXLAG_SECONDS = 5;
|
|
26
46
|
|
|
27
47
|
/**
|
|
28
|
-
* A live-lookup provider: { lookup(normTerm) -> article row | null }
|
|
48
|
+
* A live-lookup provider: { lookup(normTerm) -> article row | null }, plus
|
|
49
|
+
* the research fan-out surface: pageByTitle(title) fetches an exact title's
|
|
50
|
+
* summary in ONE round trip (no opensearch — a linked title is already
|
|
51
|
+
* exact), and linkedTitles(title) lists the namespace-0 articles the page's
|
|
52
|
+
* LEAD section links to, in document order.
|
|
29
53
|
*
|
|
30
54
|
* `fetchImpl` defaults to the global fetch; `origin` to en.wikipedia.org;
|
|
31
55
|
* `lexicon` (optional) feeds the isa extraction. The row shape is the shipped
|
|
32
56
|
* pack's own ({ term, title, text, summary, url, revid, isa? }), validated by
|
|
33
57
|
* isReferenceArticleRow before it is ever returned.
|
|
58
|
+
*
|
|
59
|
+
* Throttle posture: every public method takes one "slot" gated by the
|
|
60
|
+
* minimum interval, the single-flight guard and any open cool-off. By
|
|
61
|
+
* default a caller that asks too soon gets null (the chat's clean-miss hook
|
|
62
|
+
* must never block a turn); with `waitForSlot: true` the method WAITS for
|
|
63
|
+
* the slot instead — the research queue's posture, where a false miss would
|
|
64
|
+
* be dishonest and the caller is already paced turn-by-turn.
|
|
34
65
|
*/
|
|
35
66
|
export function createWikipediaLiveProvider({
|
|
36
67
|
fetchImpl,
|
|
@@ -38,6 +69,8 @@ export function createWikipediaLiveProvider({
|
|
|
38
69
|
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
39
70
|
minIntervalMs = DEFAULT_MIN_INTERVAL_MS,
|
|
40
71
|
lexicon = null,
|
|
72
|
+
userAgent = WIKIMEDIA_USER_AGENT,
|
|
73
|
+
waitForSlot = false,
|
|
41
74
|
} = {}) {
|
|
42
75
|
const doFetch = fetchImpl ?? ((...args) => globalThis.fetch(...args));
|
|
43
76
|
const cache = new Map(); // key -> row | null (hits AND settled misses)
|
|
@@ -45,18 +78,43 @@ export function createWikipediaLiveProvider({
|
|
|
45
78
|
let coolOffUntil = 0;
|
|
46
79
|
let inFlight = false;
|
|
47
80
|
|
|
81
|
+
const identifyingHeaders = () => {
|
|
82
|
+
if (!userAgent) return null;
|
|
83
|
+
const headers = { "Api-User-Agent": userAgent };
|
|
84
|
+
// A browser strips User-Agent as a forbidden header; only set it where
|
|
85
|
+
// no DOM says we are one (Node ships a global `navigator` these days, so
|
|
86
|
+
// `document` is the discriminating global).
|
|
87
|
+
if (typeof document === "undefined") headers["User-Agent"] = userAgent;
|
|
88
|
+
return headers;
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
function openCoolOff(retryAfterSeconds) {
|
|
92
|
+
const retryAfterMs = Number(retryAfterSeconds) * 1000;
|
|
93
|
+
coolOffUntil = Date.now() + Math.max(retryAfterMs || 0, RETRY_AFTER_FLOOR_MS);
|
|
94
|
+
}
|
|
95
|
+
|
|
48
96
|
async function fetchJson(url) {
|
|
49
97
|
const controller = typeof AbortController === "function" ? new AbortController() : null;
|
|
50
98
|
const timer = controller ? setTimeout(() => controller.abort(), timeoutMs) : null;
|
|
51
99
|
try {
|
|
52
|
-
const
|
|
100
|
+
const opts = {};
|
|
101
|
+
if (controller) opts.signal = controller.signal;
|
|
102
|
+
const headers = identifyingHeaders();
|
|
103
|
+
if (headers) opts.headers = headers;
|
|
104
|
+
const res = await doFetch(url, opts);
|
|
53
105
|
if (res.status === 429) {
|
|
54
|
-
|
|
55
|
-
coolOffUntil = Date.now() + Math.max(retryAfterMs || 0, RETRY_AFTER_FLOOR_MS);
|
|
106
|
+
openCoolOff(res.headers?.get?.("retry-after"));
|
|
56
107
|
return null;
|
|
57
108
|
}
|
|
58
109
|
if (!res.ok) return null;
|
|
59
|
-
|
|
110
|
+
const body = await res.json();
|
|
111
|
+
// A maxlag rejection arrives as HTTP 200 with an error body (and a
|
|
112
|
+
// Retry-After header) — back off exactly as a 429 asks.
|
|
113
|
+
if (body?.error?.code === "maxlag") {
|
|
114
|
+
openCoolOff(res.headers?.get?.("retry-after"));
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
return body;
|
|
60
118
|
} catch {
|
|
61
119
|
return null;
|
|
62
120
|
} finally {
|
|
@@ -64,6 +122,26 @@ export function createWikipediaLiveProvider({
|
|
|
64
122
|
}
|
|
65
123
|
}
|
|
66
124
|
|
|
125
|
+
/** When the next network slot opens: past the cool-off, past the minimum
|
|
126
|
+
* interval since the last taken slot. */
|
|
127
|
+
const slotOpensAt = () => Math.max(coolOffUntil, lastLookupAt + minIntervalMs);
|
|
128
|
+
|
|
129
|
+
/** Take the one network slot, or report it unavailable. Default posture
|
|
130
|
+
* returns false immediately (the clean-miss hook's "null, never block");
|
|
131
|
+
* `waitForSlot` sleeps until the slot opens instead. */
|
|
132
|
+
async function takeSlot() {
|
|
133
|
+
for (;;) {
|
|
134
|
+
const now = Date.now();
|
|
135
|
+
if (!inFlight && now >= slotOpensAt()) {
|
|
136
|
+
lastLookupAt = now;
|
|
137
|
+
inFlight = true;
|
|
138
|
+
return true;
|
|
139
|
+
}
|
|
140
|
+
if (!waitForSlot) return false;
|
|
141
|
+
await new Promise((resolve) => setTimeout(resolve, Math.max(slotOpensAt() - now, 25)));
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
67
145
|
/** The opensearch title whose normFactTerm fold equals or extends the key —
|
|
68
146
|
* the topic-drift guard: "quasar" may resolve to "Quasar" or "Quasars",
|
|
69
147
|
* never to a first suggestion about something else. */
|
|
@@ -76,12 +154,7 @@ export function createWikipediaLiveProvider({
|
|
|
76
154
|
return null;
|
|
77
155
|
}
|
|
78
156
|
|
|
79
|
-
async function
|
|
80
|
-
const search = await fetchJson(
|
|
81
|
-
`${origin}/w/api.php?action=opensearch&format=json&origin=*&search=${encodeURIComponent(key)}&limit=3`,
|
|
82
|
-
);
|
|
83
|
-
const title = search ? matchingTitle(key, search) : null;
|
|
84
|
-
if (!title) return null;
|
|
157
|
+
async function summaryRow(key, title) {
|
|
85
158
|
const summary = await fetchJson(
|
|
86
159
|
`${origin}/api/rest_v1/page/summary/${encodeURIComponent(title.replace(/ /g, "_"))}`,
|
|
87
160
|
);
|
|
@@ -103,25 +176,77 @@ export function createWikipediaLiveProvider({
|
|
|
103
176
|
return isReferenceArticleRow(row) ? row : null;
|
|
104
177
|
}
|
|
105
178
|
|
|
179
|
+
async function roundTrips(key) {
|
|
180
|
+
const search = await fetchJson(
|
|
181
|
+
`${origin}/w/api.php?action=opensearch&format=json&origin=*&maxlag=${MAXLAG_SECONDS}&search=${encodeURIComponent(key)}&limit=3`,
|
|
182
|
+
);
|
|
183
|
+
const title = search ? matchingTitle(key, search) : null;
|
|
184
|
+
if (!title) return null;
|
|
185
|
+
return summaryRow(key, title);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** One slot-gated, cached operation: cache first (a settled hit or miss is
|
|
189
|
+
* never refetched), then the slot, then `work()`, with every failure
|
|
190
|
+
* cached as null so it never costs a second round trip. */
|
|
191
|
+
async function cachedFetch(cacheKey, work) {
|
|
192
|
+
if (cache.has(cacheKey)) return cache.get(cacheKey);
|
|
193
|
+
if (!(await takeSlot())) return null;
|
|
194
|
+
let value = null;
|
|
195
|
+
try {
|
|
196
|
+
value = await work();
|
|
197
|
+
} catch {
|
|
198
|
+
value = null;
|
|
199
|
+
} finally {
|
|
200
|
+
inFlight = false;
|
|
201
|
+
}
|
|
202
|
+
cache.set(cacheKey, value);
|
|
203
|
+
return value;
|
|
204
|
+
}
|
|
205
|
+
|
|
106
206
|
return {
|
|
107
207
|
async lookup(normTerm) {
|
|
108
208
|
const key = String(normTerm ?? "");
|
|
109
209
|
if (!key) return null;
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
210
|
+
return cachedFetch(key, () => roundTrips(key));
|
|
211
|
+
},
|
|
212
|
+
|
|
213
|
+
/** The summary for an EXACT title — one round trip, no opensearch. The
|
|
214
|
+
* research queue's depth-1 fetch: a linked title came from the wiki
|
|
215
|
+
* itself, so the fuzzy title match would only waste a request. */
|
|
216
|
+
async pageByTitle(title) {
|
|
217
|
+
const t = String(title ?? "").trim();
|
|
218
|
+
if (!t) return null;
|
|
219
|
+
return cachedFetch(`title\0${normFactTerm(t)}`, () => summaryRow(normFactTerm(t), t));
|
|
220
|
+
},
|
|
221
|
+
|
|
222
|
+
/** The namespace-0 articles the page's LEAD section links to, in document
|
|
223
|
+
* order, capped at `limit`. One action-API round trip
|
|
224
|
+
* (action=parse&prop=links§ion=0): the lead is the smallest payload
|
|
225
|
+
* that still orders links by how the article introduces its topic —
|
|
226
|
+
* a full-page prop=links listing is alphabetical, which would make the
|
|
227
|
+
* fan-out pick by spelling instead of relevance. Null on any failure. */
|
|
228
|
+
async linkedTitles(title, { limit = 25 } = {}) {
|
|
229
|
+
const t = String(title ?? "").trim();
|
|
230
|
+
if (!t) return null;
|
|
231
|
+
const listed = await cachedFetch(`links\0${normFactTerm(t)}`, async () => {
|
|
232
|
+
const parsed = await fetchJson(
|
|
233
|
+
`${origin}/w/api.php?action=parse&format=json&formatversion=2&origin=*&maxlag=${MAXLAG_SECONDS}&prop=links&redirects=1&page=${encodeURIComponent(t)}§ion=0`,
|
|
234
|
+
);
|
|
235
|
+
const links = parsed?.parse?.links;
|
|
236
|
+
if (!Array.isArray(links)) return null;
|
|
237
|
+
const seen = new Set();
|
|
238
|
+
const out = [];
|
|
239
|
+
for (const link of links) {
|
|
240
|
+
if (!link || link.ns !== 0 || link.exists === false) continue;
|
|
241
|
+
const linkTitle = String(link.title ?? link["*"] ?? "").trim();
|
|
242
|
+
const folded = normFactTerm(linkTitle);
|
|
243
|
+
if (!linkTitle || !folded || seen.has(folded)) continue;
|
|
244
|
+
seen.add(folded);
|
|
245
|
+
out.push(linkTitle);
|
|
246
|
+
}
|
|
247
|
+
return out;
|
|
248
|
+
});
|
|
249
|
+
return Array.isArray(listed) ? listed.slice(0, Math.max(0, limit)) : null;
|
|
125
250
|
},
|
|
126
251
|
};
|
|
127
252
|
}
|
|
@@ -143,3 +268,34 @@ export function getLiveReferenceProvider() {
|
|
|
143
268
|
if (!defaultProvider) defaultProvider = createWikipediaLiveProvider();
|
|
144
269
|
return defaultProvider;
|
|
145
270
|
}
|
|
271
|
+
|
|
272
|
+
// ---- the research lane's provider: Simple English Wikipedia, waiting slots --
|
|
273
|
+
|
|
274
|
+
let defaultResearchProvider = null;
|
|
275
|
+
let registeredResearchProvider = null;
|
|
276
|
+
|
|
277
|
+
/** Swap the research lane's lookup: provider = { lookup, pageByTitle,
|
|
278
|
+
* linkedTitles } (tests and the demo pages stub it, same seam shape as
|
|
279
|
+
* registerLiveReferenceProvider). Pass null to restore the default
|
|
280
|
+
* simple.wikipedia.org provider. */
|
|
281
|
+
export function registerResearchProvider(provider) {
|
|
282
|
+
registeredResearchProvider = provider && typeof provider.lookup === "function" ? provider : null;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** The research lane's active provider — the registered one, else one lazily
|
|
286
|
+
* created singleton against simple.wikipedia.org with `waitForSlot` on: the
|
|
287
|
+
* queue is paced turn-by-turn, so a throttled step WAITS for its polite slot
|
|
288
|
+
* rather than reporting a false miss. `minIntervalMs` (first call only —
|
|
289
|
+
* the singleton keeps its throttle clock) can only ever RAISE the interval;
|
|
290
|
+
* the shipped minimum stays the floor. */
|
|
291
|
+
export function getResearchProvider({ minIntervalMs } = {}) {
|
|
292
|
+
if (registeredResearchProvider) return registeredResearchProvider;
|
|
293
|
+
if (!defaultResearchProvider) {
|
|
294
|
+
defaultResearchProvider = createWikipediaLiveProvider({
|
|
295
|
+
origin: SIMPLE_WIKIPEDIA_ORIGIN,
|
|
296
|
+
waitForSlot: true,
|
|
297
|
+
minIntervalMs: Math.max(DEFAULT_MIN_INTERVAL_MS, Number(minIntervalMs) || 0),
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
return defaultResearchProvider;
|
|
301
|
+
}
|
|
@@ -15,7 +15,7 @@ import { readFileSync } from "node:fs";
|
|
|
15
15
|
import { gunzipSync } from "node:zlib";
|
|
16
16
|
import { fileURLToPath } from "node:url";
|
|
17
17
|
import { dirname, join } from "node:path";
|
|
18
|
-
import { isWorldsIndexEntry, isWorldRow, isWorldFactRow, isWorldRuleRow, isWorldMetaRow } from "../../domain/worlds-pack.mjs";
|
|
18
|
+
import { isWorldsIndexEntry, isWorldRow, isWorldFactRow, isWorldRuleRow, isWorldMetaRow, expandWorldDefaultContents } from "../../domain/worlds-pack.mjs";
|
|
19
19
|
|
|
20
20
|
const PKG_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
|
|
21
21
|
|
|
@@ -84,7 +84,13 @@ export function loadWorld(dir, worldName) {
|
|
|
84
84
|
else if (isWorldMetaRow(row) && !meta) meta = row;
|
|
85
85
|
} catch { /* tolerated: a bad line loses one row, not the world */ }
|
|
86
86
|
}
|
|
87
|
-
|
|
87
|
+
// Class-default contents (library -> a book, kitchen -> a pan) are
|
|
88
|
+
// materialized here, the one choke point every loader path runs through
|
|
89
|
+
// (openAdventure, the site build, the tests) — never baked into the
|
|
90
|
+
// shipped shard, which stays a byte-copy of its hand-authored source.
|
|
91
|
+
if (facts.length || rules.length || meta) {
|
|
92
|
+
payload = { name: worldName, facts: expandWorldDefaultContents(facts), rules, meta };
|
|
93
|
+
}
|
|
88
94
|
}
|
|
89
95
|
worldCacheByKey.set(key, payload);
|
|
90
96
|
return payload;
|
|
@@ -125,6 +125,12 @@ export async function normalizeConfig(raw, { configDir } = {}) {
|
|
|
125
125
|
if (src.games !== undefined) cfg.games = src.games;
|
|
126
126
|
if (src.planning !== undefined) cfg.planning = src.planning;
|
|
127
127
|
|
|
128
|
+
// Research-lane knobs (src/services/research.mjs): sparse PASS-THROUGH,
|
|
129
|
+
// same discipline as [games.*] — the raw `[research]` table
|
|
130
|
+
// (fanout_limit / depth_limit / min_interval_ms, snake_case) rides through
|
|
131
|
+
// unmodified; clamping and default-filling is resolveResearchConfig's job.
|
|
132
|
+
if (src.research !== undefined) cfg.research = src.research;
|
|
133
|
+
|
|
128
134
|
const idx = src.index || {};
|
|
129
135
|
const index = {};
|
|
130
136
|
if (idx.languages !== undefined) index.languages = idx.languages;
|
package/src/domain/ask-vocab.mjs
CHANGED
|
@@ -246,6 +246,23 @@ export const VERB_TO_KIND = Object.freeze(
|
|
|
246
246
|
),
|
|
247
247
|
);
|
|
248
248
|
|
|
249
|
+
/** The bare possessive "has"/"have"/"holds"/"hold" is bucketed onto `defines`
|
|
250
|
+
* (RELATIONS.defines.verbs above) for genuine code shapes ("what modules
|
|
251
|
+
* does app.mjs have", "does createTask have tests") — but those reach the
|
|
252
|
+
* grammar/keyword-spot strategies pre-rewritten onto a different verb by
|
|
253
|
+
* normalize.mjs's PHRASING_FRAMES (has-tests -> "what tests X", members ->
|
|
254
|
+
* "what does X contain"), and the possession/property sense of "have" over a
|
|
255
|
+
* TAUGHT individual ("does whiskers have fur") is answered entirely by
|
|
256
|
+
* chat.mjs's own has-a/hasProperty readers, upstream of both strategies. So
|
|
257
|
+
* a bare have-family verb reaching either strategy's TWO-NAMED-ROLE "ask"
|
|
258
|
+
* shape ("does X have Y", free-text subject and object, no grain check) has
|
|
259
|
+
* no legitimate code question left to mean, and confidently framing it as
|
|
260
|
+
* "locate what a module/class defines" is worse than an honest miss — both
|
|
261
|
+
* grammar.mjs (T1/T2/T3) and keywords.mjs check this set to decline instead.
|
|
262
|
+
* Shared here (rather than declared per-file) so the two strategies' bundled
|
|
263
|
+
* builds never collide on the same top-level identifier. */
|
|
264
|
+
export const HAS_FAMILY_VERBS = Object.freeze(new Set(["has", "have", "holds", "hold"]));
|
|
265
|
+
|
|
249
266
|
/** "what is a kind of X" / "what is a subclass of X" collision fix: some
|
|
250
267
|
* inherits verbs are themselves phrased "is a <continuation>", which would
|
|
251
268
|
* otherwise collide with grammar.mjs's literal meta-whatis reading and
|
package/src/domain/ask.mjs
CHANGED
|
@@ -1236,6 +1236,29 @@ function reverseOverSet(graph, kind, entityType, objectIds) {
|
|
|
1236
1236
|
return [];
|
|
1237
1237
|
}
|
|
1238
1238
|
|
|
1239
|
+
const GIT_PROV_REF_RE = /^git:(.+)$/i;
|
|
1240
|
+
|
|
1241
|
+
/** How many distinct commits are attested to have touched a SET of entities —
|
|
1242
|
+
* the same "touched by N commit(s)" convention renderDescribe's attestation
|
|
1243
|
+
* line already uses (codegraph.mjs's turnRefCount), not the narrower "touches
|
|
1244
|
+
* edge count" reverseOverSet(kind="touches") returns. A commit can be recorded
|
|
1245
|
+
* in an entity's own `derived_from` provenance with no full Commit individual
|
|
1246
|
+
* of its own (the ingester's touches-edge and provenance-ref writes can drift,
|
|
1247
|
+
* e.g. a truncated commit walk) — reverseOverSet alone then undercounts, or
|
|
1248
|
+
* misses entirely when NO touches edge survived. Deduped by short sha so a
|
|
1249
|
+
* provenance ref naming a commit that DOES have a touches-edge individual
|
|
1250
|
+
* isn't double-counted. */
|
|
1251
|
+
function commitTouchCount(graph, objectIds) {
|
|
1252
|
+
const shas = new Set(reverseOverSet(graph, "touches", "Commit", objectIds).map((c) => String(c.label || "").toLowerCase()));
|
|
1253
|
+
for (const id of objectIds) {
|
|
1254
|
+
for (const ref of graph.byId.get(id)?.derived_from || []) {
|
|
1255
|
+
const m = GIT_PROV_REF_RE.exec(String(ref || ""));
|
|
1256
|
+
if (m) shas.add(m[1].toLowerCase());
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
return shas.size;
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1239
1262
|
/** Forward traversal over a SET of subject ids (the "things {X…} call/define" step). */
|
|
1240
1263
|
function forwardOverSet(graph, kind, subjectIds) {
|
|
1241
1264
|
const edges = kindsFor(kind).flatMap((k) => edgesOfKind(graph, k)).filter((e) => subjectIds.has(e.subject));
|
|
@@ -1934,6 +1957,24 @@ function evalUniversal(graph, ast, opts) {
|
|
|
1934
1957
|
};
|
|
1935
1958
|
}
|
|
1936
1959
|
|
|
1960
|
+
/** The object-id set a "how many commits touched <X>" count restricts to,
|
|
1961
|
+
* covering both AST shapes that phrasing compiles to: a flat single-object
|
|
1962
|
+
* reverse clause ("how many commits touched app/lib/a.mjs") or a composed
|
|
1963
|
+
* reverseSet over a nested inner set ("… touched the module that defines
|
|
1964
|
+
* fnAlpha"). Null for every other count shape (including an unresolved or
|
|
1965
|
+
* ambiguous object), so the caller falls back to its plain evalSet(base)
|
|
1966
|
+
* count unchanged. */
|
|
1967
|
+
function commitTouchObjectIds(graph, base, opts) {
|
|
1968
|
+
if (base.node === "reverseSet" && base.kind === "touches" && base.entityType === "Commit") {
|
|
1969
|
+
return new Set(evalSet(graph, base.inner, opts).map((i) => i.id));
|
|
1970
|
+
}
|
|
1971
|
+
if (base.node === "clause" && base.clause?.shape === "reverse" && base.clause.kind === "touches" && base.clause.entityType === "Commit") {
|
|
1972
|
+
const { objMatch, ambiguous } = traverse(graph, base.clause, opts);
|
|
1973
|
+
return objMatch && !ambiguous ? new Set([objMatch.id]) : null;
|
|
1974
|
+
}
|
|
1975
|
+
return null;
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1937
1978
|
/** Compile any compositional AST to a result object traverse() returns for the
|
|
1938
1979
|
* simple path — {matches, …} plus compositeKind/compositeMiss flags render() reads. */
|
|
1939
1980
|
function evalComposite(graph, ast, opts = {}) {
|
|
@@ -1941,7 +1982,16 @@ function evalComposite(graph, ast, opts = {}) {
|
|
|
1941
1982
|
if (ast.node === "exists") return evalExists(graph, ast);
|
|
1942
1983
|
if (ast.node === "qualCheck") return evalQualCheck(graph, ast, opts);
|
|
1943
1984
|
if (ast.node === "universal") return evalUniversal(graph, ast, opts);
|
|
1944
|
-
if (ast.node === "count")
|
|
1985
|
+
if (ast.node === "count") {
|
|
1986
|
+
// "how many commits touched <X>" — count against provenance attestation
|
|
1987
|
+
// (commitTouchCount), not the bare touches-edge set evalSet(base) would
|
|
1988
|
+
// give: see commitTouchCount's own doc for why the two can disagree.
|
|
1989
|
+
const commitTouchIds = commitTouchObjectIds(graph, ast.base, opts);
|
|
1990
|
+
if (commitTouchIds) {
|
|
1991
|
+
return { compositeKind: "count", count: commitTouchCount(graph, commitTouchIds), entityType: ast.entityType, matches: [] };
|
|
1992
|
+
}
|
|
1993
|
+
return { compositeKind: "count", count: evalSet(graph, ast.base, opts).length, entityType: ast.entityType, matches: [] };
|
|
1994
|
+
}
|
|
1945
1995
|
if (ast.node === "list") return { compositeKind: "list", matches: evalSet(graph, ast.base, opts), entityType: ast.entityType, scoped: ast.scoped };
|
|
1946
1996
|
if (ast.node === "superlative") return evalSuperlative(graph, ast);
|
|
1947
1997
|
if (ast.node === "temporal") return evalTemporal(graph, ast, opts);
|
|
@@ -106,7 +106,12 @@ function resolveNP(lexicon, tokensIn, { allowCompound = false } = {}) {
|
|
|
106
106
|
if (proper) return { term: `${ns}${proper}`, individual: true, extras: [], unknown: [] };
|
|
107
107
|
if (CODE_REF.test(t)) return { term: `${ns}${t}`, individual: true, extras: [], unknown: [] };
|
|
108
108
|
const noun = lookupNoun(lexicon, t, { singularOnly });
|
|
109
|
-
|
|
109
|
+
// `folded` marks a match that only exists because lookupNoun's own
|
|
110
|
+
// trailing-"-s" strip or irregular-plural table rewrote the surface word
|
|
111
|
+
// — never an exact lexicon hit. parseCopula (below) uses this to keep a
|
|
112
|
+
// proper name that happens to fold to an unrelated dictionary word
|
|
113
|
+
// ("whiskers" -> "whisker") from silently losing its spelling.
|
|
114
|
+
if (noun) return { term: `${ns}${noun.lemma}`, individual: false, noun, folded: noun.lemma.toLowerCase() !== t.toLowerCase(), extras: [], unknown: [] };
|
|
110
115
|
return { term: null, individual: false, extras: [], unknown: [t] };
|
|
111
116
|
}
|
|
112
117
|
if (tokens.length === 2) {
|
|
@@ -161,6 +166,26 @@ function resolveNP(lexicon, tokensIn, { allowCompound = false } = {}) {
|
|
|
161
166
|
return { term: null, individual: false, extras: [], unknown: tokens.filter((t) => !classify(t, lexicon)) };
|
|
162
167
|
}
|
|
163
168
|
|
|
169
|
+
/** A bare single-token subject (no determiner of its own) that resolveNP only
|
|
170
|
+
* resolved via a fold, immediately followed by an indefinite-article object
|
|
171
|
+
* ("whiskers is A CAT", "every whiskers is A CAT"), is the canonical
|
|
172
|
+
* individual-naming shape ("john is a man") wearing a proper name that
|
|
173
|
+
* happens to fold to an unrelated dictionary word ("whiskers" ->
|
|
174
|
+
* "whisker"). Trusting the fold here silently rewrites the taught subject's
|
|
175
|
+
* spelling; declining it and reporting the token as unknown lets the
|
|
176
|
+
* caller's own novel-individual fallback store the literal typed word
|
|
177
|
+
* instead — the same honest-miss-over-guess call this file's
|
|
178
|
+
* exact/irregular-only folds already make everywhere else. Shared by
|
|
179
|
+
* parseCopula and parseEvery, the two patterns whose subject slot can hold
|
|
180
|
+
* a bare single token immediately followed by "a"/"an". */
|
|
181
|
+
function declineFoldedBareSubject(np, subjectToks, objectHead) {
|
|
182
|
+
if (np.term != null && !np.individual && np.folded
|
|
183
|
+
&& subjectToks.length === 1 && /^an?$/i.test(objectHead)) {
|
|
184
|
+
return { term: null, individual: false, extras: [], unknown: [subjectToks[0]] };
|
|
185
|
+
}
|
|
186
|
+
return np;
|
|
187
|
+
}
|
|
188
|
+
|
|
164
189
|
/** The shared miss result: a structural fit with undeclared words returns the
|
|
165
190
|
* pattern + residue (triples empty); a fit with only declared-but-unusable
|
|
166
191
|
* phrasing returns null — the honest fall-through either way. */
|
|
@@ -349,7 +374,16 @@ function parseEvery(lexicon, toks, lower) {
|
|
|
349
374
|
if (isIdx <= 1 || isIdx === toks.length - 1) return null;
|
|
350
375
|
const rest = toks.slice(isIdx + 1);
|
|
351
376
|
const everyAdjOnly = rest.length === 1 ? lookupAdjective(lexicon, rest[0]) : null;
|
|
352
|
-
const
|
|
377
|
+
const subjectToks = toks.slice(1, isIdx);
|
|
378
|
+
// Same fold-vs-individual ambiguity parseCopula guards against — a
|
|
379
|
+
// determiner-less "every whiskers is a cat" reaches the identical
|
|
380
|
+
// single-token resolveNP fold ("whiskers" -> "whisker") assertCandidates
|
|
381
|
+
// manufactures as a candidate phrasing whenever the bare payload has no
|
|
382
|
+
// determiner of its own, so this needs the same declineFoldedBareSubject
|
|
383
|
+
// guard or that candidate alone re-opens the bug parseCopula just closed.
|
|
384
|
+
const np1 = declineFoldedBareSubject(
|
|
385
|
+
resolveNP(lexicon, subjectToks, { allowCompound: !everyAdjOnly }), subjectToks, rest[0],
|
|
386
|
+
);
|
|
353
387
|
if (everyAdjOnly) return adjectiveCopula(lexicon, PATTERN_ADJECTIVE, np1, everyAdjOnly);
|
|
354
388
|
const np2 = resolveNP(lexicon, rest, { allowCompound: true });
|
|
355
389
|
if (np1.term == null || np2.term == null) return missOrNull(PATTERN_SUB_CLASS_OF, [np1, np2]);
|
|
@@ -414,7 +448,13 @@ function parseOfForm(lexicon, toks, lower) {
|
|
|
414
448
|
function parseCopula(lexicon, toks, lower, isIdx) {
|
|
415
449
|
const rest = toks.slice(isIdx + 1);
|
|
416
450
|
if (!rest.length) return null;
|
|
417
|
-
const
|
|
451
|
+
const subjectToks = toks.slice(0, isIdx);
|
|
452
|
+
// parseAce never reaches here for "are", only singular "is" — see
|
|
453
|
+
// declineFoldedBareSubject's own docblock for why a bare single-token
|
|
454
|
+
// subject that only resolves by folding needs this guard.
|
|
455
|
+
const np1 = declineFoldedBareSubject(
|
|
456
|
+
resolveNP(lexicon, subjectToks, { allowCompound: rest.length > 1 }), subjectToks, rest[0],
|
|
457
|
+
);
|
|
418
458
|
if (rest.length === 1) {
|
|
419
459
|
const adj = lookupAdjective(lexicon, rest[0]);
|
|
420
460
|
if (adj) return adjectiveCopula(lexicon, PATTERN_ADJECTIVE, np1, adj);
|
|
@@ -128,8 +128,12 @@ const GREETING_PREAMBLE_RE = /^(?:hi|hiya|hello|hey|yo|howdy|g'?day|yeah\s+nah|g
|
|
|
128
128
|
* the "thanks" word family: "thanks so much, <Q>" -> "<Q>". */
|
|
129
129
|
const THANKS_PREAMBLE_RE = /^(?:thanks|thank\s+you|many\s+thanks|thx|ty|cheers)(?:\s+(?:so\s+much|a\s+lot|very\s+much|a\s+bunch))?\s*[,—–-]\s*(?:(?:just\s+a\s+)?quick\s+question\s*[,:—–-]?\s*)?(.+)$/i;
|
|
130
130
|
/** Acknowledgement lead-in with a delimiter ("ok cool, <Q>"), repeating (`+`)
|
|
131
|
-
* so a stack of ack-words peels in one pass.
|
|
132
|
-
|
|
131
|
+
* so a stack of ack-words peels in one pass. "one more"/"another one"/"just
|
|
132
|
+
* one more" join the ack-word alternation as the same discourse move under a
|
|
133
|
+
* different wording — a throwaway counting aside before the real content,
|
|
134
|
+
* never part of the content itself ("ok, one more, teach me: no server is a
|
|
135
|
+
* client" must peel exactly as "ok cool, <Q>" already does). */
|
|
136
|
+
const ACK_PREAMBLE_RE = /^(?:(?:ok(?:ay)?|aight|cool|alright|sure|right|fine|great|nice|got it|gotcha|sounds good|no worries|no problem|(?:just\s+)?(?:one|another)\s+more|another\s+one)[\s,]+)+(.+)$/i;
|
|
133
137
|
/** Self-orientation lead-in with a delimiter — "just poking around, <Q>",
|
|
134
138
|
* "first time using this, <Q>". */
|
|
135
139
|
const BROWSING_PREAMBLE_RE = /^(?:just\s+(?:poking\s+around|looking\s+around|browsing|exploring|checking\s+(?:this|it)\s+out)|first\s+time\s+(?:trying\s+this\s+out|using\s+this|here)|i'?m\s+new\s+(?:here|around\s+here|to\s+(?:this|all\s+this)(?:\s+(?:repo|codebase|project|app|tool|thing))?))\s*[,.—–-]\s*(.+)$/i;
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
VERB_TO_KIND, ENTITY_TO_TYPE, MODIFIER_TO_KIND,
|
|
9
9
|
META_MEANING_VERBS, WHERE_MARKERS, MENTION_MARKERS,
|
|
10
10
|
INHERITS_REVERSE_VERBS, stripTrailingScopeFiller, stripTrailingDiscourseTag,
|
|
11
|
-
ARTICLE_RELATION_CONTINUATIONS,
|
|
11
|
+
ARTICLE_RELATION_CONTINUATIONS, HAS_FAMILY_VERBS,
|
|
12
12
|
} from "../../ask-vocab.mjs";
|
|
13
13
|
import { escapeRegex } from "../normalize.mjs";
|
|
14
14
|
|
|
@@ -17,6 +17,11 @@ const ENTITY_ALT = Object.keys(ENTITY_TO_TYPE).sort((a, b) => b.length - a.lengt
|
|
|
17
17
|
const MODIFIER_ALT = Object.keys(MODIFIER_TO_KIND).sort((a, b) => b.length - a.length).map(escapeRegex).join("|");
|
|
18
18
|
const META_ALT = META_MEANING_VERBS.slice().sort((a, b) => b.length - a.length).map(escapeRegex).join("|");
|
|
19
19
|
|
|
20
|
+
// See ask-vocab.mjs's own HAS_FAMILY_VERBS for why a bare have-family verb
|
|
21
|
+
// never resolves to `defines` in this template's "ask"/"reverse"/"forward"
|
|
22
|
+
// shapes below.
|
|
23
|
+
const isHasFamilyDefines = (kind, verb) => kind === "defines" && HAS_FAMILY_VERBS.has(verb);
|
|
24
|
+
|
|
20
25
|
const TEMPLATES = [
|
|
21
26
|
// T1 ASK: "does X import Y" -> Yes/No. REVERSE VERB SWAP: a semantically-reverse
|
|
22
27
|
// verb ("superclass of") means the opposite of its forward counterpart, so
|
|
@@ -27,6 +32,7 @@ const TEMPLATES = [
|
|
|
27
32
|
build: (m) => {
|
|
28
33
|
const verb = m[2].toLowerCase();
|
|
29
34
|
const kind = VERB_TO_KIND[verb];
|
|
35
|
+
if (isHasFamilyDefines(kind, verb)) return null;
|
|
30
36
|
let subject = m[1].trim();
|
|
31
37
|
let object = m[3].trim();
|
|
32
38
|
if (INHERITS_REVERSE_VERBS.includes(verb)) [subject, object] = [object, subject];
|
|
@@ -37,23 +43,30 @@ const TEMPLATES = [
|
|
|
37
43
|
{
|
|
38
44
|
name: "reverse",
|
|
39
45
|
re: new RegExp(`^which\\s+(${ENTITY_ALT})\\s+(?:(${MODIFIER_ALT})\\s+)?(${VERB_ALT})\\s+(.+?)\\??$`, "i"),
|
|
40
|
-
build: (m) =>
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
46
|
+
build: (m) => {
|
|
47
|
+
const verb = m[3].toLowerCase();
|
|
48
|
+
const kind = VERB_TO_KIND[verb];
|
|
49
|
+
if (isHasFamilyDefines(kind, verb)) return null;
|
|
50
|
+
return {
|
|
51
|
+
shape: "reverse",
|
|
52
|
+
entityType: ENTITY_TO_TYPE[m[1].toLowerCase()],
|
|
53
|
+
modifier: m[2] ? MODIFIER_TO_KIND[m[2].toLowerCase()] : "direct",
|
|
54
|
+
kind,
|
|
55
|
+
object: m[4].trim(),
|
|
56
|
+
};
|
|
57
|
+
},
|
|
47
58
|
},
|
|
48
59
|
// T3 forward: "what does <object> <verb>" — X is given, list its R-related things.
|
|
49
60
|
// "did" joins does/do for the past-tense commit forms ("what did commit <sha> touch").
|
|
50
61
|
{
|
|
51
62
|
name: "forward",
|
|
52
63
|
re: new RegExp(`^what\\s+(?:does|do|did)\\s+(.+?)\\s+(${VERB_ALT})\\??$`, "i"),
|
|
53
|
-
build: (m) =>
|
|
54
|
-
|
|
55
|
-
kind
|
|
56
|
-
|
|
64
|
+
build: (m) => {
|
|
65
|
+
const verb = m[2].toLowerCase();
|
|
66
|
+
const kind = VERB_TO_KIND[verb];
|
|
67
|
+
if (isHasFamilyDefines(kind, verb)) return null;
|
|
68
|
+
return { shape: "forward", entityType: null, modifier: "direct", kind, object: m[1].trim() };
|
|
69
|
+
},
|
|
57
70
|
},
|
|
58
71
|
// T4 meta: "what does <term> mean" — a question about the graph's own vocabulary,
|
|
59
72
|
// not a graph traversal. VERB_ALT and META_ALT are disjoint tables, so this never
|
|
@@ -65,16 +78,33 @@ const TEMPLATES = [
|
|
|
65
78
|
},
|
|
66
79
|
// T5 meta: "what is a/an <term>" — the bare (no-article) form is restricted to
|
|
67
80
|
// the closed ENTITY_TO_TYPE vocabulary (build() -> null otherwise, falling
|
|
68
|
-
// through); the WITH-article form is unrestricted.
|
|
81
|
+
// through); the WITH-article form is unrestricted. A "the"-article form is
|
|
82
|
+
// ALSO accepted, but only for a single-token term ("the Task") — the same
|
|
83
|
+
// schema-then-code-entity lookup a bare "Task" would reach (traverse()'s
|
|
84
|
+
// shape:"meta" handling, via metaFallbackEntityAnswer). Multi-word "the …"
|
|
85
|
+
// phrases stay excluded on purpose: "what is the meaning of this codebase"/
|
|
86
|
+
// "the purpose of X" are existential framings with their own decline
|
|
87
|
+
// elsewhere, never a literal term to look up (see the out-of-grammar test
|
|
88
|
+
// this guards).
|
|
69
89
|
{
|
|
70
90
|
name: "meta-whatis",
|
|
71
|
-
re: new RegExp(`^what\\s+(?:is|are)\\s+(?:(an
|
|
91
|
+
re: new RegExp(`^what\\s+(?:is|are)\\s+(?:(an?|the)\\s+)?(.+?)\\??$`, "i"),
|
|
72
92
|
build: (m) => {
|
|
93
|
+
const article = m[1] ? m[1].toLowerCase() : null;
|
|
73
94
|
const object = stripTrailingDiscourseTag(m[2].trim());
|
|
74
|
-
|
|
75
|
-
|
|
95
|
+
const isSingleToken = !/\s/.test(object);
|
|
96
|
+
if (article === "the" && !isSingleToken) return null;
|
|
97
|
+
if (!article && !ENTITY_TO_TYPE[object.toLowerCase()]) return null; // bare form: closed-set only
|
|
76
98
|
const objLower = object.toLowerCase();
|
|
77
|
-
|
|
99
|
+
// "what is a kind/subclass of X" is an inherits phrasing, not a term to
|
|
100
|
+
// define — ARTICLE_RELATION_CONTINUATIONS only ever derives from the
|
|
101
|
+
// "is a/an <continuation>" verb forms, so it's checked only for those;
|
|
102
|
+
// "the"-definite reverse-inherits forms ("is the superclass of") are a
|
|
103
|
+
// separate, deliberately unfolded set (ask-vocab.mjs's own comment on
|
|
104
|
+
// INHERITS_REVERSE_VERB_LIST) — moot here since those are always
|
|
105
|
+
// multi-word and already excluded by the single-token check above, but
|
|
106
|
+
// named for the same reason ARTICLE_RELATION_CONTINUATIONS is.
|
|
107
|
+
if (article && article !== "the" && ARTICLE_RELATION_CONTINUATIONS.some(
|
|
78
108
|
(c) => objLower === c || objLower.startsWith(`${c} `),
|
|
79
109
|
)) return null;
|
|
80
110
|
return { shape: "meta", entityType: null, modifier: "direct", kind: "meta", object: stripTrailingScopeFiller(object) };
|