@polycode-projects/the-mechanical-code-talker 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -3
- package/ROADMAP.md +45 -0
- package/bin/tmct.mjs +8 -2
- package/corpus/conceptnet/README.md +59 -52
- package/corpus/conceptnet/filter-dump.mjs +59 -2
- package/corpus/conceptnet/slice.jsonl +31067 -0
- package/corpus/seon/relations.jsonl +8 -0
- package/data/templates/responses.jsonl +3 -2
- package/package.json +6 -3
- package/src/ask.mjs +2 -2
- package/src/chat.mjs +306 -39
- package/src/concept.mjs +393 -0
- package/src/corpus/conceptnet.mjs +18 -13
- package/src/init.mjs +12 -9
- package/src/interpret/normalize.mjs +42 -0
- package/src/interpret/pipeline.mjs +2 -2
- package/src/memory/core.mjs +84 -0
- package/src/tui/app.mjs +26 -2
package/src/concept.mjs
ADDED
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
// concept.mjs — "the concept force": compose a THREE-BAND answer to a vague
|
|
2
|
+
// "what is a X" touch, when tmct KNOWS the concept X (a curated definition) AND
|
|
3
|
+
// HAS instances of it (individuals in the code graph and/or remembered isa facts).
|
|
4
|
+
//
|
|
5
|
+
// 1. THE FACT — the definition of X (lead clause of the corpus/seon entry).
|
|
6
|
+
// 2. THE EXAMPLES — real instances of X: code-graph individuals whose class maps
|
|
7
|
+
// to X (capped ~3, stable graph order, each a real node), plus any remembered
|
|
8
|
+
// "A is a X" facts.
|
|
9
|
+
// 3. THE GUIDED FOLLOW-UP — 2-3 concrete, RUNNABLE next questions built from the
|
|
10
|
+
// real instances × the query shapes valid for that kind, EACH PRE-CHECKED by
|
|
11
|
+
// actually running it through ask() so a suggestion can never miss.
|
|
12
|
+
//
|
|
13
|
+
// PURE given (graph, term, {definition, factRows}) — the follow-up validator calls
|
|
14
|
+
// ask() (deterministic, no model), so the whole composition is reproducible. The
|
|
15
|
+
// caller (chat.mjs) owns the async edges: loading corpus/seon/definitions.jsonl and
|
|
16
|
+
// the memory fact rows, and rendering through the response template. This module
|
|
17
|
+
// never fabricates: every example is a real individual/fact and every follow-up is
|
|
18
|
+
// validated against the same graph before it is offered.
|
|
19
|
+
|
|
20
|
+
import { ask } from "./ask.mjs";
|
|
21
|
+
import { relationKind } from "./codegraph.mjs";
|
|
22
|
+
|
|
23
|
+
/** A vague concept term (normalized, singular — normFactTerm's output) → the graph
|
|
24
|
+
* individual `class` it enumerates. The closed set of code-structure concepts the
|
|
25
|
+
* seon lexicon + the graph both understand; anything outside it is not a "concept
|
|
26
|
+
* force" touch (a general-vocabulary term like "cache" has a definition but no
|
|
27
|
+
* enumerable graph class, so it falls back to the ordinary definition surface). */
|
|
28
|
+
export const CONCEPT_CLASS = Object.freeze({
|
|
29
|
+
class: "Class",
|
|
30
|
+
module: "Module",
|
|
31
|
+
function: "Function",
|
|
32
|
+
method: "Method",
|
|
33
|
+
attribute: "Attribute",
|
|
34
|
+
variable: "GlobalVariable",
|
|
35
|
+
constant: "GlobalVariable",
|
|
36
|
+
commit: "Commit",
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
/** class → [singular, plural] noun for the examples band's count ("(3 classes)"). */
|
|
40
|
+
const CLASS_NOUN = Object.freeze({
|
|
41
|
+
Class: ["class", "classes"],
|
|
42
|
+
Module: ["module", "modules"],
|
|
43
|
+
Function: ["function", "functions"],
|
|
44
|
+
Method: ["method", "methods"],
|
|
45
|
+
Attribute: ["attribute", "attributes"],
|
|
46
|
+
GlobalVariable: ["variable", "variables"],
|
|
47
|
+
Commit: ["commit", "commits"],
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
/** The isa-family predicates that make a remembered fact an INSTANCE statement
|
|
51
|
+
* ("A is a X" — rdf:type). A subclass relation (rdfs:subClassOf) is not an
|
|
52
|
+
* instance, so it never contributes an example. */
|
|
53
|
+
const ISA_INSTANCE_PREDICATES = new Set(["rdf:type"]);
|
|
54
|
+
|
|
55
|
+
/** Per graph-class, the candidate follow-up shapes in priority order. Each builder
|
|
56
|
+
* takes ONE real instance label and returns a query string; the builder is offered
|
|
57
|
+
* only after the query VALIDATES against the live graph, so a shape that can't
|
|
58
|
+
* resolve for any instance is silently dropped. Shapes are curated to be exactly
|
|
59
|
+
* the ones ask.mjs's grammar answers for that kind. */
|
|
60
|
+
const FOLLOWUP_SHAPES = Object.freeze({
|
|
61
|
+
Class: [
|
|
62
|
+
(x) => `which classes inherit from ${x}`,
|
|
63
|
+
(x) => `what does ${x} contain`,
|
|
64
|
+
(x) => `where is ${x} defined`,
|
|
65
|
+
],
|
|
66
|
+
Module: [
|
|
67
|
+
(x) => `what does ${x} import`,
|
|
68
|
+
(x) => `which modules import ${x}`,
|
|
69
|
+
(x) => `where is ${x} defined`,
|
|
70
|
+
],
|
|
71
|
+
Function: [
|
|
72
|
+
(x) => `what calls ${x}`,
|
|
73
|
+
(x) => `what does ${x} call`,
|
|
74
|
+
(x) => `where is ${x} defined`,
|
|
75
|
+
],
|
|
76
|
+
Method: [
|
|
77
|
+
(x) => `which class contains ${x}`,
|
|
78
|
+
(x) => `what calls ${x}`,
|
|
79
|
+
(x) => `where is ${x} defined`,
|
|
80
|
+
],
|
|
81
|
+
Attribute: [
|
|
82
|
+
(x) => `which class contains ${x}`,
|
|
83
|
+
(x) => `where is ${x} defined`,
|
|
84
|
+
],
|
|
85
|
+
GlobalVariable: [
|
|
86
|
+
(x) => `where is ${x} defined`,
|
|
87
|
+
(x) => `where is ${x} mentioned`,
|
|
88
|
+
],
|
|
89
|
+
Commit: [
|
|
90
|
+
(x) => `what did commit ${x} touch`,
|
|
91
|
+
(x) => `when did ${x} change`,
|
|
92
|
+
],
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
// The examples/instances listing shows up to 32 before truncating (the remainder is
|
|
96
|
+
// paginated by the chat shell's "more" mechanism); the GUIDED follow-ups stay small.
|
|
97
|
+
const MAX_EXAMPLES = 32;
|
|
98
|
+
const MAX_FOLLOWUPS = 3;
|
|
99
|
+
|
|
100
|
+
/** The lead clause of a curated definition — cut at the first "; " / ": " so the
|
|
101
|
+
* FACT band is one crisp sentence ("A class is a template that defines the
|
|
102
|
+
* structure and behaviour of objects."), the rest of the entry left implicit. */
|
|
103
|
+
function leadSentence(def) {
|
|
104
|
+
const s = String(def).trim();
|
|
105
|
+
const m = s.match(/^(.*?)[;:]\s/);
|
|
106
|
+
const head = (m ? m[1] : s).replace(/[.;:\s]+$/, "");
|
|
107
|
+
return `${head}.`;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function listJoin(a) {
|
|
111
|
+
return a.length > 1 ? `${a.slice(0, -1).join(", ")} and ${a[a.length - 1]}` : a[0];
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Does a candidate follow-up query actually resolve to a real, non-empty answer?
|
|
115
|
+
* Runs it through ask() (deterministic) and checks the honest-miss flag + matches —
|
|
116
|
+
* the exact same engine the user would hit, so a validated suggestion is guaranteed
|
|
117
|
+
* to land. Failure-tolerant: any throw counts as "does not resolve". */
|
|
118
|
+
function resolves(graph, query) {
|
|
119
|
+
try {
|
|
120
|
+
const r = ask(graph, query);
|
|
121
|
+
return !!(r && r.tmct_ask && r.tmct_ask.miss === false
|
|
122
|
+
&& Array.isArray(r.tmct_ask.matches) && r.tmct_ask.matches.length > 0);
|
|
123
|
+
} catch {
|
|
124
|
+
return false;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Build up to MAX_FOLLOWUPS validated follow-ups for a class's instances. For each
|
|
129
|
+
* shape in priority order, find the instances whose query resolves and offer it for
|
|
130
|
+
* the first such instance NOT already used by an earlier follow-up (so the set
|
|
131
|
+
* showcases DIFFERENT real nodes where possible); a shape no instance satisfies is
|
|
132
|
+
* dropped entirely. Deterministic (graph order in, first-fit out). */
|
|
133
|
+
function buildFollowups(graph, cls, instanceLabels) {
|
|
134
|
+
const shapes = FOLLOWUP_SHAPES[cls] || [];
|
|
135
|
+
const used = new Set();
|
|
136
|
+
const out = [];
|
|
137
|
+
for (const shape of shapes) {
|
|
138
|
+
if (out.length >= MAX_FOLLOWUPS) break;
|
|
139
|
+
const valid = instanceLabels.filter((x) => resolves(graph, shape(x)));
|
|
140
|
+
if (!valid.length) continue;
|
|
141
|
+
const pick = valid.find((x) => !used.has(x)) ?? valid[0];
|
|
142
|
+
used.add(pick);
|
|
143
|
+
out.push(shape(pick));
|
|
144
|
+
}
|
|
145
|
+
return out;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Compose the three bands for a concept term, or null when it is NOT a concept-force
|
|
149
|
+
* case — the term is not a known enumerable concept, has no curated definition, or
|
|
150
|
+
* has NO instances anywhere (code graph and memory both empty). Returns the pieces as
|
|
151
|
+
* strings so the caller can render them through a data template:
|
|
152
|
+
* { definition, examples, followups, instances:[{id,label,type,module}] }
|
|
153
|
+
* `examples` is always non-empty when non-null (we only fire with real instances);
|
|
154
|
+
* `followups` is "" when no validated next-question exists, else a "\nWant to go
|
|
155
|
+
* deeper? Try:\n • …" block. */
|
|
156
|
+
export function composeConcept(graph, term, { definition = null, factRows = [] } = {}) {
|
|
157
|
+
const cls = CONCEPT_CLASS[term];
|
|
158
|
+
if (!cls || !definition) return null;
|
|
159
|
+
|
|
160
|
+
const individuals = (graph && Array.isArray(graph.individuals)) ? graph.individuals : [];
|
|
161
|
+
const graphInstances = individuals.filter((i) => i && i.class === cls);
|
|
162
|
+
const graphLabels = graphInstances.map((i) => i.label);
|
|
163
|
+
const graphLower = new Set(graphLabels.map((l) => String(l).toLowerCase()));
|
|
164
|
+
|
|
165
|
+
// remembered "A is a X" instance facts (rdf:type), objects matching this term —
|
|
166
|
+
// subjects are the instance names, deduped against graph instances by label.
|
|
167
|
+
const memoryLabels = [];
|
|
168
|
+
for (const f of factRows) {
|
|
169
|
+
if (!ISA_INSTANCE_PREDICATES.has(f.predicate)) continue;
|
|
170
|
+
if (String(f.object).toLowerCase() !== term) continue;
|
|
171
|
+
const sub = String(f.subject || "").trim();
|
|
172
|
+
if (sub && !graphLower.has(sub.toLowerCase()) && !memoryLabels.includes(sub)) memoryLabels.push(sub);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
if (!graphInstances.length && !memoryLabels.length) return null; // no instances → honest miss stands
|
|
176
|
+
|
|
177
|
+
const [sing, plur] = CLASS_NOUN[cls] || [cls.toLowerCase(), `${cls.toLowerCase()}s`];
|
|
178
|
+
|
|
179
|
+
// BAND 1 — the fact.
|
|
180
|
+
const bandDefinition = leadSentence(definition);
|
|
181
|
+
|
|
182
|
+
// BAND 2 — the examples. Up to MAX_EXAMPLES individuals are listed; a longer class
|
|
183
|
+
// holds its remainder for the shell's "more" pagination (say 'more' to see them).
|
|
184
|
+
const shownGraph = graphLabels.slice(0, MAX_EXAMPLES);
|
|
185
|
+
const remainderLabels = graphLabels.slice(MAX_EXAMPLES);
|
|
186
|
+
let bandExamples = "";
|
|
187
|
+
if (shownGraph.length) {
|
|
188
|
+
const total = graphInstances.length;
|
|
189
|
+
const more = remainderLabels.length
|
|
190
|
+
? ` …and ${remainderLabels.length} more — say 'more' to see them.`
|
|
191
|
+
: "";
|
|
192
|
+
bandExamples = `In this codebase, for example: ${listJoin(shownGraph)} (${total} ${total === 1 ? sing : plur}).${more}`;
|
|
193
|
+
}
|
|
194
|
+
const shownMemory = memoryLabels.slice(0, MAX_EXAMPLES);
|
|
195
|
+
if (shownMemory.length) {
|
|
196
|
+
const lead = bandExamples ? " " : "";
|
|
197
|
+
const verb = shownMemory.length === 1 ? "is a" : "are";
|
|
198
|
+
bandExamples += `${lead}You've also told me ${listJoin(shownMemory)} ${verb} ${sing}.`;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// BAND 3 — the guided follow-ups (validated; only real code-graph instances can
|
|
202
|
+
// seed a runnable graph query, so memory-only concepts simply get no follow-ups).
|
|
203
|
+
const followupQueries = buildFollowups(graph, cls, graphLabels);
|
|
204
|
+
const bandFollowups = followupQueries.length
|
|
205
|
+
? `\nWant to go deeper? Try:\n${followupQueries.map((q) => ` • ${q}`).join("\n")}`
|
|
206
|
+
: "";
|
|
207
|
+
|
|
208
|
+
return {
|
|
209
|
+
definition: bandDefinition,
|
|
210
|
+
examples: bandExamples,
|
|
211
|
+
followups: bandFollowups,
|
|
212
|
+
followupQueries,
|
|
213
|
+
instances: graphInstances.slice(0, MAX_EXAMPLES).map((i) => ({
|
|
214
|
+
id: i.id, label: i.label, type: i.class,
|
|
215
|
+
})),
|
|
216
|
+
// the un-shown instance labels + their plural noun, for the shell's "more"
|
|
217
|
+
// pagination — empty when nothing was truncated.
|
|
218
|
+
remainder: remainderLabels,
|
|
219
|
+
noun: plur,
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// ============================================================================
|
|
224
|
+
// THE RELATION CONCEPT FORCE — the same three-band shape (definition + real
|
|
225
|
+
// example EDGES + validated follow-ups) for a vague touch on a RELATION/edge kind
|
|
226
|
+
// ("what about imports", "what are the calls", "tell me about contains"). Where
|
|
227
|
+
// composeConcept enumerates INDIVIDUALS of a class, composeRelation enumerates the
|
|
228
|
+
// EDGES of a relation kind, and seeds its follow-ups from real edge endpoints.
|
|
229
|
+
// PURE given (graph, relTerm, {definition}); every example is a real edge and every
|
|
230
|
+
// follow-up is validated via resolves() before it is offered. Never fabricates.
|
|
231
|
+
// ============================================================================
|
|
232
|
+
|
|
233
|
+
/** A vague relation term (lower-cased) → the internal concept key it enumerates.
|
|
234
|
+
* The closed set of edge-kind concepts the seon relation table + the graph both
|
|
235
|
+
* understand; a term outside it is not a relation-force touch. Nouns, gerunds and
|
|
236
|
+
* a couple of synonyms all collapse to one key. */
|
|
237
|
+
export const RELATION_TERM = Object.freeze({
|
|
238
|
+
import: "imports", imports: "imports", importing: "imports", imported: "imports",
|
|
239
|
+
call: "calls", calls: "calls", calling: "calls", called: "calls", invoke: "calls", invokes: "calls", invoking: "calls",
|
|
240
|
+
contain: "contains", contains: "contains", containing: "contains", containment: "contains", member: "contains", members: "contains",
|
|
241
|
+
inherit: "inherits", inherits: "inherits", inheriting: "inherits", inheritance: "inherits",
|
|
242
|
+
extend: "inherits", extends: "inherits", extending: "inherits", subclass: "inherits", subclasses: "inherits", subclassing: "inherits",
|
|
243
|
+
test: "tests", tests: "tests", testing: "tests", tested: "tests", coverage: "tests",
|
|
244
|
+
define: "defines", defines: "defines", defining: "defines", defined: "defines", definition: "defines", definitions: "defines", declaration: "defines",
|
|
245
|
+
touch: "touches", touches: "touches", touching: "touches", touched: "touches",
|
|
246
|
+
cochange: "cochange", "co-change": "cochange", "change-coupling": "cochange", coupled: "cochange",
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
/** concept key → the relationKind()s whose edges it enumerates. A concept can span
|
|
250
|
+
* several graph kinds (calls is both module-coarse and symbol-granular; touches is
|
|
251
|
+
* file- and symbol-level) — the force gathers edges across the whole set. */
|
|
252
|
+
const RELATION_KINDS = Object.freeze({
|
|
253
|
+
imports: ["imports"],
|
|
254
|
+
calls: ["calls", "callsSymbol"],
|
|
255
|
+
contains: ["contains"],
|
|
256
|
+
inherits: ["inherits"],
|
|
257
|
+
tests: ["tests"],
|
|
258
|
+
defines: ["defines"],
|
|
259
|
+
touches: ["touches", "touchesSymbol"],
|
|
260
|
+
cochange: ["cochange"],
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
/** concept key → the verb phrase that renders an edge as an English sentence
|
|
264
|
+
* ("a imports b"), and the singular edge-noun for the count ("(18 import edges)"). */
|
|
265
|
+
const RELATION_RENDER = Object.freeze({
|
|
266
|
+
imports: { verb: "imports", edgeNoun: "import" },
|
|
267
|
+
calls: { verb: "calls", edgeNoun: "call" },
|
|
268
|
+
contains: { verb: "contains", edgeNoun: "containment" },
|
|
269
|
+
inherits: { verb: "inherits from", edgeNoun: "inheritance" },
|
|
270
|
+
tests: { verb: "tests", edgeNoun: "test" },
|
|
271
|
+
defines: { verb: "defines", edgeNoun: "definition" },
|
|
272
|
+
touches: { verb: "touches", edgeNoun: "touch" },
|
|
273
|
+
cochange: { verb: "changes together with", edgeNoun: "change-coupling" },
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
/** Per concept key, the candidate follow-up shapes in priority order. Each shape
|
|
277
|
+
* draws a real endpoint from one SIDE of the edges (subject or object) and builds a
|
|
278
|
+
* query; a shape is offered only once the query VALIDATES against the live graph
|
|
279
|
+
* (resolves()), so a shape no endpoint satisfies is silently dropped. Curated to be
|
|
280
|
+
* exactly the shapes ask.mjs answers for that kind. */
|
|
281
|
+
const RELATION_FOLLOWUP_SHAPES = Object.freeze({
|
|
282
|
+
imports: [
|
|
283
|
+
{ side: "obj", make: (x) => `which modules import ${x}` },
|
|
284
|
+
{ side: "subj", make: (x) => `what does ${x} import` },
|
|
285
|
+
],
|
|
286
|
+
calls: [
|
|
287
|
+
{ side: "obj", make: (x) => `what calls ${x}` },
|
|
288
|
+
{ side: "subj", make: (x) => `what does ${x} call` },
|
|
289
|
+
],
|
|
290
|
+
contains: [
|
|
291
|
+
{ side: "subj", make: (x) => `what does ${x} contain` },
|
|
292
|
+
{ side: "obj", make: (x) => `which class contains ${x}` },
|
|
293
|
+
],
|
|
294
|
+
inherits: [
|
|
295
|
+
{ side: "obj", make: (x) => `which classes inherit from ${x}` },
|
|
296
|
+
{ side: "subj", make: (x) => `where is ${x} defined` },
|
|
297
|
+
],
|
|
298
|
+
tests: [
|
|
299
|
+
{ side: "obj", make: (x) => `what tests ${x}` },
|
|
300
|
+
{ side: "obj", make: (x) => `where is ${x} defined` },
|
|
301
|
+
],
|
|
302
|
+
defines: [
|
|
303
|
+
{ side: "obj", make: (x) => `where is ${x} defined` },
|
|
304
|
+
{ side: "subj", make: (x) => `what does ${x} contain` },
|
|
305
|
+
],
|
|
306
|
+
touches: [
|
|
307
|
+
{ side: "obj", make: (x) => `when did ${x} change` },
|
|
308
|
+
{ side: "subj", make: (x) => `what did commit ${x} touch` },
|
|
309
|
+
],
|
|
310
|
+
cochange: [
|
|
311
|
+
{ side: "obj", make: (x) => `where is ${x} defined` },
|
|
312
|
+
{ side: "subj", make: (x) => `which modules import ${x}` },
|
|
313
|
+
],
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
/** How many example edges the relation force shows before the remainder is held for
|
|
317
|
+
* the shell's "more" pagination (edges render as full sentences, so a much smaller
|
|
318
|
+
* page than the noun force's 32 instances reads better). */
|
|
319
|
+
const MAX_EDGE_EXAMPLES = 3;
|
|
320
|
+
|
|
321
|
+
const edgeSubjectLabel = (e) => String(e.subjectLabel || e.subject);
|
|
322
|
+
const edgeObjectLabel = (e) => String(e.objectLabel || e.object);
|
|
323
|
+
|
|
324
|
+
/** Build up to MAX_FOLLOWUPS validated follow-ups for a relation's edges. Same
|
|
325
|
+
* first-fit discipline as buildFollowups: for each shape in priority order, find the
|
|
326
|
+
* endpoints (of that shape's side) whose query resolves and offer it for the first
|
|
327
|
+
* such endpoint not already used, so the set showcases DIFFERENT real nodes. */
|
|
328
|
+
function buildRelationFollowups(graph, key, subjLabels, objLabels) {
|
|
329
|
+
const shapes = RELATION_FOLLOWUP_SHAPES[key] || [];
|
|
330
|
+
const used = new Set();
|
|
331
|
+
const out = [];
|
|
332
|
+
for (const shape of shapes) {
|
|
333
|
+
if (out.length >= MAX_FOLLOWUPS) break;
|
|
334
|
+
const pool = shape.side === "subj" ? subjLabels : objLabels;
|
|
335
|
+
const valid = pool.filter((x) => resolves(graph, shape.make(x)));
|
|
336
|
+
if (!valid.length) continue;
|
|
337
|
+
const pick = valid.find((x) => !used.has(x)) ?? valid[0];
|
|
338
|
+
used.add(pick);
|
|
339
|
+
out.push(shape.make(pick));
|
|
340
|
+
}
|
|
341
|
+
return out;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/** Compose the three bands for a RELATION concept term, or null when it is NOT a
|
|
345
|
+
* relation-force case — the term is not a known enumerable relation, has no curated
|
|
346
|
+
* definition, or the graph has NO edges of that kind (honest miss stands, never a
|
|
347
|
+
* fabricated edge). Returns the same string-band shape composeConcept does:
|
|
348
|
+
* { definition, examples, followups, followupQueries, remainder, noun }
|
|
349
|
+
* `examples` is always non-empty when non-null (we only fire with real edges);
|
|
350
|
+
* `followups` is "" when no validated next-question exists. */
|
|
351
|
+
export function composeRelation(graph, relTerm, { definition = null } = {}) {
|
|
352
|
+
const key = RELATION_TERM[String(relTerm || "").toLowerCase()];
|
|
353
|
+
if (!key || !definition) return null;
|
|
354
|
+
const kinds = RELATION_KINDS[key] || [];
|
|
355
|
+
const groups = (graph && Array.isArray(graph.relations) ? graph.relations : [])
|
|
356
|
+
.filter((g) => kinds.includes(relationKind(g)));
|
|
357
|
+
const edges = groups.flatMap((g) => (Array.isArray(g.edges) ? g.edges : []));
|
|
358
|
+
if (!edges.length) return null; // no edges of this kind → honest miss stands
|
|
359
|
+
const total = groups.reduce((s, g) => s + (Number(g.count) || (g.edges || []).length), 0);
|
|
360
|
+
|
|
361
|
+
const { verb, edgeNoun } = RELATION_RENDER[key] || { verb: key, edgeNoun: key };
|
|
362
|
+
|
|
363
|
+
// BAND 1 — the fact (the relation defined as a verb/relationship).
|
|
364
|
+
const bandDefinition = leadSentence(definition);
|
|
365
|
+
|
|
366
|
+
// BAND 2 — the example edges, rendered as English sentences with a count. The
|
|
367
|
+
// first MAX_EDGE_EXAMPLES are shown; the remainder is held for "more" pagination.
|
|
368
|
+
const rendered = edges.map((e) => `${edgeSubjectLabel(e)} ${verb} ${edgeObjectLabel(e)}`);
|
|
369
|
+
const shown = rendered.slice(0, MAX_EDGE_EXAMPLES);
|
|
370
|
+
const remainder = rendered.slice(MAX_EDGE_EXAMPLES);
|
|
371
|
+
const countNoun = `${edgeNoun} edge${total === 1 ? "" : "s"}`;
|
|
372
|
+
const more = remainder.length ? ` …and ${remainder.length} more — say 'more' to see them.` : "";
|
|
373
|
+
const bandExamples = `In this codebase, for example: ${listJoin(shown)} (${total} ${countNoun}).${more}`;
|
|
374
|
+
|
|
375
|
+
// BAND 3 — the guided follow-ups, seeded from DISTINCT real edge endpoints and
|
|
376
|
+
// validated against the live graph, so an offered follow-up can never miss.
|
|
377
|
+
const subjLabels = [...new Set(edges.map(edgeSubjectLabel))];
|
|
378
|
+
const objLabels = [...new Set(edges.map(edgeObjectLabel))];
|
|
379
|
+
const followupQueries = buildRelationFollowups(graph, key, subjLabels, objLabels);
|
|
380
|
+
const bandFollowups = followupQueries.length
|
|
381
|
+
? `\nWant to go deeper? Try:\n${followupQueries.map((q) => ` • ${q}`).join("\n")}`
|
|
382
|
+
: "";
|
|
383
|
+
|
|
384
|
+
return {
|
|
385
|
+
definition: bandDefinition,
|
|
386
|
+
examples: bandExamples,
|
|
387
|
+
followups: bandFollowups,
|
|
388
|
+
followupQueries,
|
|
389
|
+
relation: key,
|
|
390
|
+
remainder,
|
|
391
|
+
noun: countNoun,
|
|
392
|
+
};
|
|
393
|
+
}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// loadSlice(path?) stream corpus/conceptnet/slice.jsonl → assertions
|
|
5
5
|
// loadMap(path?) src/corpus/conceptnet-map.toml → Map(rel → row)
|
|
6
6
|
// toFacts(assertions,map) assertions → appendFact-shaped triples
|
|
7
|
-
// seedMemory(dir, opts) write them into <dir>/.tmct/memory via
|
|
7
|
+
// seedMemory(dir, opts) write them into <dir>/.tmct/memory via appendFacts (one batched write)
|
|
8
8
|
//
|
|
9
9
|
// The slice is committed data (one JSON object per line: {start, rel, end,
|
|
10
10
|
// surfaceText?, weight}; en→en only; CC-BY-SA 4.0 for ConceptNet-derived rows
|
|
@@ -13,11 +13,12 @@
|
|
|
13
13
|
// ace = "none" are deliberate non-emissions. A slice relation MISSING from
|
|
14
14
|
// the table is a drift error — loud, never guessed around.
|
|
15
15
|
//
|
|
16
|
-
// Seeding goes through src/memory/core.mjs
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
16
|
+
// Seeding goes through src/memory/core.mjs appendFacts() (memory is import-only
|
|
17
|
+
// here): fact ids are content-hashed from (s,p,o), so re-seeding is idempotent by
|
|
18
|
+
// construction. seedMemory pre-loads the store once and skips triples already
|
|
19
|
+
// present, then hands the survivors to appendFacts as ONE batched read-modify-
|
|
20
|
+
// write — so seeding the whole slice is O(N), not the O(N²) a per-fact appendFact
|
|
21
|
+
// loop would incur (the 6 k-fact slice: ~7 min → a couple of seconds).
|
|
21
22
|
|
|
22
23
|
import { createReadStream } from "node:fs";
|
|
23
24
|
import { readFile } from "node:fs/promises";
|
|
@@ -25,7 +26,7 @@ import { createInterface } from "node:readline";
|
|
|
25
26
|
import { fileURLToPath } from "node:url";
|
|
26
27
|
import { dirname, join } from "node:path";
|
|
27
28
|
import { parse as parseToml } from "smol-toml";
|
|
28
|
-
import {
|
|
29
|
+
import { appendFacts, loadMemory, normFactTerm } from "../memory/core.mjs";
|
|
29
30
|
|
|
30
31
|
const PKG_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
31
32
|
export const SLICE_FILE = join(PKG_ROOT, "corpus", "conceptnet", "slice.jsonl");
|
|
@@ -140,9 +141,10 @@ export function toFacts(assertions, map, provenancePrefix = "corpus:conceptnet")
|
|
|
140
141
|
* location trivia the slice happens to open with; without `prefer` the
|
|
141
142
|
* behavior is byte-identical to before.
|
|
142
143
|
*
|
|
143
|
-
* Idempotent twice over:
|
|
144
|
+
* Idempotent twice over: appendFacts' content-hashed ids make a blind
|
|
144
145
|
* re-append an upsert, and we pre-read the store once to skip triples that
|
|
145
|
-
* are already there (so re-seeding costs one read, not N rewrites).
|
|
146
|
+
* are already there (so re-seeding costs one read, not N rewrites). The
|
|
147
|
+
* survivors are written in ONE batched appendFacts call, not a per-fact loop.
|
|
146
148
|
* Returns { appended, skipped, total }. `provenancePrefix` is threaded through to
|
|
147
149
|
* toFacts (default "corpus:conceptnet" → byte-identical seed) so a seon/tier-2
|
|
148
150
|
* corpus can tag its facts "corpus:seon" / "corpus:tier2:<id>". */
|
|
@@ -169,17 +171,20 @@ export async function seedMemory(dir, { limit, slicePath = SLICE_FILE, mapPath =
|
|
|
169
171
|
existing.add(factKey(get("subject"), get("predicate"), get("object")));
|
|
170
172
|
}
|
|
171
173
|
|
|
172
|
-
let appended = 0;
|
|
173
174
|
let skipped = 0;
|
|
175
|
+
const toWrite = [];
|
|
174
176
|
for (const fact of facts) {
|
|
175
177
|
const key = factKey(fact.subject, fact.predicate, fact.object);
|
|
176
178
|
if (existing.has(key)) {
|
|
177
179
|
skipped += 1;
|
|
178
180
|
continue;
|
|
179
181
|
}
|
|
180
|
-
await appendFact(dir, fact);
|
|
181
182
|
existing.add(key);
|
|
182
|
-
|
|
183
|
+
toWrite.push(fact);
|
|
183
184
|
}
|
|
184
|
-
|
|
185
|
+
// ONE read-modify-write for the whole seed (was one per fact — O(N²) I/O, ~7 min
|
|
186
|
+
// for the 6 k-fact slice). appendFacts also skips any malformed row rather than
|
|
187
|
+
// throwing, so its skipped count folds into the dedup skips here.
|
|
188
|
+
const res = await appendFacts(dir, toWrite);
|
|
189
|
+
return { appended: res.appended, skipped: skipped + res.skipped, total: facts.length };
|
|
185
190
|
}
|
package/src/init.mjs
CHANGED
|
@@ -35,12 +35,14 @@ export const MEMORY_DIR_REL = join(".tmct", "memory");
|
|
|
35
35
|
export const SESSIONS_DIR_REL = join(".tmct", "sessions");
|
|
36
36
|
export const SEED_MARKER_REL = join(".tmct", "memory", "corpus-seed.json");
|
|
37
37
|
|
|
38
|
-
/**
|
|
39
|
-
* init-seeded repo and a bootstrap-seeded repo carry the identical
|
|
40
|
-
|
|
38
|
+
/** `undefined` = seed the WHOLE ConceptNet band (no cap) — matches chat.mjs
|
|
39
|
+
* SEED_LIMIT so an init-seeded repo and a bootstrap-seeded repo carry the identical
|
|
40
|
+
* slice. A number in `tmct.toml`'s `seed.limit` still caps (explicit user override);
|
|
41
|
+
* absent ⇒ all. */
|
|
42
|
+
export const SEED_LIMIT = undefined;
|
|
41
43
|
|
|
42
|
-
/** Predicate
|
|
43
|
-
*
|
|
44
|
+
/** Predicate order for the seed (definitional band first) — matches chat.mjs
|
|
45
|
+
* SEED_PREFER. With the cap lifted this sets ORDER only; every fact seeds. */
|
|
44
46
|
export const SEED_PREFER = ["rdfs:subClassOf", "rdf:type", "mgx:usedFor", "mgx:partOf", "mgx:capableOf"];
|
|
45
47
|
|
|
46
48
|
/** The shipped default config — the exact shape written into `tmct.toml` and
|
|
@@ -49,7 +51,7 @@ export function defaultConfig() {
|
|
|
49
51
|
return {
|
|
50
52
|
graphFile: join(".tmct", "graph.json"),
|
|
51
53
|
corpus: { tier: "tier1" },
|
|
52
|
-
seed: { enabled: true
|
|
54
|
+
seed: { enabled: true },
|
|
53
55
|
};
|
|
54
56
|
}
|
|
55
57
|
|
|
@@ -103,8 +105,9 @@ tier = ${JSON.stringify(corpus.tier)}
|
|
|
103
105
|
# Offline and deterministic. Set false, or export TMCT_NO_SEED=1, to opt out —
|
|
104
106
|
# the repo still initialises, just empty of corpus facts.
|
|
105
107
|
enabled = ${seed.enabled ? "true" : "false"}
|
|
106
|
-
#
|
|
107
|
-
|
|
108
|
+
# By default the WHOLE committed slice seeds (no cap — the operator's "seed all").
|
|
109
|
+
# To cap it, uncomment and set a number (definitional band first):
|
|
110
|
+
${seed.limit != null ? `limit = ${Number(seed.limit)}` : "# limit = 500"}
|
|
108
111
|
`;
|
|
109
112
|
}
|
|
110
113
|
|
|
@@ -183,7 +186,7 @@ export async function initRepo(dir, { force = false, seed, env = process.env } =
|
|
|
183
186
|
} else {
|
|
184
187
|
try {
|
|
185
188
|
const { seedMemory } = await import("./corpus/conceptnet.mjs");
|
|
186
|
-
const limit = Number(config.seed
|
|
189
|
+
const limit = config.seed?.limit != null ? Number(config.seed.limit) : SEED_LIMIT;
|
|
187
190
|
seedResult = await seedMemory(root, { limit, prefer: SEED_PREFER });
|
|
188
191
|
const markerNew = !(await exists(paths.marker));
|
|
189
192
|
await mkdir(dirname(paths.marker), { recursive: true });
|
|
@@ -90,6 +90,48 @@ export function applyNegationFrames(text) {
|
|
|
90
90
|
return text;
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
+
// ---- phrasing frames (SKILL_CHAT_PLAYTEST drill-down loop) — route the natural
|
|
94
|
+
// ways a developer asks a MEMBERS-of-class or a WHERE-DEFINED question onto the
|
|
95
|
+
// canonical shapes the grammar already answers, so a phrasing miss becomes a real
|
|
96
|
+
// answer (or an honest empty with a receipt) instead of the grammar wall. Same
|
|
97
|
+
// closed-pattern, first-match-wins discipline as the negation/commit frames: each
|
|
98
|
+
// frame REWRITES the whole line to a canonical query BOTH parse strategies then
|
|
99
|
+
// handle for free. Run AFTER applyNegationFrames so a sha "what's in <sha>" is
|
|
100
|
+
// already the commit-subject question before the members frame could see it. ----
|
|
101
|
+
export const PHRASING_FRAMES = Object.freeze([
|
|
102
|
+
// MEMBERS-of-class → "what does X contain".
|
|
103
|
+
// "what functions are in Task", "what methods are inside X", "what attributes are in X"
|
|
104
|
+
{ re: /^what\s+(?:functions?|methods?|members?|attributes?|fields?|properties)\s+(?:are|is)\s+(?:in|inside|within)\s+(?:the\s+)?(.+?)\??$/i, to: (m) => `what does ${m[1]} contain` },
|
|
105
|
+
// "what functions does Task have", "what methods does X have"
|
|
106
|
+
{ re: /^what\s+(?:functions?|methods?|members?|attributes?|fields?|properties)\s+(?:does|do)\s+(.+?)\s+have\??$/i, to: (m) => `what does ${m[1]} contain` },
|
|
107
|
+
// "what are the members of X", "what are the methods in X"
|
|
108
|
+
{ re: /^what\s+are\s+(?:the\s+)?(?:functions?|methods?|members?|attributes?|fields?|properties)\s+(?:of|in|inside|within)\s+(?:the\s+)?(.+?)\??$/i, to: (m) => `what does ${m[1]} contain` },
|
|
109
|
+
// "members of X", "methods of X", "contents of X"
|
|
110
|
+
{ re: /^(?:the\s+)?(?:members?|methods?|attributes?|contents)\s+of\s+(?:the\s+)?(.+?)\??$/i, to: (m) => `what does ${m[1]} contain` },
|
|
111
|
+
// "what's in X" / "what is in X" (contraction already expanded; sha handled above)
|
|
112
|
+
{ re: /^what\s+is\s+(?:in|inside)\s+(?:the\s+)?(.+?)\??$/i, to: (m) => `what does ${m[1]} contain` },
|
|
113
|
+
|
|
114
|
+
// WHERE-DEFINED → "where is X defined". PAST TENSE ONLY ("what defined X", "what
|
|
115
|
+
// declared X"): the PRESENT "what defines X" already parses as a reverse-defines
|
|
116
|
+
// query (the module defining symbol X — test/ask.test.mjs pins that), so rewriting
|
|
117
|
+
// it would change that receipt. The past-tense form is the one that hit the wall.
|
|
118
|
+
{ re: /^what\s+(?:defined|declared)\s+(?:the\s+)?(?:function\s+|method\s+|class\s+|module\s+|variable\s+|constant\s+)?(.+?)\??$/i, to: (m) => `where is ${m[1]} defined` },
|
|
119
|
+
// "where's X defined" (the "where's" contraction is not in the contraction table)
|
|
120
|
+
{ re: /^where'?s\s+(?:the\s+)?(.+?)\s+(defined|declared|located|implemented)\??$/i, to: (m) => `where is ${m[1]} ${m[2]}` },
|
|
121
|
+
]);
|
|
122
|
+
|
|
123
|
+
/** Apply the phrasing frames (members-of-class + where-defined) — first match wins
|
|
124
|
+
* and rewriting stops; unmatched text passes through unchanged. Kept SEPARATE from
|
|
125
|
+
* applyNegationFrames so the ordering (negation/commit first, then phrasing) is
|
|
126
|
+
* explicit at the call site (normalizeInput). */
|
|
127
|
+
export function applyPhrasingFrames(text) {
|
|
128
|
+
for (const frame of PHRASING_FRAMES) {
|
|
129
|
+
const m = text.match(frame.re);
|
|
130
|
+
if (m) return frame.to(m).replace(/\s+/g, " ").trim();
|
|
131
|
+
}
|
|
132
|
+
return text;
|
|
133
|
+
}
|
|
134
|
+
|
|
93
135
|
// ---- §B1 negation — the SET-COMPLEMENT frame (Cycle 5, PLAN_CYCLE_4.md). Recognizes
|
|
94
136
|
// a BARE set-negation query — "which X do not <verb> Y", "X that don't <verb> Y",
|
|
95
137
|
// "modules not importing Y", "which X are not <qualifier>" — and returns a descriptor
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
// normalization changed the input (`normalizationChanged`), so a repaired
|
|
24
24
|
// spelling/contraction is on the record, never silent.
|
|
25
25
|
|
|
26
|
-
import { normalizeQuery, applyNegationFrames } from "./normalize.mjs";
|
|
26
|
+
import { normalizeQuery, applyNegationFrames, applyPhrasingFrames } from "./normalize.mjs";
|
|
27
27
|
import { grammarStrategy } from "./strategies/grammar.mjs";
|
|
28
28
|
import { keywordSpotStrategy } from "./strategies/keywords.mjs";
|
|
29
29
|
import { noiseStripStrategy } from "./strategies/noise-strip.mjs";
|
|
@@ -47,7 +47,7 @@ export const STRATEGIES = [grammarStrategy, keywordSpotStrategy, noiseStripStrat
|
|
|
47
47
|
* before any strategy runs. Returns {raw, text, changed}. */
|
|
48
48
|
export function normalizeInput(input) {
|
|
49
49
|
const raw = String(input || "").trim().replace(/\s+/g, " ");
|
|
50
|
-
const text = raw ? applyNegationFrames(normalizeQuery(raw)) : "";
|
|
50
|
+
const text = raw ? applyPhrasingFrames(applyNegationFrames(normalizeQuery(raw))) : "";
|
|
51
51
|
return { raw, text, changed: text !== raw };
|
|
52
52
|
}
|
|
53
53
|
|