@polycode-projects/the-mechanical-code-talker 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -27
- package/ROADMAP.md +45 -0
- package/bin/tmct.mjs +64 -2
- package/corpus/README.md +77 -5
- package/corpus/conceptnet/README.md +92 -33
- package/corpus/conceptnet/filter-dump.mjs +59 -2
- package/corpus/conceptnet/quality-filter.mjs +95 -0
- package/corpus/conceptnet/slice.jsonl +31067 -378
- package/corpus/seon/LICENSE-NOTICE +37 -0
- package/corpus/seon/README.md +121 -0
- package/corpus/seon/concepts.jsonl +238 -0
- package/corpus/seon/definitions.jsonl +288 -0
- package/corpus/seon/relations.jsonl +8 -0
- package/corpus/tier2/aws.jsonl +39 -0
- package/corpus/tier2/generate.mjs +253 -0
- package/corpus/tier2/java.jsonl +31 -0
- package/corpus/tier2/manifest.json +48 -0
- package/corpus/tier2/python.jsonl +30 -0
- package/data/templates/responses.jsonl +3 -0
- package/package.json +13 -2
- package/src/ask-vocab.mjs +19 -1
- package/src/ask.mjs +13 -13
- package/src/chat.mjs +759 -66
- package/src/codegraph.mjs +28 -5
- package/src/concept.mjs +393 -0
- package/src/conformance.mjs +166 -0
- package/src/corpus/conceptnet.mjs +42 -19
- package/src/init.mjs +12 -9
- package/src/interpret/normalize.mjs +42 -0
- package/src/interpret/pipeline.mjs +2 -2
- package/src/memory/core.mjs +84 -0
- package/src/memory/inspect.mjs +25 -0
- package/src/server.mjs +88 -7
- package/src/tui/app.mjs +26 -2
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// loadSlice(path?) stream corpus/conceptnet/slice.jsonl → assertions
|
|
5
5
|
// loadMap(path?) src/corpus/conceptnet-map.toml → Map(rel → row)
|
|
6
6
|
// toFacts(assertions,map) assertions → appendFact-shaped triples
|
|
7
|
-
// seedMemory(dir, opts) write them into <dir>/.tmct/memory via
|
|
7
|
+
// seedMemory(dir, opts) write them into <dir>/.tmct/memory via appendFacts (one batched write)
|
|
8
8
|
//
|
|
9
9
|
// The slice is committed data (one JSON object per line: {start, rel, end,
|
|
10
10
|
// surfaceText?, weight}; en→en only; CC-BY-SA 4.0 for ConceptNet-derived rows
|
|
@@ -13,11 +13,12 @@
|
|
|
13
13
|
// ace = "none" are deliberate non-emissions. A slice relation MISSING from
|
|
14
14
|
// the table is a drift error — loud, never guessed around.
|
|
15
15
|
//
|
|
16
|
-
// Seeding goes through src/memory/core.mjs
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
16
|
+
// Seeding goes through src/memory/core.mjs appendFacts() (memory is import-only
|
|
17
|
+
// here): fact ids are content-hashed from (s,p,o), so re-seeding is idempotent by
|
|
18
|
+
// construction. seedMemory pre-loads the store once and skips triples already
|
|
19
|
+
// present, then hands the survivors to appendFacts as ONE batched read-modify-
|
|
20
|
+
// write — so seeding the whole slice is O(N), not the O(N²) a per-fact appendFact
|
|
21
|
+
// loop would incur (the 6 k-fact slice: ~7 min → a couple of seconds).
|
|
21
22
|
|
|
22
23
|
import { createReadStream } from "node:fs";
|
|
23
24
|
import { readFile } from "node:fs/promises";
|
|
@@ -25,12 +26,23 @@ import { createInterface } from "node:readline";
|
|
|
25
26
|
import { fileURLToPath } from "node:url";
|
|
26
27
|
import { dirname, join } from "node:path";
|
|
27
28
|
import { parse as parseToml } from "smol-toml";
|
|
28
|
-
import {
|
|
29
|
+
import { appendFacts, loadMemory, normFactTerm } from "../memory/core.mjs";
|
|
29
30
|
|
|
30
31
|
const PKG_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
31
32
|
export const SLICE_FILE = join(PKG_ROOT, "corpus", "conceptnet", "slice.jsonl");
|
|
32
33
|
export const MAP_FILE = join(PKG_ROOT, "src", "corpus", "conceptnet-map.toml");
|
|
33
34
|
|
|
35
|
+
// The tier-1 curated Software-Engineering ontology (SEON). concepts.jsonl is in
|
|
36
|
+
// the SAME slice shape as ConceptNet ({start, rel, end, weight}), so it loads +
|
|
37
|
+
// maps through the identical loadSlice/loadMap/toFacts path — just with a
|
|
38
|
+
// "corpus:seon" provenance prefix. definitions.jsonl is a separate {term,
|
|
39
|
+
// definition, sense} list the chat answer layer prefers for a lexicon term's
|
|
40
|
+
// "what is a <term>". tier-2 corpuses (aws/python/java) share the slice shape too.
|
|
41
|
+
export const SEON_CONCEPTS_FILE = join(PKG_ROOT, "corpus", "seon", "concepts.jsonl");
|
|
42
|
+
export const SEON_DEFINITIONS_FILE = join(PKG_ROOT, "corpus", "seon", "definitions.jsonl");
|
|
43
|
+
export const TIER2_DIR = join(PKG_ROOT, "corpus", "tier2");
|
|
44
|
+
export const TIER2_MANIFEST_FILE = join(TIER2_DIR, "manifest.json");
|
|
45
|
+
|
|
34
46
|
const ACE_PATTERNS = new Set(["subClassOf", "type", "ObjectProperty", "someValuesFrom", "disjointWith", "property", "none"]);
|
|
35
47
|
|
|
36
48
|
/** Load the slice JSONL as a stream (never the whole file as one string) and
|
|
@@ -92,8 +104,13 @@ export const termText = (uri) => {
|
|
|
92
104
|
* (provenance is a STRING — exactly what src/memory/core.mjs appendFact
|
|
93
105
|
* takes; it names the corpus and the originating ConceptNet relation).
|
|
94
106
|
* Rows whose relation maps ace="none" are skipped — deliberate non-emission.
|
|
95
|
-
* A relation with NO row in the map throws: that is table drift, not data.
|
|
96
|
-
|
|
107
|
+
* A relation with NO row in the map throws: that is table drift, not data.
|
|
108
|
+
*
|
|
109
|
+
* `provenancePrefix` names the corpus half of the provenance string; it defaults
|
|
110
|
+
* to "corpus:conceptnet" so the ConceptNet seed stays BYTE-IDENTICAL to before.
|
|
111
|
+
* The seon / tier-2 corpuses reuse the same slice shape, tagged "corpus:seon" or
|
|
112
|
+
* "corpus:tier2:<id>" so a reader can tell a curated SE fact from ConceptNet noise. */
|
|
113
|
+
export function toFacts(assertions, map, provenancePrefix = "corpus:conceptnet") {
|
|
97
114
|
const facts = [];
|
|
98
115
|
for (const a of assertions) {
|
|
99
116
|
const row = map.get(a.rel);
|
|
@@ -108,7 +125,7 @@ export function toFacts(assertions, map) {
|
|
|
108
125
|
subject,
|
|
109
126
|
predicate: row.predicate,
|
|
110
127
|
object,
|
|
111
|
-
provenance:
|
|
128
|
+
provenance: `${provenancePrefix} ${a.rel}`,
|
|
112
129
|
});
|
|
113
130
|
}
|
|
114
131
|
return facts;
|
|
@@ -124,13 +141,16 @@ export function toFacts(assertions, map) {
|
|
|
124
141
|
* location trivia the slice happens to open with; without `prefer` the
|
|
125
142
|
* behavior is byte-identical to before.
|
|
126
143
|
*
|
|
127
|
-
* Idempotent twice over:
|
|
144
|
+
* Idempotent twice over: appendFacts' content-hashed ids make a blind
|
|
128
145
|
* re-append an upsert, and we pre-read the store once to skip triples that
|
|
129
|
-
* are already there (so re-seeding costs one read, not N rewrites).
|
|
130
|
-
*
|
|
131
|
-
|
|
146
|
+
* are already there (so re-seeding costs one read, not N rewrites). The
|
|
147
|
+
* survivors are written in ONE batched appendFacts call, not a per-fact loop.
|
|
148
|
+
* Returns { appended, skipped, total }. `provenancePrefix` is threaded through to
|
|
149
|
+
* toFacts (default "corpus:conceptnet" → byte-identical seed) so a seon/tier-2
|
|
150
|
+
* corpus can tag its facts "corpus:seon" / "corpus:tier2:<id>". */
|
|
151
|
+
export async function seedMemory(dir, { limit, slicePath = SLICE_FILE, mapPath = MAP_FILE, prefer, provenancePrefix } = {}) {
|
|
132
152
|
const [assertions, map] = await Promise.all([loadSlice(slicePath), loadMap(mapPath)]);
|
|
133
|
-
let facts = toFacts(assertions, map);
|
|
153
|
+
let facts = toFacts(assertions, map, provenancePrefix);
|
|
134
154
|
if (Array.isArray(prefer) && prefer.length) {
|
|
135
155
|
const rank = new Map(prefer.map((p, i) => [p, i]));
|
|
136
156
|
// stable partition: Array.prototype.sort is stable in Node, so equal-rank
|
|
@@ -151,17 +171,20 @@ export async function seedMemory(dir, { limit, slicePath = SLICE_FILE, mapPath =
|
|
|
151
171
|
existing.add(factKey(get("subject"), get("predicate"), get("object")));
|
|
152
172
|
}
|
|
153
173
|
|
|
154
|
-
let appended = 0;
|
|
155
174
|
let skipped = 0;
|
|
175
|
+
const toWrite = [];
|
|
156
176
|
for (const fact of facts) {
|
|
157
177
|
const key = factKey(fact.subject, fact.predicate, fact.object);
|
|
158
178
|
if (existing.has(key)) {
|
|
159
179
|
skipped += 1;
|
|
160
180
|
continue;
|
|
161
181
|
}
|
|
162
|
-
await appendFact(dir, fact);
|
|
163
182
|
existing.add(key);
|
|
164
|
-
|
|
183
|
+
toWrite.push(fact);
|
|
165
184
|
}
|
|
166
|
-
|
|
185
|
+
// ONE read-modify-write for the whole seed (was one per fact — O(N²) I/O, ~7 min
|
|
186
|
+
// for the 6 k-fact slice). appendFacts also skips any malformed row rather than
|
|
187
|
+
// throwing, so its skipped count folds into the dedup skips here.
|
|
188
|
+
const res = await appendFacts(dir, toWrite);
|
|
189
|
+
return { appended: res.appended, skipped: skipped + res.skipped, total: facts.length };
|
|
167
190
|
}
|
package/src/init.mjs
CHANGED
|
@@ -35,12 +35,14 @@ export const MEMORY_DIR_REL = join(".tmct", "memory");
|
|
|
35
35
|
export const SESSIONS_DIR_REL = join(".tmct", "sessions");
|
|
36
36
|
export const SEED_MARKER_REL = join(".tmct", "memory", "corpus-seed.json");
|
|
37
37
|
|
|
38
|
-
/**
|
|
39
|
-
* init-seeded repo and a bootstrap-seeded repo carry the identical
|
|
40
|
-
|
|
38
|
+
/** `undefined` = seed the WHOLE ConceptNet band (no cap) — matches chat.mjs
|
|
39
|
+
* SEED_LIMIT so an init-seeded repo and a bootstrap-seeded repo carry the identical
|
|
40
|
+
* slice. A number in `tmct.toml`'s `seed.limit` still caps (explicit user override);
|
|
41
|
+
* absent ⇒ all. */
|
|
42
|
+
export const SEED_LIMIT = undefined;
|
|
41
43
|
|
|
42
|
-
/** Predicate
|
|
43
|
-
*
|
|
44
|
+
/** Predicate order for the seed (definitional band first) — matches chat.mjs
|
|
45
|
+
* SEED_PREFER. With the cap lifted this sets ORDER only; every fact seeds. */
|
|
44
46
|
export const SEED_PREFER = ["rdfs:subClassOf", "rdf:type", "mgx:usedFor", "mgx:partOf", "mgx:capableOf"];
|
|
45
47
|
|
|
46
48
|
/** The shipped default config — the exact shape written into `tmct.toml` and
|
|
@@ -49,7 +51,7 @@ export function defaultConfig() {
|
|
|
49
51
|
return {
|
|
50
52
|
graphFile: join(".tmct", "graph.json"),
|
|
51
53
|
corpus: { tier: "tier1" },
|
|
52
|
-
seed: { enabled: true
|
|
54
|
+
seed: { enabled: true },
|
|
53
55
|
};
|
|
54
56
|
}
|
|
55
57
|
|
|
@@ -103,8 +105,9 @@ tier = ${JSON.stringify(corpus.tier)}
|
|
|
103
105
|
# Offline and deterministic. Set false, or export TMCT_NO_SEED=1, to opt out —
|
|
104
106
|
# the repo still initialises, just empty of corpus facts.
|
|
105
107
|
enabled = ${seed.enabled ? "true" : "false"}
|
|
106
|
-
#
|
|
107
|
-
|
|
108
|
+
# By default the WHOLE committed slice seeds (no cap — the operator's "seed all").
|
|
109
|
+
# To cap it, uncomment and set a number (definitional band first):
|
|
110
|
+
${seed.limit != null ? `limit = ${Number(seed.limit)}` : "# limit = 500"}
|
|
108
111
|
`;
|
|
109
112
|
}
|
|
110
113
|
|
|
@@ -183,7 +186,7 @@ export async function initRepo(dir, { force = false, seed, env = process.env } =
|
|
|
183
186
|
} else {
|
|
184
187
|
try {
|
|
185
188
|
const { seedMemory } = await import("./corpus/conceptnet.mjs");
|
|
186
|
-
const limit = Number(config.seed
|
|
189
|
+
const limit = config.seed?.limit != null ? Number(config.seed.limit) : SEED_LIMIT;
|
|
187
190
|
seedResult = await seedMemory(root, { limit, prefer: SEED_PREFER });
|
|
188
191
|
const markerNew = !(await exists(paths.marker));
|
|
189
192
|
await mkdir(dirname(paths.marker), { recursive: true });
|
|
@@ -90,6 +90,48 @@ export function applyNegationFrames(text) {
|
|
|
90
90
|
return text;
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
+
// ---- phrasing frames (SKILL_CHAT_PLAYTEST drill-down loop) — route the natural
|
|
94
|
+
// ways a developer asks a MEMBERS-of-class or a WHERE-DEFINED question onto the
|
|
95
|
+
// canonical shapes the grammar already answers, so a phrasing miss becomes a real
|
|
96
|
+
// answer (or an honest empty with a receipt) instead of the grammar wall. Same
|
|
97
|
+
// closed-pattern, first-match-wins discipline as the negation/commit frames: each
|
|
98
|
+
// frame REWRITES the whole line to a canonical query BOTH parse strategies then
|
|
99
|
+
// handle for free. Run AFTER applyNegationFrames so a sha "what's in <sha>" is
|
|
100
|
+
// already the commit-subject question before the members frame could see it. ----
|
|
101
|
+
export const PHRASING_FRAMES = Object.freeze([
|
|
102
|
+
// MEMBERS-of-class → "what does X contain".
|
|
103
|
+
// "what functions are in Task", "what methods are inside X", "what attributes are in X"
|
|
104
|
+
{ re: /^what\s+(?:functions?|methods?|members?|attributes?|fields?|properties)\s+(?:are|is)\s+(?:in|inside|within)\s+(?:the\s+)?(.+?)\??$/i, to: (m) => `what does ${m[1]} contain` },
|
|
105
|
+
// "what functions does Task have", "what methods does X have"
|
|
106
|
+
{ re: /^what\s+(?:functions?|methods?|members?|attributes?|fields?|properties)\s+(?:does|do)\s+(.+?)\s+have\??$/i, to: (m) => `what does ${m[1]} contain` },
|
|
107
|
+
// "what are the members of X", "what are the methods in X"
|
|
108
|
+
{ re: /^what\s+are\s+(?:the\s+)?(?:functions?|methods?|members?|attributes?|fields?|properties)\s+(?:of|in|inside|within)\s+(?:the\s+)?(.+?)\??$/i, to: (m) => `what does ${m[1]} contain` },
|
|
109
|
+
// "members of X", "methods of X", "contents of X"
|
|
110
|
+
{ re: /^(?:the\s+)?(?:members?|methods?|attributes?|contents)\s+of\s+(?:the\s+)?(.+?)\??$/i, to: (m) => `what does ${m[1]} contain` },
|
|
111
|
+
// "what's in X" / "what is in X" (contraction already expanded; sha handled above)
|
|
112
|
+
{ re: /^what\s+is\s+(?:in|inside)\s+(?:the\s+)?(.+?)\??$/i, to: (m) => `what does ${m[1]} contain` },
|
|
113
|
+
|
|
114
|
+
// WHERE-DEFINED → "where is X defined". PAST TENSE ONLY ("what defined X", "what
|
|
115
|
+
// declared X"): the PRESENT "what defines X" already parses as a reverse-defines
|
|
116
|
+
// query (the module defining symbol X — test/ask.test.mjs pins that), so rewriting
|
|
117
|
+
// it would change that receipt. The past-tense form is the one that hit the wall.
|
|
118
|
+
{ re: /^what\s+(?:defined|declared)\s+(?:the\s+)?(?:function\s+|method\s+|class\s+|module\s+|variable\s+|constant\s+)?(.+?)\??$/i, to: (m) => `where is ${m[1]} defined` },
|
|
119
|
+
// "where's X defined" (the "where's" contraction is not in the contraction table)
|
|
120
|
+
{ re: /^where'?s\s+(?:the\s+)?(.+?)\s+(defined|declared|located|implemented)\??$/i, to: (m) => `where is ${m[1]} ${m[2]}` },
|
|
121
|
+
]);
|
|
122
|
+
|
|
123
|
+
/** Apply the phrasing frames (members-of-class + where-defined) — first match wins
|
|
124
|
+
* and rewriting stops; unmatched text passes through unchanged. Kept SEPARATE from
|
|
125
|
+
* applyNegationFrames so the ordering (negation/commit first, then phrasing) is
|
|
126
|
+
* explicit at the call site (normalizeInput). */
|
|
127
|
+
export function applyPhrasingFrames(text) {
|
|
128
|
+
for (const frame of PHRASING_FRAMES) {
|
|
129
|
+
const m = text.match(frame.re);
|
|
130
|
+
if (m) return frame.to(m).replace(/\s+/g, " ").trim();
|
|
131
|
+
}
|
|
132
|
+
return text;
|
|
133
|
+
}
|
|
134
|
+
|
|
93
135
|
// ---- §B1 negation — the SET-COMPLEMENT frame (Cycle 5, PLAN_CYCLE_4.md). Recognizes
|
|
94
136
|
// a BARE set-negation query — "which X do not <verb> Y", "X that don't <verb> Y",
|
|
95
137
|
// "modules not importing Y", "which X are not <qualifier>" — and returns a descriptor
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
// normalization changed the input (`normalizationChanged`), so a repaired
|
|
24
24
|
// spelling/contraction is on the record, never silent.
|
|
25
25
|
|
|
26
|
-
import { normalizeQuery, applyNegationFrames } from "./normalize.mjs";
|
|
26
|
+
import { normalizeQuery, applyNegationFrames, applyPhrasingFrames } from "./normalize.mjs";
|
|
27
27
|
import { grammarStrategy } from "./strategies/grammar.mjs";
|
|
28
28
|
import { keywordSpotStrategy } from "./strategies/keywords.mjs";
|
|
29
29
|
import { noiseStripStrategy } from "./strategies/noise-strip.mjs";
|
|
@@ -47,7 +47,7 @@ export const STRATEGIES = [grammarStrategy, keywordSpotStrategy, noiseStripStrat
|
|
|
47
47
|
* before any strategy runs. Returns {raw, text, changed}. */
|
|
48
48
|
export function normalizeInput(input) {
|
|
49
49
|
const raw = String(input || "").trim().replace(/\s+/g, " ");
|
|
50
|
-
const text = raw ? applyNegationFrames(normalizeQuery(raw)) : "";
|
|
50
|
+
const text = raw ? applyPhrasingFrames(applyNegationFrames(normalizeQuery(raw))) : "";
|
|
51
51
|
return { raw, text, changed: text !== raw };
|
|
52
52
|
}
|
|
53
53
|
|
package/src/memory/core.mjs
CHANGED
|
@@ -425,6 +425,14 @@ export function normFactTerm(t) {
|
|
|
425
425
|
return s.toLowerCase();
|
|
426
426
|
}
|
|
427
427
|
|
|
428
|
+
// The fact-id contract: a Fact is content-addressed by its NUL-DELIMITED
|
|
429
|
+
// (s, p, o). NUL never occurs in a normalized term or a predicate URI, so it is
|
|
430
|
+
// a collision-proof separator (a space could be forged by a term that contains
|
|
431
|
+
// one). appendFact hashes the SAME `${s}\0${p}\0${o}` inline; appendFacts routes
|
|
432
|
+
// through here so the batch path can never drift to a space and silently re-key
|
|
433
|
+
// every seeded fact — the golden-equivalence test pins the two paths together.
|
|
434
|
+
const factIdFor = (s, p, o) => `fact:${fnv1aHex(`${s}\0${p}\0${o}`)}`;
|
|
435
|
+
|
|
428
436
|
/** Append one grammar-derived OWL triple, RDF-reified: a `Fact` individual
|
|
429
437
|
* carrying rdf:subject / rdf:predicate / rdf:object (+ provenance). The
|
|
430
438
|
* Phase-2 ACE parser's write point. Same (s,p,o) → same id → upsert, never a
|
|
@@ -465,6 +473,82 @@ export async function appendFact(dir, { subject, predicate, object, provenance =
|
|
|
465
473
|
return { id };
|
|
466
474
|
}
|
|
467
475
|
|
|
476
|
+
/** Batch append of grammar/corpus-derived triples — ONE read-modify-write for a
|
|
477
|
+
* whole seed (the appendUtterances precedent, for facts). The per-fact
|
|
478
|
+
* appendFact does a full read → mutate → prose-reindex → atomic-write PER FACT,
|
|
479
|
+
* so seeding N facts is O(N²) I/O (6 k facts ≈ 7 min); this collapses it to a
|
|
480
|
+
* single mutate.
|
|
481
|
+
*
|
|
482
|
+
* Every fact is normalized + prose-tokenized OUTSIDE the mutate, then a SINGLE
|
|
483
|
+
* mutateMemory upserts each Fact through an id→individual Map (O(1) upsert, so
|
|
484
|
+
* the growing individuals array is never rescanned per fact), reconciles each
|
|
485
|
+
* touched fact's Sources + trust via the SAME syncFactSources appendFact uses,
|
|
486
|
+
* and recountClasses ONCE at the end. The result is deep-equal (modulo array
|
|
487
|
+
* order) to looping appendFact: same fact ids, same mgx:factProvenance union,
|
|
488
|
+
* same statedBy Source edges, same mgx:trustScore, same first-write-wins
|
|
489
|
+
* createdAt. Malformed facts (missing subject/predicate/object) are SKIPPED (a
|
|
490
|
+
* bad row never aborts a 6 k-fact seed), not thrown as appendFact does.
|
|
491
|
+
* Returns { ids, appended, skipped } — ids one per applied fact (in order),
|
|
492
|
+
* appended = ids.length, skipped = malformed count. */
|
|
493
|
+
export async function appendFacts(dir, facts) {
|
|
494
|
+
const prepared = [];
|
|
495
|
+
let skipped = 0;
|
|
496
|
+
for (const f of facts || []) {
|
|
497
|
+
const s = normFactTerm(f?.subject);
|
|
498
|
+
const p = normText(f?.predicate);
|
|
499
|
+
const o = normFactTerm(f?.object);
|
|
500
|
+
if (!s || !p || !o) { skipped += 1; continue; } // batch skips, never throws
|
|
501
|
+
const text = `${s} ${p} ${o}`;
|
|
502
|
+
prepared.push({
|
|
503
|
+
id: factIdFor(s, p, o), // NUL-delimited — byte-identical to appendFact's id
|
|
504
|
+
s, p, o, text,
|
|
505
|
+
tokens: proseTokensFor({ doc: text }),
|
|
506
|
+
provenance: normText(f?.provenance),
|
|
507
|
+
createdAt: f?.createdAt || "",
|
|
508
|
+
});
|
|
509
|
+
}
|
|
510
|
+
const ids = [];
|
|
511
|
+
if (!prepared.length) return { ids, appended: 0, skipped };
|
|
512
|
+
await mutateMemory(dir, (payload) => {
|
|
513
|
+
// id → individual index for O(1) upsert (the array grows to thousands).
|
|
514
|
+
const byId = new Map(payload.individuals.map((i) => [i?.id, i]));
|
|
515
|
+
const touched = [];
|
|
516
|
+
const seen = new Set();
|
|
517
|
+
for (const f of prepared) {
|
|
518
|
+
const prior = byId.get(f.id);
|
|
519
|
+
const priorProv = prior?.attributes?.find((a) => a?.prop === "mgx:factProvenance")?.value || "";
|
|
520
|
+
// Same as appendFact: the mgx:factProvenance union stays byte-identical (a
|
|
521
|
+
// compat shim); the Source edges below are DERIVED from it, purely additive.
|
|
522
|
+
const provs = [...new Set([...priorProv.split(" | "), f.provenance].filter(Boolean))];
|
|
523
|
+
const createdAtVal = firstWriteCreatedAt(prior, f.createdAt); // first-write-wins
|
|
524
|
+
const ind = {
|
|
525
|
+
id: f.id, label: labelOf(f.text), class: FACT_CLASS,
|
|
526
|
+
derived_from: [], mentions: [],
|
|
527
|
+
attributes: [
|
|
528
|
+
{ prop: "rdf:type", key: "type", value: "rdf:Statement" },
|
|
529
|
+
{ prop: "rdf:subject", key: "subject", value: f.s },
|
|
530
|
+
{ prop: "rdf:predicate", key: "predicate", value: f.p },
|
|
531
|
+
{ prop: "rdf:object", key: "object", value: f.o },
|
|
532
|
+
{ prop: CREATED_AT_PROP, key: "createdAt", value: createdAtVal },
|
|
533
|
+
...(provs.length ? [{ prop: "mgx:factProvenance", key: "provenance", value: provs.join(" | ") }] : []),
|
|
534
|
+
...(f.tokens.length ? [{ prop: "mgx:hasProseTokens", key: "prose_tokens", value: f.tokens.join(" ") }] : []),
|
|
535
|
+
],
|
|
536
|
+
};
|
|
537
|
+
// Upsert into BOTH the array (replace-in-place keeps order) and the index.
|
|
538
|
+
if (prior) payload.individuals[payload.individuals.indexOf(prior)] = ind;
|
|
539
|
+
else payload.individuals.push(ind);
|
|
540
|
+
byId.set(f.id, ind);
|
|
541
|
+
ids.push(f.id);
|
|
542
|
+
if (!seen.has(f.id)) { seen.add(f.id); touched.push(f.id); }
|
|
543
|
+
}
|
|
544
|
+
// Reconcile each touched fact's Sources + trust once (add-only, idempotent),
|
|
545
|
+
// then recount classes a SINGLE time at the end.
|
|
546
|
+
for (const id of touched) syncFactSources(payload, byId.get(id));
|
|
547
|
+
recountClasses(payload);
|
|
548
|
+
});
|
|
549
|
+
return { ids, appended: ids.length, skipped };
|
|
550
|
+
}
|
|
551
|
+
|
|
468
552
|
// ---- Chat-facing seams (W4 fact lookup + contradiction) ---------------------
|
|
469
553
|
// The W4 fact-lookup THREADING lives in chat.mjs (NOT here); these pure readers
|
|
470
554
|
// are the seam it calls so the answer layer ranks candidates by relevance ×
|
package/src/memory/inspect.mjs
CHANGED
|
@@ -128,6 +128,31 @@ export function renderMemory({ memory, blocks }, { verbose = false } = {}) {
|
|
|
128
128
|
lines.push("", "blocks — none folded yet (a session folds when it ends).");
|
|
129
129
|
}
|
|
130
130
|
|
|
131
|
+
// ---- explore hooks: real, runnable example queries built from what's actually
|
|
132
|
+
// stored, so /memory is a springboard for drilling in, not just a dump ----
|
|
133
|
+
if (individuals.length) {
|
|
134
|
+
const clean = (t) => typeof t === "string" && /^[a-z][a-z0-9]+(?: [a-z0-9]{2,}){0,2}$/.test(t) && t.length <= 22 && !/^\d+$/.test(t);
|
|
135
|
+
const facts = readFactRows(memory);
|
|
136
|
+
// Rank candidate "what is a X" terms by CATEGORY SIZE (how many facts point at
|
|
137
|
+
// them) so the hooks land on rich, recognisable categories (function, class, …),
|
|
138
|
+
// not a lone ConceptNet oddity.
|
|
139
|
+
const freq = new Map();
|
|
140
|
+
for (const f of facts) if (clean(f.object)) freq.set(f.object, (freq.get(f.object) || 0) + 1);
|
|
141
|
+
const terms = [...freq.entries()]
|
|
142
|
+
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
143
|
+
.map(([t]) => t)
|
|
144
|
+
.slice(0, 3);
|
|
145
|
+
const sample = facts.find((f) => clean(f.subject) && terms.includes(f.object));
|
|
146
|
+
const ex = terms.map((t) => ` what is a ${t}`);
|
|
147
|
+
if (sample) ex.push(` is a ${sample.subject} a ${sample.object}`);
|
|
148
|
+
if (terms[0]) ex.push(` what did i tell you about ${terms[0]}`);
|
|
149
|
+
if (ex.length) {
|
|
150
|
+
lines.push("", "explore — ask any of these (real terms from the store above):");
|
|
151
|
+
lines.push(...ex);
|
|
152
|
+
lines.push(" /memory verbose — the full store · /stats — the code-graph overview");
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
131
156
|
return lines.join("\n");
|
|
132
157
|
}
|
|
133
158
|
|
package/src/server.mjs
CHANGED
|
@@ -49,9 +49,73 @@ import {
|
|
|
49
49
|
} from "./codegraph.mjs";
|
|
50
50
|
import { ask } from "./ask.mjs";
|
|
51
51
|
import { createGraphService } from "./providers/graph-service.mjs";
|
|
52
|
+
// Read-ONLY consumers of the conversational-memory graph (src/memory/core.mjs) — the 500
|
|
53
|
+
// corpus facts live there, NOT in the code-map graph.json every tool below loads. Used by
|
|
54
|
+
// the FALL-THROUGH bridge (below): when the code-map resolves NOTHING for a concept query
|
|
55
|
+
// (/subclasses /describe /members /find), answer from the reified isa-family facts instead
|
|
56
|
+
// of a flat "no entity". Never written here; memory writes stay owned by memory/*.
|
|
57
|
+
import { loadMemory, readFactRows, normFactTerm } from "./memory/core.mjs";
|
|
52
58
|
|
|
53
59
|
const SNIPPET_MAX_LINES = 200;
|
|
54
60
|
|
|
61
|
+
// The reified isa-family predicates a memory Fact carries ("<subject> rdfs:subClassOf
|
|
62
|
+
// <object>" / "rdf:type"): subject IS-A object. Subclasses of X = facts whose OBJECT is X;
|
|
63
|
+
// superclasses of X = facts whose SUBJECT is X. (Matches chat.mjs's ISA_PREDICATES.)
|
|
64
|
+
const ISA_PREDICATES = new Set(["rdfs:subClassOf", "rdf:type"]);
|
|
65
|
+
const MEMORY_LIST_CAP = 40;
|
|
66
|
+
|
|
67
|
+
/** Load the conversational-memory Facts as trust-bearing rows, failure-tolerant (no memory
|
|
68
|
+
* store / unreadable → [], so the tool still returns its honest code-map miss). repoRoot is
|
|
69
|
+
* the dir that CONTAINS .tmct/ (graphFile = <repo>/.tmct/graph.json), which is exactly the
|
|
70
|
+
* `dir` loadMemory joins MEMORY_GRAPH_REL onto. */
|
|
71
|
+
async function memoryFactRows(config) {
|
|
72
|
+
try {
|
|
73
|
+
return readFactRows(await loadMemory(dirname(dirname(config.graphFile))));
|
|
74
|
+
} catch {
|
|
75
|
+
return [];
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** A short provenance receipt for a set of memory rows — distinct source strings, capped. */
|
|
80
|
+
function memoryProvenance(rows) {
|
|
81
|
+
const provs = [...new Set(rows.map((r) => r.provenance).filter(Boolean))];
|
|
82
|
+
if (!provs.length) return "provenance: memory/corpus facts";
|
|
83
|
+
const shown = provs.slice(0, 2).join("; ");
|
|
84
|
+
return `provenance: ${shown}${provs.length > 2 ? `, +${provs.length - 2} more source(s)` : ""}`;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** FALL-THROUGH: subclasses of a concept from the reified isa-family facts (subjects of
|
|
88
|
+
* "<subj> subClassOf <term>"). Null when the term names no such facts (so the caller can
|
|
89
|
+
* keep the honest code-map miss). Provenance is always cited. */
|
|
90
|
+
function renderMemorySubclasses(rows, term) {
|
|
91
|
+
const t = normFactTerm(term);
|
|
92
|
+
const hits = rows.filter((r) => ISA_PREDICATES.has(r.predicate) && r.object === t);
|
|
93
|
+
if (!hits.length) return null;
|
|
94
|
+
const labels = [...new Set(hits.map((r) => r.subject))].sort();
|
|
95
|
+
const shown = labels.slice(0, MEMORY_LIST_CAP);
|
|
96
|
+
const tail = labels.length > MEMORY_LIST_CAP ? `\n …+${labels.length - MEMORY_LIST_CAP} more` : "";
|
|
97
|
+
return `"${term}" is not a code-map entity — answering from memory/corpus facts. ` +
|
|
98
|
+
`${labels.length} known subclass(es):\n ${shown.join("\n ")}${tail}\n(${memoryProvenance(hits)})`;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** FALL-THROUGH: a concept's DEFINITION from the isa-family facts — its superclasses ("is
|
|
102
|
+
* a …") plus a count/sample of its known subclasses. Null when the term names no facts. */
|
|
103
|
+
function renderMemoryDefinition(rows, term) {
|
|
104
|
+
const t = normFactTerm(term);
|
|
105
|
+
const isa = rows.filter((r) => ISA_PREDICATES.has(r.predicate) && (r.subject === t || r.object === t));
|
|
106
|
+
if (!isa.length) return null;
|
|
107
|
+
const supers = [...new Set(isa.filter((r) => r.subject === t).map((r) => r.object))];
|
|
108
|
+
const subs = [...new Set(isa.filter((r) => r.object === t).map((r) => r.subject))].sort();
|
|
109
|
+
const lines = [`"${term}" is not a code-map entity — answering from memory/corpus facts.`];
|
|
110
|
+
if (supers.length) lines.push(`is a: ${supers.slice(0, MEMORY_LIST_CAP).join(", ")}`);
|
|
111
|
+
if (subs.length) {
|
|
112
|
+
const tail = subs.length > MEMORY_LIST_CAP ? `, +${subs.length - MEMORY_LIST_CAP} more` : "";
|
|
113
|
+
lines.push(`known subclasses (${subs.length}): ${subs.slice(0, MEMORY_LIST_CAP).join(", ")}${tail}`);
|
|
114
|
+
}
|
|
115
|
+
lines.push(`(${memoryProvenance(isa)})`);
|
|
116
|
+
return lines.join("\n");
|
|
117
|
+
}
|
|
118
|
+
|
|
55
119
|
// Tiered tool surface: the hot tools carry full descriptions/schemas in this
|
|
56
120
|
// catalog; every COLD tool (describe/members/impact/history/…) is still served
|
|
57
121
|
// by dispatchTool below and is reachable via the CLI `cli <tool>` route +
|
|
@@ -298,8 +362,11 @@ export async function dispatchTool(name, args, { config, source = defaultSource
|
|
|
298
362
|
if (name === "tmct_describe") {
|
|
299
363
|
const symbol = String(args?.symbol || "").trim();
|
|
300
364
|
if (!symbol) throw new ToolError("symbol is required");
|
|
301
|
-
const { match, candidates } =
|
|
302
|
-
return renderDescribe(graph, match, { candidates });
|
|
365
|
+
const { match, candidates } = resolveSymbol(svc.graph, symbol);
|
|
366
|
+
if (match) return renderDescribe(graph, match, { candidates }); // code-map wins when present
|
|
367
|
+
const fb = renderMemoryDefinition(await memoryFactRows(config), symbol);
|
|
368
|
+
if (fb) return fb;
|
|
369
|
+
resolveOrThrow(svc, symbol, "symbol"); // no code-map + no memory fact → the honest miss
|
|
303
370
|
}
|
|
304
371
|
if (name === "tmct_snippet") {
|
|
305
372
|
const symbol = String(args?.symbol || "").trim();
|
|
@@ -347,23 +414,37 @@ export async function dispatchTool(name, args, { config, source = defaultSource
|
|
|
347
414
|
const query = String(args?.query || "").trim();
|
|
348
415
|
const kind = String(args?.kind || "").trim();
|
|
349
416
|
if (!query && !kind) throw new ToolError("query is required");
|
|
350
|
-
|
|
417
|
+
const out = renderSearch(graph, query, {
|
|
351
418
|
kind,
|
|
352
419
|
decorator: String(args?.decorator || "").trim(),
|
|
353
420
|
name: String(args?.name || "").trim(),
|
|
354
421
|
});
|
|
422
|
+
// FALL-THROUGH: a code-map miss ("no module matches …") on a plain concept query still
|
|
423
|
+
// answers from the memory/corpus isa-family facts when the concept is known there.
|
|
424
|
+
if (!kind && /^no module matches/.test(out)) {
|
|
425
|
+
const fb = renderMemoryDefinition(await memoryFactRows(config), query);
|
|
426
|
+
if (fb) return fb;
|
|
427
|
+
}
|
|
428
|
+
return out;
|
|
355
429
|
}
|
|
356
430
|
if (name === "tmct_members") {
|
|
357
431
|
const symbol = String(args?.class || "").trim();
|
|
358
432
|
if (!symbol) throw new ToolError("class is required");
|
|
359
|
-
const { match } =
|
|
360
|
-
return renderMembers(graph, match);
|
|
433
|
+
const { match } = resolveSymbol(svc.graph, symbol);
|
|
434
|
+
if (match) return renderMembers(graph, match); // code-map wins when present
|
|
435
|
+
// a concept's "members" in the corpus sense are its subclasses (its instances).
|
|
436
|
+
const fb = renderMemorySubclasses(await memoryFactRows(config), symbol);
|
|
437
|
+
if (fb) return fb;
|
|
438
|
+
resolveOrThrow(svc, symbol, "class"); // the honest miss
|
|
361
439
|
}
|
|
362
440
|
if (name === "tmct_subclasses") {
|
|
363
441
|
const symbol = String(args?.class || "").trim();
|
|
364
442
|
if (!symbol) throw new ToolError("class is required");
|
|
365
|
-
const { match } =
|
|
366
|
-
return renderSubclasses(graph, match);
|
|
443
|
+
const { match } = resolveSymbol(svc.graph, symbol);
|
|
444
|
+
if (match) return renderSubclasses(graph, match); // code-map wins when present
|
|
445
|
+
const fb = renderMemorySubclasses(await memoryFactRows(config), symbol);
|
|
446
|
+
if (fb) return fb;
|
|
447
|
+
resolveOrThrow(svc, symbol, "class"); // no code-map subclass + no memory fact → honest miss
|
|
367
448
|
}
|
|
368
449
|
if (name === "tmct_architecture") {
|
|
369
450
|
return renderArchitecture(graph, { pkg: String(args?.package || "").trim() });
|
package/src/tui/app.mjs
CHANGED
|
@@ -92,6 +92,10 @@ export function App({ session }) {
|
|
|
92
92
|
const [input, setInput] = useState("");
|
|
93
93
|
const [prompt, setPrompt] = useState(session.promptFor());
|
|
94
94
|
const [busy, setBusy] = useState(false);
|
|
95
|
+
// Command history (up/down arrow recall, readline-style). `history` is oldest→newest;
|
|
96
|
+
// `histCursor` is -1 for the live input, else the offset back from the newest entry.
|
|
97
|
+
const [history, setHistory] = useState([]);
|
|
98
|
+
const [histCursor, setHistCursor] = useState(-1);
|
|
95
99
|
|
|
96
100
|
const submit = async (line) => {
|
|
97
101
|
if (line === "/exit") { exit(); return; }
|
|
@@ -111,14 +115,34 @@ export function App({ session }) {
|
|
|
111
115
|
if (busy) return; // one turn at a time — the engine is deterministic and fast
|
|
112
116
|
const line = String(raw).trim();
|
|
113
117
|
setInput("");
|
|
114
|
-
|
|
118
|
+
setHistCursor(-1); // any submit resets history navigation to the live input
|
|
119
|
+
if (line) {
|
|
120
|
+
// record for up-arrow recall; collapse an immediate duplicate of the last line
|
|
121
|
+
setHistory((h) => (h[h.length - 1] === line ? h : [...h, line]));
|
|
122
|
+
void submit(line);
|
|
123
|
+
}
|
|
115
124
|
};
|
|
116
125
|
|
|
117
126
|
useInput((ch, key) => {
|
|
118
127
|
if (key.return) { trySubmit(input); return; }
|
|
119
128
|
if (key.backspace || key.delete) { setInput((s) => s.slice(0, -1)); return; }
|
|
120
129
|
if (key.ctrl && ch === "u") { setInput(""); return; }
|
|
121
|
-
|
|
130
|
+
// Up/down arrow: recall previous prompts (readline-style), oldest→newest history.
|
|
131
|
+
if (key.upArrow) {
|
|
132
|
+
if (!history.length) return;
|
|
133
|
+
const nc = Math.min(histCursor + 1, history.length - 1);
|
|
134
|
+
setHistCursor(nc);
|
|
135
|
+
setInput(history[history.length - 1 - nc]);
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
if (key.downArrow) {
|
|
139
|
+
if (histCursor <= 0) { setHistCursor(-1); setInput(""); return; } // back to a fresh line
|
|
140
|
+
const nc = histCursor - 1;
|
|
141
|
+
setHistCursor(nc);
|
|
142
|
+
setInput(history[history.length - 1 - nc]);
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
if (key.ctrl || key.meta || key.escape || key.tab || key.leftArrow || key.rightArrow) return;
|
|
122
146
|
if (!ch) return;
|
|
123
147
|
// A PASTED chunk arrives as one multi-char event; a newline inside it means
|
|
124
148
|
// "submit this line" (one line per turn — the readline shell's per-line read).
|