@polycode-projects/the-mechanical-code-talker 1.12.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +244 -48
- package/ROADMAP.md +23 -34
- package/bin/tmct.mjs +107 -71
- package/corpus/LICENSES.json +118 -0
- package/corpus/README.md +17 -13
- package/corpus/conceptnet/README.md +5 -5
- package/corpus/conceptnet/fetch-slice.mjs +1 -1
- package/corpus/conceptnet/filter-dump.mjs +1 -1
- package/corpus/generated/README.md +9 -10
- package/corpus/namenet/README.md +39 -0
- package/corpus/seon/README.md +2 -2
- package/corpus/tier2/generate.mjs +58 -9
- package/corpus/tier2/manifest.json +44 -0
- package/corpus/wordnet/README.md +37 -0
- package/corpus/wordnet/generate.mjs +1 -1
- package/data/templates/constructions/agent-noun-relations.toml +2 -2
- package/data/templates/grammar-rules.toml +1 -1
- package/package.json +13 -22
- package/src/{ask-nlp.mjs → adapters/ask-nlp.mjs} +1 -1
- package/src/{config.mjs → adapters/config.mjs} +1 -1
- package/src/{corpus → adapters/corpus}/conceptnet-map.toml +4 -4
- package/src/{corpus → adapters/corpus}/conceptnet.mjs +3 -3
- package/src/adapters/corpus/construction-banks.mjs +43 -0
- package/src/{corpus → adapters/corpus}/templates.mjs +1 -1
- package/src/{embed.mjs → adapters/embed.mjs} +1 -11
- package/src/{graph-build.mjs → adapters/graph-build.mjs} +6 -6
- package/src/{memory → adapters/memory}/blocks.mjs +2 -2
- package/src/{memory → adapters/memory}/core.mjs +38 -94
- package/src/adapters/prose-tokens.mjs +98 -0
- package/src/{providers → adapters/providers}/bootstrap.mjs +2 -2
- package/src/{providers → adapters/providers}/fixture.mjs +3 -3
- package/src/{providers → adapters/providers}/graph-service.mjs +9 -4
- package/src/{source-slice.mjs → adapters/source-slice.mjs} +2 -2
- package/src/{source.mjs → adapters/source.mjs} +1 -1
- package/src/{toml-config.mjs → adapters/toml-config.mjs} +1 -1
- package/src/{answer-variants.json → domain/answer-variants.json} +1 -1
- package/src/domain/answer-variants.mjs +23 -0
- package/src/{ask-vocab.mjs → domain/ask-vocab.mjs} +37 -5
- package/src/{ask.mjs → domain/ask.mjs} +216 -52
- package/src/{codegraph.mjs → domain/codegraph.mjs} +31 -315
- package/src/{completions → domain/completions}/complete.mjs +16 -10
- package/src/{completions → domain/completions}/graph-adapter.mjs +10 -4
- package/src/{completions → domain/completions}/group.mjs +14 -6
- package/src/{completions → domain/completions}/infer.mjs +57 -39
- package/src/domain/completions/injected.mjs +21 -0
- package/src/{completions → domain/completions}/rank.mjs +15 -8
- package/src/{completions → domain/completions}/search.mjs +4 -2
- package/src/{grammar → domain/grammar}/ace.mjs +3 -3
- package/src/{grammar → domain/grammar}/assert.mjs +12 -8
- package/src/{grammar → domain/grammar}/lexicon-core.json +1 -1
- package/src/{grammar → domain/grammar}/lexicon.mjs +4 -6
- package/src/domain/hash.mjs +147 -0
- package/src/{interpret → domain/interpret}/fuzzy.mjs +42 -4
- package/src/domain/interpret/nlp-registry.mjs +20 -0
- package/src/{interpret → domain/interpret}/normalize.mjs +35 -5
- package/src/{interpret → domain/interpret}/pipeline.mjs +1 -5
- package/src/{interpret → domain/interpret}/strategies/ace.mjs +1 -1
- package/src/{interpret → domain/interpret}/strategies/constructions.mjs +30 -52
- package/src/{interpret → domain/interpret}/strategies/keywords.mjs +48 -26
- package/src/domain/memory/capability.mjs +235 -0
- package/src/domain/memory/fold.mjs +54 -0
- package/src/domain/memory/session-turns.mjs +7 -0
- package/src/{memory → domain/memory}/trust.mjs +48 -0
- package/src/{paraphrase.mjs → domain/paraphrase.mjs} +2 -2
- package/src/{prose.mjs → domain/prose.mjs} +1 -1
- package/src/domain/real-word-collisions.json +1 -0
- package/src/{router → domain/router}/call-validator.mjs +1 -1
- package/src/{router → domain/router}/drive.mjs +34 -25
- package/src/{router → domain/router}/goal-reasoner.mjs +1 -1
- package/src/{router → domain/router}/guardrail.mjs +1 -1
- package/src/{router → domain/router}/planner.mjs +1 -1
- package/src/{router → domain/router}/registry.mjs +5 -5
- package/src/{router → domain/router}/resolver.mjs +16 -13
- package/src/{router → domain/router}/results.mjs +1 -1
- package/src/{router → domain/router}/set-algebra.mjs +1 -1
- package/src/{router → domain/router}/taught.mjs +10 -9
- package/src/{syllogise.mjs → domain/syllogise.mjs} +21 -4
- package/src/domain/vector.mjs +12 -0
- package/src/services/chat-session.mjs +451 -0
- package/src/{chat.mjs → services/chat.mjs} +1209 -684
- package/src/{cli-args.mjs → services/cli-args.mjs} +2 -2
- package/src/services/completions.mjs +55 -0
- package/src/{extensions.mjs → services/extensions.mjs} +7 -7
- package/src/{finish.mjs → services/finish.mjs} +2 -2
- package/src/{memory → services}/fold.mjs +0 -0
- package/src/{import-file.mjs → services/import-file.mjs} +3 -3
- package/src/{index.mjs → services/index.mjs} +21 -12
- package/src/{init.mjs → services/init.mjs} +9 -9
- package/src/{ledger-viz.mjs → services/ledger-viz.mjs} +3 -3
- package/src/{plan-viz.mjs → services/plan-viz.mjs} +98 -28
- package/src/{sentences.mjs → services/sentences.mjs} +1 -1
- package/src/{sessions.mjs → services/sessions.mjs} +4 -5
- package/src/{telemetry.mjs → services/telemetry.mjs} +1 -1
- package/src/{server-http.mjs → surfaces/http/server-http.mjs} +11 -65
- package/src/{tui → surfaces/tui}/app.mjs +3 -3
- package/src/{memory-ask-browser-entry.mjs → surfaces/web/memory-ask-browser-entry.mjs} +5 -5
- package/src/{memory-ask-browser.bundle.js → surfaces/web/memory-ask-browser.bundle.js} +9465 -6366
- package/src/tools/catalog.mjs +29 -0
- package/src/{conformance.mjs → tools/conformance.mjs} +2 -2
- package/src/tools/definitions.mjs +288 -0
- package/src/tools/graph-load.mjs +20 -0
- package/src/tools/handlers/index.mjs +54 -0
- package/src/tools/handlers/kit.mjs +33 -0
- package/src/tools/handlers/tmct-architecture.mjs +7 -0
- package/src/tools/handlers/tmct-ask.mjs +14 -0
- package/src/tools/handlers/tmct-callees.mjs +6 -0
- package/src/tools/handlers/tmct-callers.mjs +6 -0
- package/src/tools/handlers/tmct-calls.mjs +6 -0
- package/src/tools/handlers/tmct-class-history.mjs +6 -0
- package/src/tools/handlers/tmct-cochanges.mjs +6 -0
- package/src/tools/handlers/tmct-context-more.mjs +9 -0
- package/src/tools/handlers/tmct-context.mjs +163 -0
- package/src/tools/handlers/tmct-describe.mjs +15 -0
- package/src/tools/handlers/tmct-exports.mjs +9 -0
- package/src/tools/handlers/tmct-file-history.mjs +6 -0
- package/src/tools/handlers/tmct-history.mjs +6 -0
- package/src/tools/handlers/tmct-impact.mjs +9 -0
- package/src/tools/handlers/tmct-members.mjs +16 -0
- package/src/tools/handlers/tmct-method-history.mjs +6 -0
- package/src/tools/handlers/tmct-search.mjs +22 -0
- package/src/tools/handlers/tmct-signature.mjs +6 -0
- package/src/tools/handlers/tmct-snippet.mjs +37 -0
- package/src/tools/handlers/tmct-subclasses.mjs +16 -0
- package/src/tools/handlers/tmct-tests-for.mjs +6 -0
- package/src/tools/handlers/tmct-untested.mjs +7 -0
- package/src/tools/memory-fallthrough.mjs +65 -0
- package/src/{schema-docs.mjs → tools/schema-docs.mjs} +1 -1
- package/src/tools/server.mjs +61 -0
- package/src/answer-variants.mjs +0 -39
- package/src/hash.mjs +0 -24
- package/src/server.mjs +0 -501
- /package/src/{corpus → adapters/corpus}/unknown-ingest.mjs +0 -0
- /package/src/{graph-merge.mjs → adapters/graph-merge.mjs} +0 -0
- /package/src/{memory → adapters/memory}/inspect.mjs +0 -0
- /package/src/{memory → adapters/memory}/shacl.mjs +0 -0
- /package/src/{prose-nlp.mjs → adapters/prose-nlp.mjs} +0 -0
- /package/src/{repository-interface.mjs → adapters/repository-interface.mjs} +0 -0
- /package/src/{uuid.mjs → adapters/uuid.mjs} +0 -0
- /package/src/{wink-model.mjs → adapters/wink-model.mjs} +0 -0
- /package/src/{completions → domain/completions}/prune.mjs +0 -0
- /package/src/{concept.mjs → domain/concept.mjs} +0 -0
- /package/src/{domain.mjs → domain/domain.mjs} +0 -0
- /package/src/{interpret → domain/interpret}/merge.mjs +0 -0
- /package/src/{interpret → domain/interpret}/strategies/grammar.mjs +0 -0
- /package/src/{interpret → domain/interpret}/strategies/noise-strip.mjs +0 -0
- /package/src/{memory → domain/memory}/bias.mjs +0 -0
- /package/src/{planning.mjs → domain/planning.mjs} +0 -0
- /package/src/{viz-theme.mjs → services/viz-theme.mjs} +0 -0
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
#
|
|
4
4
|
# One row per ConceptNet relation (the canonical closed set of 34 —
|
|
5
5
|
# docs/references/schemas/conceptnet-relations.md is the reference list this
|
|
6
|
-
# table is drift-checked against; test/corpus-conceptnet.test.mjs fails if the
|
|
6
|
+
# table is drift-checked against; test/adapters/corpus-conceptnet.test.mjs fails if the
|
|
7
7
|
# committed slice contains a relation with no row here).
|
|
8
8
|
#
|
|
9
9
|
# Fields:
|
|
@@ -13,11 +13,11 @@
|
|
|
13
13
|
# ace which ACE-OWL pattern it maps to (docs/references/schemas/
|
|
14
14
|
# ace-owl-fragment.md): subClassOf | type | ObjectProperty |
|
|
15
15
|
# someValuesFrom | disjointWith | property | none
|
|
16
|
-
# predicate the predicate URI src/corpus/conceptnet.mjs emits into memory
|
|
16
|
+
# predicate the predicate URI src/adapters/corpus/conceptnet.mjs emits into memory
|
|
17
17
|
# facts (absent when ace = "none" — no fact is emitted)
|
|
18
18
|
# note why, and what a "none" row is still good for
|
|
19
19
|
#
|
|
20
|
-
# Loader contract (src/corpus/conceptnet.mjs): a relation in the slice that is
|
|
20
|
+
# Loader contract (src/adapters/corpus/conceptnet.mjs): a relation in the slice that is
|
|
21
21
|
# MISSING here is an error (drift guard); a row with ace = "none" is a
|
|
22
22
|
# deliberate non-emission, silently skipped by toFacts().
|
|
23
23
|
|
|
@@ -189,7 +189,7 @@ rel = "/r/RelatedTo"
|
|
|
189
189
|
surface = "{start} is related to {end}"
|
|
190
190
|
ace = "ObjectProperty"
|
|
191
191
|
predicate = "mgx:relatedTo"
|
|
192
|
-
note = "weakest, undirected association — re-examined 2026-07-12 (TOO_HARD_AUDIT.md): the surface template above was already fully authored, so the prior 'too vague for an axiom' exclusion was a design call dressed as a technical one, not a real blocker. Emitted at LOWER trust (src/corpus/conceptnet.mjs routes RelatedTo facts through the corpus-weak: provenance prefix -> SOURCE_PRIOR.corpusWeak, memory/trust.mjs) rather than either full-strength or excluded."
|
|
192
|
+
note = "weakest, undirected association — re-examined 2026-07-12 (TOO_HARD_AUDIT.md): the surface template above was already fully authored, so the prior 'too vague for an axiom' exclusion was a design call dressed as a technical one, not a real blocker. Emitted at LOWER trust (src/adapters/corpus/conceptnet.mjs routes RelatedTo facts through the corpus-weak: provenance prefix -> SOURCE_PRIOR.corpusWeak, src/domain/memory/trust.mjs) rather than either full-strength or excluded."
|
|
193
193
|
|
|
194
194
|
[[relation]]
|
|
195
195
|
rel = "/r/Synonym"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// corpus/conceptnet.mjs — the ConceptNet slice loader + memory seeder.
|
|
2
2
|
//
|
|
3
3
|
// loadSlice(path?) stream corpus/conceptnet/slice.jsonl → assertions
|
|
4
|
-
// loadMap(path?) src/corpus/conceptnet-map.toml → Map(rel → row)
|
|
4
|
+
// loadMap(path?) src/adapters/corpus/conceptnet-map.toml → Map(rel → row)
|
|
5
5
|
// toFacts(assertions,map) assertions → appendFact-shaped triples
|
|
6
6
|
// seedMemory(dir, opts) write them into <dir>/.tmct/memory via appendFacts
|
|
7
7
|
//
|
|
@@ -18,9 +18,9 @@ import { dirname, join } from "node:path";
|
|
|
18
18
|
import { parse as parseToml } from "smol-toml";
|
|
19
19
|
import { appendFacts, loadMemory, normFactTerm } from "../memory/core.mjs";
|
|
20
20
|
|
|
21
|
-
const PKG_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
21
|
+
const PKG_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
|
|
22
22
|
export const SLICE_FILE = join(PKG_ROOT, "corpus", "conceptnet", "slice.jsonl");
|
|
23
|
-
export const MAP_FILE = join(PKG_ROOT, "src", "corpus", "conceptnet-map.toml");
|
|
23
|
+
export const MAP_FILE = join(PKG_ROOT, "src", "adapters", "corpus", "conceptnet-map.toml");
|
|
24
24
|
|
|
25
25
|
// The tier-1 curated Software-Engineering ontology (SEON): concepts.jsonl shares
|
|
26
26
|
// ConceptNet's slice shape and loads through the same loadSlice/loadMap/toFacts path.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// corpus/construction-banks.mjs — the filesystem side of the construction-
|
|
2
|
+
// grammar template banks: read data/templates/constructions/*.toml and hand
|
|
3
|
+
// the raw parsed tables to the domain strategy
|
|
4
|
+
// (interpret/strategies/constructions.mjs), which receives them via
|
|
5
|
+
// setConstructionBanks — the strategy itself never touches the filesystem.
|
|
6
|
+
|
|
7
|
+
import { readFileSync, readdirSync } from "node:fs";
|
|
8
|
+
import { fileURLToPath } from "node:url";
|
|
9
|
+
import { join, dirname } from "node:path";
|
|
10
|
+
import { parse as parseToml } from "smol-toml";
|
|
11
|
+
|
|
12
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
13
|
+
/** The construction-bank directory (data, not code) — every *.toml file inside
|
|
14
|
+
* is loaded, in filename order, so a future bank is a new committed file, not
|
|
15
|
+
* an edit to this loader. */
|
|
16
|
+
export const CONSTRUCTIONS_DIR = join(HERE, "..", "..", "..", "data", "templates", "constructions");
|
|
17
|
+
|
|
18
|
+
/** Read every *.toml file in `dir` (sorted, deterministic) and return the raw
|
|
19
|
+
* parsed tables concatenated: {relations:[...], constructions:[...]}. A
|
|
20
|
+
* missing directory or an unparseable file is DEFENSIVE (per-file: a broken
|
|
21
|
+
* file is skipped, not fatal to the others) — callers get whatever validly
|
|
22
|
+
* parsed, never a thrown error from a data-authoring mistake. */
|
|
23
|
+
export function readConstructionFiles(dir = CONSTRUCTIONS_DIR) {
|
|
24
|
+
let files;
|
|
25
|
+
try {
|
|
26
|
+
files = readdirSync(dir).filter((f) => f.endsWith(".toml")).sort();
|
|
27
|
+
} catch {
|
|
28
|
+
return { relations: [], constructions: [] };
|
|
29
|
+
}
|
|
30
|
+
const relations = [];
|
|
31
|
+
const constructions = [];
|
|
32
|
+
for (const file of files) {
|
|
33
|
+
let parsed;
|
|
34
|
+
try {
|
|
35
|
+
parsed = parseToml(readFileSync(join(dir, file), "utf8"));
|
|
36
|
+
} catch {
|
|
37
|
+
continue; // one malformed file never takes the others down
|
|
38
|
+
}
|
|
39
|
+
if (Array.isArray(parsed.relation)) relations.push(...parsed.relation);
|
|
40
|
+
if (Array.isArray(parsed.construction)) constructions.push(...parsed.construction);
|
|
41
|
+
}
|
|
42
|
+
return { relations, constructions };
|
|
43
|
+
}
|
|
@@ -15,7 +15,7 @@ import { readFile } from "node:fs/promises";
|
|
|
15
15
|
import { fileURLToPath } from "node:url";
|
|
16
16
|
import { join, dirname } from "node:path";
|
|
17
17
|
|
|
18
|
-
const PKG_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
18
|
+
const PKG_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
|
|
19
19
|
export const TEMPLATES_FILE = join(PKG_ROOT, "data", "templates", "responses.jsonl");
|
|
20
20
|
export const PHRASEBOOK_FILE = join(PKG_ROOT, "data", "phrasebook", "software-phrases.txt");
|
|
21
21
|
|
|
@@ -24,7 +24,7 @@ const CONFIG_FILE = "config.json";
|
|
|
24
24
|
export function defaultEmbeddingsDir() {
|
|
25
25
|
if (process.env.TMCT_EMBED_DIR) return process.env.TMCT_EMBED_DIR;
|
|
26
26
|
const here = dirname(fileURLToPath(import.meta.url)); // packages/tmct/src
|
|
27
|
-
return join(here, "..", "..", "..", "vendor", "embeddings", "potion-base-8M");
|
|
27
|
+
return join(here, "..", "..", "..", "..", "vendor", "embeddings", "potion-base-8M");
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
// ---- safetensors (hand-rolled: u64le header length + JSON header + raw tensors) -------
|
|
@@ -167,13 +167,3 @@ export function loadEmbedder({ dir = defaultEmbeddingsDir() } = {}) {
|
|
|
167
167
|
LOADED.set(dir, embedder);
|
|
168
168
|
return embedder;
|
|
169
169
|
}
|
|
170
|
-
|
|
171
|
-
/** Cosine similarity. Over L2-normalised vectors this is just the dot product, but the
|
|
172
|
-
* full form is kept so unnormalised test fixtures behave. 0 when either vector is zero. */
|
|
173
|
-
export function cosine(a, b) {
|
|
174
|
-
let dot = 0, na = 0, nb = 0;
|
|
175
|
-
const n = Math.min(a.length, b.length);
|
|
176
|
-
for (let i = 0; i < n; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
|
|
177
|
-
if (na === 0 || nb === 0) return 0;
|
|
178
|
-
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
179
|
-
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
// graph-build.mjs
|
|
1
|
+
// graph-build.mjs — the PURE assembly of the typed `entities` payload from
|
|
2
2
|
// already-parsed module + commit records. No subprocesses, no filesystem, no git:
|
|
3
|
-
// data in, graph out
|
|
3
|
+
// data in, graph out — which is why tests build in-memory graphs through it, and
|
|
4
4
|
// why it is the write-path primitive conversation memory grows on (sessions.mjs
|
|
5
5
|
// folds session records into the same shape).
|
|
6
6
|
//
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
// seon:containsCodeEntity Class -> Method/Attribute (class membership)
|
|
18
18
|
// mgx:subclassOf Class -> Class (inheritance)
|
|
19
19
|
|
|
20
|
-
import { attachProseTokens, buildProseIndex } from "./prose.mjs";
|
|
20
|
+
import { attachProseTokens, buildProseIndex } from "./prose-tokens.mjs";
|
|
21
21
|
|
|
22
22
|
const isTestPath = (p) =>
|
|
23
23
|
p.startsWith("tests/") || /(^|\/)tests?\//.test(p) || /(^|\/)test_[^/]*\.py$/.test(p) || /\.tests(\.|$)/.test(p);
|
|
@@ -287,14 +287,14 @@ export function buildEntities(modules, commits, { generatedAt = "", symbolHistor
|
|
|
287
287
|
const COCHANGE_MIN = 2; // co-occur in ≥ N commits
|
|
288
288
|
const COCHANGE_MAX_COMMIT = 50; // skip sweeping refactors (O(n²) noise)
|
|
289
289
|
const COCHANGE_PER_NODE = 12; // cap neighbours per module
|
|
290
|
-
const pairCount = new Map(); // "a
|
|
290
|
+
const pairCount = new Map(); // "a\0b" (a<b lexical) -> count
|
|
291
291
|
for (const c of commits) {
|
|
292
292
|
const mods = [...new Set((c.files || []).filter((f) => modById.has(f)))];
|
|
293
293
|
if (mods.length < 2 || mods.length > COCHANGE_MAX_COMMIT) continue;
|
|
294
294
|
for (let i = 0; i < mods.length; i += 1) {
|
|
295
295
|
for (let j = i + 1; j < mods.length; j += 1) {
|
|
296
296
|
const [a, b] = mods[i] < mods[j] ? [mods[i], mods[j]] : [mods[j], mods[i]];
|
|
297
|
-
const key = `${a}
|
|
297
|
+
const key = `${a}\0${b}`;
|
|
298
298
|
pairCount.set(key, (pairCount.get(key) || 0) + 1);
|
|
299
299
|
}
|
|
300
300
|
}
|
|
@@ -302,7 +302,7 @@ export function buildEntities(modules, commits, { generatedAt = "", symbolHistor
|
|
|
302
302
|
const cochangeEdges = [];
|
|
303
303
|
const cochangePerNode = new Map();
|
|
304
304
|
for (const [key, n] of [...pairCount.entries()].filter(([, c]) => c >= COCHANGE_MIN).sort((x, y) => y[1] - x[1])) {
|
|
305
|
-
const [a, b] = key.split("
|
|
305
|
+
const [a, b] = key.split("\0");
|
|
306
306
|
if ((cochangePerNode.get(a) || 0) >= COCHANGE_PER_NODE || (cochangePerNode.get(b) || 0) >= COCHANGE_PER_NODE) continue;
|
|
307
307
|
cochangePerNode.set(a, (cochangePerNode.get(a) || 0) + 1);
|
|
308
308
|
cochangePerNode.set(b, (cochangePerNode.get(b) || 0) + 1);
|
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
|
|
7
7
|
import { mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
|
|
8
8
|
import { join } from "node:path";
|
|
9
|
-
import { splitIdentifierWords, tokenizeProse } from "../prose.mjs";
|
|
10
|
-
import { SOURCE_PRIOR } from "
|
|
9
|
+
import { splitIdentifierWords, tokenizeProse } from "../prose-tokens.mjs";
|
|
10
|
+
import { SOURCE_PRIOR } from "../../domain/memory/trust.mjs";
|
|
11
11
|
|
|
12
12
|
// A block inherits its Source's trust (operator 1.0, corpus 0.7); retrieval
|
|
13
13
|
// weights relevance × trust via a bounded factor (0.5 + trust, ~[0.5, 1.5]).
|
|
@@ -7,11 +7,24 @@
|
|
|
7
7
|
|
|
8
8
|
import { mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
|
|
9
9
|
import { dirname, join } from "node:path";
|
|
10
|
-
import { proseTokensFor, buildProseIndex } from "../prose.mjs";
|
|
11
|
-
import { fnv1aHex } from "
|
|
12
|
-
|
|
10
|
+
import { proseTokensFor, buildProseIndex } from "../prose-tokens.mjs";
|
|
11
|
+
import { fnv1aHex, normText, normFactTerm, normFactPredicate, factIdFor, factIdForTriple } from "../../domain/hash.mjs";
|
|
12
|
+
|
|
13
|
+
// Fact identity (normalization + id derivation) lives in hash.mjs — the one
|
|
14
|
+
// content-address contract — and is re-exported here so store consumers keep
|
|
15
|
+
// a single import site for read/write plus identity.
|
|
16
|
+
export { normFactTerm, normFactPredicate, factIdForTriple } from "../../domain/hash.mjs";
|
|
17
|
+
import {
|
|
18
|
+
computeTrust, sessionReliabilityFrom, TRUST_SCORE_PROP, TRUST_INPUTS_PROP,
|
|
19
|
+
CREATED_AT_PROP, UPDATED_AT_PROP, provenanceTagToSource,
|
|
20
|
+
} from "../../domain/memory/trust.mjs";
|
|
21
|
+
|
|
22
|
+
// The createdAt/updatedAt vocabulary and the provenance-tag Source parser live
|
|
23
|
+
// with the trust layer (they are its inputs); re-exported here so store
|
|
24
|
+
// consumers keep one import site.
|
|
25
|
+
export { CREATED_AT_PROP, UPDATED_AT_PROP, provenanceTagToSource } from "../../domain/memory/trust.mjs";
|
|
26
|
+
import { NEG_PREDICATE_PREFIX, negatedPredicate } from "../../domain/memory/capability.mjs";
|
|
13
27
|
import { assertIndividualValid } from "./shacl.mjs";
|
|
14
|
-
import { findActionPath, findReachableSet } from "../planning.mjs";
|
|
15
28
|
|
|
16
29
|
export const MEMORY_DIR_REL = join(".tmct", "memory");
|
|
17
30
|
export const MEMORY_GRAPH_REL = join(MEMORY_DIR_REL, "graph.json");
|
|
@@ -33,11 +46,6 @@ export const IN_REPLY_TO_PROP = "mgx:inReplyTo";
|
|
|
33
46
|
export const DERIVED_FROM_PROP = "mgx:derivedFrom"; // umbrella: Fact → Source|Fact
|
|
34
47
|
export const STATED_BY_PROP = "mgx:statedBy"; // a Source directly asserts a Fact
|
|
35
48
|
export const CANONICALISED_FROM_PROP = "mgx:canonicalisedFrom"; // a canonical Fact ← its raw form
|
|
36
|
-
export const CREATED_AT_PROP = "mgx:createdAt"; // first-write-wins ISO-8601 on every individual
|
|
37
|
-
// For call sites that mutate an individual's own attributes without touching
|
|
38
|
-
// an edge (upsertSession, recomputeFactTrust, recomputeSourceReliability),
|
|
39
|
-
// where codegraph.mjs's derived-updatedAt rule alone can't see the change.
|
|
40
|
-
export const UPDATED_AT_PROP = "mgx:updatedAt";
|
|
41
49
|
export const SOURCE_RELIABILITY_PROP = "mgx:sourceReliability"; // actor-level (session-scoped) trust nudge on a Source, [0.5,1.5]
|
|
42
50
|
|
|
43
51
|
// Bare (session-less) singleton Source ids — fallback for a provenance tag
|
|
@@ -48,7 +56,6 @@ export const TEACH_SOURCE_ID = "src:teach-chat";
|
|
|
48
56
|
|
|
49
57
|
const ROLES = new Set(["visitor", "tmct"]);
|
|
50
58
|
const LABEL_CAP = 48; // utterance/fact labels stay skimmable in renders
|
|
51
|
-
const TEXT_CAP = 2000; // an utterance's stored text (a whole answer fits; a pasted book doesn't)
|
|
52
59
|
|
|
53
60
|
/** The memory graph's vocabulary — documented in-payload exactly like
|
|
54
61
|
* graph-build.mjs documents the code graph's. */
|
|
@@ -85,6 +92,7 @@ const MEMORY_VOCABULARY = [
|
|
|
85
92
|
{ prop: TRUST_INPUTS_PROP, note: "JSON of the inputs the trust score was computed from (source-type multiset, corroboration count, createdAt, recency) — makes the score auditable" },
|
|
86
93
|
{ prop: "mgx:hasProseTokens", note: "prose tokens (prose.mjs tokenizer) backing the payload's proseIndex" },
|
|
87
94
|
{ prop: "mgx:sessionStarted", note: "session anchor: when the session started, ISO-8601" },
|
|
95
|
+
{ prop: "rdf:predicate", prefix: NEG_PREDICATE_PREFIX, note: "a reified fact's predicate carries its POLARITY: mgxneg:capableOf is the negative twin of mgx:capableOf ('a penguin cannot fly'). Polarity cannot be a separate property — the fact id hashes (subject, predicate, object), so both polarities would share one id and union their statedBy edges (memory/capability.mjs)" },
|
|
88
96
|
];
|
|
89
97
|
|
|
90
98
|
/** A fresh, empty memory payload — the buildEntities shape, plus the OWL/RDF
|
|
@@ -99,6 +107,7 @@ export function emptyMemory() {
|
|
|
99
107
|
rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
|
100
108
|
rdfs: "http://www.w3.org/2000/01/rdf-schema#",
|
|
101
109
|
mgx: "urn:tmct:mgx#",
|
|
110
|
+
mgxneg: "urn:tmct:mgxneg#",
|
|
102
111
|
},
|
|
103
112
|
vocabulary: MEMORY_VOCABULARY.map((v) => ({ ...v })),
|
|
104
113
|
classes: [],
|
|
@@ -580,7 +589,6 @@ async function mutateMemory(dir, fn) {
|
|
|
580
589
|
return out;
|
|
581
590
|
}
|
|
582
591
|
|
|
583
|
-
const normText = (t) => String(t ?? "").replace(/\s+/g, " ").trim().slice(0, TEXT_CAP);
|
|
584
592
|
const labelOf = (text) => (text.length > LABEL_CAP ? text.slice(0, LABEL_CAP - 1) + "…" : text);
|
|
585
593
|
const nowIso = () => new Date().toISOString();
|
|
586
594
|
|
|
@@ -644,48 +652,6 @@ function upsertSource(payload, desc, createdAtCandidate) {
|
|
|
644
652
|
return info.id;
|
|
645
653
|
}
|
|
646
654
|
|
|
647
|
-
/** Parse the "chat" shape both provenanceTag and teachProvenanceTag emit —
|
|
648
|
-
* `<source>[:<sessionId>][@<ts>]` — into { createdAt, sessionId? }. */
|
|
649
|
-
function parseChatTagRest(rest) {
|
|
650
|
-
const at = rest.indexOf("@");
|
|
651
|
-
const beforeAt = at >= 0 ? rest.slice(0, at) : rest;
|
|
652
|
-
const createdAt = at >= 0 ? rest.slice(at + 1) : "";
|
|
653
|
-
const colon = beforeAt.indexOf(":");
|
|
654
|
-
const sessionId = colon >= 0 ? beforeAt.slice(colon + 1) : "";
|
|
655
|
-
return { createdAt, ...(sessionId ? { sessionId } : {}) };
|
|
656
|
-
}
|
|
657
|
-
|
|
658
|
-
/**
|
|
659
|
-
* Parse one legacy provenance TAG into a Source descriptor over the closed
|
|
660
|
-
* kind set:
|
|
661
|
-
* corpus:conceptnet /r/IsA -> { kind:"corpus", name:"conceptnet" }
|
|
662
|
-
* corpus-weak:conceptnet /r/RelatedTo -> { kind:"corpusWeak", name:"conceptnet" }
|
|
663
|
-
* ace:chat:<session>@<ts> -> { kind:"operator", createdAt:<ts>, sessionId:<session> }
|
|
664
|
-
* teach:chat:<session>@<ts> -> { kind:"teach", createdAt:<ts>, sessionId:<session> }
|
|
665
|
-
* web:<url> | url:<url> -> { kind:"web", url:<url> }
|
|
666
|
-
* extracted:<file-basename> -> { kind:"extracted", name:<file-basename> }
|
|
667
|
-
* entailed:<rule> -> { kind:"entailed", rule:<rule> }
|
|
668
|
-
* chat:/session: refs map to the operator; an unknown tag -> null (no Source).
|
|
669
|
-
*/
|
|
670
|
-
export function provenanceTagToSource(tag) {
|
|
671
|
-
const t = String(tag || "").trim();
|
|
672
|
-
if (!t) return null;
|
|
673
|
-
const head = t.split(/\s+/)[0]; // drop trailing " /r/IsA" etc.
|
|
674
|
-
if (head.startsWith("corpus-weak:")) return { kind: "corpusWeak", name: head.slice("corpus-weak:".length) || "unknown" };
|
|
675
|
-
if (head.startsWith("corpus:")) return { kind: "corpus", name: head.slice("corpus:".length) || "unknown" };
|
|
676
|
-
if (head.startsWith("ace:")) return { kind: "operator", ...parseChatTagRest(head.slice("ace:".length)) };
|
|
677
|
-
if (head.startsWith("teach:")) {
|
|
678
|
-
// the chat teach lane's natural frames — chat.mjs's teachProvenanceTag
|
|
679
|
-
return { kind: "teach", ...parseChatTagRest(head.slice("teach:".length)) };
|
|
680
|
-
}
|
|
681
|
-
if (head.startsWith("web:")) return { kind: "web", url: head.slice("web:".length) };
|
|
682
|
-
if (head.startsWith("url:")) return { kind: "web", url: head.slice("url:".length) };
|
|
683
|
-
if (head.startsWith("extracted:")) return { kind: "extracted", name: head.slice("extracted:".length) || "unknown" };
|
|
684
|
-
if (head.startsWith("entailed:")) return { kind: "entailed", rule: head.slice("entailed:".length) };
|
|
685
|
-
if (head.startsWith("chat:") || head.startsWith("session:") || head.startsWith("operator")) return { kind: "operator" };
|
|
686
|
-
return null;
|
|
687
|
-
}
|
|
688
|
-
|
|
689
655
|
/** Map a payload's Source individuals into the { id: Source } shape computeTrust
|
|
690
656
|
* resolves against. */
|
|
691
657
|
function sourcesByIdMap(payload) {
|
|
@@ -846,7 +812,7 @@ function upsertIndividual(payload, ind) {
|
|
|
846
812
|
* on the edge, first-write-wins over the same (subject,object) pair — mirrors
|
|
847
813
|
* `firstWriteCreatedAt`'s discipline: a re-upserted edge keeps its original creation time
|
|
848
814
|
* rather than resetting to "now" on every write. This is the only place in the codebase edges
|
|
849
|
-
* get a timestamp at all
|
|
815
|
+
* get a timestamp at all. */
|
|
850
816
|
function upsertEdge(payload, { predicate, prop }, edge) {
|
|
851
817
|
let group = payload.objectProperties.find((g) => g?.prop === prop);
|
|
852
818
|
if (!group) {
|
|
@@ -986,33 +952,6 @@ export async function appendUtterances(dir, utterances) {
|
|
|
986
952
|
return { ids };
|
|
987
953
|
}
|
|
988
954
|
|
|
989
|
-
/** Normalize a fact TERM (subject/object) so every writer converges on one
|
|
990
|
-
* spelling: ConceptNet's /c/en/foo_bar, tmct:Foo_bar, and bare "Foo bar" all
|
|
991
|
-
* become "foo bar". Also strips a leading "the"/"a"/"an" (idempotent — safe
|
|
992
|
-
* for storage too). The predicate is never normalized this way — its casing
|
|
993
|
-
* is meaningful controlled vocabulary. */
|
|
994
|
-
export function normFactTerm(t) {
|
|
995
|
-
let s = normText(t);
|
|
996
|
-
s = s.replace(/^\/c\/[a-z]{2,3}\//i, "");
|
|
997
|
-
s = s.replace(/^[a-z][\w.-]*:/i, "");
|
|
998
|
-
s = s.replace(/_/g, " ").replace(/\s+/g, " ").trim();
|
|
999
|
-
s = s.replace(/^(?:the|an?)\s+/i, "");
|
|
1000
|
-
return s.toLowerCase();
|
|
1001
|
-
}
|
|
1002
|
-
|
|
1003
|
-
// A Fact is content-addressed by its NUL-delimited (s, p, o) — NUL never
|
|
1004
|
-
// occurs in a normalized term/predicate, so it's collision-proof unlike a
|
|
1005
|
-
// space. appendFact hashes the same `${s}\0${p}\0${o}` inline; keep both in sync.
|
|
1006
|
-
const factIdFor = (s, p, o) => `fact:${fnv1aHex(`${s}\0${p}\0${o}`)}`;
|
|
1007
|
-
|
|
1008
|
-
/** Content-address a fact's id from (subject, predicate, object) without
|
|
1009
|
-
* writing it — same contract as factIdFor. Lets a caller (e.g.
|
|
1010
|
-
* syllogise.mjs's retraction machinery) name a not-yet-written fact's id
|
|
1011
|
-
* deterministically, without an extra read. Pure, no I/O. */
|
|
1012
|
-
export function factIdForTriple(subject, predicate, object) {
|
|
1013
|
-
return factIdFor(normFactTerm(subject), normText(predicate), normFactTerm(object));
|
|
1014
|
-
}
|
|
1015
|
-
|
|
1016
955
|
/** Append one grammar-derived OWL triple, RDF-reified as a `Fact` individual.
|
|
1017
956
|
* Same (s,p,o) -> same id -> upsert, never a duplicate. `premiseTrusts`/
|
|
1018
957
|
* `ruleConfidence` optionally engage trust.mjs's entailed hook. Validated
|
|
@@ -1020,10 +959,10 @@ export function factIdForTriple(subject, predicate, object) {
|
|
|
1020
959
|
* Returns { id }. */
|
|
1021
960
|
export async function appendFact(dir, { subject, predicate, object, provenance = "", createdAt = "", quantifier = "", premiseTrusts, ruleConfidence } = {}) {
|
|
1022
961
|
const s = normFactTerm(subject);
|
|
1023
|
-
const p =
|
|
962
|
+
const p = normFactPredicate(predicate);
|
|
1024
963
|
const o = normFactTerm(object);
|
|
1025
964
|
if (!s || !p || !o) throw new Error("a fact needs subject, predicate and object");
|
|
1026
|
-
const id =
|
|
965
|
+
const id = factIdFor(s, p, o);
|
|
1027
966
|
const text = `${s} ${p} ${o}`;
|
|
1028
967
|
const tokens = proseTokensFor({ doc: text });
|
|
1029
968
|
const q = normText(quantifier);
|
|
@@ -1074,7 +1013,7 @@ export async function appendFacts(dir, facts) {
|
|
|
1074
1013
|
let skipped = 0;
|
|
1075
1014
|
for (const f of facts || []) {
|
|
1076
1015
|
const s = normFactTerm(f?.subject);
|
|
1077
|
-
const p =
|
|
1016
|
+
const p = normFactPredicate(f?.predicate);
|
|
1078
1017
|
const o = normFactTerm(f?.object);
|
|
1079
1018
|
if (!s || !p || !o) { skipped += 1; continue; } // batch skips, never throws
|
|
1080
1019
|
const text = `${s} ${p} ${o}`;
|
|
@@ -1206,7 +1145,7 @@ const RULE_SLOT_SPEC = {
|
|
|
1206
1145
|
["objectRole", "mgx:ruleActionEffectObject"],
|
|
1207
1146
|
],
|
|
1208
1147
|
// "the <left> may not be with the <right> without the <guard>" — each slot
|
|
1209
|
-
// names a class whose sole member src/domain.mjs resolves at compile time.
|
|
1148
|
+
// names a class whose sole member src/domain/domain.mjs resolves at compile time.
|
|
1210
1149
|
[RULE_KIND_ACTION_CONSTRAINT]: [
|
|
1211
1150
|
["left", "mgx:ruleActionConstraintLeft"], ["right", "mgx:ruleActionConstraintRight"],
|
|
1212
1151
|
["guard", "mgx:ruleActionConstraintGuard"],
|
|
@@ -1217,7 +1156,7 @@ const RULE_SLOT_SPEC = {
|
|
|
1217
1156
|
// mirroring factIdFor's NUL-delimited discipline: identical rules upsert,
|
|
1218
1157
|
// different ones coexist. For 2-slot kinds the joined string is byte-identical
|
|
1219
1158
|
// to the historical (kind, name, slot1, slot2) template, so pre-existing rule
|
|
1220
|
-
// ids never change (pinned by test/memory-rules-action.test.mjs).
|
|
1159
|
+
// ids never change (pinned by test/adapters/memory-rules-action.test.mjs).
|
|
1221
1160
|
const ruleIdFor = (kind, name, slotValues) => `rule:${fnv1aHex([kind, name, ...slotValues].join("\0"))}`;
|
|
1222
1161
|
|
|
1223
1162
|
/** Append one taught RULE — a sibling of appendFact storing a `Rule`
|
|
@@ -1290,7 +1229,7 @@ export function findRulesByName(memory, name) {
|
|
|
1290
1229
|
}
|
|
1291
1230
|
|
|
1292
1231
|
/** Every taught Rule as a plain row {id, name, kind, slots, provenance} —
|
|
1293
|
-
* the sibling of readFactRows, so consumers (src/domain.mjs) never touch
|
|
1232
|
+
* the sibling of readFactRows, so consumers (src/domain/domain.mjs) never touch
|
|
1294
1233
|
* raw individuals. Rules whose kind has no RULE_SLOT_SPEC entry are
|
|
1295
1234
|
* skipped (unreadable without a slot contract). Sorted by name, kind, id. */
|
|
1296
1235
|
export function readRuleRows(memory) {
|
|
@@ -1322,9 +1261,10 @@ export function readRuleRows(memory) {
|
|
|
1322
1261
|
// importable functions taking an already-loaded `memory` (a loadMemory()
|
|
1323
1262
|
// payload — callers load it once, not per recursive call) and a `helpers`
|
|
1324
1263
|
// bag (`relationFactsFor`, `renderFactLine`, `factPhrase`, `factTermVariants`,
|
|
1325
|
-
// `byTrust`, the trust-bearing `rows` array,
|
|
1326
|
-
//
|
|
1327
|
-
//
|
|
1264
|
+
// `byTrust`, the trust-bearing `rows` array, `HAS_PROPERTY_PREDICATE`, and
|
|
1265
|
+
// the search kernels `findActionPath`/`findReachableSet` from planning.mjs —
|
|
1266
|
+
// injected so this store module never imports the domain layer), so callers
|
|
1267
|
+
// outside chat.mjs's own dispatch context can reuse the same resolution logic.
|
|
1328
1268
|
//
|
|
1329
1269
|
// Dispatch order: direct/alias fact hit → compose2 rule chase → filter rule
|
|
1330
1270
|
// chase → honest miss (OWA discipline: null / [] on a miss, never a guessed
|
|
@@ -1337,7 +1277,7 @@ export function readRuleRows(memory) {
|
|
|
1337
1277
|
* Rule chase. Returns `{ citation: string[] }` on a hit, null on an honest miss.
|
|
1338
1278
|
*/
|
|
1339
1279
|
export async function resolveRelationChase(memory, name, subjectTerm, objectTerm, helpers) {
|
|
1340
|
-
const { relationFactsFor, renderFactLine, factPhrase, factTermVariants, byTrust, rows, HAS_PROPERTY_PREDICATE } = helpers;
|
|
1280
|
+
const { relationFactsFor, renderFactLine, factPhrase, factTermVariants, byTrust, rows, HAS_PROPERTY_PREDICATE, findActionPath } = helpers;
|
|
1341
1281
|
const target = String(name || "").trim().toLowerCase();
|
|
1342
1282
|
// (i)+(ii): direct hit or alias-chased hit for this exact (subject, object)
|
|
1343
1283
|
// pair under the queried relation name.
|
|
@@ -1413,7 +1353,7 @@ export async function resolveRelationChase(memory, name, subjectTerm, objectTerm
|
|
|
1413
1353
|
* `{ subject, citation }` pair that satisfies it, instead of one yes/no.
|
|
1414
1354
|
*/
|
|
1415
1355
|
export async function resolveRelationChaseReverse(memory, name, objectTerm, helpers) {
|
|
1416
|
-
const { relationFactsFor, renderFactLine, factPhrase, factTermVariants, byTrust, rows, HAS_PROPERTY_PREDICATE } = helpers;
|
|
1356
|
+
const { relationFactsFor, renderFactLine, factPhrase, factTermVariants, byTrust, rows, HAS_PROPERTY_PREDICATE, findReachableSet } = helpers;
|
|
1417
1357
|
const target = String(name || "").trim().toLowerCase();
|
|
1418
1358
|
const ov = factTermVariants(normFactTerm, objectTerm);
|
|
1419
1359
|
// (i)+(ii): every direct/alias-chased fact under this name whose object
|
|
@@ -1582,8 +1522,12 @@ export const CAPABLE_OF_PREDICATE = "mgx:capableOf";
|
|
|
1582
1522
|
/** Predicates whose real-world semantics allow many objects at once ("a dog
|
|
1583
1523
|
* has legs" AND "a dog has a tail"; "a bird can fly" AND "a bird can sing"),
|
|
1584
1524
|
* so a second object is a second fact, never a disagreement. A closed list:
|
|
1585
|
-
* every predicate outside it keeps the full contradiction contract.
|
|
1586
|
-
|
|
1525
|
+
* every predicate outside it keeps the full contradiction contract. Each
|
|
1526
|
+
* entry's negative twin joins it — "a penguin cannot fly" and "a penguin
|
|
1527
|
+
* cannot sing" are two claims, not a self-contradiction. */
|
|
1528
|
+
export const MULTI_VALUED_PREDICATES = new Set(
|
|
1529
|
+
[HAS_A_PREDICATE, CAPABLE_OF_PREDICATE].flatMap((p) => [p, negatedPredicate(p)]),
|
|
1530
|
+
);
|
|
1587
1531
|
|
|
1588
1532
|
/** Facts that CONTRADICT: same (subject, predicate), different object, each
|
|
1589
1533
|
* above the trust floor. Returns groups (trust-desc) so callers surface both,
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
// prose-tokens.mjs — the adapters layer's own tokenizer, shared by every
|
|
2
|
+
// adapter that writes prose tokens (the memory store, graph-build). Adapters
|
|
3
|
+
// may not import the domain layer, while prose.mjs (the graph/ask side's
|
|
4
|
+
// canonical tokenizer) is domain and may not import adapters — so this layer
|
|
5
|
+
// carries its own copy of the primitives it stores tokens with. The two copies
|
|
6
|
+
// must stay byte-identical: the parity suite in
|
|
7
|
+
// test/adapters/prose-tokens.test.mjs pins every function here to its
|
|
8
|
+
// prose.mjs twin, so a change to either side fails loudly until both move
|
|
9
|
+
// together.
|
|
10
|
+
|
|
11
|
+
const STOPWORDS = new Set(
|
|
12
|
+
("a an and or but the of to in on at for with from by as is are was were be been being " +
|
|
13
|
+
"it its this that these those i you he she they we me my your our do does did not no " +
|
|
14
|
+
"yes if then else than so such can will would should could may might about into over " +
|
|
15
|
+
"under out up down off again more most some any all what which who whom whose when " +
|
|
16
|
+
"where why how").split(/\s+/),
|
|
17
|
+
);
|
|
18
|
+
|
|
19
|
+
const MAX_TOKEN_LEN = 40; // drops hash-like/garbage tokens
|
|
20
|
+
const MAX_TOKENS_PER_DOC = 120; // bounds cost on a pathologically long docstring/name
|
|
21
|
+
|
|
22
|
+
/** Split an identifier or a path-like name into lowercase word tokens.
|
|
23
|
+
* Handles camelCase, PascalCase, snake_case, kebab-case, dotted names, path
|
|
24
|
+
* separators, and acronym runs ("HTTPSConnection" -> https/connection,
|
|
25
|
+
* "parseXML" -> parse/xml). Filters single-character tokens (loop-variable noise). */
|
|
26
|
+
export function splitIdentifierWords(raw) {
|
|
27
|
+
if (!raw) return [];
|
|
28
|
+
let s = String(raw).replace(/\.[A-Za-z0-9]+$/, ""); // strip a trailing file extension only
|
|
29
|
+
s = s
|
|
30
|
+
.replace(/[/\\]/g, " ") // path separators
|
|
31
|
+
.replace(/([a-z0-9])([A-Z])/g, "$1 $2") // camelCase / word|Digit boundary
|
|
32
|
+
.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") // acronym run -> TitleCase (HTTPSConnection)
|
|
33
|
+
.replace(/([A-Za-z])([0-9])/g, "$1 $2")
|
|
34
|
+
.replace(/([0-9])([A-Za-z])/g, "$1 $2")
|
|
35
|
+
.replace(/[_\-.]+/g, " ");
|
|
36
|
+
return s.split(/\s+/).map((w) => w.toLowerCase()).filter((w) => w.length > 1 && w.length <= MAX_TOKEN_LEN);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Tokenize free prose (a docstring/doc-comment) — lowercase words, punctuation stripped,
|
|
40
|
+
* common stopwords and single-/over-length tokens dropped, capped at MAX_TOKENS_PER_DOC. */
|
|
41
|
+
export function tokenizeProse(text) {
|
|
42
|
+
if (!text) return [];
|
|
43
|
+
const out = [];
|
|
44
|
+
const seen = new Set();
|
|
45
|
+
for (const raw of String(text).toLowerCase().split(/[^a-z0-9]+/)) {
|
|
46
|
+
if (raw.length < 2 || raw.length > MAX_TOKEN_LEN || STOPWORDS.has(raw)) continue;
|
|
47
|
+
if (seen.has(raw)) continue;
|
|
48
|
+
seen.add(raw);
|
|
49
|
+
out.push(raw);
|
|
50
|
+
if (out.length >= MAX_TOKENS_PER_DOC) break;
|
|
51
|
+
}
|
|
52
|
+
return out;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** The combined, deduped, sorted token set for one individual: its (decomposed) name
|
|
56
|
+
* plus any captured doc text. Returns [] if there's nothing to index (never null). */
|
|
57
|
+
export function proseTokensFor({ name, doc } = {}) {
|
|
58
|
+
const set = new Set([...splitIdentifierWords(name), ...tokenizeProse(doc)]);
|
|
59
|
+
return [...set].sort();
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Attach a `prose_tokens` attribute to every individual, from its (decomposed)
|
|
63
|
+
* name and captured doc text — except Commit, whose `label` is a truncated
|
|
64
|
+
* SHA, not a decomposable identifier: it tokenizes `message` instead. Mutates
|
|
65
|
+
* and returns the same array; `enabled=false` is a no-op. */
|
|
66
|
+
export function attachProseTokens(individuals, { enabled = true } = {}) {
|
|
67
|
+
if (!enabled) return individuals;
|
|
68
|
+
for (const ind of individuals) {
|
|
69
|
+
const attrs = ind.attributes || [];
|
|
70
|
+
const isCommit = ind.class === "Commit";
|
|
71
|
+
const name = isCommit ? null : ind.label;
|
|
72
|
+
const doc = isCommit
|
|
73
|
+
? attrs.find((a) => a.key === "message")?.value
|
|
74
|
+
: attrs.find((a) => a.key === "doc")?.value;
|
|
75
|
+
const tokens = proseTokensFor({ name, doc });
|
|
76
|
+
if (tokens.length) {
|
|
77
|
+
ind.attributes = [...(ind.attributes || []), { prop: "mgx:hasProseTokens", key: "prose_tokens", value: tokens.join(" ") }];
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return individuals;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Build the inverted index (word -> sorted, deduped [individual ids]) from individuals
|
|
84
|
+
* that already carry a `prose_tokens` attribute. Plain object, JSON-serializable —
|
|
85
|
+
* this is what lands as the payload's `proseIndex`. */
|
|
86
|
+
export function buildProseIndex(individuals) {
|
|
87
|
+
const index = Object.create(null);
|
|
88
|
+
for (const ind of individuals) {
|
|
89
|
+
const tokAttr = (ind.attributes || []).find((a) => a.key === "prose_tokens");
|
|
90
|
+
if (!tokAttr?.value) continue;
|
|
91
|
+
for (const word of tokAttr.value.split(" ")) {
|
|
92
|
+
if (!index[word]) index[word] = [];
|
|
93
|
+
index[word].push(ind.id);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
for (const word of Object.keys(index)) index[word].sort();
|
|
97
|
+
return index;
|
|
98
|
+
}
|
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
// "contains" before anything is indexed.
|
|
3
3
|
//
|
|
4
4
|
// It implements every Repository-Interface service over the empty bootstrap
|
|
5
|
-
// payload (src/source.mjs emptyEntities): every id-taking service returns
|
|
5
|
+
// payload (src/adapters/source.mjs emptyEntities): every id-taking service returns
|
|
6
6
|
// miss(UNRESOLVED_TERM) — there are no individuals — and every aggregate returns
|
|
7
7
|
// an honest empty (stats.total = 0, untested.modules = [], …). Nothing throws.
|
|
8
8
|
// This is the other end of the compatibility kit: the provider that has no data
|
|
9
9
|
// must still CONFORM.
|
|
10
10
|
|
|
11
|
-
import { parseEntities } from "
|
|
11
|
+
import { parseEntities } from "../../domain/codegraph.mjs";
|
|
12
12
|
import { emptyEntities } from "../source.mjs";
|
|
13
13
|
import { createGraphService } from "./graph-service.mjs";
|
|
14
14
|
|
|
@@ -3,13 +3,13 @@
|
|
|
3
3
|
//
|
|
4
4
|
// It is a degenerate provider in the sense that its graph is tiny and its source
|
|
5
5
|
// bodies are absent (snippet/context answer NO_SOURCE) — but every OTHER service
|
|
6
|
-
// returns real graph truth. The contract suite (test/repository-interface.test.mjs)
|
|
6
|
+
// returns real graph truth. The contract suite (test/adapters/repository-interface.test.mjs)
|
|
7
7
|
// runs the whole compatibility kit against it.
|
|
8
8
|
//
|
|
9
9
|
// The payload is embedded (not read from test/) so this ships as a runnable spec
|
|
10
10
|
// inside the library. Its shape is exactly a parseEntities() input.
|
|
11
11
|
|
|
12
|
-
import { parseEntities } from "
|
|
12
|
+
import { parseEntities } from "../../domain/codegraph.mjs";
|
|
13
13
|
import { createGraphService } from "./graph-service.mjs";
|
|
14
14
|
|
|
15
15
|
/** A compact but type-complete entities payload: modules, a class hierarchy
|
|
@@ -114,7 +114,7 @@ export function fixtureGraph() {
|
|
|
114
114
|
/** The fixture provider: a Repository-Interface service over the small real graph.
|
|
115
115
|
* `opts` passes straight through to createGraphService — e.g. `{ sourceAccess: true,
|
|
116
116
|
* repoRoot, readFile }` to construct a source-capable fixture provider for testing
|
|
117
|
-
* (see test/repository-interface.test.mjs's third runConformance call, which is the
|
|
117
|
+
* (see test/adapters/repository-interface.test.mjs's third runConformance call, which is the
|
|
118
118
|
* only thing that exercises the conformance kit's source-capable branch). */
|
|
119
119
|
export function fixtureProvider(opts = {}) {
|
|
120
120
|
return createGraphService(fixtureGraph(), opts);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// The reference Repository-Interface service over a parsed code graph.
|
|
2
2
|
//
|
|
3
3
|
// createGraphService(graph) returns a typed service object implementing EVERY
|
|
4
|
-
// service in src/repository-interface.mjs over the `{ individuals, byId,
|
|
4
|
+
// service in src/adapters/repository-interface.mjs over the `{ individuals, byId,
|
|
5
5
|
// relations, … }` shape parseEntities() yields. Every method returns a typed
|
|
6
6
|
// Result (hit/miss) — a clean miss is a value, never a throw. The two providers
|
|
7
7
|
// tmct ships (fixture, bootstrap) are this same builder over a small real graph
|
|
@@ -25,9 +25,8 @@ import {
|
|
|
25
25
|
sizeBundle,
|
|
26
26
|
bundleMask,
|
|
27
27
|
renderGraphOnlyBundle,
|
|
28
|
-
} from "
|
|
28
|
+
} from "../../domain/codegraph.mjs";
|
|
29
29
|
import { readSpanSafe, sliceSpan } from "../source-slice.mjs";
|
|
30
|
-
import { ask } from "../ask.mjs";
|
|
31
30
|
import {
|
|
32
31
|
hit,
|
|
33
32
|
miss,
|
|
@@ -126,9 +125,14 @@ async function renderSourceBodies(plan, mask, { readFile, repoRoot }) {
|
|
|
126
125
|
* @param {object|null} [opts.tel] an optional telemetry sink ({ record(fields) }). When
|
|
127
126
|
* present, every service is wrapped once to time it and record counts only, never raw
|
|
128
127
|
* text/body.
|
|
128
|
+
* @param {Function} [opts.ask] the natural-language answerer backing the `ask` service
|
|
129
|
+
* (src/domain/ask.mjs's `ask` in the live wiring) — injected at construction like fs access,
|
|
130
|
+
* never an ambient import. Constructing without it makes `.ask()` an honest
|
|
131
|
+
* CAPABILITY_ABSENT miss, the same negotiation snippet/context use for missing
|
|
132
|
+
* source access.
|
|
129
133
|
* @returns the typed service object
|
|
130
134
|
*/
|
|
131
|
-
export function createGraphService(graph, { sourceAccess = false, repoRoot = null, readFile = null, tel = null } = {}) {
|
|
135
|
+
export function createGraphService(graph, { sourceAccess = false, repoRoot = null, readFile = null, tel = null, ask = null } = {}) {
|
|
132
136
|
const byId = graph.byId;
|
|
133
137
|
if (sourceAccess && (!repoRoot || typeof readFile !== "function")) {
|
|
134
138
|
throw new TypeError(
|
|
@@ -401,6 +405,7 @@ export function createGraphService(graph, { sourceAccess = false, repoRoot = nul
|
|
|
401
405
|
},
|
|
402
406
|
|
|
403
407
|
ask(query) {
|
|
408
|
+
if (typeof ask !== "function") return miss(MISS_REASONS.CAPABILITY_ABSENT, "this provider was constructed without an ask answerer");
|
|
404
409
|
const { content, tmct_ask } = ask(graph, String(query || ""));
|
|
405
410
|
return hit({ content, tmct_ask });
|
|
406
411
|
},
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
// Shared, safe source-span slicing for the tool layer (src/server.mjs) and the
|
|
2
|
-
// source-capable Repository Interface provider (src/providers/graph-service.mjs).
|
|
1
|
+
// Shared, safe source-span slicing for the tool layer (src/tools/server.mjs) and the
|
|
2
|
+
// source-capable Repository Interface provider (src/adapters/providers/graph-service.mjs).
|
|
3
3
|
//
|
|
4
4
|
// Two halves:
|
|
5
5
|
// - sliceSpan — PURE. Given an in-memory `lines` array, extracts + line-numbers
|