@polycode-projects/the-mechanical-code-talker 2.0.3 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/ROADMAP.md +27 -4
- package/corpus/generated/ace-surface-variants.jsonl +1 -0
- package/corpus/generated/manifest.json +3 -3
- package/package.json +1 -1
- package/src/adapters/toml-config.mjs +0 -1
- package/src/domain/codegraph.mjs +4 -68
- package/src/domain/markdown-links.mjs +55 -0
- package/src/surfaces/web/memory-ask-browser.bundle.js +3 -76
- package/src/adapters/embed.mjs +0 -169
- package/src/domain/vector.mjs +0 -12
package/README.md
CHANGED
|
@@ -180,7 +180,7 @@ resolves to a real graph traversal or declines honestly:
|
|
|
180
180
|
(*because/although/while*), conditionals, and false-premise flags ("why
|
|
181
181
|
does X still import Y" when it no longer does).
|
|
182
182
|
|
|
183
|
-
The full catalog with measured coverage lives in `
|
|
183
|
+
The full catalog with measured coverage lives in `CAPABILITIES_2.0.3.md` and
|
|
184
184
|
the `BENCHMARK_*.md` reports.
|
|
185
185
|
|
|
186
186
|
**Response finishing.** Before an answer prints, it is segmented into typed
|
package/ROADMAP.md
CHANGED
|
@@ -100,7 +100,30 @@ blending web-sourced facts with graph/operator facts.
|
|
|
100
100
|
|
|
101
101
|
## Design docs
|
|
102
102
|
|
|
103
|
-
Every substantial design lives in its own `PLAN_*.md` at the repo root
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
103
|
+
Every substantial design lives in its own `PLAN_*.md` at the repo root; `archive/` holds the shipped
|
|
104
|
+
and closed ones. This file points to them, it doesn't repeat their content. Each plan states its own
|
|
105
|
+
status in its opening lines — read it there, because a status quoted here would rot.
|
|
106
|
+
|
|
107
|
+
| Plan | What it's for |
|
|
108
|
+
| --- | --- |
|
|
109
|
+
| [PLAN_ADVENTURE.md](PLAN_ADVENTURE.md) | a text adventure as an architectural stretch: imperative command grammar, NPC turn scheduler, room-look digest |
|
|
110
|
+
| [PLAN_AGENTS.md](PLAN_AGENTS.md) | the governing plan for the multi-repo arc (marginalia, seonix, a pluggable LLM rung), with its own phase sequencing |
|
|
111
|
+
| [PLAN_CHILD_CORPUS.md](PLAN_CHILD_CORPUS.md) | a wider default seed corpus, chosen by age of acquisition |
|
|
112
|
+
| [PLAN_CLASS_QUERY.md](PLAN_CLASS_QUERY.md) | "list/count all X of class Y", reconciled against what already shipped |
|
|
113
|
+
| [PLAN_CODE.md](PLAN_CODE.md) | program synthesis over tmct's closed DSLs, plus JS/HTML/CSS fragments and goal-directed program repair |
|
|
114
|
+
| [PLAN_CONSISTENCY_CHECK.md](PLAN_CONSISTENCY_CHECK.md) | tmct as a consistency service for an LLM tool loop |
|
|
115
|
+
| [PLAN_EMBEDDINGS.md](PLAN_EMBEDDINGS.md) | the semantic-similarity axis, and the way back to it |
|
|
116
|
+
| [PLAN_GRAPH_SCAN.md](PLAN_GRAPH_SCAN.md) | seed and query cost at `init:xl`/`init:xxl` corpus scale |
|
|
117
|
+
| [PLAN_GUESS_NUMBER.md](PLAN_GUESS_NUMBER.md) | closed-loop planning over hidden state, via belief-interval bisection |
|
|
118
|
+
| [PLAN_MUD.md](PLAN_MUD.md) | persistent, shared tmct worlds over a `server:` memory backend |
|
|
119
|
+
| [PLAN_NLU_BENCHMARKS.md](PLAN_NLU_BENCHMARKS.md) | scoring tmct on the CLINC150 and HWU64 intent sets |
|
|
120
|
+
| [PLAN_OPEN_ITEMS.md](PLAN_OPEN_ITEMS.md) | the build order closing the backlog `HANDOVER.md` carries |
|
|
121
|
+
| [PLAN_PARAPHRASE_VERIFICATION.md](PLAN_PARAPHRASE_VERIFICATION.md) | checking a paraphrase against the graph before it prints |
|
|
122
|
+
| [PLAN_PURGE.md](PLAN_PURGE.md) | promote the load-bearing code, delete the dead weight |
|
|
123
|
+
| [PLAN_REPO_INDEX.md](PLAN_REPO_INDEX.md) | tmct grows its own code parsers, ported from seonix |
|
|
124
|
+
| [PLAN_SYLLOGIST.md](PLAN_SYLLOGIST.md) | the reasoning engine's incrementality and retraction horizon |
|
|
125
|
+
| [PLAN_SYLLOGIST_EL_DL.md](PLAN_SYLLOGIST_EL_DL.md) | beyond OWL 2 RL: an EL classifier, then a DL tableau prover |
|
|
126
|
+
|
|
127
|
+
`SKILL_*.md` docs specify the repeatable measurement and build cycles (the benchmarks, the capability
|
|
128
|
+
audit, the background strategy advisor, plain-prose writing). `HANDOVER.md` is the single
|
|
129
|
+
current-open-items list.
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
{"kind":"rescue","sentence":"Code search is a retrieval task","rescued":"Code hunt is a retrieval task","from":"search","to":"hunt","pos":"noun","synsetId":"00947217-n","sourceFile":"PLAN_EMBEDDINGS.md","provenance":"wordnet:00947217-n"}
|
|
1
2
|
{"kind":"variant","seed":"the ace of trumps is a sure winner","generated":"the one of trumps is a sure winner","from":"ace","to":"one","pos":"noun","synsetId":"13764713-n","sourceCorpus":"corpus/tier2/human-examples-large.jsonl","provenance":"wordnet:13764713-n"}
|
|
2
3
|
{"kind":"variant","seed":"the ace of trumps is a sure winner","generated":"the single of trumps is a sure winner","from":"ace","to":"single","pos":"noun","synsetId":"13764713-n","sourceCorpus":"corpus/tier2/human-examples-large.jsonl","provenance":"wordnet:13764713-n"}
|
|
3
4
|
{"kind":"variant","seed":"the ace of trumps is a sure winner","generated":"the unity of trumps is a sure winner","from":"ace","to":"unity","pos":"noun","synsetId":"13764713-n","sourceCorpus":"corpus/tier2/human-examples-large.jsonl","provenance":"wordnet:13764713-n"}
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
"version": 1,
|
|
3
3
|
"generated": "by scripts/generate-template-variants.mjs",
|
|
4
4
|
"file": "ace-surface-variants.jsonl",
|
|
5
|
-
"rows":
|
|
6
|
-
"bytes":
|
|
7
|
-
"sha256": "
|
|
5
|
+
"rows": 18,
|
|
6
|
+
"bytes": 5012,
|
|
7
|
+
"sha256": "28fae5a3353f5dd1df365b4407e0ac2a5c66b98b232d8bf454fe3573f2c16760",
|
|
8
8
|
"license": "CC-BY-4.0 (WordNet-derived synonym substitutions of Open English WordNet / SemCor example sentences and this repo's own MPL-2.0 docs prose — see corpus/generated/README.md)"
|
|
9
9
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@polycode-projects/the-mechanical-code-talker",
|
|
3
|
-
"version": "2.0
|
|
3
|
+
"version": "2.2.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "The Mechanical Code Talker (tmct) — a tolerant, offline, $0 chat surface that guides you toward precision queries about a software repository. ELIZA/PARRY-style but domain-obsessed with code. No model calls; no codebase index of its own.",
|
|
@@ -139,7 +139,6 @@ export async function normalizeConfig(raw, { configDir } = {}) {
|
|
|
139
139
|
if (t.impl_of_interface !== undefined) tune.implOfInterface = t.impl_of_interface;
|
|
140
140
|
if (t.beam_search !== undefined) tune.beamSearch = t.beam_search;
|
|
141
141
|
if (t.beam_width !== undefined) tune.beamWidth = t.beam_width;
|
|
142
|
-
if (t.embed_rank !== undefined) tune.embedRank = t.embed_rank;
|
|
143
142
|
if (t.prose_layers !== undefined) tune.proseLayers = t.prose_layers;
|
|
144
143
|
const exp = t.expansion || {};
|
|
145
144
|
const expansion = {};
|
package/src/domain/codegraph.mjs
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { lookupByProseTokens, proseLayerHits, splitIdentifierWords } from "./prose.mjs";
|
|
2
|
-
import { cosine } from "./vector.mjs";
|
|
3
2
|
import { CREATED_AT_PROP, UPDATED_AT_PROP } from "./memory/trust.mjs";
|
|
4
3
|
|
|
5
4
|
// Pure (no-network, no-fs) query logic over the typed `entities` payload that the
|
|
@@ -511,15 +510,6 @@ const LIT_COMP_CAP = 4;
|
|
|
511
510
|
const LIT_FRAC = 1.0;
|
|
512
511
|
const LIT_CAP_FRAC = 0.9;
|
|
513
512
|
|
|
514
|
-
// opt-in via embedRank + an injected embedder: static-embedding re-rank over
|
|
515
|
-
// path/symbol/doc text read from the graph (never source), cached per-process in EMB_CACHE
|
|
516
|
-
const EMB_FRAC = 0.2;
|
|
517
|
-
const EMB_CAP_FRAC = 0.35;
|
|
518
|
-
const EMB_TEXT_SYMBOL_CAP = 64;
|
|
519
|
-
const EMB_TEXT_DOC_CAP = 12;
|
|
520
|
-
const EMB_CACHE = new WeakMap(); // graph -> { embedder, texts, vecs: Map<moduleId, Float32Array> }
|
|
521
|
-
let embedWarned = false;
|
|
522
|
-
|
|
523
513
|
// opt-in via beamSearch: multi-ply adaptive expansion of the proximity nudge above.
|
|
524
514
|
// Beam width is a margin relative to each ply's best score (not a fixed count), so a
|
|
525
515
|
// weak-then-strong candidate isn't prematurely discarded. Successors are generated
|
|
@@ -544,30 +534,6 @@ const SPIRAL_HOP_DECAY = 0.6;
|
|
|
544
534
|
const SPIRAL_PROX_FRAC = 0.2;
|
|
545
535
|
const SPIRAL_PROX_CAP_FRAC = 0.35;
|
|
546
536
|
|
|
547
|
-
/** embedRank: per-module embeddable text from path components + defined symbol
|
|
548
|
-
* names + doc first-lines, cached alongside the vectors in EMB_CACHE. */
|
|
549
|
-
function moduleEmbedTexts(graph) {
|
|
550
|
-
const texts = new Map(); // moduleId -> text
|
|
551
|
-
const defIdx = definesIndex(graph);
|
|
552
|
-
const docs = new Map(); // moduleId -> [doc first-lines]
|
|
553
|
-
for (const ind of graph.individuals) {
|
|
554
|
-
const doc = (ind.attributes || []).find((a) => a.key === "doc")?.value;
|
|
555
|
-
if (!doc) continue;
|
|
556
|
-
const modId = (ind.class || "") === "Module" ? ind.id : moduleIdOf(graph, ind);
|
|
557
|
-
if (!modId) continue;
|
|
558
|
-
let arr = docs.get(modId);
|
|
559
|
-
if (!arr) docs.set(modId, (arr = []));
|
|
560
|
-
if (arr.length < EMB_TEXT_DOC_CAP) arr.push(String(doc).split("\n")[0]);
|
|
561
|
-
}
|
|
562
|
-
for (const ind of graph.individuals) {
|
|
563
|
-
if ((ind.class || "") !== "Module") continue;
|
|
564
|
-
const parts = String(ind.label).split(/[^a-zA-Z0-9_]+/).filter(Boolean);
|
|
565
|
-
const syms = (defIdx.get(ind.id) || []).slice(0, EMB_TEXT_SYMBOL_CAP);
|
|
566
|
-
texts.set(ind.id, [...parts, ...syms, ...(docs.get(ind.id) || [])].join(" "));
|
|
567
|
-
}
|
|
568
|
-
return texts;
|
|
569
|
-
}
|
|
570
|
-
|
|
571
537
|
/** Split a lowercased path label into boundary components: django/utils/text.py →
|
|
572
538
|
* {django,utils,text,py}. Component equality (not substring) stops "text" matching "ci<text>". */
|
|
573
539
|
function pathComponents(labelLc) {
|
|
@@ -773,7 +739,7 @@ export function spiralExpand(graph, scored = [], {
|
|
|
773
739
|
* IDF-weights each query token, scores path/symbol/exact-symbol matches, and
|
|
774
740
|
* re-ranks with a bounded import-proximity bonus. Pure; deterministic. */
|
|
775
741
|
function scoreModules(graph, tokens, opts = {}) {
|
|
776
|
-
const { demoteNonProd = false, callAdjacency = false, implOfInterface = false, beamSearch = false, spiral = false, proseBoost = false, proseLayers = false, literalMention = false,
|
|
742
|
+
const { demoteNonProd = false, callAdjacency = false, implOfInterface = false, beamSearch = false, spiral = false, proseBoost = false, proseLayers = false, literalMention = false, rawQuery = "" } = opts;
|
|
777
743
|
const beamWidth = Number.isFinite(opts.beamWidth) && opts.beamWidth > 0 ? opts.beamWidth : 8;
|
|
778
744
|
const defIdx = definesIndex(graph);
|
|
779
745
|
// Precompute each module's path components + defined-symbol exact/component sets, once.
|
|
@@ -967,36 +933,6 @@ function scoreModules(graph, tokens, opts = {}) {
|
|
|
967
933
|
s.score += Math.min(signal * PROSE_LAYER_FRAC, s.score * PROSE_LAYER_CAP_FRAC);
|
|
968
934
|
}
|
|
969
935
|
}
|
|
970
|
-
// embedRank: the embedder is injected so this module stays fs-free; absent -> a one-time
|
|
971
|
-
// stderr note, never a failure
|
|
972
|
-
if (embedRank) {
|
|
973
|
-
if (!opts.embedder) {
|
|
974
|
-
if (!embedWarned) {
|
|
975
|
-
embedWarned = true;
|
|
976
|
-
process.stderr.write("tmct: embedRank requested but no embedder available (weights not fetched? see `npm run refs:embeddings`) — flag is a no-op\n");
|
|
977
|
-
}
|
|
978
|
-
} else if (scored.length) {
|
|
979
|
-
const embedder = opts.embedder;
|
|
980
|
-
let cache = EMB_CACHE.get(graph);
|
|
981
|
-
if (!cache || cache.embedder !== embedder) {
|
|
982
|
-
cache = { embedder, texts: moduleEmbedTexts(graph), vecs: new Map() };
|
|
983
|
-
EMB_CACHE.set(graph, cache);
|
|
984
|
-
}
|
|
985
|
-
const qv = embedder.embed(rawQuery || tokens.join(" "));
|
|
986
|
-
let maxBase = 0;
|
|
987
|
-
for (const s of scored) maxBase = Math.max(maxBase, s.score);
|
|
988
|
-
for (const s of scored) {
|
|
989
|
-
let v = cache.vecs.get(s.ind.id);
|
|
990
|
-
if (!v) {
|
|
991
|
-
v = embedder.embed(cache.texts.get(s.ind.id) || String(s.ind.label));
|
|
992
|
-
cache.vecs.set(s.ind.id, v);
|
|
993
|
-
}
|
|
994
|
-
const sim = Math.max(0, cosine(qv, v)); // negative similarity never penalises
|
|
995
|
-
if (!sim) continue;
|
|
996
|
-
s.score += Math.min(sim * maxBase * EMB_FRAC, s.score * EMB_CAP_FRAC);
|
|
997
|
-
}
|
|
998
|
-
}
|
|
999
|
-
}
|
|
1000
936
|
// beamSearch (opt-in): multi-ply generalization of the single-hop families above.
|
|
1001
937
|
if (beamSearch && scored.length > 1) beamExpand(graph, scored, beamWidth);
|
|
1002
938
|
// SPIRAL (opt-in): bounded-radius ego walk that may introduce lexically-invisible modules — runs
|
|
@@ -1024,9 +960,9 @@ export function searchModulesRanked(graph, query, opts = {}) {
|
|
|
1024
960
|
const tokens = raw.toLowerCase().split(/[^a-z0-9_]+/).filter(Boolean);
|
|
1025
961
|
if (!tokens.length) return [];
|
|
1026
962
|
// literalMention needs the query BEFORE tokenization (the tokenizer destroys the dotted refs
|
|
1027
|
-
// it matches on)
|
|
1028
|
-
//
|
|
1029
|
-
const effOpts =
|
|
963
|
+
// it matches on); threaded only when a flag that consumes it is on, so the OFF path is
|
|
964
|
+
// provably unchanged.
|
|
965
|
+
const effOpts = opts.literalMention ? { ...opts, rawQuery: raw } : opts;
|
|
1030
966
|
return scoreModules(graph, tokens, effOpts).map((s) => ({ path: String(s.ind.label), score: s.score }));
|
|
1031
967
|
}
|
|
1032
968
|
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
// markdown-links.mjs — pull the relative link targets out of a markdown string.
|
|
2
|
+
// Pure: a string in, `[{ target, line }]` out, no filesystem and no imports, so
|
|
3
|
+
// the CI jobs that run without `npm ci` can reach it.
|
|
4
|
+
//
|
|
5
|
+
// Code is not prose. A doc that writes `[text](target)` inside backticks is
|
|
6
|
+
// showing you what a link looks like, not making one, and a checker that cannot
|
|
7
|
+
// tell the difference reports the example as a broken link to a file named
|
|
8
|
+
// "target". So the spans are blanked before the link patterns run — blanked
|
|
9
|
+
// rather than cut, because every offset behind them still has to name the right
|
|
10
|
+
// line number.
|
|
11
|
+
|
|
12
|
+
const blank = (line) => line.replace(/[^\n]/g, " ");
|
|
13
|
+
const FENCE = /^[ \t]{0,3}(`{3,}|~{3,})/;
|
|
14
|
+
|
|
15
|
+
/** Replace every fenced block and inline code span with spaces, keeping the
|
|
16
|
+
* string's length and its newlines so later offsets still map to their line.
|
|
17
|
+
* Fences are walked line by line: a lazy multiline regex stops at the end of
|
|
18
|
+
* the opening fence's own line and blanks only the markers. */
|
|
19
|
+
export function blankCodeSpans(markdown) {
|
|
20
|
+
let fence = null;
|
|
21
|
+
const lines = markdown.split("\n").map((line) => {
|
|
22
|
+
const marker = line.match(FENCE)?.[1];
|
|
23
|
+
if (fence) {
|
|
24
|
+
if (marker && marker[0] === fence[0] && marker.length >= fence.length) fence = null;
|
|
25
|
+
return blank(line);
|
|
26
|
+
}
|
|
27
|
+
if (marker) fence = marker;
|
|
28
|
+
return marker ? blank(line) : line;
|
|
29
|
+
});
|
|
30
|
+
return lines.join("\n").replace(/(`+)[\s\S]*?\1/g, blank);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// Inline links/images: [text](target "title") — target ends at the first
|
|
34
|
+
// whitespace or closing paren. Reference definitions: [label]: target.
|
|
35
|
+
const INLINE_LINK = /!?\[[^\]]*\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
|
|
36
|
+
const REFERENCE_DEF = /^\s{0,3}\[[^\]]+\]:\s+(\S+)/gm;
|
|
37
|
+
|
|
38
|
+
/** Every repo-relative link target in `markdown`, with the 1-based line it sits
|
|
39
|
+
* on. External URLs, bare #anchors and absolute paths are out of scope. */
|
|
40
|
+
export function relativeTargets(markdown) {
|
|
41
|
+
const prose = blankCodeSpans(markdown);
|
|
42
|
+
const targets = [];
|
|
43
|
+
for (const regex of [INLINE_LINK, REFERENCE_DEF]) {
|
|
44
|
+
for (const match of prose.matchAll(regex)) {
|
|
45
|
+
let target = match[1];
|
|
46
|
+
if (/^(https?|mailto|ftp):/i.test(target)) continue;
|
|
47
|
+
if (target.startsWith("#") || target.startsWith("/") || target.startsWith("<")) continue;
|
|
48
|
+
target = decodeURIComponent(target.split("#")[0].split("?")[0]);
|
|
49
|
+
if (!target) continue;
|
|
50
|
+
const line = prose.slice(0, match.index).split("\n").length;
|
|
51
|
+
targets.push({ target, line });
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return targets;
|
|
55
|
+
}
|
|
@@ -936,23 +936,6 @@
|
|
|
936
936
|
}
|
|
937
937
|
});
|
|
938
938
|
|
|
939
|
-
// src/domain/vector.mjs
|
|
940
|
-
function cosine(a, b) {
|
|
941
|
-
let dot = 0, na = 0, nb = 0;
|
|
942
|
-
const n = Math.min(a.length, b.length);
|
|
943
|
-
for (let i = 0; i < n; i++) {
|
|
944
|
-
dot += a[i] * b[i];
|
|
945
|
-
na += a[i] * a[i];
|
|
946
|
-
nb += b[i] * b[i];
|
|
947
|
-
}
|
|
948
|
-
if (na === 0 || nb === 0) return 0;
|
|
949
|
-
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
950
|
-
}
|
|
951
|
-
var init_vector = __esm({
|
|
952
|
-
"src/domain/vector.mjs"() {
|
|
953
|
-
}
|
|
954
|
-
});
|
|
955
|
-
|
|
956
939
|
// src/domain/memory/trust.mjs
|
|
957
940
|
function parseChatTagRest(rest) {
|
|
958
941
|
const at = rest.indexOf("@");
|
|
@@ -1307,27 +1290,6 @@
|
|
|
1307
1290
|
}
|
|
1308
1291
|
return idx;
|
|
1309
1292
|
}
|
|
1310
|
-
function moduleEmbedTexts(graph) {
|
|
1311
|
-
const texts = /* @__PURE__ */ new Map();
|
|
1312
|
-
const defIdx = definesIndex(graph);
|
|
1313
|
-
const docs = /* @__PURE__ */ new Map();
|
|
1314
|
-
for (const ind of graph.individuals) {
|
|
1315
|
-
const doc = (ind.attributes || []).find((a) => a.key === "doc")?.value;
|
|
1316
|
-
if (!doc) continue;
|
|
1317
|
-
const modId = (ind.class || "") === "Module" ? ind.id : moduleIdOf(graph, ind);
|
|
1318
|
-
if (!modId) continue;
|
|
1319
|
-
let arr = docs.get(modId);
|
|
1320
|
-
if (!arr) docs.set(modId, arr = []);
|
|
1321
|
-
if (arr.length < EMB_TEXT_DOC_CAP) arr.push(String(doc).split("\n")[0]);
|
|
1322
|
-
}
|
|
1323
|
-
for (const ind of graph.individuals) {
|
|
1324
|
-
if ((ind.class || "") !== "Module") continue;
|
|
1325
|
-
const parts = String(ind.label).split(/[^a-zA-Z0-9_]+/).filter(Boolean);
|
|
1326
|
-
const syms = (defIdx.get(ind.id) || []).slice(0, EMB_TEXT_SYMBOL_CAP);
|
|
1327
|
-
texts.set(ind.id, [...parts, ...syms, ...docs.get(ind.id) || []].join(" "));
|
|
1328
|
-
}
|
|
1329
|
-
return texts;
|
|
1330
|
-
}
|
|
1331
1293
|
function identComponents(name) {
|
|
1332
1294
|
return new Set(String(name).replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().split(/[^a-z0-9]+/).filter(Boolean));
|
|
1333
1295
|
}
|
|
@@ -1499,7 +1461,7 @@
|
|
|
1499
1461
|
return results;
|
|
1500
1462
|
}
|
|
1501
1463
|
function scoreModules(graph, tokens, opts = {}) {
|
|
1502
|
-
const { demoteNonProd = false, callAdjacency = false, implOfInterface = false, beamSearch = false, spiral = false, proseBoost = false, proseLayers = false, literalMention = false,
|
|
1464
|
+
const { demoteNonProd = false, callAdjacency = false, implOfInterface = false, beamSearch = false, spiral = false, proseBoost = false, proseLayers = false, literalMention = false, rawQuery = "" } = opts;
|
|
1503
1465
|
const beamWidth = Number.isFinite(opts.beamWidth) && opts.beamWidth > 0 ? opts.beamWidth : 8;
|
|
1504
1466
|
const defIdx = definesIndex(graph);
|
|
1505
1467
|
const modules = [];
|
|
@@ -1691,34 +1653,6 @@
|
|
|
1691
1653
|
s.score += Math.min(signal * PROSE_LAYER_FRAC, s.score * PROSE_LAYER_CAP_FRAC);
|
|
1692
1654
|
}
|
|
1693
1655
|
}
|
|
1694
|
-
if (embedRank) {
|
|
1695
|
-
if (!opts.embedder) {
|
|
1696
|
-
if (!embedWarned) {
|
|
1697
|
-
embedWarned = true;
|
|
1698
|
-
process.stderr.write("tmct: embedRank requested but no embedder available (weights not fetched? see `npm run refs:embeddings`) \u2014 flag is a no-op\n");
|
|
1699
|
-
}
|
|
1700
|
-
} else if (scored.length) {
|
|
1701
|
-
const embedder = opts.embedder;
|
|
1702
|
-
let cache2 = EMB_CACHE.get(graph);
|
|
1703
|
-
if (!cache2 || cache2.embedder !== embedder) {
|
|
1704
|
-
cache2 = { embedder, texts: moduleEmbedTexts(graph), vecs: /* @__PURE__ */ new Map() };
|
|
1705
|
-
EMB_CACHE.set(graph, cache2);
|
|
1706
|
-
}
|
|
1707
|
-
const qv = embedder.embed(rawQuery || tokens.join(" "));
|
|
1708
|
-
let maxBase = 0;
|
|
1709
|
-
for (const s of scored) maxBase = Math.max(maxBase, s.score);
|
|
1710
|
-
for (const s of scored) {
|
|
1711
|
-
let v = cache2.vecs.get(s.ind.id);
|
|
1712
|
-
if (!v) {
|
|
1713
|
-
v = embedder.embed(cache2.texts.get(s.ind.id) || String(s.ind.label));
|
|
1714
|
-
cache2.vecs.set(s.ind.id, v);
|
|
1715
|
-
}
|
|
1716
|
-
const sim = Math.max(0, cosine(qv, v));
|
|
1717
|
-
if (!sim) continue;
|
|
1718
|
-
s.score += Math.min(sim * maxBase * EMB_FRAC, s.score * EMB_CAP_FRAC);
|
|
1719
|
-
}
|
|
1720
|
-
}
|
|
1721
|
-
}
|
|
1722
1656
|
if (beamSearch && scored.length > 1) beamExpand(graph, scored, beamWidth);
|
|
1723
1657
|
if (spiral && scored.length) spiralExpand(graph, scored, {
|
|
1724
1658
|
depth: Number.isFinite(opts.spiralDepth) && opts.spiralDepth > 0 ? opts.spiralDepth : SPIRAL_DEPTH_DEFAULT,
|
|
@@ -1732,7 +1666,7 @@
|
|
|
1732
1666
|
const raw = String(query || "");
|
|
1733
1667
|
const tokens = raw.toLowerCase().split(/[^a-z0-9_]+/).filter(Boolean);
|
|
1734
1668
|
if (!tokens.length) return [];
|
|
1735
|
-
const effOpts = opts.literalMention
|
|
1669
|
+
const effOpts = opts.literalMention ? { ...opts, rawQuery: raw } : opts;
|
|
1736
1670
|
return scoreModules(graph, tokens, effOpts).map((s) => ({ path: String(s.ind.label), score: s.score }));
|
|
1737
1671
|
}
|
|
1738
1672
|
function renderSearch(graph, query, { limit = SEARCH_LIMIT, kind = "", decorator = "", name = "" } = {}) {
|
|
@@ -2414,11 +2348,10 @@ ${shown.join("\n")}${tail}`;
|
|
|
2414
2348
|
${list.join("\n ")}` + (edges.length > EXPORTS_CAP ? `
|
|
2415
2349
|
\u2026+${edges.length - EXPORTS_CAP} more` : "");
|
|
2416
2350
|
}
|
|
2417
|
-
var PROP_KIND, isProvRef, DESCRIBE_EDGE_CAP, PROV_CAP, IMPACT_DEPTHS_LISTED, IMPACT_PER_DEPTH, IMPACT_TESTS_PER_DEP, SEARCH_LIMIT, SEARCH_SYMBOLS_SHOWN, PATH_W, SYM_W, EXACT_W, SYM_MATCH_CAP, PROX_FRAC, PROX_CAP_FRAC, isTestLabel, NONPROD_DEMOTE, isNonProdLabel, CALL_PROX_FRAC, CALL_PROX_CAP_FRAC, IMPL_PROX_FRAC, IMPL_PROX_CAP_FRAC, isCsModuleLabel, looksLikeCsInterface, PROSE_PROX_FRAC, PROSE_PROX_CAP_FRAC, PROSE_LOOKUP_LIMIT, PROSE_LAYER_FRAC, PROSE_LAYER_CAP_FRAC, PROSE_LAYER_DISCOUNT, LIT_W, LIT_MIN_COMPONENTS, LIT_COMP_CAP, LIT_FRAC, LIT_CAP_FRAC,
|
|
2351
|
+
var PROP_KIND, isProvRef, DESCRIBE_EDGE_CAP, PROV_CAP, IMPACT_DEPTHS_LISTED, IMPACT_PER_DEPTH, IMPACT_TESTS_PER_DEP, SEARCH_LIMIT, SEARCH_SYMBOLS_SHOWN, PATH_W, SYM_W, EXACT_W, SYM_MATCH_CAP, PROX_FRAC, PROX_CAP_FRAC, isTestLabel, NONPROD_DEMOTE, isNonProdLabel, CALL_PROX_FRAC, CALL_PROX_CAP_FRAC, IMPL_PROX_FRAC, IMPL_PROX_CAP_FRAC, isCsModuleLabel, looksLikeCsInterface, PROSE_PROX_FRAC, PROSE_PROX_CAP_FRAC, PROSE_LOOKUP_LIMIT, PROSE_LAYER_FRAC, PROSE_LAYER_CAP_FRAC, PROSE_LAYER_DISCOUNT, LIT_W, LIT_MIN_COMPONENTS, LIT_COMP_CAP, LIT_FRAC, LIT_CAP_FRAC, BEAM_MARGIN_FRAC, BEAM_PROX_FRAC, BEAM_PROX_CAP_FRAC, BEAM_OVERFLOW_CAP, BEAM_PLIES, BEAM_EDGE_GROUPS, SPIRAL_DEPTH_DEFAULT, SPIRAL_NODE_LIMIT_DEFAULT, SPIRAL_Q_DEFAULT, SPIRAL_EXPAND_KINDS, SPIRAL_EMIT_FRAC, SPIRAL_HOP_DECAY, SPIRAL_PROX_FRAC, SPIRAL_PROX_CAP_FRAC, edgesOfKindCache, MEMBERS_CAP, SUBCLASS_CAP, CALL_CAP, attrVal, ARCH_PKG_CAP, ARCH_HUB_CAP, COVERAGE_CAP, TESTS_GRAIN_NOTE, HISTORY_CAP, CALL_SYMBOL_CLASSES, CALL_HINT_CAP, SYMBOL_CLASSES, CONTEXT_SIBLING_CAP, CLASS_MEMBER_CAP, COCHANGE_MID_CAP, CONTEXT_TESTS_CAP, TINY_MAX_LOC, TINY_MAX_ARITY, LARGE_CLASS_MEMBERS, INLINE_CALLEE_CAP, splitDecs, tokenize, countParams, modeOf, COCHANGE_CAP, EXPORTS_CAP;
|
|
2418
2352
|
var init_codegraph = __esm({
|
|
2419
2353
|
"src/domain/codegraph.mjs"() {
|
|
2420
2354
|
init_prose();
|
|
2421
|
-
init_vector();
|
|
2422
2355
|
init_trust();
|
|
2423
2356
|
PROP_KIND = {
|
|
2424
2357
|
// v2.0 faithful tokens (SEON-faithful realign)
|
|
@@ -2484,12 +2417,6 @@ ${shown.join("\n")}${tail}`;
|
|
|
2484
2417
|
LIT_COMP_CAP = 4;
|
|
2485
2418
|
LIT_FRAC = 1;
|
|
2486
2419
|
LIT_CAP_FRAC = 0.9;
|
|
2487
|
-
EMB_FRAC = 0.2;
|
|
2488
|
-
EMB_CAP_FRAC = 0.35;
|
|
2489
|
-
EMB_TEXT_SYMBOL_CAP = 64;
|
|
2490
|
-
EMB_TEXT_DOC_CAP = 12;
|
|
2491
|
-
EMB_CACHE = /* @__PURE__ */ new WeakMap();
|
|
2492
|
-
embedWarned = false;
|
|
2493
2420
|
BEAM_MARGIN_FRAC = 0.5;
|
|
2494
2421
|
BEAM_PROX_FRAC = 0.2;
|
|
2495
2422
|
BEAM_PROX_CAP_FRAC = 0.35;
|
package/src/adapters/embed.mjs
DELETED
|
@@ -1,169 +0,0 @@
|
|
|
1
|
-
// embed.mjs — deterministic static-embedding lookup (model2vec potion-base-8M,
|
|
2
|
-
// 29,528 WordPiece subwords × 256 fp32 dims). Pure table lookup + float
|
|
3
|
-
// arithmetic, no ONNX runtime, no network after the one-time fetch — the same
|
|
4
|
-
// text always embeds to the same vector.
|
|
5
|
-
//
|
|
6
|
-
// The safetensors reader and WordPiece tokenizer below are hand-rolled: both
|
|
7
|
-
// formats are simple enough to parse directly, avoiding an ONNX/HF tokenizer
|
|
8
|
-
// dependency for it.
|
|
9
|
-
//
|
|
10
|
-
// Weights are gitignored (vendor/, fetched by scripts/fetch-embeddings.mjs)
|
|
11
|
-
// and never in the npm package; loadEmbedder() returns null when absent so
|
|
12
|
-
// CI/tests never require the download.
|
|
13
|
-
|
|
14
|
-
import { readFileSync, existsSync } from "node:fs";
|
|
15
|
-
import { join, dirname } from "node:path";
|
|
16
|
-
import { fileURLToPath } from "node:url";
|
|
17
|
-
|
|
18
|
-
const MODEL_FILE = "model.safetensors";
|
|
19
|
-
const TOKENIZER_FILE = "tokenizer.json";
|
|
20
|
-
const CONFIG_FILE = "config.json";
|
|
21
|
-
|
|
22
|
-
/** Default artifact dir: $TMCT_EMBED_DIR, else <repo>/vendor/embeddings/potion-base-8M
|
|
23
|
-
* (gitignored via vendor/ — the location scripts/fetch-embeddings.mjs writes). */
|
|
24
|
-
export function defaultEmbeddingsDir() {
|
|
25
|
-
if (process.env.TMCT_EMBED_DIR) return process.env.TMCT_EMBED_DIR;
|
|
26
|
-
const here = dirname(fileURLToPath(import.meta.url)); // packages/tmct/src
|
|
27
|
-
return join(here, "..", "..", "..", "..", "vendor", "embeddings", "potion-base-8M");
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
// ---- safetensors (hand-rolled: u64le header length + JSON header + raw tensors) -------
|
|
31
|
-
|
|
32
|
-
/** Read the single 2-D F32 embedding tensor from a safetensors file →
|
|
33
|
-
* { matrix: Float32Array (row-major), rows, dim }. model2vec exports exactly one
|
|
34
|
-
* tensor named "embeddings"; any lone 2-D F32 tensor is accepted for test fixtures. */
|
|
35
|
-
function readSafetensors(file) {
|
|
36
|
-
const buf = readFileSync(file);
|
|
37
|
-
const headerLen = Number(buf.readBigUInt64LE(0));
|
|
38
|
-
const header = JSON.parse(buf.subarray(8, 8 + headerLen).toString("utf8"));
|
|
39
|
-
const name = header.embeddings
|
|
40
|
-
? "embeddings"
|
|
41
|
-
: Object.keys(header).find((k) => k !== "__metadata__" && header[k]?.shape?.length === 2);
|
|
42
|
-
const t = name && header[name];
|
|
43
|
-
if (!t) throw new Error(`no 2-D tensor in ${file}`);
|
|
44
|
-
if (t.dtype !== "F32") throw new Error(`unsupported dtype ${t.dtype} in ${file} (only F32)`);
|
|
45
|
-
const [rows, dim] = t.shape;
|
|
46
|
-
const [start, end] = t.data_offsets;
|
|
47
|
-
const bytes = buf.subarray(8 + headerLen + start, 8 + headerLen + end);
|
|
48
|
-
if (bytes.byteLength !== rows * dim * 4) throw new Error(`tensor size mismatch in ${file}`);
|
|
49
|
-
// Copy into a fresh ArrayBuffer: the slice's byteOffset inside the file buffer is not
|
|
50
|
-
// guaranteed 4-byte aligned, and Float32Array requires alignment.
|
|
51
|
-
const matrix = new Float32Array(rows * dim);
|
|
52
|
-
new Uint8Array(matrix.buffer).set(bytes);
|
|
53
|
-
return { matrix, rows, dim };
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
// ---- Bert-style WordPiece tokenizer (from tokenizer.json) -----------------------------
|
|
57
|
-
|
|
58
|
-
const isPunct = (ch) => {
|
|
59
|
-
const c = ch.codePointAt(0);
|
|
60
|
-
// ASCII punctuation ranges (Bert treats these as standalone tokens) + general unicode P/S.
|
|
61
|
-
return (c >= 33 && c <= 47) || (c >= 58 && c <= 64) || (c >= 91 && c <= 96) || (c >= 123 && c <= 126) ||
|
|
62
|
-
/[\p{P}\p{S}]/u.test(ch);
|
|
63
|
-
};
|
|
64
|
-
const isCjk = (ch) => {
|
|
65
|
-
const c = ch.codePointAt(0);
|
|
66
|
-
return (c >= 0x4e00 && c <= 0x9fff) || (c >= 0x3400 && c <= 0x4dbf) ||
|
|
67
|
-
(c >= 0xf900 && c <= 0xfaff) || (c >= 0x20000 && c <= 0x2ffff);
|
|
68
|
-
};
|
|
69
|
-
|
|
70
|
-
function makeTokenizer(spec) {
|
|
71
|
-
if (spec?.model?.type !== "WordPiece") {
|
|
72
|
-
throw new Error(`unsupported tokenizer model "${spec?.model?.type}" (only WordPiece)`);
|
|
73
|
-
}
|
|
74
|
-
const vocab = spec.model.vocab; // token -> id
|
|
75
|
-
const contPrefix = spec.model.continuing_subword_prefix ?? "##";
|
|
76
|
-
const maxChars = spec.model.max_input_chars_per_word ?? 100;
|
|
77
|
-
const unkId = vocab[spec.model.unk_token] ?? null;
|
|
78
|
-
const lowercase = spec.normalizer?.lowercase !== false;
|
|
79
|
-
|
|
80
|
-
// BertNormalizer: clean control chars, pad CJK, lowercase (+ strip accents when lowercasing).
|
|
81
|
-
const normalize = (text) => {
|
|
82
|
-
let s = String(text).replace(/[\u0000\ufffd]/g, "").replace(/[\u0001-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, " ");
|
|
83
|
-
s = [...s].map((ch) => (isCjk(ch) ? ` ${ch} ` : ch)).join("");
|
|
84
|
-
if (lowercase) s = s.toLowerCase().normalize("NFD").replace(/\p{Mn}/gu, "");
|
|
85
|
-
return s;
|
|
86
|
-
};
|
|
87
|
-
// BertPreTokenizer: split on whitespace; every punctuation char is its own word.
|
|
88
|
-
const preTokenize = (s) => {
|
|
89
|
-
const words = [];
|
|
90
|
-
for (const chunk of s.split(/\s+/)) {
|
|
91
|
-
if (!chunk) continue;
|
|
92
|
-
let cur = "";
|
|
93
|
-
for (const ch of chunk) {
|
|
94
|
-
if (isPunct(ch)) {
|
|
95
|
-
if (cur) { words.push(cur); cur = ""; }
|
|
96
|
-
words.push(ch);
|
|
97
|
-
} else cur += ch;
|
|
98
|
-
}
|
|
99
|
-
if (cur) words.push(cur);
|
|
100
|
-
}
|
|
101
|
-
return words;
|
|
102
|
-
};
|
|
103
|
-
// Greedy longest-match WordPiece per word; a word with no valid segmentation → [UNK].
|
|
104
|
-
const wordPiece = (word) => {
|
|
105
|
-
if (word.length > maxChars) return unkId == null ? [] : [unkId];
|
|
106
|
-
const ids = [];
|
|
107
|
-
let start = 0;
|
|
108
|
-
while (start < word.length) {
|
|
109
|
-
let end = word.length;
|
|
110
|
-
let id = null;
|
|
111
|
-
while (end > start) {
|
|
112
|
-
const piece = (start > 0 ? contPrefix : "") + word.slice(start, end);
|
|
113
|
-
if (vocab[piece] !== undefined) { id = vocab[piece]; break; }
|
|
114
|
-
end--;
|
|
115
|
-
}
|
|
116
|
-
if (id == null) return unkId == null ? [] : [unkId];
|
|
117
|
-
ids.push(id);
|
|
118
|
-
start = end;
|
|
119
|
-
}
|
|
120
|
-
return ids;
|
|
121
|
-
};
|
|
122
|
-
// No [CLS]/[SEP] template — model2vec pools content subwords only.
|
|
123
|
-
return (text) => preTokenize(normalize(text)).flatMap(wordPiece);
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
// ---- embedder ---------------------------------------------------------------------------
|
|
127
|
-
|
|
128
|
-
const LOADED = new Map(); // dir -> embedder | null (per-process cache; weights load once)
|
|
129
|
-
|
|
130
|
-
/** Load tokenizer + embedding matrix from `dir` (default: defaultEmbeddingsDir()).
|
|
131
|
-
* Returns null — silently — when the artifacts are absent (the one-time
|
|
132
|
-
* `npm run refs:embeddings` fetch has not run): callers no-op, never fail.
|
|
133
|
-
* The returned embedder is { dim, dir, embed(text) → L2-normalised Float32Array }. */
|
|
134
|
-
export function loadEmbedder({ dir = defaultEmbeddingsDir() } = {}) {
|
|
135
|
-
if (LOADED.has(dir)) return LOADED.get(dir);
|
|
136
|
-
const modelFile = join(dir, MODEL_FILE);
|
|
137
|
-
const tokFile = join(dir, TOKENIZER_FILE);
|
|
138
|
-
if (!existsSync(modelFile) || !existsSync(tokFile)) {
|
|
139
|
-
LOADED.set(dir, null);
|
|
140
|
-
return null;
|
|
141
|
-
}
|
|
142
|
-
const { matrix, rows, dim } = readSafetensors(modelFile);
|
|
143
|
-
const tokenize = makeTokenizer(JSON.parse(readFileSync(tokFile, "utf8")));
|
|
144
|
-
let cfg = {};
|
|
145
|
-
try { cfg = JSON.parse(readFileSync(join(dir, CONFIG_FILE), "utf8")); } catch { /* optional */ }
|
|
146
|
-
const doNormalize = cfg.normalize !== false;
|
|
147
|
-
|
|
148
|
-
const embed = (text) => {
|
|
149
|
-
const ids = tokenize(text);
|
|
150
|
-
const v = new Float32Array(dim);
|
|
151
|
-
if (!ids.length) return v; // zero vector: cosine 0 against everything
|
|
152
|
-
for (const id of ids) {
|
|
153
|
-
if (id < 0 || id >= rows) continue;
|
|
154
|
-
const off = id * dim;
|
|
155
|
-
for (let j = 0; j < dim; j++) v[j] += matrix[off + j];
|
|
156
|
-
}
|
|
157
|
-
for (let j = 0; j < dim; j++) v[j] /= ids.length; // mean pool
|
|
158
|
-
if (doNormalize) {
|
|
159
|
-
let norm = 0;
|
|
160
|
-
for (let j = 0; j < dim; j++) norm += v[j] * v[j];
|
|
161
|
-
norm = Math.sqrt(norm);
|
|
162
|
-
if (norm > 0) for (let j = 0; j < dim; j++) v[j] /= norm;
|
|
163
|
-
}
|
|
164
|
-
return v;
|
|
165
|
-
};
|
|
166
|
-
const embedder = { dim, dir, embed };
|
|
167
|
-
LOADED.set(dir, embedder);
|
|
168
|
-
return embedder;
|
|
169
|
-
}
|
package/src/domain/vector.mjs
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
// vector.mjs — pure vector arithmetic over embedding vectors. No model, no fs:
|
|
2
|
-
// the loader that reads weights off disk lives in src/adapters/embed.mjs.
|
|
3
|
-
|
|
4
|
-
/** Cosine similarity. Over L2-normalised vectors this is just the dot product, but the
|
|
5
|
-
* full form is kept so unnormalised test fixtures behave. 0 when either vector is zero. */
|
|
6
|
-
export function cosine(a, b) {
|
|
7
|
-
let dot = 0, na = 0, nb = 0;
|
|
8
|
-
const n = Math.min(a.length, b.length);
|
|
9
|
-
for (let i = 0; i < n; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
|
|
10
|
-
if (na === 0 || nb === 0) return 0;
|
|
11
|
-
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
12
|
-
}
|