@polycode-projects/the-mechanical-code-talker 2.0.3 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -180,7 +180,7 @@ resolves to a real graph traversal or declines honestly:
180
180
  (*because/although/while*), conditionals, and false-premise flags ("why
181
181
  does X still import Y" when it no longer does).
182
182
 
183
- The full catalog with measured coverage lives in `CAPABILITIES_1.7.3.md` and
183
+ The full catalog with measured coverage lives in `CAPABILITIES_2.0.3.md` and
184
184
  the `BENCHMARK_*.md` reports.
185
185
 
186
186
  **Response finishing.** Before an answer prints, it is segmented into typed
package/ROADMAP.md CHANGED
@@ -100,7 +100,30 @@ blending web-sourced facts with graph/operator facts.
100
100
 
101
101
  ## Design docs
102
102
 
103
- Every substantial design lives in its own `PLAN_*.md` at the repo root (active) or `archive/`
104
- (shipped and closed) this file points to them, it doesn't repeat their content. `SKILL_*.md` docs
105
- specify the repeatable measurement/build cycles (benchmarks, the background strategy advisor,
106
- plain-prose writing). `HANDOVER.md` is the single current-open-items list.
103
+ Every substantial design lives in its own `PLAN_*.md` at the repo root; `archive/` holds the shipped
104
+ and closed ones. This file points to them, it doesn't repeat their content. Each plan states its own
105
+ status in its opening lines — read it there, because a status quoted here would rot.
106
+
107
+ | Plan | What it's for |
108
+ | --- | --- |
109
+ | [PLAN_ADVENTURE.md](PLAN_ADVENTURE.md) | a text adventure as an architectural stretch: imperative command grammar, NPC turn scheduler, room-look digest |
110
+ | [PLAN_AGENTS.md](PLAN_AGENTS.md) | the governing plan for the multi-repo arc (marginalia, seonix, a pluggable LLM rung), with its own phase sequencing |
111
+ | [PLAN_CHILD_CORPUS.md](PLAN_CHILD_CORPUS.md) | a wider default seed corpus, chosen by age of acquisition |
112
+ | [PLAN_CLASS_QUERY.md](PLAN_CLASS_QUERY.md) | "list/count all X of class Y", reconciled against what already shipped |
113
+ | [PLAN_CODE.md](PLAN_CODE.md) | program synthesis over tmct's closed DSLs, plus JS/HTML/CSS fragments and goal-directed program repair |
114
+ | [PLAN_CONSISTENCY_CHECK.md](PLAN_CONSISTENCY_CHECK.md) | tmct as a consistency service for an LLM tool loop |
115
+ | [PLAN_EMBEDDINGS.md](PLAN_EMBEDDINGS.md) | the semantic-similarity axis, and the way back to it |
116
+ | [PLAN_GRAPH_SCAN.md](PLAN_GRAPH_SCAN.md) | seed and query cost at `init:xl`/`init:xxl` corpus scale |
117
+ | [PLAN_GUESS_NUMBER.md](PLAN_GUESS_NUMBER.md) | closed-loop planning over hidden state, via belief-interval bisection |
118
+ | [PLAN_MUD.md](PLAN_MUD.md) | persistent, shared tmct worlds over a `server:` memory backend |
119
+ | [PLAN_NLU_BENCHMARKS.md](PLAN_NLU_BENCHMARKS.md) | scoring tmct on the CLINC150 and HWU64 intent sets |
120
+ | [PLAN_OPEN_ITEMS.md](PLAN_OPEN_ITEMS.md) | the build order closing the backlog `HANDOVER.md` carries |
121
+ | [PLAN_PARAPHRASE_VERIFICATION.md](PLAN_PARAPHRASE_VERIFICATION.md) | checking a paraphrase against the graph before it prints |
122
+ | [PLAN_PURGE.md](PLAN_PURGE.md) | promote the load-bearing code, delete the dead weight |
123
+ | [PLAN_REPO_INDEX.md](PLAN_REPO_INDEX.md) | tmct grows its own code parsers, ported from seonix |
124
+ | [PLAN_SYLLOGIST.md](PLAN_SYLLOGIST.md) | the reasoning engine's incrementality and retraction horizon |
125
+ | [PLAN_SYLLOGIST_EL_DL.md](PLAN_SYLLOGIST_EL_DL.md) | beyond OWL 2 RL: an EL classifier, then a DL tableau prover |
126
+
127
+ `SKILL_*.md` docs specify the repeatable measurement and build cycles (the benchmarks, the capability
128
+ audit, the background strategy advisor, plain-prose writing). `HANDOVER.md` is the single
129
+ current-open-items list.
@@ -1,3 +1,4 @@
1
+ {"kind":"rescue","sentence":"Code search is a retrieval task","rescued":"Code hunt is a retrieval task","from":"search","to":"hunt","pos":"noun","synsetId":"00947217-n","sourceFile":"PLAN_EMBEDDINGS.md","provenance":"wordnet:00947217-n"}
1
2
  {"kind":"variant","seed":"the ace of trumps is a sure winner","generated":"the one of trumps is a sure winner","from":"ace","to":"one","pos":"noun","synsetId":"13764713-n","sourceCorpus":"corpus/tier2/human-examples-large.jsonl","provenance":"wordnet:13764713-n"}
2
3
  {"kind":"variant","seed":"the ace of trumps is a sure winner","generated":"the single of trumps is a sure winner","from":"ace","to":"single","pos":"noun","synsetId":"13764713-n","sourceCorpus":"corpus/tier2/human-examples-large.jsonl","provenance":"wordnet:13764713-n"}
3
4
  {"kind":"variant","seed":"the ace of trumps is a sure winner","generated":"the unity of trumps is a sure winner","from":"ace","to":"unity","pos":"noun","synsetId":"13764713-n","sourceCorpus":"corpus/tier2/human-examples-large.jsonl","provenance":"wordnet:13764713-n"}
@@ -2,8 +2,8 @@
2
2
  "version": 1,
3
3
  "generated": "by scripts/generate-template-variants.mjs",
4
4
  "file": "ace-surface-variants.jsonl",
5
- "rows": 17,
6
- "bytes": 4774,
7
- "sha256": "5364415f73f7a366371f1fdcc072b5059f81871ef48b106088f05b1441def569",
5
+ "rows": 18,
6
+ "bytes": 5012,
7
+ "sha256": "28fae5a3353f5dd1df365b4407e0ac2a5c66b98b232d8bf454fe3573f2c16760",
8
8
  "license": "CC-BY-4.0 (WordNet-derived synonym substitutions of Open English WordNet / SemCor example sentences and this repo's own MPL-2.0 docs prose — see corpus/generated/README.md)"
9
9
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@polycode-projects/the-mechanical-code-talker",
3
- "version": "2.0.3",
3
+ "version": "2.2.0",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "description": "The Mechanical Code Talker (tmct) — a tolerant, offline, $0 chat surface that guides you toward precision queries about a software repository. ELIZA/PARRY-style but domain-obsessed with code. No model calls; no codebase index of its own.",
@@ -139,7 +139,6 @@ export async function normalizeConfig(raw, { configDir } = {}) {
139
139
  if (t.impl_of_interface !== undefined) tune.implOfInterface = t.impl_of_interface;
140
140
  if (t.beam_search !== undefined) tune.beamSearch = t.beam_search;
141
141
  if (t.beam_width !== undefined) tune.beamWidth = t.beam_width;
142
- if (t.embed_rank !== undefined) tune.embedRank = t.embed_rank;
143
142
  if (t.prose_layers !== undefined) tune.proseLayers = t.prose_layers;
144
143
  const exp = t.expansion || {};
145
144
  const expansion = {};
@@ -1,5 +1,4 @@
1
1
  import { lookupByProseTokens, proseLayerHits, splitIdentifierWords } from "./prose.mjs";
2
- import { cosine } from "./vector.mjs";
3
2
  import { CREATED_AT_PROP, UPDATED_AT_PROP } from "./memory/trust.mjs";
4
3
 
5
4
  // Pure (no-network, no-fs) query logic over the typed `entities` payload that the
@@ -511,15 +510,6 @@ const LIT_COMP_CAP = 4;
511
510
  const LIT_FRAC = 1.0;
512
511
  const LIT_CAP_FRAC = 0.9;
513
512
 
514
- // opt-in via embedRank + an injected embedder: static-embedding re-rank over
515
- // path/symbol/doc text read from the graph (never source), cached per-process in EMB_CACHE
516
- const EMB_FRAC = 0.2;
517
- const EMB_CAP_FRAC = 0.35;
518
- const EMB_TEXT_SYMBOL_CAP = 64;
519
- const EMB_TEXT_DOC_CAP = 12;
520
- const EMB_CACHE = new WeakMap(); // graph -> { embedder, texts, vecs: Map<moduleId, Float32Array> }
521
- let embedWarned = false;
522
-
523
513
  // opt-in via beamSearch: multi-ply adaptive expansion of the proximity nudge above.
524
514
  // Beam width is a margin relative to each ply's best score (not a fixed count), so a
525
515
  // weak-then-strong candidate isn't prematurely discarded. Successors are generated
@@ -544,30 +534,6 @@ const SPIRAL_HOP_DECAY = 0.6;
544
534
  const SPIRAL_PROX_FRAC = 0.2;
545
535
  const SPIRAL_PROX_CAP_FRAC = 0.35;
546
536
 
547
- /** embedRank: per-module embeddable text from path components + defined symbol
548
- * names + doc first-lines, cached alongside the vectors in EMB_CACHE. */
549
- function moduleEmbedTexts(graph) {
550
- const texts = new Map(); // moduleId -> text
551
- const defIdx = definesIndex(graph);
552
- const docs = new Map(); // moduleId -> [doc first-lines]
553
- for (const ind of graph.individuals) {
554
- const doc = (ind.attributes || []).find((a) => a.key === "doc")?.value;
555
- if (!doc) continue;
556
- const modId = (ind.class || "") === "Module" ? ind.id : moduleIdOf(graph, ind);
557
- if (!modId) continue;
558
- let arr = docs.get(modId);
559
- if (!arr) docs.set(modId, (arr = []));
560
- if (arr.length < EMB_TEXT_DOC_CAP) arr.push(String(doc).split("\n")[0]);
561
- }
562
- for (const ind of graph.individuals) {
563
- if ((ind.class || "") !== "Module") continue;
564
- const parts = String(ind.label).split(/[^a-zA-Z0-9_]+/).filter(Boolean);
565
- const syms = (defIdx.get(ind.id) || []).slice(0, EMB_TEXT_SYMBOL_CAP);
566
- texts.set(ind.id, [...parts, ...syms, ...(docs.get(ind.id) || [])].join(" "));
567
- }
568
- return texts;
569
- }
570
-
571
537
  /** Split a lowercased path label into boundary components: django/utils/text.py →
572
538
  * {django,utils,text,py}. Component equality (not substring) stops "text" matching "ci<text>". */
573
539
  function pathComponents(labelLc) {
@@ -773,7 +739,7 @@ export function spiralExpand(graph, scored = [], {
773
739
  * IDF-weights each query token, scores path/symbol/exact-symbol matches, and
774
740
  * re-ranks with a bounded import-proximity bonus. Pure; deterministic. */
775
741
  function scoreModules(graph, tokens, opts = {}) {
776
- const { demoteNonProd = false, callAdjacency = false, implOfInterface = false, beamSearch = false, spiral = false, proseBoost = false, proseLayers = false, literalMention = false, embedRank = false, rawQuery = "" } = opts;
742
+ const { demoteNonProd = false, callAdjacency = false, implOfInterface = false, beamSearch = false, spiral = false, proseBoost = false, proseLayers = false, literalMention = false, rawQuery = "" } = opts;
777
743
  const beamWidth = Number.isFinite(opts.beamWidth) && opts.beamWidth > 0 ? opts.beamWidth : 8;
778
744
  const defIdx = definesIndex(graph);
779
745
  // Precompute each module's path components + defined-symbol exact/component sets, once.
@@ -967,36 +933,6 @@ function scoreModules(graph, tokens, opts = {}) {
967
933
  s.score += Math.min(signal * PROSE_LAYER_FRAC, s.score * PROSE_LAYER_CAP_FRAC);
968
934
  }
969
935
  }
970
- // embedRank: the embedder is injected so this module stays fs-free; absent -> a one-time
971
- // stderr note, never a failure
972
- if (embedRank) {
973
- if (!opts.embedder) {
974
- if (!embedWarned) {
975
- embedWarned = true;
976
- process.stderr.write("tmct: embedRank requested but no embedder available (weights not fetched? see `npm run refs:embeddings`) — flag is a no-op\n");
977
- }
978
- } else if (scored.length) {
979
- const embedder = opts.embedder;
980
- let cache = EMB_CACHE.get(graph);
981
- if (!cache || cache.embedder !== embedder) {
982
- cache = { embedder, texts: moduleEmbedTexts(graph), vecs: new Map() };
983
- EMB_CACHE.set(graph, cache);
984
- }
985
- const qv = embedder.embed(rawQuery || tokens.join(" "));
986
- let maxBase = 0;
987
- for (const s of scored) maxBase = Math.max(maxBase, s.score);
988
- for (const s of scored) {
989
- let v = cache.vecs.get(s.ind.id);
990
- if (!v) {
991
- v = embedder.embed(cache.texts.get(s.ind.id) || String(s.ind.label));
992
- cache.vecs.set(s.ind.id, v);
993
- }
994
- const sim = Math.max(0, cosine(qv, v)); // negative similarity never penalises
995
- if (!sim) continue;
996
- s.score += Math.min(sim * maxBase * EMB_FRAC, s.score * EMB_CAP_FRAC);
997
- }
998
- }
999
- }
1000
936
  // beamSearch (opt-in): multi-ply generalization of the single-hop families above.
1001
937
  if (beamSearch && scored.length > 1) beamExpand(graph, scored, beamWidth);
1002
938
  // SPIRAL (opt-in): bounded-radius ego walk that may introduce lexically-invisible modules — runs
@@ -1024,9 +960,9 @@ export function searchModulesRanked(graph, query, opts = {}) {
1024
960
  const tokens = raw.toLowerCase().split(/[^a-z0-9_]+/).filter(Boolean);
1025
961
  if (!tokens.length) return [];
1026
962
  // literalMention needs the query BEFORE tokenization (the tokenizer destroys the dotted refs
1027
- // it matches on) and embedRank embeds the raw phrasing; threaded only when a flag that
1028
- // consumes it is on, so the OFF path is provably unchanged.
1029
- const effOpts = (opts.literalMention || opts.embedRank) ? { ...opts, rawQuery: raw } : opts;
963
+ // it matches on); threaded only when a flag that consumes it is on, so the OFF path is
964
+ // provably unchanged.
965
+ const effOpts = opts.literalMention ? { ...opts, rawQuery: raw } : opts;
1030
966
  return scoreModules(graph, tokens, effOpts).map((s) => ({ path: String(s.ind.label), score: s.score }));
1031
967
  }
1032
968
 
@@ -0,0 +1,55 @@
1
+ // markdown-links.mjs — pull the relative link targets out of a markdown string.
2
+ // Pure: a string in, `[{ target, line }]` out, no filesystem and no imports, so
3
+ // the CI jobs that run without `npm ci` can reach it.
4
+ //
5
+ // Code is not prose. A doc that writes `[text](target)` inside backticks is
6
+ // showing you what a link looks like, not making one, and a checker that cannot
7
+ // tell the difference reports the example as a broken link to a file named
8
+ // "target". So the spans are blanked before the link patterns run — blanked
9
+ // rather than cut, because every offset behind them still has to name the right
10
+ // line number.
11
+
12
+ const blank = (line) => line.replace(/[^\n]/g, " ");
13
+ const FENCE = /^[ \t]{0,3}(`{3,}|~{3,})/;
14
+
15
+ /** Replace every fenced block and inline code span with spaces, keeping the
16
+ * string's length and its newlines so later offsets still map to their line.
17
+ * Fences are walked line by line: a lazy multiline regex stops at the end of
18
+ * the opening fence's own line and blanks only the markers. */
19
+ export function blankCodeSpans(markdown) {
20
+ let fence = null;
21
+ const lines = markdown.split("\n").map((line) => {
22
+ const marker = line.match(FENCE)?.[1];
23
+ if (fence) {
24
+ if (marker && marker[0] === fence[0] && marker.length >= fence.length) fence = null;
25
+ return blank(line);
26
+ }
27
+ if (marker) fence = marker;
28
+ return marker ? blank(line) : line;
29
+ });
30
+ return lines.join("\n").replace(/(`+)[\s\S]*?\1/g, blank);
31
+ }
32
+
33
+ // Inline links/images: [text](target "title") — target ends at the first
34
+ // whitespace or closing paren. Reference definitions: [label]: target.
35
+ const INLINE_LINK = /!?\[[^\]]*\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
36
+ const REFERENCE_DEF = /^\s{0,3}\[[^\]]+\]:\s+(\S+)/gm;
37
+
38
+ /** Every repo-relative link target in `markdown`, with the 1-based line it sits
39
+ * on. External URLs, bare #anchors and absolute paths are out of scope. */
40
+ export function relativeTargets(markdown) {
41
+ const prose = blankCodeSpans(markdown);
42
+ const targets = [];
43
+ for (const regex of [INLINE_LINK, REFERENCE_DEF]) {
44
+ for (const match of prose.matchAll(regex)) {
45
+ let target = match[1];
46
+ if (/^(https?|mailto|ftp):/i.test(target)) continue;
47
+ if (target.startsWith("#") || target.startsWith("/") || target.startsWith("<")) continue;
48
+ target = decodeURIComponent(target.split("#")[0].split("?")[0]);
49
+ if (!target) continue;
50
+ const line = prose.slice(0, match.index).split("\n").length;
51
+ targets.push({ target, line });
52
+ }
53
+ }
54
+ return targets;
55
+ }
@@ -936,23 +936,6 @@
936
936
  }
937
937
  });
938
938
 
939
- // src/domain/vector.mjs
940
- function cosine(a, b) {
941
- let dot = 0, na = 0, nb = 0;
942
- const n = Math.min(a.length, b.length);
943
- for (let i = 0; i < n; i++) {
944
- dot += a[i] * b[i];
945
- na += a[i] * a[i];
946
- nb += b[i] * b[i];
947
- }
948
- if (na === 0 || nb === 0) return 0;
949
- return dot / (Math.sqrt(na) * Math.sqrt(nb));
950
- }
951
- var init_vector = __esm({
952
- "src/domain/vector.mjs"() {
953
- }
954
- });
955
-
956
939
  // src/domain/memory/trust.mjs
957
940
  function parseChatTagRest(rest) {
958
941
  const at = rest.indexOf("@");
@@ -1307,27 +1290,6 @@
1307
1290
  }
1308
1291
  return idx;
1309
1292
  }
1310
- function moduleEmbedTexts(graph) {
1311
- const texts = /* @__PURE__ */ new Map();
1312
- const defIdx = definesIndex(graph);
1313
- const docs = /* @__PURE__ */ new Map();
1314
- for (const ind of graph.individuals) {
1315
- const doc = (ind.attributes || []).find((a) => a.key === "doc")?.value;
1316
- if (!doc) continue;
1317
- const modId = (ind.class || "") === "Module" ? ind.id : moduleIdOf(graph, ind);
1318
- if (!modId) continue;
1319
- let arr = docs.get(modId);
1320
- if (!arr) docs.set(modId, arr = []);
1321
- if (arr.length < EMB_TEXT_DOC_CAP) arr.push(String(doc).split("\n")[0]);
1322
- }
1323
- for (const ind of graph.individuals) {
1324
- if ((ind.class || "") !== "Module") continue;
1325
- const parts = String(ind.label).split(/[^a-zA-Z0-9_]+/).filter(Boolean);
1326
- const syms = (defIdx.get(ind.id) || []).slice(0, EMB_TEXT_SYMBOL_CAP);
1327
- texts.set(ind.id, [...parts, ...syms, ...docs.get(ind.id) || []].join(" "));
1328
- }
1329
- return texts;
1330
- }
1331
1293
  function identComponents(name) {
1332
1294
  return new Set(String(name).replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase().split(/[^a-z0-9]+/).filter(Boolean));
1333
1295
  }
@@ -1499,7 +1461,7 @@
1499
1461
  return results;
1500
1462
  }
1501
1463
  function scoreModules(graph, tokens, opts = {}) {
1502
- const { demoteNonProd = false, callAdjacency = false, implOfInterface = false, beamSearch = false, spiral = false, proseBoost = false, proseLayers = false, literalMention = false, embedRank = false, rawQuery = "" } = opts;
1464
+ const { demoteNonProd = false, callAdjacency = false, implOfInterface = false, beamSearch = false, spiral = false, proseBoost = false, proseLayers = false, literalMention = false, rawQuery = "" } = opts;
1503
1465
  const beamWidth = Number.isFinite(opts.beamWidth) && opts.beamWidth > 0 ? opts.beamWidth : 8;
1504
1466
  const defIdx = definesIndex(graph);
1505
1467
  const modules = [];
@@ -1691,34 +1653,6 @@
1691
1653
  s.score += Math.min(signal * PROSE_LAYER_FRAC, s.score * PROSE_LAYER_CAP_FRAC);
1692
1654
  }
1693
1655
  }
1694
- if (embedRank) {
1695
- if (!opts.embedder) {
1696
- if (!embedWarned) {
1697
- embedWarned = true;
1698
- process.stderr.write("tmct: embedRank requested but no embedder available (weights not fetched? see `npm run refs:embeddings`) \u2014 flag is a no-op\n");
1699
- }
1700
- } else if (scored.length) {
1701
- const embedder = opts.embedder;
1702
- let cache2 = EMB_CACHE.get(graph);
1703
- if (!cache2 || cache2.embedder !== embedder) {
1704
- cache2 = { embedder, texts: moduleEmbedTexts(graph), vecs: /* @__PURE__ */ new Map() };
1705
- EMB_CACHE.set(graph, cache2);
1706
- }
1707
- const qv = embedder.embed(rawQuery || tokens.join(" "));
1708
- let maxBase = 0;
1709
- for (const s of scored) maxBase = Math.max(maxBase, s.score);
1710
- for (const s of scored) {
1711
- let v = cache2.vecs.get(s.ind.id);
1712
- if (!v) {
1713
- v = embedder.embed(cache2.texts.get(s.ind.id) || String(s.ind.label));
1714
- cache2.vecs.set(s.ind.id, v);
1715
- }
1716
- const sim = Math.max(0, cosine(qv, v));
1717
- if (!sim) continue;
1718
- s.score += Math.min(sim * maxBase * EMB_FRAC, s.score * EMB_CAP_FRAC);
1719
- }
1720
- }
1721
- }
1722
1656
  if (beamSearch && scored.length > 1) beamExpand(graph, scored, beamWidth);
1723
1657
  if (spiral && scored.length) spiralExpand(graph, scored, {
1724
1658
  depth: Number.isFinite(opts.spiralDepth) && opts.spiralDepth > 0 ? opts.spiralDepth : SPIRAL_DEPTH_DEFAULT,
@@ -1732,7 +1666,7 @@
1732
1666
  const raw = String(query || "");
1733
1667
  const tokens = raw.toLowerCase().split(/[^a-z0-9_]+/).filter(Boolean);
1734
1668
  if (!tokens.length) return [];
1735
- const effOpts = opts.literalMention || opts.embedRank ? { ...opts, rawQuery: raw } : opts;
1669
+ const effOpts = opts.literalMention ? { ...opts, rawQuery: raw } : opts;
1736
1670
  return scoreModules(graph, tokens, effOpts).map((s) => ({ path: String(s.ind.label), score: s.score }));
1737
1671
  }
1738
1672
  function renderSearch(graph, query, { limit = SEARCH_LIMIT, kind = "", decorator = "", name = "" } = {}) {
@@ -2414,11 +2348,10 @@ ${shown.join("\n")}${tail}`;
2414
2348
  ${list.join("\n ")}` + (edges.length > EXPORTS_CAP ? `
2415
2349
  \u2026+${edges.length - EXPORTS_CAP} more` : "");
2416
2350
  }
2417
- var PROP_KIND, isProvRef, DESCRIBE_EDGE_CAP, PROV_CAP, IMPACT_DEPTHS_LISTED, IMPACT_PER_DEPTH, IMPACT_TESTS_PER_DEP, SEARCH_LIMIT, SEARCH_SYMBOLS_SHOWN, PATH_W, SYM_W, EXACT_W, SYM_MATCH_CAP, PROX_FRAC, PROX_CAP_FRAC, isTestLabel, NONPROD_DEMOTE, isNonProdLabel, CALL_PROX_FRAC, CALL_PROX_CAP_FRAC, IMPL_PROX_FRAC, IMPL_PROX_CAP_FRAC, isCsModuleLabel, looksLikeCsInterface, PROSE_PROX_FRAC, PROSE_PROX_CAP_FRAC, PROSE_LOOKUP_LIMIT, PROSE_LAYER_FRAC, PROSE_LAYER_CAP_FRAC, PROSE_LAYER_DISCOUNT, LIT_W, LIT_MIN_COMPONENTS, LIT_COMP_CAP, LIT_FRAC, LIT_CAP_FRAC, EMB_FRAC, EMB_CAP_FRAC, EMB_TEXT_SYMBOL_CAP, EMB_TEXT_DOC_CAP, EMB_CACHE, embedWarned, BEAM_MARGIN_FRAC, BEAM_PROX_FRAC, BEAM_PROX_CAP_FRAC, BEAM_OVERFLOW_CAP, BEAM_PLIES, BEAM_EDGE_GROUPS, SPIRAL_DEPTH_DEFAULT, SPIRAL_NODE_LIMIT_DEFAULT, SPIRAL_Q_DEFAULT, SPIRAL_EXPAND_KINDS, SPIRAL_EMIT_FRAC, SPIRAL_HOP_DECAY, SPIRAL_PROX_FRAC, SPIRAL_PROX_CAP_FRAC, edgesOfKindCache, MEMBERS_CAP, SUBCLASS_CAP, CALL_CAP, attrVal, ARCH_PKG_CAP, ARCH_HUB_CAP, COVERAGE_CAP, TESTS_GRAIN_NOTE, HISTORY_CAP, CALL_SYMBOL_CLASSES, CALL_HINT_CAP, SYMBOL_CLASSES, CONTEXT_SIBLING_CAP, CLASS_MEMBER_CAP, COCHANGE_MID_CAP, CONTEXT_TESTS_CAP, TINY_MAX_LOC, TINY_MAX_ARITY, LARGE_CLASS_MEMBERS, INLINE_CALLEE_CAP, splitDecs, tokenize, countParams, modeOf, COCHANGE_CAP, EXPORTS_CAP;
2351
+ var PROP_KIND, isProvRef, DESCRIBE_EDGE_CAP, PROV_CAP, IMPACT_DEPTHS_LISTED, IMPACT_PER_DEPTH, IMPACT_TESTS_PER_DEP, SEARCH_LIMIT, SEARCH_SYMBOLS_SHOWN, PATH_W, SYM_W, EXACT_W, SYM_MATCH_CAP, PROX_FRAC, PROX_CAP_FRAC, isTestLabel, NONPROD_DEMOTE, isNonProdLabel, CALL_PROX_FRAC, CALL_PROX_CAP_FRAC, IMPL_PROX_FRAC, IMPL_PROX_CAP_FRAC, isCsModuleLabel, looksLikeCsInterface, PROSE_PROX_FRAC, PROSE_PROX_CAP_FRAC, PROSE_LOOKUP_LIMIT, PROSE_LAYER_FRAC, PROSE_LAYER_CAP_FRAC, PROSE_LAYER_DISCOUNT, LIT_W, LIT_MIN_COMPONENTS, LIT_COMP_CAP, LIT_FRAC, LIT_CAP_FRAC, BEAM_MARGIN_FRAC, BEAM_PROX_FRAC, BEAM_PROX_CAP_FRAC, BEAM_OVERFLOW_CAP, BEAM_PLIES, BEAM_EDGE_GROUPS, SPIRAL_DEPTH_DEFAULT, SPIRAL_NODE_LIMIT_DEFAULT, SPIRAL_Q_DEFAULT, SPIRAL_EXPAND_KINDS, SPIRAL_EMIT_FRAC, SPIRAL_HOP_DECAY, SPIRAL_PROX_FRAC, SPIRAL_PROX_CAP_FRAC, edgesOfKindCache, MEMBERS_CAP, SUBCLASS_CAP, CALL_CAP, attrVal, ARCH_PKG_CAP, ARCH_HUB_CAP, COVERAGE_CAP, TESTS_GRAIN_NOTE, HISTORY_CAP, CALL_SYMBOL_CLASSES, CALL_HINT_CAP, SYMBOL_CLASSES, CONTEXT_SIBLING_CAP, CLASS_MEMBER_CAP, COCHANGE_MID_CAP, CONTEXT_TESTS_CAP, TINY_MAX_LOC, TINY_MAX_ARITY, LARGE_CLASS_MEMBERS, INLINE_CALLEE_CAP, splitDecs, tokenize, countParams, modeOf, COCHANGE_CAP, EXPORTS_CAP;
2418
2352
  var init_codegraph = __esm({
2419
2353
  "src/domain/codegraph.mjs"() {
2420
2354
  init_prose();
2421
- init_vector();
2422
2355
  init_trust();
2423
2356
  PROP_KIND = {
2424
2357
  // v2.0 faithful tokens (SEON-faithful realign)
@@ -2484,12 +2417,6 @@ ${shown.join("\n")}${tail}`;
2484
2417
  LIT_COMP_CAP = 4;
2485
2418
  LIT_FRAC = 1;
2486
2419
  LIT_CAP_FRAC = 0.9;
2487
- EMB_FRAC = 0.2;
2488
- EMB_CAP_FRAC = 0.35;
2489
- EMB_TEXT_SYMBOL_CAP = 64;
2490
- EMB_TEXT_DOC_CAP = 12;
2491
- EMB_CACHE = /* @__PURE__ */ new WeakMap();
2492
- embedWarned = false;
2493
2420
  BEAM_MARGIN_FRAC = 0.5;
2494
2421
  BEAM_PROX_FRAC = 0.2;
2495
2422
  BEAM_PROX_CAP_FRAC = 0.35;
@@ -1,169 +0,0 @@
1
- // embed.mjs — deterministic static-embedding lookup (model2vec potion-base-8M,
2
- // 29,528 WordPiece subwords × 256 fp32 dims). Pure table lookup + float
3
- // arithmetic, no ONNX runtime, no network after the one-time fetch — the same
4
- // text always embeds to the same vector.
5
- //
6
- // The safetensors reader and WordPiece tokenizer below are hand-rolled: both
7
- // formats are simple enough to parse directly, avoiding an ONNX/HF tokenizer
8
- // dependency for it.
9
- //
10
- // Weights are gitignored (vendor/, fetched by scripts/fetch-embeddings.mjs)
11
- // and never in the npm package; loadEmbedder() returns null when absent so
12
- // CI/tests never require the download.
13
-
14
- import { readFileSync, existsSync } from "node:fs";
15
- import { join, dirname } from "node:path";
16
- import { fileURLToPath } from "node:url";
17
-
18
- const MODEL_FILE = "model.safetensors";
19
- const TOKENIZER_FILE = "tokenizer.json";
20
- const CONFIG_FILE = "config.json";
21
-
22
- /** Default artifact dir: $TMCT_EMBED_DIR, else <repo>/vendor/embeddings/potion-base-8M
23
- * (gitignored via vendor/ — the location scripts/fetch-embeddings.mjs writes). */
24
- export function defaultEmbeddingsDir() {
25
- if (process.env.TMCT_EMBED_DIR) return process.env.TMCT_EMBED_DIR;
26
- const here = dirname(fileURLToPath(import.meta.url)); // packages/tmct/src
27
- return join(here, "..", "..", "..", "..", "vendor", "embeddings", "potion-base-8M");
28
- }
29
-
30
- // ---- safetensors (hand-rolled: u64le header length + JSON header + raw tensors) -------
31
-
32
- /** Read the single 2-D F32 embedding tensor from a safetensors file →
33
- * { matrix: Float32Array (row-major), rows, dim }. model2vec exports exactly one
34
- * tensor named "embeddings"; any lone 2-D F32 tensor is accepted for test fixtures. */
35
- function readSafetensors(file) {
36
- const buf = readFileSync(file);
37
- const headerLen = Number(buf.readBigUInt64LE(0));
38
- const header = JSON.parse(buf.subarray(8, 8 + headerLen).toString("utf8"));
39
- const name = header.embeddings
40
- ? "embeddings"
41
- : Object.keys(header).find((k) => k !== "__metadata__" && header[k]?.shape?.length === 2);
42
- const t = name && header[name];
43
- if (!t) throw new Error(`no 2-D tensor in ${file}`);
44
- if (t.dtype !== "F32") throw new Error(`unsupported dtype ${t.dtype} in ${file} (only F32)`);
45
- const [rows, dim] = t.shape;
46
- const [start, end] = t.data_offsets;
47
- const bytes = buf.subarray(8 + headerLen + start, 8 + headerLen + end);
48
- if (bytes.byteLength !== rows * dim * 4) throw new Error(`tensor size mismatch in ${file}`);
49
- // Copy into a fresh ArrayBuffer: the slice's byteOffset inside the file buffer is not
50
- // guaranteed 4-byte aligned, and Float32Array requires alignment.
51
- const matrix = new Float32Array(rows * dim);
52
- new Uint8Array(matrix.buffer).set(bytes);
53
- return { matrix, rows, dim };
54
- }
55
-
56
- // ---- Bert-style WordPiece tokenizer (from tokenizer.json) -----------------------------
57
-
58
- const isPunct = (ch) => {
59
- const c = ch.codePointAt(0);
60
- // ASCII punctuation ranges (Bert treats these as standalone tokens) + general unicode P/S.
61
- return (c >= 33 && c <= 47) || (c >= 58 && c <= 64) || (c >= 91 && c <= 96) || (c >= 123 && c <= 126) ||
62
- /[\p{P}\p{S}]/u.test(ch);
63
- };
64
- const isCjk = (ch) => {
65
- const c = ch.codePointAt(0);
66
- return (c >= 0x4e00 && c <= 0x9fff) || (c >= 0x3400 && c <= 0x4dbf) ||
67
- (c >= 0xf900 && c <= 0xfaff) || (c >= 0x20000 && c <= 0x2ffff);
68
- };
69
-
70
- function makeTokenizer(spec) {
71
- if (spec?.model?.type !== "WordPiece") {
72
- throw new Error(`unsupported tokenizer model "${spec?.model?.type}" (only WordPiece)`);
73
- }
74
- const vocab = spec.model.vocab; // token -> id
75
- const contPrefix = spec.model.continuing_subword_prefix ?? "##";
76
- const maxChars = spec.model.max_input_chars_per_word ?? 100;
77
- const unkId = vocab[spec.model.unk_token] ?? null;
78
- const lowercase = spec.normalizer?.lowercase !== false;
79
-
80
- // BertNormalizer: clean control chars, pad CJK, lowercase (+ strip accents when lowercasing).
81
- const normalize = (text) => {
82
- let s = String(text).replace(/[\u0000\ufffd]/g, "").replace(/[\u0001-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, " ");
83
- s = [...s].map((ch) => (isCjk(ch) ? ` ${ch} ` : ch)).join("");
84
- if (lowercase) s = s.toLowerCase().normalize("NFD").replace(/\p{Mn}/gu, "");
85
- return s;
86
- };
87
- // BertPreTokenizer: split on whitespace; every punctuation char is its own word.
88
- const preTokenize = (s) => {
89
- const words = [];
90
- for (const chunk of s.split(/\s+/)) {
91
- if (!chunk) continue;
92
- let cur = "";
93
- for (const ch of chunk) {
94
- if (isPunct(ch)) {
95
- if (cur) { words.push(cur); cur = ""; }
96
- words.push(ch);
97
- } else cur += ch;
98
- }
99
- if (cur) words.push(cur);
100
- }
101
- return words;
102
- };
103
- // Greedy longest-match WordPiece per word; a word with no valid segmentation → [UNK].
104
- const wordPiece = (word) => {
105
- if (word.length > maxChars) return unkId == null ? [] : [unkId];
106
- const ids = [];
107
- let start = 0;
108
- while (start < word.length) {
109
- let end = word.length;
110
- let id = null;
111
- while (end > start) {
112
- const piece = (start > 0 ? contPrefix : "") + word.slice(start, end);
113
- if (vocab[piece] !== undefined) { id = vocab[piece]; break; }
114
- end--;
115
- }
116
- if (id == null) return unkId == null ? [] : [unkId];
117
- ids.push(id);
118
- start = end;
119
- }
120
- return ids;
121
- };
122
- // No [CLS]/[SEP] template — model2vec pools content subwords only.
123
- return (text) => preTokenize(normalize(text)).flatMap(wordPiece);
124
- }
125
-
126
- // ---- embedder ---------------------------------------------------------------------------
127
-
128
- const LOADED = new Map(); // dir -> embedder | null (per-process cache; weights load once)
129
-
130
- /** Load tokenizer + embedding matrix from `dir` (default: defaultEmbeddingsDir()).
131
- * Returns null — silently — when the artifacts are absent (the one-time
132
- * `npm run refs:embeddings` fetch has not run): callers no-op, never fail.
133
- * The returned embedder is { dim, dir, embed(text) → L2-normalised Float32Array }. */
134
- export function loadEmbedder({ dir = defaultEmbeddingsDir() } = {}) {
135
- if (LOADED.has(dir)) return LOADED.get(dir);
136
- const modelFile = join(dir, MODEL_FILE);
137
- const tokFile = join(dir, TOKENIZER_FILE);
138
- if (!existsSync(modelFile) || !existsSync(tokFile)) {
139
- LOADED.set(dir, null);
140
- return null;
141
- }
142
- const { matrix, rows, dim } = readSafetensors(modelFile);
143
- const tokenize = makeTokenizer(JSON.parse(readFileSync(tokFile, "utf8")));
144
- let cfg = {};
145
- try { cfg = JSON.parse(readFileSync(join(dir, CONFIG_FILE), "utf8")); } catch { /* optional */ }
146
- const doNormalize = cfg.normalize !== false;
147
-
148
- const embed = (text) => {
149
- const ids = tokenize(text);
150
- const v = new Float32Array(dim);
151
- if (!ids.length) return v; // zero vector: cosine 0 against everything
152
- for (const id of ids) {
153
- if (id < 0 || id >= rows) continue;
154
- const off = id * dim;
155
- for (let j = 0; j < dim; j++) v[j] += matrix[off + j];
156
- }
157
- for (let j = 0; j < dim; j++) v[j] /= ids.length; // mean pool
158
- if (doNormalize) {
159
- let norm = 0;
160
- for (let j = 0; j < dim; j++) norm += v[j] * v[j];
161
- norm = Math.sqrt(norm);
162
- if (norm > 0) for (let j = 0; j < dim; j++) v[j] /= norm;
163
- }
164
- return v;
165
- };
166
- const embedder = { dim, dir, embed };
167
- LOADED.set(dir, embedder);
168
- return embedder;
169
- }
@@ -1,12 +0,0 @@
1
- // vector.mjs — pure vector arithmetic over embedding vectors. No model, no fs:
2
- // the loader that reads weights off disk lives in src/adapters/embed.mjs.
3
-
4
- /** Cosine similarity. Over L2-normalised vectors this is just the dot product, but the
5
- * full form is kept so unnormalised test fixtures behave. 0 when either vector is zero. */
6
- export function cosine(a, b) {
7
- let dot = 0, na = 0, nb = 0;
8
- const n = Math.min(a.length, b.length);
9
- for (let i = 0; i < n; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
10
- if (na === 0 || nb === 0) return 0;
11
- return dot / (Math.sqrt(na) * Math.sqrt(nb));
12
- }