@maxgfr/codeindex 2.9.0 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +75 -0
- package/docs/MIGRATION.md +29 -0
- package/package.json +1 -1
- package/scripts/engine.d.mts +73 -3
- package/scripts/engine.mjs +635 -31
- package/src/embed/encode.ts +143 -0
- package/src/embed/endpoint.ts +144 -0
- package/src/embed/index.ts +141 -0
- package/src/embed/model.ts +117 -0
- package/src/embed/search.ts +96 -0
- package/src/engine-cli.ts +184 -5
- package/src/engine.ts +33 -0
- package/src/extract/code.ts +15 -3
- package/src/mcp.ts +61 -10
- package/src/types.ts +6 -3
package/src/engine-cli.ts
CHANGED
|
@@ -19,6 +19,16 @@ import { symbolComplexity, riskHotspots } from "./complexity.js";
|
|
|
19
19
|
import { renderMermaid } from "./viz.js";
|
|
20
20
|
import { searchIndex } from "./bm25.js";
|
|
21
21
|
import { checkRules, parseRules } from "./rules.js";
|
|
22
|
+
import { EMBED_VERSION, resolveEmbedModelDir, loadEmbedModel, resolveEmbedPullUrl } from "./embed/model.js";
|
|
23
|
+
import { buildEmbeddingIndex, serializeEmbeddings } from "./embed/index.js";
|
|
24
|
+
import { searchSemantic } from "./embed/search.js";
|
|
25
|
+
import {
|
|
26
|
+
resolveEmbedEndpoint,
|
|
27
|
+
buildEndpointIndex,
|
|
28
|
+
encodeQueryViaEndpoint,
|
|
29
|
+
probeEndpoint,
|
|
30
|
+
} from "./embed/endpoint.js";
|
|
31
|
+
import { have, sh } from "./util.js";
|
|
22
32
|
|
|
23
33
|
const HELP = `codeindex engine v${ENGINE_VERSION} — deterministic repo indexing
|
|
24
34
|
|
|
@@ -37,7 +47,18 @@ Commands:
|
|
|
37
47
|
churn Per-file git commit counts (JSON; --since <ref> to bound)
|
|
38
48
|
grep Search: cli.mjs grep <pattern> --repo <dir> (JSON hits)
|
|
39
49
|
search Keyless BM25 lexical search over symbol names, path segments,
|
|
40
|
-
markdown headings and summaries: cli.mjs search "<query>" --repo <dir
|
|
50
|
+
markdown headings and summaries: cli.mjs search "<query>" --repo <dir>.
|
|
51
|
+
--semantic fuses in an embedding tier (RRF) — the HTTP endpoint
|
|
52
|
+
(CODEINDEX_EMBED_ENDPOINT) if set, else a local static model;
|
|
53
|
+
degrades to lexical (exit 0) when neither is available/reachable
|
|
54
|
+
embed Embedding tiers (opt-in). Precedence: endpoint > static model:
|
|
55
|
+
embed status Effective mode (none/static/endpoint), model +
|
|
56
|
+
EMBED_VERSION, and endpoint reachability (JSON)
|
|
57
|
+
embed build Write embeddings.bin into --out <dir> (static tier)
|
|
58
|
+
embed pull Fetch the model asset into CODEINDEX_EMBED_DIR (or
|
|
59
|
+
<repo>/.codeindex/models/) — needs CODEINDEX_EMBED_URL
|
|
60
|
+
embed serve Print (or --run) the docker command that starts the
|
|
61
|
+
containerized embedding server (rich tier)
|
|
41
62
|
rules Architecture rules (forbidden edges, cycles, orphans) validated
|
|
42
63
|
against the link-graph: --config <codeindex.rules.json>; exits 1
|
|
43
64
|
on any error-severity violation (a CI gate)
|
|
@@ -65,6 +86,10 @@ Flags:
|
|
|
65
86
|
--limit <n> Max results for \`search\` (default 20)
|
|
66
87
|
--no-fuzzy \`search\`: disable trigram fuzzy fallback for query terms
|
|
67
88
|
with zero document frequency (default: enabled)
|
|
89
|
+
--semantic \`search\`: RRF-fuse an embedding tier with lexical — the
|
|
90
|
+
HTTP endpoint if CODEINDEX_EMBED_ENDPOINT is set, else a
|
|
91
|
+
local static model (lexical-only when neither is available)
|
|
92
|
+
--run \`embed serve\`: run the docker command instead of printing it
|
|
68
93
|
--recall \`callers\`: recall-oriented binding (issue #7) — relaxes
|
|
69
94
|
the JS/TS import gate to unique repo-wide names and labels
|
|
70
95
|
each site corroborated|unique-name
|
|
@@ -87,13 +112,15 @@ interface CliFlags {
|
|
|
87
112
|
config?: string; // rules config path
|
|
88
113
|
limit?: number; // search result cap
|
|
89
114
|
fuzzy: boolean; // search: trigram fuzzy fallback for df==0 terms (default true)
|
|
115
|
+
semantic: boolean; // search: RRF-fuse the static-embedding tier (default false)
|
|
90
116
|
recall?: boolean; // callers: recall-oriented binding
|
|
117
|
+
run?: boolean; // `embed serve`: actually run the docker command (default: print)
|
|
91
118
|
projectRoot?: string; // scip: override Metadata.project_root
|
|
92
119
|
positional?: string; // e.g. the grep pattern or search query
|
|
93
120
|
}
|
|
94
121
|
|
|
95
122
|
function parseFlags(args: string[]): CliFlags {
|
|
96
|
-
const flags: CliFlags = { repo: process.cwd(), include: [], exclude: [], gitignore: true, noAst: false, fuzzy: true };
|
|
123
|
+
const flags: CliFlags = { repo: process.cwd(), include: [], exclude: [], gitignore: true, noAst: false, fuzzy: true, semantic: false };
|
|
97
124
|
for (let i = 0; i < args.length; i++) {
|
|
98
125
|
const a = args[i]!;
|
|
99
126
|
const next = (): string => {
|
|
@@ -126,7 +153,9 @@ function parseFlags(args: string[]): CliFlags {
|
|
|
126
153
|
else if (a === "--config") flags.config = resolve(next());
|
|
127
154
|
else if (a === "--limit") flags.limit = num();
|
|
128
155
|
else if (a === "--no-fuzzy") flags.fuzzy = false;
|
|
156
|
+
else if (a === "--semantic") flags.semantic = true;
|
|
129
157
|
else if (a === "--recall") flags.recall = true;
|
|
158
|
+
else if (a === "--run") flags.run = true;
|
|
130
159
|
else if (!a.startsWith("--") && flags.positional === undefined) flags.positional = a;
|
|
131
160
|
else throw new Error(`unknown flag: ${a}`);
|
|
132
161
|
}
|
|
@@ -203,7 +232,18 @@ export async function runCli(argv: string[]): Promise<void> {
|
|
|
203
232
|
cachePath,
|
|
204
233
|
JSON.stringify({ schemaVersion: SCHEMA_VERSION, extractorVersion: EXTRACTOR_VERSION, files }) + "\n",
|
|
205
234
|
);
|
|
206
|
-
|
|
235
|
+
// Deterministic embeddings sidecar: written next to graph.json ONLY when a
|
|
236
|
+
// model asset is present (opt-in). Silently skipped otherwise — no model, no
|
|
237
|
+
// embeddings.bin, no impact on the graph/symbols consumers.
|
|
238
|
+
let embedNote = "";
|
|
239
|
+
const modelDir = resolveEmbedModelDir(flags.repo);
|
|
240
|
+
const model = modelDir ? loadEmbedModel(modelDir) : undefined;
|
|
241
|
+
if (model) {
|
|
242
|
+
const index = buildEmbeddingIndex(scan, model);
|
|
243
|
+
writeFileSync(join(outDir, "embeddings.bin"), serializeEmbeddings(index));
|
|
244
|
+
embedNote = ` + embeddings.bin (${index.records.length} records, model ${model.modelId})`;
|
|
245
|
+
}
|
|
246
|
+
process.stderr.write(`codeindex: ${scan.files.length} files → ${outDir}/graph.json + symbols.json${embedNote}${scan.capped ? " (capped)" : ""}\n`);
|
|
207
247
|
} else if (cmd === "scan") {
|
|
208
248
|
const { scan } = buildIndexArtifacts(flags.repo, scanOptions(flags));
|
|
209
249
|
const summary = {
|
|
@@ -238,8 +278,147 @@ export async function runCli(argv: string[]): Promise<void> {
|
|
|
238
278
|
} else if (cmd === "search") {
|
|
239
279
|
if (!flags.positional) throw new Error('search needs a query: cli.mjs search "<query>" --repo <dir>');
|
|
240
280
|
const scan = scanRepo(flags.repo, scanOptions(flags));
|
|
241
|
-
|
|
242
|
-
|
|
281
|
+
if (flags.semantic) {
|
|
282
|
+
const endpoint = resolveEmbedEndpoint();
|
|
283
|
+
const lexical = (): void => {
|
|
284
|
+
const results = searchIndex(scan, flags.positional!, { limit: flags.limit, fuzzy: flags.fuzzy });
|
|
285
|
+
emit(JSON.stringify(results, null, 2) + "\n", flags.out);
|
|
286
|
+
};
|
|
287
|
+
if (endpoint) {
|
|
288
|
+
// Rich tier. The endpoint takes PRECEDENCE over a local static model:
|
|
289
|
+
// configuring CODEINDEX_EMBED_ENDPOINT is an explicit user intent. An
|
|
290
|
+
// unreachable/timed-out/malformed endpoint degrades straight to lexical
|
|
291
|
+
// (a stderr note, exit 0) — NOT to the static model.
|
|
292
|
+
try {
|
|
293
|
+
const index = await buildEndpointIndex(scan);
|
|
294
|
+
const queryVec = await encodeQueryViaEndpoint(flags.positional);
|
|
295
|
+
const results = searchSemantic(scan, flags.positional, index, { queryVec, limit: flags.limit, fuzzy: flags.fuzzy });
|
|
296
|
+
emit(JSON.stringify(results, null, 2) + "\n", flags.out);
|
|
297
|
+
} catch (e) {
|
|
298
|
+
process.stderr.write(
|
|
299
|
+
`codeindex: embedding endpoint ${endpoint} unavailable (${e instanceof Error ? e.message : e}) — returning lexical results\n`,
|
|
300
|
+
);
|
|
301
|
+
lexical();
|
|
302
|
+
}
|
|
303
|
+
} else {
|
|
304
|
+
const modelDir = resolveEmbedModelDir(flags.repo);
|
|
305
|
+
const model = modelDir ? loadEmbedModel(modelDir) : undefined;
|
|
306
|
+
if (!model) {
|
|
307
|
+
// Degradation: --semantic without a model or endpoint → lexical results
|
|
308
|
+
// + a stderr note, exit 0. The results shape is a superset of lexical.
|
|
309
|
+
process.stderr.write(
|
|
310
|
+
"codeindex: semantic search unavailable (no embedding model or endpoint) — returning lexical results; run `codeindex embed pull` or set CODEINDEX_EMBED_ENDPOINT to enable it\n",
|
|
311
|
+
);
|
|
312
|
+
lexical();
|
|
313
|
+
} else {
|
|
314
|
+
const index = buildEmbeddingIndex(scan, model);
|
|
315
|
+
const results = searchSemantic(scan, flags.positional, index, { model, limit: flags.limit, fuzzy: flags.fuzzy });
|
|
316
|
+
emit(JSON.stringify(results, null, 2) + "\n", flags.out);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
} else {
|
|
320
|
+
const results = searchIndex(scan, flags.positional, { limit: flags.limit, fuzzy: flags.fuzzy });
|
|
321
|
+
emit(JSON.stringify(results, null, 2) + "\n", flags.out);
|
|
322
|
+
}
|
|
323
|
+
} else if (cmd === "embed") {
|
|
324
|
+
const sub = flags.positional;
|
|
325
|
+
const modelDir = resolveEmbedModelDir(flags.repo);
|
|
326
|
+
if (sub === "status") {
|
|
327
|
+
const model = modelDir ? loadEmbedModel(modelDir) : undefined;
|
|
328
|
+
const endpoint = resolveEmbedEndpoint();
|
|
329
|
+
// Effective mode with precedence: endpoint > static model > none.
|
|
330
|
+
const mode: "none" | "static" | "endpoint" = endpoint ? "endpoint" : model ? "static" : "none";
|
|
331
|
+
const status: Record<string, unknown> = {
|
|
332
|
+
embedVersion: EMBED_VERSION,
|
|
333
|
+
mode,
|
|
334
|
+
model: model
|
|
335
|
+
? { present: true, dir: modelDir, modelId: model.modelId, dim: model.dim, vocabSize: model.vocabSize }
|
|
336
|
+
: { present: false },
|
|
337
|
+
endpoint: endpoint ?? null,
|
|
338
|
+
};
|
|
339
|
+
// When an endpoint is configured, actually probe its reachability.
|
|
340
|
+
if (endpoint) status.endpointReachable = await probeEndpoint(endpoint);
|
|
341
|
+
emit(JSON.stringify(status, null, 2) + "\n", flags.out);
|
|
342
|
+
} else if (sub === "serve") {
|
|
343
|
+
// Convenience only — the LIBRARY never orchestrates docker (engine.ts is
|
|
344
|
+
// side-effect-free). This lives in the CLI: it prints (or, with --run,
|
|
345
|
+
// executes) the docker command that starts the embedding server image.
|
|
346
|
+
const dockerArgs = ["run", "-d", "-p", "8756:8756", "ghcr.io/maxgfr/codeindex-embed:latest"];
|
|
347
|
+
const oneLiner = `docker ${dockerArgs.join(" ")}`;
|
|
348
|
+
if (!have("docker")) {
|
|
349
|
+
process.stderr.write(
|
|
350
|
+
"codeindex: docker not found on PATH. Install Docker, then run:\n " + oneLiner + "\n",
|
|
351
|
+
);
|
|
352
|
+
process.exitCode = 1;
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
if (flags.run) {
|
|
356
|
+
process.stderr.write(`codeindex: starting embedding server → ${oneLiner}\n`);
|
|
357
|
+
const res = sh("docker", dockerArgs);
|
|
358
|
+
if (res.stdout.trim()) process.stdout.write(res.stdout.trim() + "\n"); // container id
|
|
359
|
+
if (!res.ok) {
|
|
360
|
+
process.stderr.write(res.stderr || "codeindex: docker run failed\n");
|
|
361
|
+
process.exitCode = 1;
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
process.stderr.write(
|
|
365
|
+
"codeindex: server starting on http://localhost:8756 — then:\n" +
|
|
366
|
+
" CODEINDEX_EMBED_ENDPOINT=http://localhost:8756 codeindex search \"<query>\" --repo . --semantic\n",
|
|
367
|
+
);
|
|
368
|
+
} else {
|
|
369
|
+
// Print the command for the user to run (default; no side effects).
|
|
370
|
+
process.stdout.write(oneLiner + "\n");
|
|
371
|
+
process.stderr.write(
|
|
372
|
+
"codeindex: run the line above to start the embedding server (or `embed serve --run`), then:\n" +
|
|
373
|
+
" CODEINDEX_EMBED_ENDPOINT=http://localhost:8756 codeindex search \"<query>\" --repo . --semantic\n",
|
|
374
|
+
);
|
|
375
|
+
}
|
|
376
|
+
} else if (sub === "build") {
|
|
377
|
+
if (!flags.out) throw new Error("embed build needs --out <dir>");
|
|
378
|
+
if (!modelDir) {
|
|
379
|
+
process.stderr.write("codeindex: no embedding model present — run `codeindex embed pull` first (nothing written)\n");
|
|
380
|
+
process.exitCode = 1;
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
383
|
+
const model = loadEmbedModel(modelDir)!;
|
|
384
|
+
mkdirSync(flags.out, { recursive: true });
|
|
385
|
+
const scan = scanRepo(flags.repo, scanOptions(flags));
|
|
386
|
+
const index = buildEmbeddingIndex(scan, model);
|
|
387
|
+
writeFileSync(join(flags.out, "embeddings.bin"), serializeEmbeddings(index));
|
|
388
|
+
process.stderr.write(`codeindex: ${index.records.length} embedding records → ${flags.out}/embeddings.bin (model ${model.modelId})\n`);
|
|
389
|
+
} else if (sub === "pull") {
|
|
390
|
+
const url = resolveEmbedPullUrl();
|
|
391
|
+
if (!url) {
|
|
392
|
+
// Clean, actionable failure — the official asset is not published yet.
|
|
393
|
+
process.stderr.write(
|
|
394
|
+
"codeindex: no model URL configured. The official static-embedding asset is not published yet.\n" +
|
|
395
|
+
"Set CODEINDEX_EMBED_URL to a model.json URL (optionally CODEINDEX_EMBED_DIR as the destination), then re-run `codeindex embed pull`.\n",
|
|
396
|
+
);
|
|
397
|
+
process.exitCode = 1;
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
const destDir = process.env.CODEINDEX_EMBED_DIR ?? join(flags.repo, ".codeindex", "models");
|
|
401
|
+
mkdirSync(destDir, { recursive: true });
|
|
402
|
+
process.stderr.write(`codeindex: fetching model from ${url} → ${join(destDir, "model.json")}\n`);
|
|
403
|
+
const res = await fetch(url);
|
|
404
|
+
if (!res.ok) {
|
|
405
|
+
process.stderr.write(`codeindex: pull failed — HTTP ${res.status} from ${url}\n`);
|
|
406
|
+
process.exitCode = 1;
|
|
407
|
+
return;
|
|
408
|
+
}
|
|
409
|
+
const body = await res.text();
|
|
410
|
+
try {
|
|
411
|
+
JSON.parse(body);
|
|
412
|
+
} catch {
|
|
413
|
+
process.stderr.write("codeindex: pull failed — response is not a valid model.json (expected JSON)\n");
|
|
414
|
+
process.exitCode = 1;
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
writeFileSync(join(destDir, "model.json"), body);
|
|
418
|
+
process.stderr.write(`codeindex: model written to ${join(destDir, "model.json")}\n`);
|
|
419
|
+
} else {
|
|
420
|
+
throw new Error("embed needs a subcommand: status | build | pull | serve");
|
|
421
|
+
}
|
|
243
422
|
} else if (cmd === "rules") {
|
|
244
423
|
if (!flags.config) throw new Error("rules needs --config <codeindex.rules.json>");
|
|
245
424
|
const rules = parseRules(JSON.parse(readFileSync(flags.config, "utf8")));
|
package/src/engine.ts
CHANGED
|
@@ -103,6 +103,39 @@ export type { SearchHit, GrepOptions } from "./grep.js";
|
|
|
103
103
|
export { searchIndex, subtokens } from "./bm25.js";
|
|
104
104
|
export type { SearchOptions, SearchResult } from "./bm25.js";
|
|
105
105
|
|
|
106
|
+
// Deterministic static-embedding tier (v2.10.0): a keyless, byte-deterministic
|
|
107
|
+
// semantic search, opt-in by model-asset presence (models NEVER ship in the
|
|
108
|
+
// tarball). EMBED_VERSION is dedicated to the embeddings.bin sidecar and is
|
|
109
|
+
// independent of SCHEMA_VERSION / EXTRACTOR_VERSION.
|
|
110
|
+
export {
|
|
111
|
+
EMBED_VERSION,
|
|
112
|
+
resolveEmbedModelDir,
|
|
113
|
+
hasEmbedModel,
|
|
114
|
+
loadEmbedModel,
|
|
115
|
+
resolveEmbedPullUrl,
|
|
116
|
+
} from "./embed/model.js";
|
|
117
|
+
export type { StaticEmbedModel } from "./embed/model.js";
|
|
118
|
+
export { encode, quantize, tokenize, wordpiece, basicTokenize, roundHalfToEven, intDot } from "./embed/encode.js";
|
|
119
|
+
export { buildEmbeddingIndex, serializeEmbeddings, deserializeEmbeddings, embeddingUnits } from "./embed/index.js";
|
|
120
|
+
export type { EmbeddingIndex, EmbeddingRecord, EmbeddingUnit } from "./embed/index.js";
|
|
121
|
+
export { searchSemantic } from "./embed/search.js";
|
|
122
|
+
export type { SemanticSearchOptions, SemanticSearchResult } from "./embed/search.js";
|
|
123
|
+
// HTTP endpoint tier (v2.11.0 — the "rich" tier). The engine is a fetch consumer
|
|
124
|
+
// of a containerized embedding server (CODEINDEX_EMBED_ENDPOINT): float vectors
|
|
125
|
+
// run through the SAME L2+int8 quantize pipeline, then the same integer ranking.
|
|
126
|
+
// Deterministic PER IMAGE DIGEST (not byte-golden). The library never
|
|
127
|
+
// orchestrates docker — `codeindex embed serve` (CLI) only prints/runs it.
|
|
128
|
+
export {
|
|
129
|
+
embedViaEndpoint,
|
|
130
|
+
resolveEmbedEndpoint,
|
|
131
|
+
embedEndpointUrl,
|
|
132
|
+
healthzUrl,
|
|
133
|
+
probeEndpoint,
|
|
134
|
+
encodeQueryViaEndpoint,
|
|
135
|
+
buildEndpointIndex,
|
|
136
|
+
} from "./embed/endpoint.js";
|
|
137
|
+
export type { EmbedEndpointOptions } from "./embed/endpoint.js";
|
|
138
|
+
|
|
106
139
|
// Architecture rules: forbidden edges + cycles/orphans builtins (issue #4).
|
|
107
140
|
export { checkRules, parseRules } from "./rules.js";
|
|
108
141
|
export type { ArchRule, ForbiddenEdgeRule, BuiltinRule, RuleSeverity, RuleViolation } from "./rules.js";
|
package/src/extract/code.ts
CHANGED
|
@@ -256,12 +256,22 @@ function extractImports(ext: string, content: string): RawRef[] {
|
|
|
256
256
|
// Barrel re-exports (`export { A, B as C } from './x'`, `export * from './y'`).
|
|
257
257
|
// The line-based lang extractor can't capture multi-name lists, but these ARE
|
|
258
258
|
// the public facade of a module — so list them as exported symbols here.
|
|
259
|
-
|
|
259
|
+
//
|
|
260
|
+
// An ALIAS with no `from` clause (`export { b as c }`) renames an in-file
|
|
261
|
+
// declaration — `localSymbols` (already extracted by the AST or regex tier)
|
|
262
|
+
// lets us resolve `b` and mirror ITS kind onto `c` (e.g. "function"), so the
|
|
263
|
+
// alias reads as the real symbol it is rather than the generic "reexport".
|
|
264
|
+
// A true cross-module re-export (`export { b as c } from "./mod"`) has no
|
|
265
|
+
// local `b` to resolve — and an alias the local pass genuinely can't see
|
|
266
|
+
// (destructured/ambient/etc.) falls back the same way — both keep "reexport".
|
|
267
|
+
function extractReexports(rel: string, content: string, localSymbols: CodeSymbol[]): CodeSymbol[] {
|
|
260
268
|
if (!JS_TS.has(rel.slice(rel.lastIndexOf(".")))) return [];
|
|
261
269
|
const lang = /\.(ts|tsx|mts|cts)$/.test(rel) ? "typescript" : "javascript";
|
|
262
270
|
const out: CodeSymbol[] = [];
|
|
263
271
|
const seen = new Set<string>();
|
|
264
272
|
const lineAt = (idx: number): number => content.slice(0, idx).split(/\r?\n/).length;
|
|
273
|
+
const localKindOf = new Map<string, string>();
|
|
274
|
+
for (const s of localSymbols) if (!localKindOf.has(s.name)) localKindOf.set(s.name, s.kind);
|
|
265
275
|
|
|
266
276
|
const named = /export\s*\{([\s\S]*?)\}\s*(?:from\s*['"]([^'"]+)['"])?\s*;?/g;
|
|
267
277
|
let m: RegExpExecArray | null;
|
|
@@ -270,11 +280,13 @@ function extractReexports(rel: string, content: string): CodeSymbol[] {
|
|
|
270
280
|
for (const part of m[1]!.split(",")) {
|
|
271
281
|
const p = part.trim().replace(/^type\s+/, "");
|
|
272
282
|
const as = /^(\S+)\s+as\s+([A-Za-z_$][\w$]*)$/.exec(p);
|
|
283
|
+
const orig = as ? as[1]! : p;
|
|
273
284
|
const name = as ? as[2]! : p;
|
|
274
285
|
if (!/^[A-Za-z_$][\w$]*$/.test(name) || name === "default" || seen.has(name)) continue;
|
|
275
286
|
seen.add(name);
|
|
287
|
+
const mirroredKind = !from ? localKindOf.get(orig) : undefined;
|
|
276
288
|
out.push({
|
|
277
|
-
name, kind: "reexport", file: rel, line: lineAt(m.index),
|
|
289
|
+
name, kind: mirroredKind ?? "reexport", file: rel, line: lineAt(m.index),
|
|
278
290
|
signature: from ? `export { ${name} } from "${from}"` : `export { ${name} }`,
|
|
279
291
|
exported: true, lang,
|
|
280
292
|
});
|
|
@@ -354,7 +366,7 @@ export function extractCode(rel: string, ext: string, content: string): CodeInfo
|
|
|
354
366
|
const symbols = (ast ? ast.symbols : extractSymbols(rel, ext, content)).slice(0, 400);
|
|
355
367
|
// Add barrel re-exports the local def didn't already cover.
|
|
356
368
|
const known = new Set(symbols.map((s) => s.name));
|
|
357
|
-
const reexports = extractReexports(rel, content).filter((s) => !known.has(s.name));
|
|
369
|
+
const reexports = extractReexports(rel, content, symbols).filter((s) => !known.has(s.name));
|
|
358
370
|
return {
|
|
359
371
|
symbols: [...symbols, ...reexports],
|
|
360
372
|
summary: topDocComment(content),
|
package/src/mcp.ts
CHANGED
|
@@ -25,6 +25,10 @@ import { replaceSymbolBody, insertAfterSymbol, insertBeforeSymbol } from "./edit
|
|
|
25
25
|
import { writeMemory, readMemory, deleteMemory, listMemories } from "./memory.js";
|
|
26
26
|
import { searchIndex } from "./bm25.js";
|
|
27
27
|
import { checkRules, parseRules } from "./rules.js";
|
|
28
|
+
import { EMBED_VERSION, resolveEmbedModelDir, loadEmbedModel } from "./embed/model.js";
|
|
29
|
+
import { buildEmbeddingIndex } from "./embed/index.js";
|
|
30
|
+
import { searchSemantic } from "./embed/search.js";
|
|
31
|
+
import { resolveEmbedEndpoint, buildEndpointIndex, encodeQueryViaEndpoint, probeEndpoint } from "./embed/endpoint.js";
|
|
28
32
|
|
|
29
33
|
interface RpcRequest {
|
|
30
34
|
jsonrpc: "2.0";
|
|
@@ -266,7 +270,7 @@ const TOOLS = [
|
|
|
266
270
|
{
|
|
267
271
|
name: "search",
|
|
268
272
|
description:
|
|
269
|
-
'Natural-language-ish lexical search: BM25 ranking (k1=1.2, b=0.75) over symbol names (camelCase/snake_case subtokens), file path segments, markdown headings and summary lines. NOT embeddings — deterministic, diacritic-folded, zero API keys. Answers "where is auth handled?"-style queries with ranked files, matched terms and top symbols. Query terms with zero document frequency get a deterministic trigram-fuzzy fallback (typo-tolerant) unless `fuzzy: false`.',
|
|
273
|
+
'Natural-language-ish lexical search: BM25 ranking (k1=1.2, b=0.75) over symbol names (camelCase/snake_case subtokens), file path segments, markdown headings and summary lines. NOT embeddings by default — deterministic, diacritic-folded, zero API keys. Answers "where is auth handled?"-style queries with ranked files, matched terms and top symbols. Query terms with zero document frequency get a deterministic trigram-fuzzy fallback (typo-tolerant) unless `fuzzy: false`. Set `semantic: true` to RRF-fuse the deterministic static-embedding tier when a model asset is present (degrades to lexical otherwise — see embed_status).',
|
|
270
274
|
inputSchema: {
|
|
271
275
|
type: "object",
|
|
272
276
|
properties: {
|
|
@@ -279,10 +283,21 @@ const TOOLS = [
|
|
|
279
283
|
description:
|
|
280
284
|
"Trigram fuzzy fallback for query terms with zero document frequency (default true)",
|
|
281
285
|
},
|
|
286
|
+
semantic: {
|
|
287
|
+
type: "boolean",
|
|
288
|
+
description:
|
|
289
|
+
"RRF-fuse an embedding tier with lexical (default false). Precedence: the HTTP endpoint (CODEINDEX_EMBED_ENDPOINT) if set, else a local static model. Degrades silently to lexical-only when neither is available/reachable — see embed_status.",
|
|
290
|
+
},
|
|
282
291
|
},
|
|
283
292
|
required: ["repo", "query"],
|
|
284
293
|
},
|
|
285
294
|
},
|
|
295
|
+
{
|
|
296
|
+
name: "embed_status",
|
|
297
|
+
description:
|
|
298
|
+
"Report the embedding tier: the effective mode (none/static/endpoint; endpoint > static model), the resolved model (opt-in, never shipped in the package) with its modelId/dim, EMBED_VERSION, and the configured HTTP endpoint with its reachability. Use to check whether `search` with semantic:true will fuse embeddings or degrade to lexical.",
|
|
299
|
+
inputSchema: { type: "object", properties: { ...repoProp }, required: ["repo"] },
|
|
300
|
+
},
|
|
286
301
|
{
|
|
287
302
|
name: "check_rules",
|
|
288
303
|
description:
|
|
@@ -306,7 +321,7 @@ function strArray(v: unknown): string[] | undefined {
|
|
|
306
321
|
return Array.isArray(v) && v.every((x) => typeof x === "string") && v.length ? (v as string[]) : undefined;
|
|
307
322
|
}
|
|
308
323
|
|
|
309
|
-
function callTool(name: string, args: Record<string, unknown>): string {
|
|
324
|
+
async function callTool(name: string, args: Record<string, unknown>): Promise<string> {
|
|
310
325
|
const repo = str(args.repo);
|
|
311
326
|
if (!repo) throw new Error("`repo` is required (absolute path to the repository root)");
|
|
312
327
|
const scanOpts = { scope: str(args.scope), include: strArray(args.include), exclude: strArray(args.exclude) };
|
|
@@ -440,11 +455,47 @@ function callTool(name: string, args: Record<string, unknown>): string {
|
|
|
440
455
|
if (name === "search") {
|
|
441
456
|
const query = str(args.query);
|
|
442
457
|
if (!query) throw new Error("`query` is required");
|
|
443
|
-
const
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
458
|
+
const scan = scanRepo(repo, scanOpts);
|
|
459
|
+
const limit = typeof args.limit === "number" ? args.limit : undefined;
|
|
460
|
+
const fuzzy = typeof args.fuzzy === "boolean" ? args.fuzzy : undefined;
|
|
461
|
+
if (args.semantic === true) {
|
|
462
|
+
const endpoint = resolveEmbedEndpoint();
|
|
463
|
+
if (endpoint) {
|
|
464
|
+
// Rich tier — endpoint takes PRECEDENCE over a local static model. An
|
|
465
|
+
// unreachable/malformed endpoint degrades to lexical (no throw).
|
|
466
|
+
try {
|
|
467
|
+
const index = await buildEndpointIndex(scan);
|
|
468
|
+
const queryVec = await encodeQueryViaEndpoint(query);
|
|
469
|
+
return JSON.stringify(searchSemantic(scan, query, index, { queryVec, limit, fuzzy }), null, 2);
|
|
470
|
+
} catch {
|
|
471
|
+
return JSON.stringify(searchIndex(scan, query, { limit, fuzzy }), null, 2);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
const modelDir = resolveEmbedModelDir(repo);
|
|
475
|
+
const model = modelDir ? loadEmbedModel(modelDir) : undefined;
|
|
476
|
+
if (model) {
|
|
477
|
+
const index = buildEmbeddingIndex(scan, model);
|
|
478
|
+
return JSON.stringify(searchSemantic(scan, query, index, { model, limit, fuzzy }), null, 2);
|
|
479
|
+
}
|
|
480
|
+
// Degrade to lexical (opt-in tier not activated) — no throw.
|
|
481
|
+
}
|
|
482
|
+
return JSON.stringify(searchIndex(scan, query, { limit, fuzzy }), null, 2);
|
|
483
|
+
}
|
|
484
|
+
if (name === "embed_status") {
|
|
485
|
+
const modelDir = resolveEmbedModelDir(repo);
|
|
486
|
+
const model = modelDir ? loadEmbedModel(modelDir) : undefined;
|
|
487
|
+
const endpoint = resolveEmbedEndpoint();
|
|
488
|
+
const mode: "none" | "static" | "endpoint" = endpoint ? "endpoint" : model ? "static" : "none";
|
|
489
|
+
const status: Record<string, unknown> = {
|
|
490
|
+
embedVersion: EMBED_VERSION,
|
|
491
|
+
mode,
|
|
492
|
+
model: model
|
|
493
|
+
? { present: true, dir: modelDir, modelId: model.modelId, dim: model.dim, vocabSize: model.vocabSize }
|
|
494
|
+
: { present: false },
|
|
495
|
+
endpoint: endpoint ?? null,
|
|
496
|
+
};
|
|
497
|
+
if (endpoint) status.endpointReachable = await probeEndpoint(endpoint);
|
|
498
|
+
return JSON.stringify(status, null, 2);
|
|
448
499
|
}
|
|
449
500
|
if (name === "check_rules") {
|
|
450
501
|
const rules = parseRules(args.rules); // throws a descriptive error on a malformed payload
|
|
@@ -475,10 +526,10 @@ export async function runMcpServer(): Promise<void> {
|
|
|
475
526
|
// JSON-RPC 2.0 batch: answer each member (a batching client would
|
|
476
527
|
// otherwise hang forever on a silently dropped array).
|
|
477
528
|
const requests = Array.isArray(parsed) ? (parsed as RpcRequest[]) : [parsed as RpcRequest];
|
|
478
|
-
for (const req of requests) handle(req);
|
|
529
|
+
for (const req of requests) await handle(req);
|
|
479
530
|
}
|
|
480
531
|
|
|
481
|
-
function handle(req: RpcRequest): void {
|
|
532
|
+
async function handle(req: RpcRequest): Promise<void> {
|
|
482
533
|
if (req.id === undefined || req.id === null) return; // notification — no response
|
|
483
534
|
|
|
484
535
|
try {
|
|
@@ -500,7 +551,7 @@ export async function runMcpServer(): Promise<void> {
|
|
|
500
551
|
const name = str(params.name) ?? "";
|
|
501
552
|
const args = (params.arguments ?? {}) as Record<string, unknown>;
|
|
502
553
|
try {
|
|
503
|
-
const text = callTool(name, args);
|
|
554
|
+
const text = await callTool(name, args);
|
|
504
555
|
send({ id: req.id, result: { content: [{ type: "text", text }] } });
|
|
505
556
|
} catch (e) {
|
|
506
557
|
send({
|
package/src/types.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Single source of truth for the engine version the bundle reports. Kept in
|
|
2
2
|
// lockstep with package.json by the release pipeline. Do not edit by hand
|
|
3
3
|
// outside a release.
|
|
4
|
-
export const ENGINE_VERSION = "2.
|
|
4
|
+
export const ENGINE_VERSION = "2.11.0";
|
|
5
5
|
|
|
6
6
|
// Bumped whenever the on-disk artifact shape changes, so a consumer can reject
|
|
7
7
|
// an index written by an incompatible engine instead of misreading it. The
|
|
@@ -24,8 +24,11 @@ export const SCHEMA_VERSION = 4;
|
|
|
24
24
|
// (the immediate receiver of a qualified call, both tiers) and JS/TS export
|
|
25
25
|
// parity with ultradoc (CJS `exports.foo =` / `module.exports = {…}` named
|
|
26
26
|
// exports, `export { a, b as c }` local marking, anonymous `export default`
|
|
27
|
-
// named after the file stem, `export default Foo` marking the declaration)
|
|
28
|
-
export
|
|
27
|
+
// named after the file stem, `export default Foo` marking the declaration);
|
|
28
|
+
// v7 makes an export-alias symbol (`export { b as c }`) mirror the aliased
|
|
29
|
+
// local declaration's own kind (e.g. "function") instead of the generic
|
|
30
|
+
// "reexport" when it resolves in-file.
|
|
31
|
+
export const EXTRACTOR_VERSION = 7;
|
|
29
32
|
|
|
30
33
|
// How a file is classified. `code` gets symbol/import extraction; `doc` gets
|
|
31
34
|
// link/heading extraction; the rest are catalogued but not deeply parsed.
|