@maxgfr/codeindex 2.8.1 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/docs/MIGRATION.md +9 -0
- package/package.json +1 -1
- package/scripts/engine.d.mts +3 -1
- package/scripts/engine.mjs +92 -15
- package/src/bm25.ts +118 -9
- package/src/engine-cli.ts +6 -2
- package/src/mcp.ts +7 -1
- package/src/types.ts +1 -1
package/README.md
CHANGED
|
@@ -75,6 +75,20 @@ codeindex callers --repo . # per-symbol caller index
|
|
|
75
75
|
codeindex grep 'pattern' --repo .
|
|
76
76
|
```
|
|
77
77
|
|
|
78
|
+
## Search
|
|
79
|
+
|
|
80
|
+
`codeindex search "<query>" --repo .` ranks files with keyless BM25 over symbol
|
|
81
|
+
names, path segments, markdown headings and summaries. A query term that
|
|
82
|
+
matches nothing in the corpus (zero document frequency) gets a deterministic
|
|
83
|
+
**trigram fuzzy fallback** — typo tolerance without embeddings: the term is
|
|
84
|
+
compared to the corpus vocabulary by character-trigram Dice similarity
|
|
85
|
+
(threshold 0.6, top-3 candidates, contribution scaled by the Dice score so a
|
|
86
|
+
near-miss always ranks below an exact hit). Terms that already match anything
|
|
87
|
+
are never touched, so an existing query stays byte-identical. Enabled by
|
|
88
|
+
default; disable with `--no-fuzzy` (CLI) or `fuzzy: false` (library/MCP
|
|
89
|
+
`SearchOptions.fuzzy`); results carry an additive `fuzzyTerms` field when the
|
|
90
|
+
fallback contributed.
|
|
91
|
+
|
|
78
92
|
## Use as an MCP server
|
|
79
93
|
|
|
80
94
|
`codeindex mcp` (or `node scripts/cli.mjs mcp`) serves the engine over stdio —
|
package/docs/MIGRATION.md
CHANGED
|
@@ -55,6 +55,15 @@ does; nobody else should).
|
|
|
55
55
|
`meta: { version, schemaVersion }` so a consumer stamps its own identity into
|
|
56
56
|
artifacts it persists (ultraindex does this to keep its graph.json lineage).
|
|
57
57
|
|
|
58
|
+
## v2.9.0 — `search` trigram fuzzy fallback
|
|
59
|
+
|
|
60
|
+
New, purely additive: `SearchOptions.fuzzy?: boolean` (default `true`) and
|
|
61
|
+
`SearchResult.fuzzyTerms?: string[]` (present only when the fallback
|
|
62
|
+
contributed). A query term is only ever expanded when it has zero document
|
|
63
|
+
frequency in the corpus, so any query where every term already matched keeps
|
|
64
|
+
producing byte-identical output — **no re-pin required**, no action needed
|
|
65
|
+
from existing consumers. Pass `fuzzy: false` (or CLI `--no-fuzzy`) to opt out.
|
|
66
|
+
|
|
58
67
|
## Per-skill mapping (what to replace with what)
|
|
59
68
|
|
|
60
69
|
| Skill | Replace | With (engine export) |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@maxgfr/codeindex",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.9.0",
|
|
4
4
|
"description": "Self-contained, deterministic repo-indexing engine: walk + language detection + symbol/import extraction (tree-sitter AST with regex fallback) + import resolution + typed cross-file link-graph + analytics. Ships as a single zero-dependency engine.mjs that consumer tools vendor.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"packageManager": "pnpm@10.33.0",
|
package/scripts/engine.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
declare const ENGINE_VERSION = "2.
|
|
1
|
+
declare const ENGINE_VERSION = "2.9.0";
|
|
2
2
|
declare const SCHEMA_VERSION = 4;
|
|
3
3
|
declare const EXTRACTOR_VERSION = 6;
|
|
4
4
|
type FileKind = "code" | "doc" | "config" | "asset" | "other";
|
|
@@ -487,12 +487,14 @@ declare function grepRepo(root: string, pattern: string, opts?: GrepOptions): Se
|
|
|
487
487
|
|
|
488
488
|
interface SearchOptions {
|
|
489
489
|
limit?: number;
|
|
490
|
+
fuzzy?: boolean;
|
|
490
491
|
}
|
|
491
492
|
interface SearchResult {
|
|
492
493
|
file: string;
|
|
493
494
|
score: number;
|
|
494
495
|
matchedTerms: string[];
|
|
495
496
|
topSymbols: string[];
|
|
497
|
+
fuzzyTerms?: string[];
|
|
496
498
|
}
|
|
497
499
|
declare function subtokens(raw: string): string[];
|
|
498
500
|
declare function searchIndex(scan: RepoScan, query: string, opts?: SearchOptions): SearchResult[];
|
package/scripts/engine.mjs
CHANGED
|
@@ -14,7 +14,7 @@ var ENGINE_VERSION, SCHEMA_VERSION, EXTRACTOR_VERSION;
|
|
|
14
14
|
var init_types = __esm({
|
|
15
15
|
"src/types.ts"() {
|
|
16
16
|
"use strict";
|
|
17
|
-
ENGINE_VERSION = "2.
|
|
17
|
+
ENGINE_VERSION = "2.9.0";
|
|
18
18
|
SCHEMA_VERSION = 4;
|
|
19
19
|
EXTRACTOR_VERSION = 6;
|
|
20
20
|
}
|
|
@@ -9210,6 +9210,27 @@ function buildDocs(scan2) {
|
|
|
9210
9210
|
}
|
|
9211
9211
|
return docs;
|
|
9212
9212
|
}
|
|
9213
|
+
function charTrigrams(term) {
|
|
9214
|
+
const padded = `^^${term}$$`;
|
|
9215
|
+
const grams = /* @__PURE__ */ new Set();
|
|
9216
|
+
for (let i2 = 0; i2 + 3 <= padded.length; i2++) grams.add(padded.slice(i2, i2 + 3));
|
|
9217
|
+
return grams;
|
|
9218
|
+
}
|
|
9219
|
+
function diceCoefficient(a, b) {
|
|
9220
|
+
if (!a.size || !b.size) return 0;
|
|
9221
|
+
let inter = 0;
|
|
9222
|
+
for (const g of a) if (b.has(g)) inter++;
|
|
9223
|
+
return 2 * inter / (a.size + b.size);
|
|
9224
|
+
}
|
|
9225
|
+
function buildTrigramIndex(docs) {
|
|
9226
|
+
const index = /* @__PURE__ */ new Map();
|
|
9227
|
+
for (const d of docs) {
|
|
9228
|
+
for (const term of d.tf.keys()) {
|
|
9229
|
+
if (!index.has(term)) index.set(term, charTrigrams(term));
|
|
9230
|
+
}
|
|
9231
|
+
}
|
|
9232
|
+
return index;
|
|
9233
|
+
}
|
|
9213
9234
|
function searchIndex(scan2, query, opts = {}) {
|
|
9214
9235
|
const terms = [];
|
|
9215
9236
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -9233,35 +9254,81 @@ function searchIndex(scan2, query, opts = {}) {
|
|
|
9233
9254
|
for (const d of docs) if (d.tf.has(t)) count++;
|
|
9234
9255
|
df.set(t, count);
|
|
9235
9256
|
}
|
|
9257
|
+
const fuzzyEnabled = opts.fuzzy ?? true;
|
|
9258
|
+
const fuzzyCandidates = /* @__PURE__ */ new Map();
|
|
9259
|
+
if (fuzzyEnabled) {
|
|
9260
|
+
const unmatched = terms.filter((t) => df.get(t) === 0);
|
|
9261
|
+
if (unmatched.length) {
|
|
9262
|
+
const trigramIndex = buildTrigramIndex(docs);
|
|
9263
|
+
for (const t of unmatched) {
|
|
9264
|
+
const grams = charTrigrams(t);
|
|
9265
|
+
const candidates = [];
|
|
9266
|
+
for (const [vocabTerm, vocabGrams] of trigramIndex) {
|
|
9267
|
+
const dice = diceCoefficient(grams, vocabGrams);
|
|
9268
|
+
if (dice >= FUZZY_DICE_THRESHOLD) candidates.push({ term: vocabTerm, dice });
|
|
9269
|
+
}
|
|
9270
|
+
candidates.sort((a, b) => b.dice - a.dice || byStr(a.term, b.term));
|
|
9271
|
+
fuzzyCandidates.set(t, candidates.slice(0, FUZZY_CAP));
|
|
9272
|
+
}
|
|
9273
|
+
}
|
|
9274
|
+
}
|
|
9275
|
+
const vocabDf = /* @__PURE__ */ new Map();
|
|
9276
|
+
const dfOfVocabTerm = (term) => {
|
|
9277
|
+
const known = df.get(term) ?? vocabDf.get(term);
|
|
9278
|
+
if (known !== void 0) return known;
|
|
9279
|
+
let count = 0;
|
|
9280
|
+
for (const d of docs) if (d.tf.has(term)) count++;
|
|
9281
|
+
vocabDf.set(term, count);
|
|
9282
|
+
return count;
|
|
9283
|
+
};
|
|
9236
9284
|
const results = [];
|
|
9237
9285
|
for (const d of docs) {
|
|
9238
9286
|
let score = 0;
|
|
9239
9287
|
const matched = [];
|
|
9288
|
+
const symbolTerms = /* @__PURE__ */ new Set();
|
|
9289
|
+
const fuzzyHit = /* @__PURE__ */ new Set();
|
|
9240
9290
|
for (const t of terms) {
|
|
9241
9291
|
const tf = d.tf.get(t);
|
|
9242
|
-
if (
|
|
9243
|
-
|
|
9244
|
-
|
|
9245
|
-
|
|
9292
|
+
if (tf) {
|
|
9293
|
+
matched.push(t);
|
|
9294
|
+
symbolTerms.add(t);
|
|
9295
|
+
const idf = Math.log(1 + (n - df.get(t) + 0.5) / (df.get(t) + 0.5));
|
|
9296
|
+
score += idf * (tf * (K1 + 1)) / (tf + K1 * (1 - B + B * d.len / avgLen));
|
|
9297
|
+
continue;
|
|
9298
|
+
}
|
|
9299
|
+
const candidates = fuzzyCandidates.get(t);
|
|
9300
|
+
if (!candidates) continue;
|
|
9301
|
+
for (const cand of candidates) {
|
|
9302
|
+
const ctf = d.tf.get(cand.term);
|
|
9303
|
+
if (!ctf) continue;
|
|
9304
|
+
const cdf = dfOfVocabTerm(cand.term);
|
|
9305
|
+
const idf = Math.log(1 + (n - cdf + 0.5) / (cdf + 0.5));
|
|
9306
|
+
const contribution = idf * (ctf * (K1 + 1)) / (ctf + K1 * (1 - B + B * d.len / avgLen));
|
|
9307
|
+
score += contribution * cand.dice;
|
|
9308
|
+
symbolTerms.add(cand.term);
|
|
9309
|
+
fuzzyHit.add(t);
|
|
9310
|
+
}
|
|
9246
9311
|
}
|
|
9247
|
-
if (!matched.length) continue;
|
|
9312
|
+
if (!matched.length && !fuzzyHit.size) continue;
|
|
9248
9313
|
const scored = d.symbols.map((name2) => {
|
|
9249
9314
|
const toks = new Set(subtokens(name2));
|
|
9250
9315
|
let hits = 0;
|
|
9251
|
-
for (const t of
|
|
9316
|
+
for (const t of symbolTerms) if (toks.has(t)) hits++;
|
|
9252
9317
|
return { name: name2, hits };
|
|
9253
9318
|
}).filter((s) => s.hits > 0).sort((a, b) => b.hits - a.hits || byStr(a.name, b.name));
|
|
9254
|
-
|
|
9319
|
+
const result = {
|
|
9255
9320
|
file: d.file,
|
|
9256
9321
|
score: Number(score.toFixed(4)),
|
|
9257
9322
|
matchedTerms: matched.sort(byStr),
|
|
9258
9323
|
topSymbols: scored.slice(0, TOP_SYMBOLS).map((s) => s.name)
|
|
9259
|
-
}
|
|
9324
|
+
};
|
|
9325
|
+
if (fuzzyHit.size) result.fuzzyTerms = [...fuzzyHit].sort(byStr);
|
|
9326
|
+
results.push(result);
|
|
9260
9327
|
}
|
|
9261
9328
|
results.sort((a, b) => b.score - a.score || byStr(a.file, b.file));
|
|
9262
9329
|
return results.slice(0, opts.limit ?? DEFAULT_LIMIT);
|
|
9263
9330
|
}
|
|
9264
|
-
var K1, B, DEFAULT_LIMIT, TOP_SYMBOLS;
|
|
9331
|
+
var K1, B, DEFAULT_LIMIT, TOP_SYMBOLS, FUZZY_DICE_THRESHOLD, FUZZY_CAP;
|
|
9265
9332
|
var init_bm25 = __esm({
|
|
9266
9333
|
"src/bm25.ts"() {
|
|
9267
9334
|
"use strict";
|
|
@@ -9271,6 +9338,8 @@ var init_bm25 = __esm({
|
|
|
9271
9338
|
B = 0.75;
|
|
9272
9339
|
DEFAULT_LIMIT = 20;
|
|
9273
9340
|
TOP_SYMBOLS = 5;
|
|
9341
|
+
FUZZY_DICE_THRESHOLD = 0.6;
|
|
9342
|
+
FUZZY_CAP = 3;
|
|
9274
9343
|
}
|
|
9275
9344
|
});
|
|
9276
9345
|
|
|
@@ -9803,7 +9872,8 @@ function callTool(name2, args2) {
|
|
|
9803
9872
|
const query = str(args2.query);
|
|
9804
9873
|
if (!query) throw new Error("`query` is required");
|
|
9805
9874
|
const results = searchIndex(scanRepo(repo, scanOpts), query, {
|
|
9806
|
-
limit: typeof args2.limit === "number" ? args2.limit : void 0
|
|
9875
|
+
limit: typeof args2.limit === "number" ? args2.limit : void 0,
|
|
9876
|
+
fuzzy: typeof args2.fuzzy === "boolean" ? args2.fuzzy : void 0
|
|
9807
9877
|
});
|
|
9808
9878
|
return JSON.stringify(results, null, 2);
|
|
9809
9879
|
}
|
|
@@ -10105,14 +10175,18 @@ var init_mcp = __esm({
|
|
|
10105
10175
|
},
|
|
10106
10176
|
{
|
|
10107
10177
|
name: "search",
|
|
10108
|
-
description: 'Natural-language-ish lexical search: BM25 ranking (k1=1.2, b=0.75) over symbol names (camelCase/snake_case subtokens), file path segments, markdown headings and summary lines. NOT embeddings \u2014 deterministic, diacritic-folded, zero API keys. Answers "where is auth handled?"-style queries with ranked files, matched terms and top symbols.',
|
|
10178
|
+
description: 'Natural-language-ish lexical search: BM25 ranking (k1=1.2, b=0.75) over symbol names (camelCase/snake_case subtokens), file path segments, markdown headings and summary lines. NOT embeddings \u2014 deterministic, diacritic-folded, zero API keys. Answers "where is auth handled?"-style queries with ranked files, matched terms and top symbols. Query terms with zero document frequency get a deterministic trigram-fuzzy fallback (typo-tolerant) unless `fuzzy: false`.',
|
|
10109
10179
|
inputSchema: {
|
|
10110
10180
|
type: "object",
|
|
10111
10181
|
properties: {
|
|
10112
10182
|
...repoProp,
|
|
10113
10183
|
...scopeProps,
|
|
10114
10184
|
query: { type: "string", description: "Natural-language or identifier query" },
|
|
10115
|
-
limit: { type: "number", description: "Max results (default 20)" }
|
|
10185
|
+
limit: { type: "number", description: "Max results (default 20)" },
|
|
10186
|
+
fuzzy: {
|
|
10187
|
+
type: "boolean",
|
|
10188
|
+
description: "Trigram fuzzy fallback for query terms with zero document frequency (default true)"
|
|
10189
|
+
}
|
|
10116
10190
|
},
|
|
10117
10191
|
required: ["repo", "query"]
|
|
10118
10192
|
}
|
|
@@ -10606,12 +10680,14 @@ Flags:
|
|
|
10606
10680
|
--no-ast Skip tree-sitter grammars even when present (regex tier)
|
|
10607
10681
|
--config <file> Rules config for \`rules\` (JSON: [{name, from, to, \u2026}])
|
|
10608
10682
|
--limit <n> Max results for \`search\` (default 20)
|
|
10683
|
+
--no-fuzzy \`search\`: disable trigram fuzzy fallback for query terms
|
|
10684
|
+
with zero document frequency (default: enabled)
|
|
10609
10685
|
--recall \`callers\`: recall-oriented binding (issue #7) \u2014 relaxes
|
|
10610
10686
|
the JS/TS import gate to unique repo-wide names and labels
|
|
10611
10687
|
each site corroborated|unique-name
|
|
10612
10688
|
`;
|
|
10613
10689
|
function parseFlags(args2) {
|
|
10614
|
-
const flags2 = { repo: process.cwd(), include: [], exclude: [], gitignore: true, noAst: false };
|
|
10690
|
+
const flags2 = { repo: process.cwd(), include: [], exclude: [], gitignore: true, noAst: false, fuzzy: true };
|
|
10615
10691
|
for (let i2 = 0; i2 < args2.length; i2++) {
|
|
10616
10692
|
const a = args2[i2];
|
|
10617
10693
|
const next = () => {
|
|
@@ -10643,6 +10719,7 @@ function parseFlags(args2) {
|
|
|
10643
10719
|
else if (a === "--since") flags2.since = next();
|
|
10644
10720
|
else if (a === "--config") flags2.config = resolve(next());
|
|
10645
10721
|
else if (a === "--limit") flags2.limit = num();
|
|
10722
|
+
else if (a === "--no-fuzzy") flags2.fuzzy = false;
|
|
10646
10723
|
else if (a === "--recall") flags2.recall = true;
|
|
10647
10724
|
else if (!a.startsWith("--") && flags2.positional === void 0) flags2.positional = a;
|
|
10648
10725
|
else throw new Error(`unknown flag: ${a}`);
|
|
@@ -10745,7 +10822,7 @@ async function runCli(argv) {
|
|
|
10745
10822
|
} else if (cmd === "search") {
|
|
10746
10823
|
if (!flags2.positional) throw new Error('search needs a query: cli.mjs search "<query>" --repo <dir>');
|
|
10747
10824
|
const scan2 = scanRepo(flags2.repo, scanOptions(flags2));
|
|
10748
|
-
const results = searchIndex(scan2, flags2.positional, { limit: flags2.limit });
|
|
10825
|
+
const results = searchIndex(scan2, flags2.positional, { limit: flags2.limit, fuzzy: flags2.fuzzy });
|
|
10749
10826
|
emit(JSON.stringify(results, null, 2) + "\n", flags2.out);
|
|
10750
10827
|
} else if (cmd === "rules") {
|
|
10751
10828
|
if (!flags2.config) throw new Error("rules needs --config <codeindex.rules.json>");
|
package/src/bm25.ts
CHANGED
|
@@ -8,6 +8,14 @@
|
|
|
8
8
|
// the query reuses util keywords, so query and haystack always tokenize alike.
|
|
9
9
|
// Deterministic: files are scored in scan order (sorted by rel), scores are
|
|
10
10
|
// fixed to 4 decimal places, and ties break by path.
|
|
11
|
+
//
|
|
12
|
+
// Trigram fuzzy fallback (v2.9.0): a query term that matches NOTHING in the
|
|
13
|
+
// corpus (document frequency == 0 — checked STRICTLY, so any term that
|
|
14
|
+
// already matches anywhere is never touched) is expanded against the corpus
|
|
15
|
+
// vocabulary via character-trigram Dice similarity (threshold 0.6, top-3
|
|
16
|
+
// candidates, deterministic tie-break). This keeps every currently-matching
|
|
17
|
+
// query byte-identical: the expansion only ever engages on terms that would
|
|
18
|
+
// otherwise contribute nothing.
|
|
11
19
|
import type { RepoScan } from "./scan.js";
|
|
12
20
|
import { foldText, keywords } from "./util.js";
|
|
13
21
|
import { byStr } from "./sort.js";
|
|
@@ -16,10 +24,17 @@ const K1 = 1.2;
|
|
|
16
24
|
const B = 0.75;
|
|
17
25
|
const DEFAULT_LIMIT = 20;
|
|
18
26
|
const TOP_SYMBOLS = 5;
|
|
27
|
+
const FUZZY_DICE_THRESHOLD = 0.6;
|
|
28
|
+
const FUZZY_CAP = 3;
|
|
19
29
|
|
|
20
30
|
export interface SearchOptions {
|
|
21
31
|
// Maximum results returned (default 20).
|
|
22
32
|
limit?: number;
|
|
33
|
+
// Trigram fuzzy fallback for query terms with zero document frequency
|
|
34
|
+
// (default true). Safe as an always-on default: the df==0 gate means it
|
|
35
|
+
// only ever engages on terms that would otherwise match nothing, so a
|
|
36
|
+
// query where every term already hits is completely unaffected.
|
|
37
|
+
fuzzy?: boolean;
|
|
23
38
|
}
|
|
24
39
|
|
|
25
40
|
export interface SearchResult {
|
|
@@ -27,6 +42,10 @@ export interface SearchResult {
|
|
|
27
42
|
score: number; // BM25 score, fixed to 4 decimal places
|
|
28
43
|
matchedTerms: string[]; // query tokens present in this file's document, sorted
|
|
29
44
|
topSymbols: string[]; // symbols whose name matches the most query tokens (cap 5)
|
|
45
|
+
// Query terms (df==0) resolved via trigram fuzzy fallback that contributed
|
|
46
|
+
// to this result, sorted. Present only when >=1 term used the fallback —
|
|
47
|
+
// purely additive, never present for an all-exact-match result.
|
|
48
|
+
fuzzyTerms?: string[];
|
|
30
49
|
}
|
|
31
50
|
|
|
32
51
|
// Split an identifier/phrase into lowercase, diacritic-folded subtokens:
|
|
@@ -87,6 +106,39 @@ function buildDocs(scan: RepoScan): Doc[] {
|
|
|
87
106
|
return docs;
|
|
88
107
|
}
|
|
89
108
|
|
|
109
|
+
// Character trigrams of a token, padded with two boundary sentinels on each
|
|
110
|
+
// side (pg_trgm-style: "^^t…m$$") so short prefix/suffix runs still produce
|
|
111
|
+
// shared grams. Deduplicated into a Set — a repeated gram doesn't inflate
|
|
112
|
+
// Dice similarity.
|
|
113
|
+
export function charTrigrams(term: string): Set<string> {
|
|
114
|
+
const padded = `^^${term}$$`;
|
|
115
|
+
const grams = new Set<string>();
|
|
116
|
+
for (let i = 0; i + 3 <= padded.length; i++) grams.add(padded.slice(i, i + 3));
|
|
117
|
+
return grams;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Dice coefficient between two trigram sets: 2|A∩B| / (|A|+|B|). 0 when
|
|
121
|
+
// either side is empty (no divide-by-zero).
|
|
122
|
+
export function diceCoefficient(a: ReadonlySet<string>, b: ReadonlySet<string>): number {
|
|
123
|
+
if (!a.size || !b.size) return 0;
|
|
124
|
+
let inter = 0;
|
|
125
|
+
for (const g of a) if (b.has(g)) inter++;
|
|
126
|
+
return (2 * inter) / (a.size + b.size);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Trigram index of the corpus vocabulary: every distinct doc token mapped to
|
|
130
|
+
// its trigram set. Built LAZILY by searchIndex — only when >=1 query term has
|
|
131
|
+
// df==0 — so a fully-matched query never pays this cost.
|
|
132
|
+
function buildTrigramIndex(docs: Doc[]): Map<string, Set<string>> {
|
|
133
|
+
const index = new Map<string, Set<string>>();
|
|
134
|
+
for (const d of docs) {
|
|
135
|
+
for (const term of d.tf.keys()) {
|
|
136
|
+
if (!index.has(term)) index.set(term, charTrigrams(term));
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return index;
|
|
140
|
+
}
|
|
141
|
+
|
|
90
142
|
// Rank the scanned files against a natural-language (or identifier) query.
|
|
91
143
|
// Pure and deterministic: same scan + query → the same results, byte-for-byte.
|
|
92
144
|
export function searchIndex(scan: RepoScan, query: string, opts: SearchOptions = {}): SearchResult[] {
|
|
@@ -118,36 +170,93 @@ export function searchIndex(scan: RepoScan, query: string, opts: SearchOptions =
|
|
|
118
170
|
df.set(t, count);
|
|
119
171
|
}
|
|
120
172
|
|
|
173
|
+
// Fuzzy fallback: STRICT df==0 gate — a term that matches anywhere, even
|
|
174
|
+
// once, is never expanded. The trigram index of the corpus vocabulary is
|
|
175
|
+
// built lazily, only when at least one term needs it, so a fully-matched
|
|
176
|
+
// query (the common case) pays zero extra cost and stays byte-identical.
|
|
177
|
+
const fuzzyEnabled = opts.fuzzy ?? true;
|
|
178
|
+
const fuzzyCandidates = new Map<string, { term: string; dice: number }[]>();
|
|
179
|
+
if (fuzzyEnabled) {
|
|
180
|
+
const unmatched = terms.filter((t) => df.get(t) === 0);
|
|
181
|
+
if (unmatched.length) {
|
|
182
|
+
const trigramIndex = buildTrigramIndex(docs);
|
|
183
|
+
for (const t of unmatched) {
|
|
184
|
+
const grams = charTrigrams(t);
|
|
185
|
+
const candidates: { term: string; dice: number }[] = [];
|
|
186
|
+
for (const [vocabTerm, vocabGrams] of trigramIndex) {
|
|
187
|
+
const dice = diceCoefficient(grams, vocabGrams);
|
|
188
|
+
if (dice >= FUZZY_DICE_THRESHOLD) candidates.push({ term: vocabTerm, dice });
|
|
189
|
+
}
|
|
190
|
+
// Deterministic: similarity desc, then vocab term asc.
|
|
191
|
+
candidates.sort((a, b) => b.dice - a.dice || byStr(a.term, b.term));
|
|
192
|
+
fuzzyCandidates.set(t, candidates.slice(0, FUZZY_CAP));
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
// df cache for expanded vocab terms (distinct from query-term df above).
|
|
197
|
+
const vocabDf = new Map<string, number>();
|
|
198
|
+
const dfOfVocabTerm = (term: string): number => {
|
|
199
|
+
const known = df.get(term) ?? vocabDf.get(term);
|
|
200
|
+
if (known !== undefined) return known;
|
|
201
|
+
let count = 0;
|
|
202
|
+
for (const d of docs) if (d.tf.has(term)) count++;
|
|
203
|
+
vocabDf.set(term, count);
|
|
204
|
+
return count;
|
|
205
|
+
};
|
|
206
|
+
|
|
121
207
|
const results: SearchResult[] = [];
|
|
122
208
|
for (const d of docs) {
|
|
123
209
|
let score = 0;
|
|
124
210
|
const matched: string[] = [];
|
|
211
|
+
const symbolTerms = new Set<string>(); // matched ∪ fuzzy-expanded vocab terms, for topSymbols ranking
|
|
212
|
+
const fuzzyHit = new Set<string>(); // original query terms resolved via fuzzy fallback, for this doc
|
|
125
213
|
for (const t of terms) {
|
|
126
214
|
const tf = d.tf.get(t);
|
|
127
|
-
if (
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
215
|
+
if (tf) {
|
|
216
|
+
matched.push(t);
|
|
217
|
+
symbolTerms.add(t);
|
|
218
|
+
const idf = Math.log(1 + (n - df.get(t)! + 0.5) / (df.get(t)! + 0.5));
|
|
219
|
+
score += (idf * (tf * (K1 + 1))) / (tf + K1 * (1 - B + (B * d.len) / avgLen));
|
|
220
|
+
continue;
|
|
221
|
+
}
|
|
222
|
+
// Only ever reached for a term with df==0 (or absent from THIS doc but
|
|
223
|
+
// matched elsewhere — fuzzyCandidates has no entry for those, so the
|
|
224
|
+
// lookup below is a no-op and behavior is identical to before v2.9.0).
|
|
225
|
+
const candidates = fuzzyCandidates.get(t);
|
|
226
|
+
if (!candidates) continue;
|
|
227
|
+
for (const cand of candidates) {
|
|
228
|
+
const ctf = d.tf.get(cand.term);
|
|
229
|
+
if (!ctf) continue;
|
|
230
|
+
const cdf = dfOfVocabTerm(cand.term);
|
|
231
|
+
const idf = Math.log(1 + (n - cdf + 0.5) / (cdf + 0.5));
|
|
232
|
+
const contribution = (idf * (ctf * (K1 + 1))) / (ctf + K1 * (1 - B + (B * d.len) / avgLen));
|
|
233
|
+
score += contribution * cand.dice; // near-miss always scores below an exact hit (dice < 1)
|
|
234
|
+
symbolTerms.add(cand.term);
|
|
235
|
+
fuzzyHit.add(t);
|
|
236
|
+
}
|
|
131
237
|
}
|
|
132
|
-
if (!matched.length) continue;
|
|
238
|
+
if (!matched.length && !fuzzyHit.size) continue;
|
|
133
239
|
|
|
134
|
-
// Symbols ranked by how many query tokens
|
|
240
|
+
// Symbols ranked by how many query tokens (exact or fuzzy-expanded) their
|
|
241
|
+
// name carries, then by name.
|
|
135
242
|
const scored = d.symbols
|
|
136
243
|
.map((name) => {
|
|
137
244
|
const toks = new Set(subtokens(name));
|
|
138
245
|
let hits = 0;
|
|
139
|
-
for (const t of
|
|
246
|
+
for (const t of symbolTerms) if (toks.has(t)) hits++;
|
|
140
247
|
return { name, hits };
|
|
141
248
|
})
|
|
142
249
|
.filter((s) => s.hits > 0)
|
|
143
250
|
.sort((a, b) => b.hits - a.hits || byStr(a.name, b.name));
|
|
144
251
|
|
|
145
|
-
|
|
252
|
+
const result: SearchResult = {
|
|
146
253
|
file: d.file,
|
|
147
254
|
score: Number(score.toFixed(4)),
|
|
148
255
|
matchedTerms: matched.sort(byStr),
|
|
149
256
|
topSymbols: scored.slice(0, TOP_SYMBOLS).map((s) => s.name),
|
|
150
|
-
}
|
|
257
|
+
};
|
|
258
|
+
if (fuzzyHit.size) result.fuzzyTerms = [...fuzzyHit].sort(byStr);
|
|
259
|
+
results.push(result);
|
|
151
260
|
}
|
|
152
261
|
|
|
153
262
|
// Rounded score first (so 4-dp ties resolve stably), then path.
|
package/src/engine-cli.ts
CHANGED
|
@@ -63,6 +63,8 @@ Flags:
|
|
|
63
63
|
--no-ast Skip tree-sitter grammars even when present (regex tier)
|
|
64
64
|
--config <file> Rules config for \`rules\` (JSON: [{name, from, to, …}])
|
|
65
65
|
--limit <n> Max results for \`search\` (default 20)
|
|
66
|
+
--no-fuzzy \`search\`: disable trigram fuzzy fallback for query terms
|
|
67
|
+
with zero document frequency (default: enabled)
|
|
66
68
|
--recall \`callers\`: recall-oriented binding (issue #7) — relaxes
|
|
67
69
|
the JS/TS import gate to unique repo-wide names and labels
|
|
68
70
|
each site corroborated|unique-name
|
|
@@ -84,13 +86,14 @@ interface CliFlags {
|
|
|
84
86
|
budgetTokens?: number;
|
|
85
87
|
config?: string; // rules config path
|
|
86
88
|
limit?: number; // search result cap
|
|
89
|
+
fuzzy: boolean; // search: trigram fuzzy fallback for df==0 terms (default true)
|
|
87
90
|
recall?: boolean; // callers: recall-oriented binding
|
|
88
91
|
projectRoot?: string; // scip: override Metadata.project_root
|
|
89
92
|
positional?: string; // e.g. the grep pattern or search query
|
|
90
93
|
}
|
|
91
94
|
|
|
92
95
|
function parseFlags(args: string[]): CliFlags {
|
|
93
|
-
const flags: CliFlags = { repo: process.cwd(), include: [], exclude: [], gitignore: true, noAst: false };
|
|
96
|
+
const flags: CliFlags = { repo: process.cwd(), include: [], exclude: [], gitignore: true, noAst: false, fuzzy: true };
|
|
94
97
|
for (let i = 0; i < args.length; i++) {
|
|
95
98
|
const a = args[i]!;
|
|
96
99
|
const next = (): string => {
|
|
@@ -122,6 +125,7 @@ function parseFlags(args: string[]): CliFlags {
|
|
|
122
125
|
else if (a === "--since") flags.since = next();
|
|
123
126
|
else if (a === "--config") flags.config = resolve(next());
|
|
124
127
|
else if (a === "--limit") flags.limit = num();
|
|
128
|
+
else if (a === "--no-fuzzy") flags.fuzzy = false;
|
|
125
129
|
else if (a === "--recall") flags.recall = true;
|
|
126
130
|
else if (!a.startsWith("--") && flags.positional === undefined) flags.positional = a;
|
|
127
131
|
else throw new Error(`unknown flag: ${a}`);
|
|
@@ -234,7 +238,7 @@ export async function runCli(argv: string[]): Promise<void> {
|
|
|
234
238
|
} else if (cmd === "search") {
|
|
235
239
|
if (!flags.positional) throw new Error('search needs a query: cli.mjs search "<query>" --repo <dir>');
|
|
236
240
|
const scan = scanRepo(flags.repo, scanOptions(flags));
|
|
237
|
-
const results = searchIndex(scan, flags.positional, { limit: flags.limit });
|
|
241
|
+
const results = searchIndex(scan, flags.positional, { limit: flags.limit, fuzzy: flags.fuzzy });
|
|
238
242
|
emit(JSON.stringify(results, null, 2) + "\n", flags.out);
|
|
239
243
|
} else if (cmd === "rules") {
|
|
240
244
|
if (!flags.config) throw new Error("rules needs --config <codeindex.rules.json>");
|
package/src/mcp.ts
CHANGED
|
@@ -266,7 +266,7 @@ const TOOLS = [
|
|
|
266
266
|
{
|
|
267
267
|
name: "search",
|
|
268
268
|
description:
|
|
269
|
-
'Natural-language-ish lexical search: BM25 ranking (k1=1.2, b=0.75) over symbol names (camelCase/snake_case subtokens), file path segments, markdown headings and summary lines. NOT embeddings — deterministic, diacritic-folded, zero API keys. Answers "where is auth handled?"-style queries with ranked files, matched terms and top symbols.',
|
|
269
|
+
'Natural-language-ish lexical search: BM25 ranking (k1=1.2, b=0.75) over symbol names (camelCase/snake_case subtokens), file path segments, markdown headings and summary lines. NOT embeddings — deterministic, diacritic-folded, zero API keys. Answers "where is auth handled?"-style queries with ranked files, matched terms and top symbols. Query terms with zero document frequency get a deterministic trigram-fuzzy fallback (typo-tolerant) unless `fuzzy: false`.',
|
|
270
270
|
inputSchema: {
|
|
271
271
|
type: "object",
|
|
272
272
|
properties: {
|
|
@@ -274,6 +274,11 @@ const TOOLS = [
|
|
|
274
274
|
...scopeProps,
|
|
275
275
|
query: { type: "string", description: "Natural-language or identifier query" },
|
|
276
276
|
limit: { type: "number", description: "Max results (default 20)" },
|
|
277
|
+
fuzzy: {
|
|
278
|
+
type: "boolean",
|
|
279
|
+
description:
|
|
280
|
+
"Trigram fuzzy fallback for query terms with zero document frequency (default true)",
|
|
281
|
+
},
|
|
277
282
|
},
|
|
278
283
|
required: ["repo", "query"],
|
|
279
284
|
},
|
|
@@ -437,6 +442,7 @@ function callTool(name: string, args: Record<string, unknown>): string {
|
|
|
437
442
|
if (!query) throw new Error("`query` is required");
|
|
438
443
|
const results = searchIndex(scanRepo(repo, scanOpts), query, {
|
|
439
444
|
limit: typeof args.limit === "number" ? args.limit : undefined,
|
|
445
|
+
fuzzy: typeof args.fuzzy === "boolean" ? args.fuzzy : undefined,
|
|
440
446
|
});
|
|
441
447
|
return JSON.stringify(results, null, 2);
|
|
442
448
|
}
|
package/src/types.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Single source of truth for the engine version the bundle reports. Kept in
|
|
2
2
|
// lockstep with package.json by the release pipeline. Do not edit by hand
|
|
3
3
|
// outside a release.
|
|
4
|
-
export const ENGINE_VERSION = "2.
|
|
4
|
+
export const ENGINE_VERSION = "2.9.0";
|
|
5
5
|
|
|
6
6
|
// Bumped whenever the on-disk artifact shape changes, so a consumer can reject
|
|
7
7
|
// an index written by an incompatible engine instead of misreading it. The
|