@zvec/zvec-grep 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +222 -0
- package/README_CN.md +222 -0
- package/dist/cli/args.d.ts +2 -0
- package/dist/cli/args.js +611 -0
- package/dist/cli/args.js.map +1 -0
- package/dist/cli/commands.d.ts +2 -0
- package/dist/cli/commands.js +270 -0
- package/dist/cli/commands.js.map +1 -0
- package/dist/cli/errors.d.ts +6 -0
- package/dist/cli/errors.js +97 -0
- package/dist/cli/errors.js.map +1 -0
- package/dist/cli/format/context.d.ts +5 -0
- package/dist/cli/format/context.js +490 -0
- package/dist/cli/format/context.js.map +1 -0
- package/dist/cli/format/debug.d.ts +4 -0
- package/dist/cli/format/debug.js +42 -0
- package/dist/cli/format/debug.js.map +1 -0
- package/dist/cli/format/highlight.d.ts +3 -0
- package/dist/cli/format/highlight.js +106 -0
- package/dist/cli/format/highlight.js.map +1 -0
- package/dist/cli/format/progress.d.ts +9 -0
- package/dist/cli/format/progress.js +92 -0
- package/dist/cli/format/progress.js.map +1 -0
- package/dist/cli/format/range.d.ts +3 -0
- package/dist/cli/format/range.js +24 -0
- package/dist/cli/format/range.js.map +1 -0
- package/dist/cli/format/status.d.ts +7 -0
- package/dist/cli/format/status.js +200 -0
- package/dist/cli/format/status.js.map +1 -0
- package/dist/cli/help.d.ts +1 -0
- package/dist/cli/help.js +88 -0
- package/dist/cli/help.js.map +1 -0
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.js +32 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/types.d.ts +56 -0
- package/dist/cli/types.js +10 -0
- package/dist/cli/types.js.map +1 -0
- package/dist/cli/version.d.ts +1 -0
- package/dist/cli/version.js +11 -0
- package/dist/cli/version.js.map +1 -0
- package/dist/engine/collection/index.d.ts +50 -0
- package/dist/engine/collection/index.js +388 -0
- package/dist/engine/collection/index.js.map +1 -0
- package/dist/engine/errors/details.d.ts +6 -0
- package/dist/engine/errors/details.js +27 -0
- package/dist/engine/errors/details.js.map +1 -0
- package/dist/engine/errors/index.d.ts +14 -0
- package/dist/engine/errors/index.js +16 -0
- package/dist/engine/errors/index.js.map +1 -0
- package/dist/engine/models/catalog.d.ts +47 -0
- package/dist/engine/models/catalog.js +68 -0
- package/dist/engine/models/catalog.js.map +1 -0
- package/dist/engine/models/embeddings.d.ts +25 -0
- package/dist/engine/models/embeddings.js +100 -0
- package/dist/engine/models/embeddings.js.map +1 -0
- package/dist/engine/models/factory.d.ts +5 -0
- package/dist/engine/models/factory.js +66 -0
- package/dist/engine/models/factory.js.map +1 -0
- package/dist/engine/models/index.d.ts +6 -0
- package/dist/engine/models/index.js +5 -0
- package/dist/engine/models/index.js.map +1 -0
- package/dist/engine/models/providers/index.d.ts +2 -0
- package/dist/engine/models/providers/index.js +3 -0
- package/dist/engine/models/providers/index.js.map +1 -0
- package/dist/engine/models/providers/llama-cpp/embedding.d.ts +94 -0
- package/dist/engine/models/providers/llama-cpp/embedding.js +459 -0
- package/dist/engine/models/providers/llama-cpp/embedding.js.map +1 -0
- package/dist/engine/models/providers/llama-cpp/index.d.ts +1 -0
- package/dist/engine/models/providers/llama-cpp/index.js +2 -0
- package/dist/engine/models/providers/llama-cpp/index.js.map +1 -0
- package/dist/engine/models/providers/qwen/embedding.d.ts +38 -0
- package/dist/engine/models/providers/qwen/embedding.js +322 -0
- package/dist/engine/models/providers/qwen/embedding.js.map +1 -0
- package/dist/engine/models/providers/qwen/index.d.ts +1 -0
- package/dist/engine/models/providers/qwen/index.js +2 -0
- package/dist/engine/models/providers/qwen/index.js.map +1 -0
- package/dist/engine/models/ranking.d.ts +15 -0
- package/dist/engine/models/ranking.js +2 -0
- package/dist/engine/models/ranking.js.map +1 -0
- package/dist/engine/models/types.d.ts +30 -0
- package/dist/engine/models/types.js +2 -0
- package/dist/engine/models/types.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/adapter.d.ts +19 -0
- package/dist/engine/pipeline/indexing/extractor/code/adapter.js +24 -0
- package/dist/engine/pipeline/indexing/extractor/code/adapter.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/extractor.d.ts +14 -0
- package/dist/engine/pipeline/indexing/extractor/code/extractor.js +594 -0
- package/dist/engine/pipeline/indexing/extractor/code/extractor.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/families/c-family.d.ts +2 -0
- package/dist/engine/pipeline/indexing/extractor/code/families/c-family.js +116 -0
- package/dist/engine/pipeline/indexing/extractor/code/families/c-family.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/families/js-ts.d.ts +12 -0
- package/dist/engine/pipeline/indexing/extractor/code/families/js-ts.js +112 -0
- package/dist/engine/pipeline/indexing/extractor/code/families/js-ts.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/families/metadata.d.ts +8 -0
- package/dist/engine/pipeline/indexing/extractor/code/families/metadata.js +83 -0
- package/dist/engine/pipeline/indexing/extractor/code/families/metadata.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/families/name-field.d.ts +3 -0
- package/dist/engine/pipeline/indexing/extractor/code/families/name-field.js +22 -0
- package/dist/engine/pipeline/indexing/extractor/code/families/name-field.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/c.d.ts +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/c.js +15 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/c.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/cpp.d.ts +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/cpp.js +18 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/cpp.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/go.d.ts +2 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/go.js +72 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/go.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/java.d.ts +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/java.js +17 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/java.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/javascript.d.ts +2 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/javascript.js +25 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/javascript.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/python.d.ts +2 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/python.js +53 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/python.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/rust.d.ts +2 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/rust.js +29 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/rust.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/typescript.d.ts +2 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/typescript.js +37 -0
- package/dist/engine/pipeline/indexing/extractor/code/languages/typescript.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/index.d.ts +44 -0
- package/dist/engine/pipeline/indexing/extractor/index.js +281 -0
- package/dist/engine/pipeline/indexing/extractor/index.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/markdown/extractor.d.ts +13 -0
- package/dist/engine/pipeline/indexing/extractor/markdown/extractor.js +339 -0
- package/dist/engine/pipeline/indexing/extractor/markdown/extractor.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/tree-sitter/grammar.d.ts +4 -0
- package/dist/engine/pipeline/indexing/extractor/tree-sitter/grammar.js +49 -0
- package/dist/engine/pipeline/indexing/extractor/tree-sitter/grammar.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/tree-sitter/nodes.d.ts +3 -0
- package/dist/engine/pipeline/indexing/extractor/tree-sitter/nodes.js +26 -0
- package/dist/engine/pipeline/indexing/extractor/tree-sitter/nodes.js.map +1 -0
- package/dist/engine/pipeline/indexing/extractor/tree-sitter/parser.d.ts +2 -0
- package/dist/engine/pipeline/indexing/extractor/tree-sitter/parser.js +31 -0
- package/dist/engine/pipeline/indexing/extractor/tree-sitter/parser.js.map +1 -0
- package/dist/engine/pipeline/indexing/ids.d.ts +1 -0
- package/dist/engine/pipeline/indexing/ids.js +5 -0
- package/dist/engine/pipeline/indexing/ids.js.map +1 -0
- package/dist/engine/pipeline/indexing/index.d.ts +12 -0
- package/dist/engine/pipeline/indexing/index.js +538 -0
- package/dist/engine/pipeline/indexing/index.js.map +1 -0
- package/dist/engine/pipeline/indexing/root-paths.d.ts +7 -0
- package/dist/engine/pipeline/indexing/root-paths.js +151 -0
- package/dist/engine/pipeline/indexing/root-paths.js.map +1 -0
- package/dist/engine/pipeline/indexing/scanner/index.d.ts +5 -0
- package/dist/engine/pipeline/indexing/scanner/index.js +504 -0
- package/dist/engine/pipeline/indexing/scanner/index.js.map +1 -0
- package/dist/engine/pipeline/search/index.d.ts +12 -0
- package/dist/engine/pipeline/search/index.js +761 -0
- package/dist/engine/pipeline/search/index.js.map +1 -0
- package/dist/engine/service/index.d.ts +2 -0
- package/dist/engine/service/index.js +2 -0
- package/dist/engine/service/index.js.map +1 -0
- package/dist/engine/service/lexical.d.ts +29 -0
- package/dist/engine/service/lexical.js +482 -0
- package/dist/engine/service/lexical.js.map +1 -0
- package/dist/engine/service/root.d.ts +15 -0
- package/dist/engine/service/root.js +44 -0
- package/dist/engine/service/root.js.map +1 -0
- package/dist/engine/service/types.d.ts +157 -0
- package/dist/engine/service/types.js +2 -0
- package/dist/engine/service/types.js.map +1 -0
- package/dist/engine/service/zvec-grep.d.ts +2 -0
- package/dist/engine/service/zvec-grep.js +935 -0
- package/dist/engine/service/zvec-grep.js.map +1 -0
- package/dist/engine/storage/index.d.ts +42 -0
- package/dist/engine/storage/index.js +2 -0
- package/dist/engine/storage/index.js.map +1 -0
- package/dist/engine/storage/zvec.d.ts +51 -0
- package/dist/engine/storage/zvec.js +706 -0
- package/dist/engine/storage/zvec.js.map +1 -0
- package/dist/engine/types.d.ts +266 -0
- package/dist/engine/types.js +14 -0
- package/dist/engine/types.js.map +1 -0
- package/dist/engine/utils/glob.d.ts +6 -0
- package/dist/engine/utils/glob.js +110 -0
- package/dist/engine/utils/glob.js.map +1 -0
- package/dist/engine/utils/hash.d.ts +2 -0
- package/dist/engine/utils/hash.js +8 -0
- package/dist/engine/utils/hash.js.map +1 -0
- package/dist/engine/utils/json.d.ts +4 -0
- package/dist/engine/utils/json.js +45 -0
- package/dist/engine/utils/json.js.map +1 -0
- package/dist/engine/utils/lock.d.ts +19 -0
- package/dist/engine/utils/lock.js +254 -0
- package/dist/engine/utils/lock.js.map +1 -0
- package/dist/engine/utils/path.d.ts +4 -0
- package/dist/engine/utils/path.js +18 -0
- package/dist/engine/utils/path.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -0
- package/package.json +47 -0
- package/skills/zvec-grep/SKILL.md +141 -0
- package/skills/zvec-grep/agents/openai.yaml +4 -0
|
@@ -0,0 +1,761 @@
|
|
|
1
|
+
import { collectionDetail, detail, EngineError, errorDetails, } from "../../errors/index.js";
|
|
2
|
+
import { hasPathGlob, isAbsolutePathPattern, normalizePathForMatch, normalizePathPattern, pathPatternMatches, } from "../../utils/glob.js";
|
|
3
|
+
const DEFAULT_LIMIT = 7;
|
|
4
|
+
const RRF_K = 60;
|
|
5
|
+
const RECALL_INITIAL_DEPTH = 200;
|
|
6
|
+
const RECALL_MAX_DEPTH = 2000;
|
|
7
|
+
const RECALL_GROWTH_FACTOR = 2;
|
|
8
|
+
const RECALL_TARGET_FACTOR = 5;
|
|
9
|
+
const RECALL_MIN_TARGET_CANDIDATES = 50;
|
|
10
|
+
export async function searchPlanCollection(plan, ctx) {
|
|
11
|
+
const normalized = validateSearchPlan(plan);
|
|
12
|
+
const limit = normalized.limit ?? DEFAULT_LIMIT;
|
|
13
|
+
const trace = normalized.trace === true || normalized.trackEntityId !== undefined;
|
|
14
|
+
const filter = searchPlanToStorageFilter(normalized, ctx.storage);
|
|
15
|
+
const hasSearchableFiles = !filterMatchesNoFiles(filter);
|
|
16
|
+
const candidates = new Map();
|
|
17
|
+
const vectorByRoute = hasSearchableFiles && planUsesVector(normalized)
|
|
18
|
+
? await embedVectorRoutes(normalized.routes, requireEmbeddingModel(ctx, "searchPlan"))
|
|
19
|
+
: new Map();
|
|
20
|
+
let recallDepth = RECALL_INITIAL_DEPTH;
|
|
21
|
+
if (hasSearchableFiles) {
|
|
22
|
+
recallDepth = collectAdaptiveRecall({
|
|
23
|
+
routes: normalized.routes,
|
|
24
|
+
filter,
|
|
25
|
+
preferSymbol: normalized.preferSymbol === true,
|
|
26
|
+
vectorByRoute,
|
|
27
|
+
limit,
|
|
28
|
+
storage: ctx.storage,
|
|
29
|
+
candidates,
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
if (normalized.trackEntityId) {
|
|
33
|
+
forceTrackEntity({
|
|
34
|
+
entityId: normalized.trackEntityId,
|
|
35
|
+
routes: normalized.routes,
|
|
36
|
+
vectorByRoute,
|
|
37
|
+
recallDepth,
|
|
38
|
+
filter,
|
|
39
|
+
storage: ctx.storage,
|
|
40
|
+
candidates,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
const fused = fuseCandidates(candidates);
|
|
44
|
+
const visible = fused.slice(0, limit);
|
|
45
|
+
const tracked = normalized.trackEntityId
|
|
46
|
+
? fused.find((candidate) => candidate.id === normalized.trackEntityId)
|
|
47
|
+
: undefined;
|
|
48
|
+
if (tracked && !visible.some((candidate) => candidate.id === tracked.id)) {
|
|
49
|
+
visible.push(tracked);
|
|
50
|
+
}
|
|
51
|
+
const hits = visible.map((candidate) => candidateToHit(candidate, limit, trace));
|
|
52
|
+
const trackedHit = normalized.trackEntityId
|
|
53
|
+
? hits.find((hit) => hit.entity.id === normalized.trackEntityId)
|
|
54
|
+
: undefined;
|
|
55
|
+
return {
|
|
56
|
+
collectionId: ctx.collection.id,
|
|
57
|
+
collectionName: ctx.collection.name,
|
|
58
|
+
plan: normalized,
|
|
59
|
+
hits,
|
|
60
|
+
trackedHit,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
export async function diagnoseEntitySearch(query, entityId, ctx) {
|
|
64
|
+
const stored = ctx.storage.getEntity(entityId);
|
|
65
|
+
if (!stored) {
|
|
66
|
+
throw new Error(`Entity not found: ${entityId}`);
|
|
67
|
+
}
|
|
68
|
+
const search = await searchPlanCollection({
|
|
69
|
+
routes: [
|
|
70
|
+
{ mode: "fts", query },
|
|
71
|
+
{ mode: "vector", query },
|
|
72
|
+
],
|
|
73
|
+
trace: true,
|
|
74
|
+
trackEntityId: entityId,
|
|
75
|
+
}, ctx);
|
|
76
|
+
return {
|
|
77
|
+
query,
|
|
78
|
+
entityId,
|
|
79
|
+
file: stored.file,
|
|
80
|
+
entity: stored.entity,
|
|
81
|
+
search,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
export async function diagnoseFileSearch(query, absolutePath, ctx) {
|
|
85
|
+
const file = ctx.storage.getFileByPath(absolutePath);
|
|
86
|
+
if (!file) {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
const entityId = await chooseBestEntityInFile(query, file, ctx);
|
|
90
|
+
if (!entityId) {
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
return diagnoseEntitySearch(query, entityId, ctx);
|
|
94
|
+
}
|
|
95
|
+
function validateSearchPlan(plan) {
|
|
96
|
+
if (!Array.isArray(plan.routes) || plan.routes.length === 0) {
|
|
97
|
+
throw new EngineError("Search plan requires at least one route", {
|
|
98
|
+
code: "ZVEC_GREP.ENGINE.SEARCH_PLAN.EMPTY_ROUTES",
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
const routeIds = new Set();
|
|
102
|
+
const routeCounts = new Map();
|
|
103
|
+
const routes = plan.routes.map((route, index) => {
|
|
104
|
+
const query = typeof route.query === "string" ? route.query.trim() : "";
|
|
105
|
+
if (route.mode !== "fts" && route.mode !== "vector") {
|
|
106
|
+
throw new EngineError("Search plan route has an unsupported mode", {
|
|
107
|
+
code: "ZVEC_GREP.ENGINE.SEARCH_PLAN.UNSUPPORTED_ROUTE_MODE",
|
|
108
|
+
context: `routeIndex=${index} mode=${String(route.mode)}`,
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
const id = makeDefaultRouteId(route.mode, routeCounts, routeIds);
|
|
112
|
+
if (query.length === 0) {
|
|
113
|
+
throw new EngineError("Search plan route requires a non-empty query", {
|
|
114
|
+
code: "ZVEC_GREP.ENGINE.SEARCH_PLAN.EMPTY_ROUTE_QUERY",
|
|
115
|
+
context: `routeId=${id}`,
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
routeIds.add(id);
|
|
119
|
+
return {
|
|
120
|
+
id,
|
|
121
|
+
mode: route.mode,
|
|
122
|
+
query,
|
|
123
|
+
};
|
|
124
|
+
});
|
|
125
|
+
const modifiedAfter = normalizeModifiedTime(plan.modifiedAfter, "modifiedAfter");
|
|
126
|
+
const modifiedBefore = normalizeModifiedTime(plan.modifiedBefore, "modifiedBefore");
|
|
127
|
+
if (modifiedAfter !== undefined
|
|
128
|
+
&& modifiedBefore !== undefined
|
|
129
|
+
&& modifiedAfter > modifiedBefore) {
|
|
130
|
+
throw new EngineError("Search plan modified-after filter must not be later than modified-before", {
|
|
131
|
+
code: "ZVEC_GREP.ENGINE.SEARCH_PLAN.INVALID_MODIFIED_TIME_RANGE",
|
|
132
|
+
context: `modifiedAfter=${modifiedAfter} modifiedBefore=${modifiedBefore}`,
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
return {
|
|
136
|
+
...plan,
|
|
137
|
+
routes,
|
|
138
|
+
includePaths: normalizePathFilters(plan.includePaths, "includePaths"),
|
|
139
|
+
excludePaths: normalizePathFilters(plan.excludePaths, "excludePaths"),
|
|
140
|
+
modifiedAfter,
|
|
141
|
+
modifiedBefore,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
function makeDefaultRouteId(mode, routeCounts, usedRouteIds) {
|
|
145
|
+
let count = routeCounts.get(mode) ?? 0;
|
|
146
|
+
while (true) {
|
|
147
|
+
count++;
|
|
148
|
+
const id = count === 1 ? mode : `${mode}-${count}`;
|
|
149
|
+
if (!usedRouteIds.has(id)) {
|
|
150
|
+
routeCounts.set(mode, count);
|
|
151
|
+
return id;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
function planUsesVector(plan) {
|
|
156
|
+
return plan.routes.some((route) => route.mode === "vector");
|
|
157
|
+
}
|
|
158
|
+
function requireEmbeddingModel(ctx, operation) {
|
|
159
|
+
if (!ctx.embeddingModel) {
|
|
160
|
+
throw new EngineError("Search operation requires an embedding model", {
|
|
161
|
+
code: "ZVEC_GREP.ENGINE.SEARCH.EMBEDDING_MODEL_REQUIRED",
|
|
162
|
+
context: errorDetails([
|
|
163
|
+
collectionDetail(ctx.collection.name),
|
|
164
|
+
detail("operation", operation),
|
|
165
|
+
]),
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
return ctx.embeddingModel;
|
|
169
|
+
}
|
|
170
|
+
function normalizePathFilters(value, field) {
|
|
171
|
+
if (value === undefined) {
|
|
172
|
+
return undefined;
|
|
173
|
+
}
|
|
174
|
+
if (!Array.isArray(value)) {
|
|
175
|
+
throw new EngineError("Search plan path filters must be arrays", {
|
|
176
|
+
code: "ZVEC_GREP.ENGINE.SEARCH_PLAN.INVALID_PATH_FILTERS",
|
|
177
|
+
context: `field=${field}`,
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
const patterns = [];
|
|
181
|
+
for (const [index, item] of value.entries()) {
|
|
182
|
+
if (typeof item !== "string") {
|
|
183
|
+
throw new EngineError("Search plan path filters must contain strings", {
|
|
184
|
+
code: "ZVEC_GREP.ENGINE.SEARCH_PLAN.INVALID_PATH_FILTER",
|
|
185
|
+
context: `field=${field} index=${index}`,
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
const pattern = normalizePathFilterPattern(item);
|
|
189
|
+
if (pattern.length > 0) {
|
|
190
|
+
patterns.push(pattern);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
return patterns.length > 0 ? patterns : undefined;
|
|
194
|
+
}
|
|
195
|
+
function normalizeModifiedTime(value, field) {
|
|
196
|
+
if (value === undefined) {
|
|
197
|
+
return undefined;
|
|
198
|
+
}
|
|
199
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
200
|
+
throw new EngineError("Search plan modified time filters must be non-negative epoch milliseconds", {
|
|
201
|
+
code: "ZVEC_GREP.ENGINE.SEARCH_PLAN.INVALID_MODIFIED_TIME_FILTER",
|
|
202
|
+
context: `field=${field} value=${String(value)}`,
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
return value;
|
|
206
|
+
}
|
|
207
|
+
async function embedVectorRoutes(routes, model) {
|
|
208
|
+
const vectorRoutes = routes.filter((route) => route.mode === "vector");
|
|
209
|
+
const vectorsByRoute = new Map();
|
|
210
|
+
for (let start = 0; start < vectorRoutes.length; start += model.limits.maxBatchSize) {
|
|
211
|
+
const batch = vectorRoutes.slice(start, start + model.limits.maxBatchSize);
|
|
212
|
+
const vectors = await model.embed(batch.map((route) => ({
|
|
213
|
+
kind: "text",
|
|
214
|
+
text: route.query,
|
|
215
|
+
})), { purpose: "query" });
|
|
216
|
+
for (const [index, route] of batch.entries()) {
|
|
217
|
+
vectorsByRoute.set(route.id, vectors[index]);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return vectorsByRoute;
|
|
221
|
+
}
|
|
222
|
+
function addRecallHits(candidates, hits, route, storage, options = {}) {
|
|
223
|
+
const startIndex = Math.max(0, options.startIndex ?? 0);
|
|
224
|
+
for (let index = startIndex; index < hits.length; index++) {
|
|
225
|
+
const hit = hits[index];
|
|
226
|
+
if (!hit) {
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
const rank = index + 1;
|
|
230
|
+
const entityId = publicEntityId(hit.fragment);
|
|
231
|
+
const resolved = candidates.get(entityId) ?? resolveHitEntity(hit, storage);
|
|
232
|
+
if (!resolved) {
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
const candidate = candidates.get(entityId) ?? {
|
|
236
|
+
id: entityId,
|
|
237
|
+
entity: resolved.entity,
|
|
238
|
+
file: resolved.file,
|
|
239
|
+
sources: new Set(),
|
|
240
|
+
recall: [],
|
|
241
|
+
evidence: [],
|
|
242
|
+
score: 0,
|
|
243
|
+
rank: Number.POSITIVE_INFINITY,
|
|
244
|
+
forced: false,
|
|
245
|
+
};
|
|
246
|
+
candidate.sources.add(route.mode);
|
|
247
|
+
candidate.evidence.push({
|
|
248
|
+
fragment: hit.fragment,
|
|
249
|
+
path: route.mode,
|
|
250
|
+
routeId: route.id,
|
|
251
|
+
query: route.query,
|
|
252
|
+
rank,
|
|
253
|
+
score: hit.score,
|
|
254
|
+
});
|
|
255
|
+
addOrUpdateRecall(candidate, {
|
|
256
|
+
path: route.mode,
|
|
257
|
+
routeId: route.id,
|
|
258
|
+
query: route.query,
|
|
259
|
+
found: true,
|
|
260
|
+
rank,
|
|
261
|
+
score: hit.score,
|
|
262
|
+
});
|
|
263
|
+
candidates.set(candidate.id, candidate);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
function collectAdaptiveRecall(input) {
|
|
267
|
+
const routes = buildRecallRoutes(input.routes, input.filter, input.preferSymbol);
|
|
268
|
+
const targetCandidates = recallTargetCandidateCount(input.limit);
|
|
269
|
+
let previousDepth = 0;
|
|
270
|
+
let depth = RECALL_INITIAL_DEPTH;
|
|
271
|
+
while (true) {
|
|
272
|
+
const saturated = collectRecallPass({
|
|
273
|
+
routes,
|
|
274
|
+
vectorByRoute: input.vectorByRoute,
|
|
275
|
+
depth,
|
|
276
|
+
previousDepth,
|
|
277
|
+
storage: input.storage,
|
|
278
|
+
candidates: input.candidates,
|
|
279
|
+
});
|
|
280
|
+
if (input.candidates.size >= targetCandidates
|
|
281
|
+
|| !saturated
|
|
282
|
+
|| depth >= RECALL_MAX_DEPTH) {
|
|
283
|
+
return depth;
|
|
284
|
+
}
|
|
285
|
+
previousDepth = depth;
|
|
286
|
+
depth = Math.min(depth * RECALL_GROWTH_FACTOR, RECALL_MAX_DEPTH);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
function buildRecallRoutes(routes, filter, preferSymbol) {
|
|
290
|
+
const output = routes.map((route) => ({
|
|
291
|
+
...route,
|
|
292
|
+
filter,
|
|
293
|
+
vectorRouteId: route.mode === "vector" ? route.id : undefined,
|
|
294
|
+
}));
|
|
295
|
+
if (!preferSymbol) {
|
|
296
|
+
return output;
|
|
297
|
+
}
|
|
298
|
+
const seen = new Set();
|
|
299
|
+
for (const route of routes) {
|
|
300
|
+
const symbolNames = extractSymbolNames(route.query);
|
|
301
|
+
if (symbolNames.length === 0) {
|
|
302
|
+
continue;
|
|
303
|
+
}
|
|
304
|
+
const key = `${route.id}\0${symbolNames.join("\0")}`;
|
|
305
|
+
if (seen.has(key)) {
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
seen.add(key);
|
|
309
|
+
output.push({
|
|
310
|
+
id: `${route.id}.prefer-symbol`,
|
|
311
|
+
mode: "fts",
|
|
312
|
+
query: route.query,
|
|
313
|
+
filter: {
|
|
314
|
+
...filter,
|
|
315
|
+
symbolNames,
|
|
316
|
+
},
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
return output;
|
|
320
|
+
}
|
|
321
|
+
function collectRecallPass(input) {
|
|
322
|
+
let saturated = false;
|
|
323
|
+
for (const route of input.routes) {
|
|
324
|
+
const hits = recallRouteHits(route, input);
|
|
325
|
+
saturated = saturated || hits.length >= input.depth;
|
|
326
|
+
addRecallHits(input.candidates, hits, route, input.storage, {
|
|
327
|
+
startIndex: input.previousDepth,
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
return saturated;
|
|
331
|
+
}
|
|
332
|
+
function recallRouteHits(route, input) {
|
|
333
|
+
if (route.mode === "fts") {
|
|
334
|
+
return input.storage.searchFts(route.query, input.depth, route.filter);
|
|
335
|
+
}
|
|
336
|
+
const vector = input.vectorByRoute.get(route.vectorRouteId ?? route.id);
|
|
337
|
+
return vector
|
|
338
|
+
? input.storage.searchVector(vector, input.depth, route.filter)
|
|
339
|
+
: [];
|
|
340
|
+
}
|
|
341
|
+
function recallTargetCandidateCount(limit) {
|
|
342
|
+
return Math.max(limit * RECALL_TARGET_FACTOR, RECALL_MIN_TARGET_CANDIDATES);
|
|
343
|
+
}
|
|
344
|
+
function resolveHitEntity(hit, storage) {
|
|
345
|
+
if (!hit.fragment.group || hit.fragment.group === hit.fragment.id) {
|
|
346
|
+
return {
|
|
347
|
+
entity: fragmentToEntity(hit.fragment),
|
|
348
|
+
file: hit.file,
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
return storage.getEntity(hit.fragment.group);
|
|
352
|
+
}
|
|
353
|
+
function fragmentToEntity(fragment) {
|
|
354
|
+
return {
|
|
355
|
+
id: publicEntityId(fragment),
|
|
356
|
+
fileId: fragment.fileId,
|
|
357
|
+
range: fragment.range,
|
|
358
|
+
content: fragment.content,
|
|
359
|
+
metadata: fragment.metadata,
|
|
360
|
+
};
|
|
361
|
+
}
|
|
362
|
+
function publicEntityId(fragment) {
|
|
363
|
+
return fragment.group ?? fragment.id;
|
|
364
|
+
}
|
|
365
|
+
function addOrUpdateRecall(candidate, recall) {
|
|
366
|
+
const existing = candidate.recall.find((item) => item.path === recall.path && item.routeId === recall.routeId);
|
|
367
|
+
if (!existing) {
|
|
368
|
+
candidate.recall.push(recall);
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
if (!recall.found) {
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
if (!existing.found
|
|
375
|
+
|| existing.rank === undefined
|
|
376
|
+
|| (recall.rank !== undefined && recall.rank < existing.rank)) {
|
|
377
|
+
Object.assign(existing, recall, {
|
|
378
|
+
forced: existing.forced || recall.forced || undefined,
|
|
379
|
+
});
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
if (recall.forced) {
|
|
383
|
+
existing.forced = true;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
function extractSymbolNames(query) {
|
|
387
|
+
const keywords = new Set([
|
|
388
|
+
"class",
|
|
389
|
+
"struct",
|
|
390
|
+
"enum",
|
|
391
|
+
"interface",
|
|
392
|
+
"function",
|
|
393
|
+
"method",
|
|
394
|
+
"type",
|
|
395
|
+
"const",
|
|
396
|
+
"let",
|
|
397
|
+
"var",
|
|
398
|
+
"namespace",
|
|
399
|
+
"where",
|
|
400
|
+
"find",
|
|
401
|
+
"explain",
|
|
402
|
+
]);
|
|
403
|
+
const names = new Set();
|
|
404
|
+
for (const match of query.matchAll(/[A-Za-z_~][A-Za-z0-9_:~]*/g)) {
|
|
405
|
+
const token = match[0];
|
|
406
|
+
if (keywords.has(token.toLowerCase())) {
|
|
407
|
+
continue;
|
|
408
|
+
}
|
|
409
|
+
const name = symbolNameFromToken(token);
|
|
410
|
+
if (name) {
|
|
411
|
+
names.add(name);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
return [...names];
|
|
415
|
+
}
|
|
416
|
+
function symbolNameFromToken(token) {
|
|
417
|
+
const parts = token.split("::").filter((part) => part.length > 0);
|
|
418
|
+
const name = parts[parts.length - 1] ?? token;
|
|
419
|
+
if (!/^[A-Za-z_~][A-Za-z0-9_~]*$/.test(name)) {
|
|
420
|
+
return null;
|
|
421
|
+
}
|
|
422
|
+
if (parts.length < 2) {
|
|
423
|
+
return name;
|
|
424
|
+
}
|
|
425
|
+
const owner = parts[parts.length - 2] ?? "";
|
|
426
|
+
if (/^[A-Z_~]/.test(owner)) {
|
|
427
|
+
return `${owner}::${name}`;
|
|
428
|
+
}
|
|
429
|
+
return name;
|
|
430
|
+
}
|
|
431
|
+
function forceTrackEntity(input) {
|
|
432
|
+
const tracked = input.storage.getEntity(input.entityId);
|
|
433
|
+
if (!tracked) {
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
const candidate = input.candidates.get(input.entityId) ?? {
|
|
437
|
+
id: input.entityId,
|
|
438
|
+
entity: tracked.entity,
|
|
439
|
+
file: tracked.file,
|
|
440
|
+
sources: new Set(),
|
|
441
|
+
recall: [],
|
|
442
|
+
evidence: [],
|
|
443
|
+
score: 0,
|
|
444
|
+
rank: Number.POSITIVE_INFINITY,
|
|
445
|
+
forced: true,
|
|
446
|
+
};
|
|
447
|
+
const seenRoutes = new Set(candidate.recall.map((trace) => trace.routeId));
|
|
448
|
+
for (const route of input.routes) {
|
|
449
|
+
if (seenRoutes.has(route.id)) {
|
|
450
|
+
continue;
|
|
451
|
+
}
|
|
452
|
+
if (filterMatchesNoFiles(input.filter)) {
|
|
453
|
+
forceTrackNoMatchingFilesRoute(candidate, route);
|
|
454
|
+
continue;
|
|
455
|
+
}
|
|
456
|
+
if (filterExcludesFile(input.filter, tracked.file.id)) {
|
|
457
|
+
forceTrackPathExcludedRoute(candidate, route);
|
|
458
|
+
continue;
|
|
459
|
+
}
|
|
460
|
+
if (route.mode === "fts") {
|
|
461
|
+
forceTrackFtsRoute(candidate, {
|
|
462
|
+
...input,
|
|
463
|
+
targetFileId: tracked.file.id,
|
|
464
|
+
}, route);
|
|
465
|
+
}
|
|
466
|
+
else {
|
|
467
|
+
forceTrackVectorRoute(candidate, {
|
|
468
|
+
...input,
|
|
469
|
+
targetFileId: tracked.file.id,
|
|
470
|
+
}, route);
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
input.candidates.set(input.entityId, candidate);
|
|
474
|
+
}
|
|
475
|
+
function filterExcludesFile(filter, fileId) {
|
|
476
|
+
return filter?.fileIds !== undefined && !filter.fileIds.includes(fileId);
|
|
477
|
+
}
|
|
478
|
+
function forceTrackNoMatchingFilesRoute(candidate, route) {
|
|
479
|
+
candidate.recall.push({
|
|
480
|
+
path: route.mode,
|
|
481
|
+
routeId: route.id,
|
|
482
|
+
query: route.query,
|
|
483
|
+
found: false,
|
|
484
|
+
forced: true,
|
|
485
|
+
reason: "No files matched the path filters",
|
|
486
|
+
});
|
|
487
|
+
}
|
|
488
|
+
function forceTrackPathExcludedRoute(candidate, route) {
|
|
489
|
+
candidate.recall.push({
|
|
490
|
+
path: route.mode,
|
|
491
|
+
routeId: route.id,
|
|
492
|
+
query: route.query,
|
|
493
|
+
found: false,
|
|
494
|
+
forced: true,
|
|
495
|
+
reason: "Target entity file was excluded by the path filters",
|
|
496
|
+
});
|
|
497
|
+
}
|
|
498
|
+
function forceTrackFtsRoute(candidate, input, route) {
|
|
499
|
+
const hit = searchTrackedEntityFts(input, route);
|
|
500
|
+
if (hit) {
|
|
501
|
+
candidate.sources.add("fts");
|
|
502
|
+
candidate.evidence.push({
|
|
503
|
+
fragment: hit.fragment,
|
|
504
|
+
path: "fts",
|
|
505
|
+
routeId: route.id,
|
|
506
|
+
query: route.query,
|
|
507
|
+
rank: input.recallDepth + 1,
|
|
508
|
+
score: hit.score,
|
|
509
|
+
forced: true,
|
|
510
|
+
});
|
|
511
|
+
addOrUpdateRecall(candidate, {
|
|
512
|
+
path: "fts",
|
|
513
|
+
routeId: route.id,
|
|
514
|
+
query: route.query,
|
|
515
|
+
found: true,
|
|
516
|
+
forced: true,
|
|
517
|
+
rank: input.recallDepth + 1,
|
|
518
|
+
score: hit.score,
|
|
519
|
+
});
|
|
520
|
+
return;
|
|
521
|
+
}
|
|
522
|
+
candidate.recall.push({
|
|
523
|
+
path: "fts",
|
|
524
|
+
routeId: route.id,
|
|
525
|
+
query: route.query,
|
|
526
|
+
found: false,
|
|
527
|
+
forced: true,
|
|
528
|
+
reason: "Target entity did not match the FTS query",
|
|
529
|
+
});
|
|
530
|
+
}
|
|
531
|
+
function forceTrackVectorRoute(candidate, input, route) {
|
|
532
|
+
const vector = input.vectorByRoute.get(route.id);
|
|
533
|
+
if (!vector) {
|
|
534
|
+
candidate.recall.push({
|
|
535
|
+
path: "vector",
|
|
536
|
+
routeId: route.id,
|
|
537
|
+
query: route.query,
|
|
538
|
+
found: false,
|
|
539
|
+
forced: true,
|
|
540
|
+
reason: "Vector route was not available for this query",
|
|
541
|
+
});
|
|
542
|
+
return;
|
|
543
|
+
}
|
|
544
|
+
const hit = searchTrackedEntityVector(input, vector);
|
|
545
|
+
if (hit) {
|
|
546
|
+
candidate.sources.add("vector");
|
|
547
|
+
candidate.evidence.push({
|
|
548
|
+
fragment: hit.fragment,
|
|
549
|
+
path: "vector",
|
|
550
|
+
routeId: route.id,
|
|
551
|
+
query: route.query,
|
|
552
|
+
rank: input.recallDepth + 1,
|
|
553
|
+
score: hit.score,
|
|
554
|
+
forced: true,
|
|
555
|
+
});
|
|
556
|
+
addOrUpdateRecall(candidate, {
|
|
557
|
+
path: "vector",
|
|
558
|
+
routeId: route.id,
|
|
559
|
+
query: route.query,
|
|
560
|
+
found: true,
|
|
561
|
+
forced: true,
|
|
562
|
+
rank: input.recallDepth + 1,
|
|
563
|
+
score: hit.score,
|
|
564
|
+
});
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
567
|
+
candidate.recall.push({
|
|
568
|
+
path: "vector",
|
|
569
|
+
routeId: route.id,
|
|
570
|
+
query: route.query,
|
|
571
|
+
found: false,
|
|
572
|
+
forced: true,
|
|
573
|
+
reason: "Target entity could not be scored by vector search",
|
|
574
|
+
});
|
|
575
|
+
}
|
|
576
|
+
function searchTrackedEntityFts(input, route) {
|
|
577
|
+
const [groupHit] = input.storage.searchFts(route.query, 1, {
|
|
578
|
+
...input.filter,
|
|
579
|
+
groupIds: [input.entityId],
|
|
580
|
+
});
|
|
581
|
+
if (groupHit) {
|
|
582
|
+
return groupHit;
|
|
583
|
+
}
|
|
584
|
+
return input.storage.searchFts(route.query, input.recallDepth, restrictFilterToFile(input.filter, input.targetFileId)).find((hit) => publicEntityId(hit.fragment) === input.entityId);
|
|
585
|
+
}
|
|
586
|
+
function searchTrackedEntityVector(input, vector) {
|
|
587
|
+
const [groupHit] = input.storage.searchVector(vector, 1, {
|
|
588
|
+
...input.filter,
|
|
589
|
+
groupIds: [input.entityId],
|
|
590
|
+
});
|
|
591
|
+
if (groupHit) {
|
|
592
|
+
return groupHit;
|
|
593
|
+
}
|
|
594
|
+
return input.storage.searchVector(vector, input.recallDepth, restrictFilterToFile(input.filter, input.targetFileId)).find((hit) => publicEntityId(hit.fragment) === input.entityId);
|
|
595
|
+
}
|
|
596
|
+
function restrictFilterToFile(filter, fileId) {
|
|
597
|
+
return {
|
|
598
|
+
...filter,
|
|
599
|
+
fileIds: [fileId],
|
|
600
|
+
};
|
|
601
|
+
}
|
|
602
|
+
function searchPlanToStorageFilter(plan, storage) {
|
|
603
|
+
const fileIds = resolveFilteredFileIds(plan, storage.listFiles());
|
|
604
|
+
const symbolTypes = plan.symbolTypes && plan.symbolTypes.length > 0
|
|
605
|
+
? plan.symbolTypes
|
|
606
|
+
: undefined;
|
|
607
|
+
if (fileIds === undefined
|
|
608
|
+
&& symbolTypes === undefined) {
|
|
609
|
+
return undefined;
|
|
610
|
+
}
|
|
611
|
+
return {
|
|
612
|
+
...(fileIds !== undefined ? { fileIds } : {}),
|
|
613
|
+
...(symbolTypes !== undefined ? { symbolTypes } : {}),
|
|
614
|
+
};
|
|
615
|
+
}
|
|
616
|
+
function filterMatchesNoFiles(filter) {
|
|
617
|
+
return filter?.fileIds !== undefined && filter.fileIds.length === 0;
|
|
618
|
+
}
|
|
619
|
+
function resolveFilteredFileIds(plan, files) {
|
|
620
|
+
const includeMatchers = (plan.includePaths ?? []).map(compilePathFilter);
|
|
621
|
+
const excludeMatchers = (plan.excludePaths ?? []).map(compilePathFilter);
|
|
622
|
+
const hasModifiedFilter = plan.modifiedAfter !== undefined || plan.modifiedBefore !== undefined;
|
|
623
|
+
if (includeMatchers.length === 0 && excludeMatchers.length === 0 && !hasModifiedFilter) {
|
|
624
|
+
return undefined;
|
|
625
|
+
}
|
|
626
|
+
return files
|
|
627
|
+
.filter((file) => {
|
|
628
|
+
const included = includeMatchers.length === 0
|
|
629
|
+
|| includeMatchers.some((matcher) => matcher(file));
|
|
630
|
+
const excluded = excludeMatchers.some((matcher) => matcher(file));
|
|
631
|
+
return included && !excluded && matchesModifiedTimeFilter(file, plan);
|
|
632
|
+
})
|
|
633
|
+
.map((file) => file.id);
|
|
634
|
+
}
|
|
635
|
+
function matchesModifiedTimeFilter(file, plan) {
|
|
636
|
+
if (plan.modifiedAfter !== undefined && file.lastModifiedTime < plan.modifiedAfter) {
|
|
637
|
+
return false;
|
|
638
|
+
}
|
|
639
|
+
if (plan.modifiedBefore !== undefined && file.lastModifiedTime > plan.modifiedBefore) {
|
|
640
|
+
return false;
|
|
641
|
+
}
|
|
642
|
+
return true;
|
|
643
|
+
}
|
|
644
|
+
function compilePathFilter(pattern) {
|
|
645
|
+
const pathTarget = isAbsolutePathPattern(pattern) ? "absolutePath" : "relativePath";
|
|
646
|
+
if (hasPathGlob(pattern)) {
|
|
647
|
+
return (file) => pathPatternMatches(pattern, normalizePathForMatch(file[pathTarget]));
|
|
648
|
+
}
|
|
649
|
+
return (file) => {
|
|
650
|
+
const path = normalizePathForMatch(file[pathTarget]);
|
|
651
|
+
return pathPatternMatches(pattern, path);
|
|
652
|
+
};
|
|
653
|
+
}
|
|
654
|
+
function normalizePathFilterPattern(pattern) {
|
|
655
|
+
return normalizePathPattern(pattern);
|
|
656
|
+
}
|
|
657
|
+
function fuseCandidates(candidates) {
|
|
658
|
+
for (const candidate of candidates.values()) {
|
|
659
|
+
candidate.score = 0;
|
|
660
|
+
candidate.forced = candidate.recall.some((trace) => trace.forced);
|
|
661
|
+
for (const recall of candidate.recall) {
|
|
662
|
+
if (recall.found && recall.rank !== undefined) {
|
|
663
|
+
candidate.score += 1 / (RRF_K + recall.rank);
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
const fused = [...candidates.values()].sort((left, right) => {
|
|
668
|
+
if (right.score !== left.score) {
|
|
669
|
+
return right.score - left.score;
|
|
670
|
+
}
|
|
671
|
+
return left.id.localeCompare(right.id);
|
|
672
|
+
});
|
|
673
|
+
for (const [index, candidate] of fused.entries()) {
|
|
674
|
+
candidate.rank = index + 1;
|
|
675
|
+
}
|
|
676
|
+
return fused;
|
|
677
|
+
}
|
|
678
|
+
function candidateToHit(candidate, limit, trace) {
|
|
679
|
+
return {
|
|
680
|
+
entity: candidate.entity,
|
|
681
|
+
file: candidate.file,
|
|
682
|
+
evidence: sortEvidence(candidate.evidence).map(evidenceToSearchHitEvidence),
|
|
683
|
+
rank: candidate.rank,
|
|
684
|
+
score: candidate.score,
|
|
685
|
+
matchedBy: deriveMatchedBy(candidate.sources),
|
|
686
|
+
trace: trace ? candidateToTrace(candidate, limit) : undefined,
|
|
687
|
+
};
|
|
688
|
+
}
|
|
689
|
+
function evidenceToSearchHitEvidence(evidence) {
|
|
690
|
+
return {
|
|
691
|
+
range: evidence.fragment.range,
|
|
692
|
+
content: evidence.fragment.content,
|
|
693
|
+
metadata: evidence.fragment.metadata,
|
|
694
|
+
isEntity: evidence.fragment.id === publicEntityId(evidence.fragment),
|
|
695
|
+
path: evidence.path,
|
|
696
|
+
routeId: evidence.routeId,
|
|
697
|
+
query: evidence.query,
|
|
698
|
+
rank: evidence.rank,
|
|
699
|
+
score: evidence.score,
|
|
700
|
+
forced: evidence.forced,
|
|
701
|
+
};
|
|
702
|
+
}
|
|
703
|
+
function sortEvidence(evidence) {
|
|
704
|
+
return [...evidence].sort((left, right) => {
|
|
705
|
+
const leftRank = left.rank ?? Number.POSITIVE_INFINITY;
|
|
706
|
+
const rightRank = right.rank ?? Number.POSITIVE_INFINITY;
|
|
707
|
+
if (leftRank !== rightRank) {
|
|
708
|
+
return leftRank - rightRank;
|
|
709
|
+
}
|
|
710
|
+
if (left.path !== right.path) {
|
|
711
|
+
return left.path.localeCompare(right.path);
|
|
712
|
+
}
|
|
713
|
+
return left.fragment.id.localeCompare(right.fragment.id);
|
|
714
|
+
});
|
|
715
|
+
}
|
|
716
|
+
function candidateToTrace(candidate, limit) {
|
|
717
|
+
const final = {
|
|
718
|
+
returnedByLimit: candidate.rank <= limit,
|
|
719
|
+
cutoffRank: limit,
|
|
720
|
+
};
|
|
721
|
+
return {
|
|
722
|
+
recall: candidate.recall,
|
|
723
|
+
fusion: {
|
|
724
|
+
rank: candidate.rank,
|
|
725
|
+
score: candidate.score,
|
|
726
|
+
forced: candidate.forced || undefined,
|
|
727
|
+
},
|
|
728
|
+
final,
|
|
729
|
+
};
|
|
730
|
+
}
|
|
731
|
+
async function chooseBestEntityInFile(query, file, ctx) {
|
|
732
|
+
const candidates = new Map();
|
|
733
|
+
addRecallHits(candidates, ctx.storage.searchFts(query, 10, {
|
|
734
|
+
fileIds: [file.id],
|
|
735
|
+
}), {
|
|
736
|
+
id: "fts",
|
|
737
|
+
mode: "fts",
|
|
738
|
+
query,
|
|
739
|
+
}, ctx.storage);
|
|
740
|
+
const [queryVector] = await requireEmbeddingModel(ctx, "diagnose").embed([{ kind: "text", text: query }], { purpose: "query" });
|
|
741
|
+
addRecallHits(candidates, ctx.storage.searchVector(queryVector, 10, {
|
|
742
|
+
fileIds: [file.id],
|
|
743
|
+
}), {
|
|
744
|
+
id: "vector",
|
|
745
|
+
mode: "vector",
|
|
746
|
+
query,
|
|
747
|
+
}, ctx.storage);
|
|
748
|
+
const [best] = fuseCandidates(candidates);
|
|
749
|
+
if (best) {
|
|
750
|
+
return best.id;
|
|
751
|
+
}
|
|
752
|
+
const [first] = ctx.storage.listEntitiesByFile(file.id, { limit: 1 });
|
|
753
|
+
return first?.entity.id ?? null;
|
|
754
|
+
}
|
|
755
|
+
function deriveMatchedBy(sources) {
|
|
756
|
+
if (sources.has("fts") && sources.has("vector")) {
|
|
757
|
+
return "fts+vector";
|
|
758
|
+
}
|
|
759
|
+
return sources.has("vector") ? "vector" : "fts";
|
|
760
|
+
}
|
|
761
|
+
//# sourceMappingURL=index.js.map
|