@amsterdamdatalabs/enact-docs-search 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -4
- package/dist/cli.js +182 -54
- package/dist/cli.js.map +1 -1
- package/dist/dependency-errors.d.ts +3 -0
- package/dist/dependency-errors.d.ts.map +1 -0
- package/dist/dependency-errors.js +26 -0
- package/dist/dependency-errors.js.map +1 -0
- package/dist/doctor.d.ts +17 -0
- package/dist/doctor.d.ts.map +1 -0
- package/dist/doctor.js +88 -0
- package/dist/doctor.js.map +1 -0
- package/dist/embeddings/embedding-service.d.ts.map +1 -1
- package/dist/embeddings/embedding-service.js +2 -3
- package/dist/embeddings/embedding-service.js.map +1 -1
- package/dist/indexing/chunker.d.ts +6 -1
- package/dist/indexing/chunker.d.ts.map +1 -1
- package/dist/indexing/chunker.js +17 -11
- package/dist/indexing/chunker.js.map +1 -1
- package/dist/indexing/embedding-text.d.ts +22 -0
- package/dist/indexing/embedding-text.d.ts.map +1 -0
- package/dist/indexing/embedding-text.js +27 -0
- package/dist/indexing/embedding-text.js.map +1 -0
- package/dist/indexing/frontmatter.d.ts +48 -0
- package/dist/indexing/frontmatter.d.ts.map +1 -0
- package/dist/indexing/frontmatter.js +162 -0
- package/dist/indexing/frontmatter.js.map +1 -0
- package/dist/indexing/indexing-service.d.ts +1 -4
- package/dist/indexing/indexing-service.d.ts.map +1 -1
- package/dist/indexing/indexing-service.js +24 -12
- package/dist/indexing/indexing-service.js.map +1 -1
- package/dist/search/authority.d.ts +31 -0
- package/dist/search/authority.d.ts.map +1 -0
- package/dist/search/authority.js +50 -0
- package/dist/search/authority.js.map +1 -0
- package/dist/search/relevance.d.ts +80 -0
- package/dist/search/relevance.d.ts.map +1 -0
- package/dist/search/relevance.js +119 -0
- package/dist/search/relevance.js.map +1 -0
- package/dist/search/search-service.d.ts +44 -2
- package/dist/search/search-service.d.ts.map +1 -1
- package/dist/search/search-service.js +181 -23
- package/dist/search/search-service.js.map +1 -1
- package/dist/search/snippet.d.ts +11 -0
- package/dist/search/snippet.d.ts.map +1 -1
- package/dist/search/snippet.js +102 -17
- package/dist/search/snippet.js.map +1 -1
- package/dist/storage/document-store.d.ts +47 -5
- package/dist/storage/document-store.d.ts.map +1 -1
- package/dist/storage/document-store.js +94 -16
- package/dist/storage/document-store.js.map +1 -1
- package/dist/storage/persistence.d.ts +6 -10
- package/dist/storage/persistence.d.ts.map +1 -1
- package/dist/storage/persistence.js +20 -20
- package/dist/storage/persistence.js.map +1 -1
- package/dist/storage/vector-store.d.ts +2 -3
- package/dist/storage/vector-store.d.ts.map +1 -1
- package/dist/storage/vector-store.js +5 -15
- package/dist/storage/vector-store.js.map +1 -1
- package/dist/types.d.ts +2 -7
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +2 -20
- package/dist/types.js.map +1 -1
- package/package.json +9 -2
|
@@ -7,20 +7,22 @@
|
|
|
7
7
|
* keeps the plugin process alive). This CLI process exits after every
|
|
8
8
|
* command, so this version is built around a manifest-driven incremental
|
|
9
9
|
* diff (new/changed/removed files by mtime) against a store that is loaded
|
|
10
|
-
* from and saved back to disk on every run
|
|
10
|
+
* from and saved back to disk on every run for each target repo.
|
|
11
11
|
*/
|
|
12
12
|
import { DocumentStore } from "../storage/document-store.js";
|
|
13
13
|
import { VectorStore } from "../storage/vector-store.js";
|
|
14
14
|
import { EmbeddingService, EMBEDDING_DIMENSIONS } from "../embeddings/embedding-service.js";
|
|
15
|
+
import { parseFrontmatter } from "./frontmatter.js";
|
|
16
|
+
import { buildEmbeddingText } from "./embedding-text.js";
|
|
15
17
|
import { Chunker } from "./chunker.js";
|
|
16
18
|
import { listMarkdownFiles, readRepoFile } from "../adapter/repo-walker.js";
|
|
17
19
|
import { ensureRepoCache, readManifest, writeManifest, emptyManifest, readDocumentStoreExport, writeDocumentStoreExport, readVectorSidecar, writeVectorSidecar, removeVectorCache, vectorIndexExists, repoCacheDir, repoModelCacheDir, paths, EMBEDDING_MODEL_ID, } from "../storage/persistence.js";
|
|
18
|
-
import {
|
|
20
|
+
import { DOCS_ROOT } from "../types.js";
|
|
19
21
|
const CHUNK_SIZE = 3200;
|
|
20
22
|
const CHUNK_OVERLAP = 0.15;
|
|
21
23
|
export async function runIndex(options) {
|
|
22
24
|
const start = Date.now();
|
|
23
|
-
const { repoPath,
|
|
25
|
+
const { repoPath, onProgress } = options;
|
|
24
26
|
const cacheDir = repoCacheDir(repoPath);
|
|
25
27
|
await ensureRepoCache(repoPath);
|
|
26
28
|
const persistedManifest = await readManifest(cacheDir);
|
|
@@ -33,8 +35,7 @@ export async function runIndex(options) {
|
|
|
33
35
|
await removeVectorCache(cacheDir);
|
|
34
36
|
}
|
|
35
37
|
onProgress?.({ phase: "scanning", filesProcessed: 0, totalFiles: 0 });
|
|
36
|
-
const
|
|
37
|
-
const files = await listMarkdownFiles(repoPath, roots, { exclude: options.exclude });
|
|
38
|
+
const files = await listMarkdownFiles(repoPath, [DOCS_ROOT], { exclude: options.exclude });
|
|
38
39
|
const docStore = new DocumentStore();
|
|
39
40
|
await docStore.initialize(cacheRequiresRebuild ? undefined : (await readDocumentStoreExport(cacheDir)) ?? undefined);
|
|
40
41
|
let vectorStore = null;
|
|
@@ -67,11 +68,10 @@ export async function runIndex(options) {
|
|
|
67
68
|
if (textIsStale || vectorIsStale)
|
|
68
69
|
filesToProcess.push(f);
|
|
69
70
|
}
|
|
70
|
-
// Files that were indexed
|
|
71
|
+
// Files that were indexed previously but no longer exist under docs/.
|
|
71
72
|
const removed = Object.keys(manifest.files).filter((p) => {
|
|
72
|
-
const inScope = roots.some((r) => p === r || p.startsWith(r + "/"));
|
|
73
73
|
const entry = manifest.files[p];
|
|
74
|
-
return
|
|
74
|
+
return !currentPaths.has(p) &&
|
|
75
75
|
(entry.textMtimeMs !== null || entry.vectorMtimeMs !== null || entry.vectorStale);
|
|
76
76
|
});
|
|
77
77
|
onProgress?.({
|
|
@@ -113,7 +113,10 @@ export async function runIndex(options) {
|
|
|
113
113
|
await docStore.deleteByFilePath(file.path);
|
|
114
114
|
if (vectorStore)
|
|
115
115
|
await vectorStore.deleteByFilePath(file.path);
|
|
116
|
-
|
|
116
|
+
// Frontmatter is metadata, not prose: it is parsed into fields and removed
|
|
117
|
+
// from the body, so it never reaches the keyword index or the embedding.
|
|
118
|
+
const { meta, body, frontmatterLineCount } = parseFrontmatter(content);
|
|
119
|
+
const chunks = chunker.chunk(body, file.path, frontmatterLineCount);
|
|
117
120
|
const docs = chunks.map((chunk, idx) => ({
|
|
118
121
|
id: `${file.path}:${idx}`,
|
|
119
122
|
filePath: file.path,
|
|
@@ -122,6 +125,12 @@ export async function runIndex(options) {
|
|
|
122
125
|
mtimeMs: file.mtimeMs,
|
|
123
126
|
startLine: chunk.startLine,
|
|
124
127
|
headingBreadcrumb: chunk.headingBreadcrumb,
|
|
128
|
+
title: meta.title,
|
|
129
|
+
description: meta.description,
|
|
130
|
+
headings: chunk.headingBreadcrumb.join(" > "),
|
|
131
|
+
status: meta.status,
|
|
132
|
+
documentType: meta.documentType,
|
|
133
|
+
lastUpdate: meta.lastUpdate,
|
|
125
134
|
}));
|
|
126
135
|
await docStore.indexDocuments(docs);
|
|
127
136
|
chunksIndexed += docs.length;
|
|
@@ -162,7 +171,12 @@ export async function runIndex(options) {
|
|
|
162
171
|
const BATCH = 16;
|
|
163
172
|
for (let i = 0; i < changedDocsForEmbedding.length; i += BATCH) {
|
|
164
173
|
const batch = changedDocsForEmbedding.slice(i, i + BATCH);
|
|
165
|
-
const embeddings = await embeddingService.embedBatch(batch.map((d) =>
|
|
174
|
+
const embeddings = await embeddingService.embedBatch(batch.map((d) => buildEmbeddingText({
|
|
175
|
+
title: d.title,
|
|
176
|
+
description: d.description,
|
|
177
|
+
headingBreadcrumb: d.headingBreadcrumb,
|
|
178
|
+
content: d.content,
|
|
179
|
+
})));
|
|
166
180
|
for (let j = 0; j < batch.length; j++) {
|
|
167
181
|
const doc = batch[j];
|
|
168
182
|
await vectorStore.add({
|
|
@@ -191,13 +205,11 @@ export async function runIndex(options) {
|
|
|
191
205
|
const updatedManifest = {
|
|
192
206
|
...manifest,
|
|
193
207
|
embeddingModel: options.skipEmbeddings ? manifest.embeddingModel : EMBEDDING_MODEL_ID,
|
|
194
|
-
indexedScopes: Array.from(new Set([...manifest.indexedScopes, scope])),
|
|
195
208
|
lastIndexedAt: Date.now(),
|
|
196
209
|
};
|
|
197
210
|
await writeManifest(cacheDir, updatedManifest);
|
|
198
211
|
onProgress?.({ phase: "done", filesProcessed: files.length, totalFiles: files.length });
|
|
199
212
|
return {
|
|
200
|
-
scope,
|
|
201
213
|
filesScanned: files.length,
|
|
202
214
|
filesChanged: textChanged.length,
|
|
203
215
|
filesRemoved: removed.length,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexing-service.js","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,aAAa,EAAwB,MAAM,8BAA8B,CAAC;AACnF,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,oCAAoC,CAAC;AAC5F,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAiB,MAAM,2BAA2B,CAAC;AAC3F,OAAO,EACL,eAAe,EACf,YAAY,EACZ,aAAa,EACb,aAAa,EACb,uBAAuB,EACvB,wBAAwB,EACxB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,KAAK,EACL,kBAAkB,GAEnB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"indexing-service.js","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,aAAa,EAAwB,MAAM,8BAA8B,CAAC;AACnF,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,oCAAoC,CAAC;AAC5F,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAiB,MAAM,2BAA2B,CAAC;AAC3F,OAAO,EACL,eAAe,EACf,YAAY,EACZ,aAAa,EACb,aAAa,EACb,uBAAuB,EACvB,wBAAwB,EACxB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,KAAK,EACL,kBAAkB,GAEnB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAgCxC,MAAM,UAAU,GAAG,IAAI,CAAC;AACxB,MAAM,aAAa,GAAG,IAAI,CAAC;AAE3B,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAqB;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IACzC,MAAM,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAC;IAEhC,MAAM,iBAAiB,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,aAAa,EAAE,CAAC;IACtD,MAAM,oBAAoB,GAAG,iBAAiB,KAAK,IAAI,CAAC;IACxD,IAAI,oBAAoB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QACnD,wEAAwE;QACxE,sEAAsE;QACtE,uEAAuE;QACvE,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAE3F,MAAM,QAAQ,GAAG,IAAI,aAAa,EAAE,CAAC;IACrC,MAAM,QAAQ,CAAC,UAAU,CACvB,oBAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,uBAAuB,CAAC,QAAQ,CAAC,CAAC,IAAI,SAAS,CAC1F,CAAC;IAEF,IAAI,WAAW,GAAuB,IAAI,CAAC;IAC3C,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QAC5B,WAAW,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACpE,MAAM,kBAAkB,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC7D,OAAO,GAAG,CAAC,oBAAoB,IAAI,kBAAkB;YACnD,CAAC,CAAC,MAAM,iBAAiB,CAAC,QAAQ,CAAC;YACnC,CAAC,CAAC,IAAI,CAAC;QACT,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC9B,MAAM,WAAW,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACvE,CAAC;aAAM,CAAC;YACN,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC;QACjC,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC;IAEzF,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,MAAM,WAAW,GAAe,EAAE,CAAC;IACnC,MAAM,cAAc,GAAe,EAAE,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,CAAC,CAAC,OAAO,CAAC;QAC/E,MAAM,aAAa,GACjB,CAAC,OAAO,CAAC,cAAc;YACvB,CAAC,QAAQ,CAAC,cAAc,KAAK,kBAAkB;gBAC7C,CAAC,OAAO,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;QAClF,IAAI,WAAW;YAAE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,WAAW,IAAI,aAAa;YAAE,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,sEAAsE;IACtE,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACvD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;QACjC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YACzB,CAAC,KAAK,CAAC,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC;IACtF,CAAC,CAAC,CAAC;IAEH,UAAU,EAAE,CAAC;QACX,KAAK,EAAE,eAAe;QACtB,cAAc,EAAE,CAAC;QACjB,UAAU,EAAE,cAAc,CAAC,MAAM;KAClC,CAAC,CAAC;IAEH,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,MAAM,uBAAuB,GAAsB,EAAE,CAAC;IAEtD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,WAAW;YAAE,MAAM,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QACpC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;QACzB,IAAI,WAAW,EAAE,CAAC;YAChB,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC;YAC3B,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC;QAC5B,CAAC;aAAM,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YACxC,mEAAmE;YACnE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC;QAC5B,CAAC;QACD,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI;YAAE,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9F,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC,CAAE,CAAC;QAChC,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,eAAe;YACtB,cAAc,EAAE,CAAC;YACjB,UAAU,EAAE,cAAc,CAAC,MAAM;YACjC,WAAW,EAAE,IAAI,CAAC,IAAI;SACvB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,WAAW;YAAE,MAAM,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE/D,2EAA2E;QAC3E,yEAAyE;QACzE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,oBAAoB,EAAE,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAEvE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;QACpE,MAAM,IAAI,GAAsB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YAC1D,EAAE,EAAE,GAAG,IAAI,CAAC,IAAI,IAAI,GAAG,EAAE;YACzB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,GAAG;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;YAC1C,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,QAAQ,EAAE,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC;YAC7C,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACpC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC;QAC7B,IAAI,WAAW;YAAE,uBAAuB,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;QAEvD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YACzC,WAAW,EAAE,IAAI;YACjB,aAAa,EAAE,IAAI;YACnB,WAAW,EAAE,KAAK;SACnB,CAAC;QACF,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC;QACjC,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,wEAAwE;YACxE,wEAAwE;YACxE,mCAAmC;YACnC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;QAC3B,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,sEAAsE;YACtE,6DAA6D;YAC7D,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC;YACnC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC;QAC5B,CAAC;QACD,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IACpC,CAAC;IAED,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,IAAI,WAAW,IAAI,uBAAuB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,CAAC;YAC5C,aAAa,EAAE,iBAAiB,CAAC,QAAQ,CAAC;YAC1C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI;SACrD,CAAC,CAAC;QACH,MAAM,gBAAgB,CAAC,UAAU,EAAE,CAAC;QAEpC,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,WAAW;YAClB,cAAc,EAAE,CAAC;YACjB,UAAU,EAAE,uBAAuB,CAAC,MAAM;SAC3C,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,uBAAuB,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;YAC1D,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,UAAU,CAClD,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACd,kBAAkB,CAAC;gBACjB,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,iBAAiB,EAAE,CAAC,CAAC,iBAAiB;gBACtC,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CACH,CACF,CAAC;YACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;gBACtB,MAAM,WAAW,CAAC,GAAG,CAAC;oBACpB,EAAE,EAAE,GAAG,CAAC,EAAE;oBACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,UAAU,EAAE,GAAG,CAAC,UAAU;oBAC1B,SAAS,EAAE,UAAU,CAAC,CAAC,CAAE;iBAC1B,CAAC,CAAC;gBACH,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,aAAa,GAAG,GAAG,CAAC,OAAO,CAAC;gBAC1D,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,WAAW,GAAG,KAAK,CAAC;gBAClD,mBAAmB,EAAE,CAAC;YACxB,CAAC;YACD,UAAU,EAAE,CAAC;gBACX,KAAK,EAAE,WAAW;gBAClB,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,uBAAuB,CAAC,MAAM,CAAC;gBACnE,UAAU,EAAE,uBAAuB,CAAC,MAAM;aAC3C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,WAAW;IACX,MAAM,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5D,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,cAAc,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC,CAAC;QACjF,MAAM,kBAAkB,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,eAAe,GAAa;QAChC,GAAG,QAAQ;QACX,cAAc,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,kBAAkB;QACrF,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE;KAC1B,CAAC;IACF,MAAM,aAAa,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAE/C,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAExF,OAAO;QACL,YAAY,EAAE,KAAK,CAAC,MAAM;QAC1B,YAAY,EAAE,WAAW,CAAC,MAAM;QAChC,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,aAAa;QACb,mBAAmB;QACnB,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KACpC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Document authority: how much a document's own declared state should move it
|
|
3
|
+
* up or down the page, independent of how well it matched.
|
|
4
|
+
*
|
|
5
|
+
* Two signals, both taken from frontmatter the document wrote about itself:
|
|
6
|
+
* whether it claims to be stable, and when it was last touched. Both are
|
|
7
|
+
* multiplicative on the fused score, and both are neutral when the document
|
|
8
|
+
* makes no claim -- silence is not evidence of staleness.
|
|
9
|
+
*/
|
|
10
|
+
/** Age at which a document's score is halved. */
|
|
11
|
+
export declare const RECENCY_HALF_LIFE_DAYS = 180;
|
|
12
|
+
/** Decay never drives a document below this, however old it is. */
|
|
13
|
+
export declare const RECENCY_FLOOR = 0.25;
|
|
14
|
+
export interface AuthorityInput {
|
|
15
|
+
status?: string;
|
|
16
|
+
lastUpdate?: string;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Multiplier for a declared status. An unrecognised or absent status returns
|
|
20
|
+
* 1: only an explicit claim is acted on, so a document is never penalised for
|
|
21
|
+
* carrying no frontmatter.
|
|
22
|
+
*/
|
|
23
|
+
export declare function statusFactor(status?: string): number;
|
|
24
|
+
/**
|
|
25
|
+
* Exponential decay on document age, floored. An absent, unparseable, or
|
|
26
|
+
* future timestamp is neutral -- a clock skew must not hand out a bonus.
|
|
27
|
+
*/
|
|
28
|
+
export declare function recencyFactor(lastUpdate: string | undefined, now: number): number;
|
|
29
|
+
/** Combined multiplier applied to a fused score. */
|
|
30
|
+
export declare function authorityFactor(input: AuthorityInput, now: number): number;
|
|
31
|
+
//# sourceMappingURL=authority.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"authority.d.ts","sourceRoot":"","sources":["../../src/search/authority.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAQH,iDAAiD;AACjD,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C,mEAAmE;AACnE,eAAO,MAAM,aAAa,OAAO,CAAC;AAIlC,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAGpD;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAWjF;AAED,oDAAoD;AACpD,wBAAgB,eAAe,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAE1E"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Document authority: how much a document's own declared state should move it
|
|
3
|
+
* up or down the page, independent of how well it matched.
|
|
4
|
+
*
|
|
5
|
+
* Two signals, both taken from frontmatter the document wrote about itself:
|
|
6
|
+
* whether it claims to be stable, and when it was last touched. Both are
|
|
7
|
+
* multiplicative on the fused score, and both are neutral when the document
|
|
8
|
+
* makes no claim -- silence is not evidence of staleness.
|
|
9
|
+
*/
|
|
10
|
+
/** Explicit status values and their multipliers. Anything else is neutral. */
|
|
11
|
+
const STATUS_FACTOR = {
|
|
12
|
+
stable: 1,
|
|
13
|
+
draft: 0.3,
|
|
14
|
+
};
|
|
15
|
+
/** Age at which a document's score is halved. */
|
|
16
|
+
export const RECENCY_HALF_LIFE_DAYS = 180;
|
|
17
|
+
/** Decay never drives a document below this, however old it is. */
|
|
18
|
+
export const RECENCY_FLOOR = 0.25;
|
|
19
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
20
|
+
/**
|
|
21
|
+
* Multiplier for a declared status. An unrecognised or absent status returns
|
|
22
|
+
* 1: only an explicit claim is acted on, so a document is never penalised for
|
|
23
|
+
* carrying no frontmatter.
|
|
24
|
+
*/
|
|
25
|
+
export function statusFactor(status) {
|
|
26
|
+
if (!status)
|
|
27
|
+
return 1;
|
|
28
|
+
return STATUS_FACTOR[status.trim().toLowerCase()] ?? 1;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Exponential decay on document age, floored. An absent, unparseable, or
|
|
32
|
+
* future timestamp is neutral -- a clock skew must not hand out a bonus.
|
|
33
|
+
*/
|
|
34
|
+
export function recencyFactor(lastUpdate, now) {
|
|
35
|
+
if (!lastUpdate)
|
|
36
|
+
return 1;
|
|
37
|
+
const updatedAt = Date.parse(lastUpdate);
|
|
38
|
+
if (Number.isNaN(updatedAt))
|
|
39
|
+
return 1;
|
|
40
|
+
const ageDays = (now - updatedAt) / DAY_MS;
|
|
41
|
+
if (ageDays <= 0)
|
|
42
|
+
return 1;
|
|
43
|
+
const decayed = Math.pow(2, -ageDays / RECENCY_HALF_LIFE_DAYS);
|
|
44
|
+
return Math.max(RECENCY_FLOOR, decayed);
|
|
45
|
+
}
|
|
46
|
+
/** Combined multiplier applied to a fused score. */
|
|
47
|
+
export function authorityFactor(input, now) {
|
|
48
|
+
return statusFactor(input.status) * recencyFactor(input.lastUpdate, now);
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=authority.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"authority.js","sourceRoot":"","sources":["../../src/search/authority.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,8EAA8E;AAC9E,MAAM,aAAa,GAA2B;IAC5C,MAAM,EAAE,CAAC;IACT,KAAK,EAAE,GAAG;CACX,CAAC;AAEF,iDAAiD;AACjD,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C,mEAAmE;AACnE,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,CAAC;AAElC,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAOnC;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,MAAe;IAC1C,IAAI,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IACtB,OAAO,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC;AACzD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,UAA8B,EAAE,GAAW;IACvE,IAAI,CAAC,UAAU;QAAE,OAAO,CAAC,CAAC;IAE1B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,CAAC,CAAC;IAEtC,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,SAAS,CAAC,GAAG,MAAM,CAAC;IAC3C,IAAI,OAAO,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAE3B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,OAAO,GAAG,sBAAsB,CAAC,CAAC;IAC/D,OAAO,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;AAC1C,CAAC;AAED,oDAAoD;AACpD,MAAM,UAAU,eAAe,CAAC,KAAqB,EAAE,GAAW;IAChE,OAAO,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;AAC3E,CAAC"}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cosine floor for semantic hits.
|
|
3
|
+
*
|
|
4
|
+
* Re-measured against docs/ in enact-m5 (203 chunks), taking the top cosine
|
|
5
|
+
* over the whole corpus per query:
|
|
6
|
+
*
|
|
7
|
+
* junk zzqzz qqxxw 0.131 | florbnax vimtuzzle 0.134
|
|
8
|
+
* purple giraffe tapdance 0.103 | qwertyuiop asdfghjkl 0.132
|
|
9
|
+
* genuine invariants 0.260 | brownfield reference implementation 0.322
|
|
10
|
+
* database migrations 0.374 | websocket routes 0.520
|
|
11
|
+
* sandbox lifecycle 0.531 | coverage hooks 0.693
|
|
12
|
+
*
|
|
13
|
+
* So the real gap is [0.134, 0.260], and 0.20 sits in the middle of it with
|
|
14
|
+
* roughly equal headroom on each side. An earlier value of 0.35 was set from
|
|
15
|
+
* figures this measurement could not reproduce, and it silently discarded the
|
|
16
|
+
* best hit for real queries -- "invariants" returned nothing at all.
|
|
17
|
+
*
|
|
18
|
+
* Reproduce with:
|
|
19
|
+
* node dist/cli.js "<query>" --repo <repo> --mode semantic --min-score 0 --json
|
|
20
|
+
*/
|
|
21
|
+
export declare const DEFAULT_MIN_VECTOR_SCORE = 0.2;
|
|
22
|
+
/**
|
|
23
|
+
* Fraction of the query's *weight* a document must cover (see
|
|
24
|
+
* `idfWeightedCoverage`), not a fraction of its terms.
|
|
25
|
+
*
|
|
26
|
+
* Measured separation on this corpus:
|
|
27
|
+
* 0.09 "presence is not armedness" -- absent term dominates the weight
|
|
28
|
+
* 0.47 "mobile device" matching only the commoner term
|
|
29
|
+
* 0.53 "mobile device" matching only the rarer term
|
|
30
|
+
* 0.73 a five-term query missing one term
|
|
31
|
+
* 1.00 full match
|
|
32
|
+
*
|
|
33
|
+
* 0.5 sits below every genuine partial match and far above the junk floor.
|
|
34
|
+
*/
|
|
35
|
+
export declare const DEFAULT_MIN_TERM_COVERAGE = 0.5;
|
|
36
|
+
/**
|
|
37
|
+
* Fraction of the query's content terms that appear in `text`.
|
|
38
|
+
*
|
|
39
|
+
* Returns 1 for a query with no content terms (all stopwords or too short):
|
|
40
|
+
* there is nothing to be missing, so the gate must not reject on that basis.
|
|
41
|
+
*/
|
|
42
|
+
export declare function termCoverage(query: string, text: string): number;
|
|
43
|
+
/** Per-term document frequencies for the query, plus the corpus size. */
|
|
44
|
+
export interface CorpusStats {
|
|
45
|
+
totalDocs: number;
|
|
46
|
+
/** term -> number of indexed chunks containing it. 0 means absent entirely. */
|
|
47
|
+
documentFrequency: Map<string, number>;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Normalise a raw query string.
|
|
51
|
+
*
|
|
52
|
+
* Agents type `mobile | devices` out of grep habit. The pipe is treated as a
|
|
53
|
+
* separator, exactly like a space -- `mobile | devices` and `mobile devices`
|
|
54
|
+
* are the same query. It is deliberately not an OR operator: one query syntax
|
|
55
|
+
* is easier to use correctly than two, and an unquoted pipe never reaches this
|
|
56
|
+
* process anyway -- the shell consumes it as a pipeline first.
|
|
57
|
+
*/
|
|
58
|
+
export declare function normalizeQuery(query: string): string;
|
|
59
|
+
/**
|
|
60
|
+
* Coverage weighted by term rarity.
|
|
61
|
+
*
|
|
62
|
+
* Flat coverage treats every term as equally important, which makes a two-term
|
|
63
|
+
* query behave like AND -- "mobile device" then drops a document about a
|
|
64
|
+
* "mobile target" for lacking the word "device". Weighting by IDF lets the
|
|
65
|
+
* term that actually carries the query decide, in both directions: matching
|
|
66
|
+
* only the rare term passes, matching only the common one does not.
|
|
67
|
+
*
|
|
68
|
+
* Falls back to flat coverage when no stats are supplied.
|
|
69
|
+
*/
|
|
70
|
+
export declare function idfWeightedCoverage(query: string, text: string, stats?: CorpusStats): number;
|
|
71
|
+
/**
|
|
72
|
+
* Keyword-side gate. Uses term coverage rather than a BM25 threshold: BM25
|
|
73
|
+
* scores here are unnormalised and scale with query length and corpus size,
|
|
74
|
+
* so any absolute cutoff would be a fact about one repo. Coverage is a ratio
|
|
75
|
+
* and travels.
|
|
76
|
+
*/
|
|
77
|
+
export declare function passesKeywordGate(query: string, text: string, minCoverage?: number, stats?: CorpusStats): boolean;
|
|
78
|
+
/** Semantic-side gate. Cosine is bounded, so an absolute floor is portable. */
|
|
79
|
+
export declare function passesVectorGate(score: number, minScore?: number): boolean;
|
|
80
|
+
//# sourceMappingURL=relevance.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"relevance.d.ts","sourceRoot":"","sources":["../../src/search/relevance.ts"],"names":[],"mappings":"AAYA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,wBAAwB,MAAM,CAAC;AAE5C;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAE7C;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAMhE;AAED,yEAAyE;AACzE,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,+EAA+E;IAC/E,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACxC;AAUD;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,KAAK,CAAC,EAAE,WAAW,GAClB,MAAM,CAgBR;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,WAAW,GAAE,MAAkC,EAC/C,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAET;AAED,+EAA+E;AAC/E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,MAAM,EACb,QAAQ,GAAE,MAAiC,GAC1C,OAAO,CAET"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Relevance gating.
|
|
3
|
+
*
|
|
4
|
+
* Reciprocal rank fusion scores a result by its *position*, not by how well it
|
|
5
|
+
* matched, so the top hit for a nonsense query scores about as high as the top
|
|
6
|
+
* hit for a perfect one. Measured on this repo: gibberish topped out at 0.0664
|
|
7
|
+
* against 0.0815 for a well-matched query. Nothing downstream of fusion can
|
|
8
|
+
* tell those apart, so the gate has to run on the raw per-modality evidence
|
|
9
|
+
* before fusion throws it away.
|
|
10
|
+
*/
|
|
11
|
+
import { extractQueryTerms, containsTerm } from "./snippet.js";
|
|
12
|
+
/**
|
|
13
|
+
* Cosine floor for semantic hits.
|
|
14
|
+
*
|
|
15
|
+
* Re-measured against docs/ in enact-m5 (203 chunks), taking the top cosine
|
|
16
|
+
* over the whole corpus per query:
|
|
17
|
+
*
|
|
18
|
+
* junk zzqzz qqxxw 0.131 | florbnax vimtuzzle 0.134
|
|
19
|
+
* purple giraffe tapdance 0.103 | qwertyuiop asdfghjkl 0.132
|
|
20
|
+
* genuine invariants 0.260 | brownfield reference implementation 0.322
|
|
21
|
+
* database migrations 0.374 | websocket routes 0.520
|
|
22
|
+
* sandbox lifecycle 0.531 | coverage hooks 0.693
|
|
23
|
+
*
|
|
24
|
+
* So the real gap is [0.134, 0.260], and 0.20 sits in the middle of it with
|
|
25
|
+
* roughly equal headroom on each side. An earlier value of 0.35 was set from
|
|
26
|
+
* figures this measurement could not reproduce, and it silently discarded the
|
|
27
|
+
* best hit for real queries -- "invariants" returned nothing at all.
|
|
28
|
+
*
|
|
29
|
+
* Reproduce with:
|
|
30
|
+
* node dist/cli.js "<query>" --repo <repo> --mode semantic --min-score 0 --json
|
|
31
|
+
*/
|
|
32
|
+
export const DEFAULT_MIN_VECTOR_SCORE = 0.2;
|
|
33
|
+
/**
|
|
34
|
+
* Fraction of the query's *weight* a document must cover (see
|
|
35
|
+
* `idfWeightedCoverage`), not a fraction of its terms.
|
|
36
|
+
*
|
|
37
|
+
* Measured separation on this corpus:
|
|
38
|
+
* 0.09 "presence is not armedness" -- absent term dominates the weight
|
|
39
|
+
* 0.47 "mobile device" matching only the commoner term
|
|
40
|
+
* 0.53 "mobile device" matching only the rarer term
|
|
41
|
+
* 0.73 a five-term query missing one term
|
|
42
|
+
* 1.00 full match
|
|
43
|
+
*
|
|
44
|
+
* 0.5 sits below every genuine partial match and far above the junk floor.
|
|
45
|
+
*/
|
|
46
|
+
export const DEFAULT_MIN_TERM_COVERAGE = 0.5;
|
|
47
|
+
/**
|
|
48
|
+
* Fraction of the query's content terms that appear in `text`.
|
|
49
|
+
*
|
|
50
|
+
* Returns 1 for a query with no content terms (all stopwords or too short):
|
|
51
|
+
* there is nothing to be missing, so the gate must not reject on that basis.
|
|
52
|
+
*/
|
|
53
|
+
export function termCoverage(query, text) {
|
|
54
|
+
const terms = extractQueryTerms(query);
|
|
55
|
+
if (terms.length === 0)
|
|
56
|
+
return 1;
|
|
57
|
+
const matched = terms.filter((term) => containsTerm(text, term)).length;
|
|
58
|
+
return matched / terms.length;
|
|
59
|
+
}
|
|
60
|
+
/** Smoothed inverse document frequency. A term absent from the corpus scores
|
|
61
|
+
* highest: a query term nothing contains is the most load-bearing one there
|
|
62
|
+
* is, and a result that misses it has missed the point of the query. */
|
|
63
|
+
function idf(term, stats) {
|
|
64
|
+
const df = stats.documentFrequency.get(term) ?? 0;
|
|
65
|
+
return Math.log((stats.totalDocs + 1) / (df + 1)) + 1;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Normalise a raw query string.
|
|
69
|
+
*
|
|
70
|
+
* Agents type `mobile | devices` out of grep habit. The pipe is treated as a
|
|
71
|
+
* separator, exactly like a space -- `mobile | devices` and `mobile devices`
|
|
72
|
+
* are the same query. It is deliberately not an OR operator: one query syntax
|
|
73
|
+
* is easier to use correctly than two, and an unquoted pipe never reaches this
|
|
74
|
+
* process anyway -- the shell consumes it as a pipeline first.
|
|
75
|
+
*/
|
|
76
|
+
export function normalizeQuery(query) {
|
|
77
|
+
return query.replace(/\|/g, " ").replace(/\s+/g, " ").trim();
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Coverage weighted by term rarity.
|
|
81
|
+
*
|
|
82
|
+
* Flat coverage treats every term as equally important, which makes a two-term
|
|
83
|
+
* query behave like AND -- "mobile device" then drops a document about a
|
|
84
|
+
* "mobile target" for lacking the word "device". Weighting by IDF lets the
|
|
85
|
+
* term that actually carries the query decide, in both directions: matching
|
|
86
|
+
* only the rare term passes, matching only the common one does not.
|
|
87
|
+
*
|
|
88
|
+
* Falls back to flat coverage when no stats are supplied.
|
|
89
|
+
*/
|
|
90
|
+
export function idfWeightedCoverage(query, text, stats) {
|
|
91
|
+
if (!stats)
|
|
92
|
+
return termCoverage(query, text);
|
|
93
|
+
const terms = extractQueryTerms(query);
|
|
94
|
+
if (terms.length === 0)
|
|
95
|
+
return 1;
|
|
96
|
+
let matchedWeight = 0;
|
|
97
|
+
let totalWeight = 0;
|
|
98
|
+
for (const term of terms) {
|
|
99
|
+
const weight = idf(term, stats);
|
|
100
|
+
totalWeight += weight;
|
|
101
|
+
if (containsTerm(text, term))
|
|
102
|
+
matchedWeight += weight;
|
|
103
|
+
}
|
|
104
|
+
return totalWeight === 0 ? 1 : matchedWeight / totalWeight;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Keyword-side gate. Uses term coverage rather than a BM25 threshold: BM25
|
|
108
|
+
* scores here are unnormalised and scale with query length and corpus size,
|
|
109
|
+
* so any absolute cutoff would be a fact about one repo. Coverage is a ratio
|
|
110
|
+
* and travels.
|
|
111
|
+
*/
|
|
112
|
+
export function passesKeywordGate(query, text, minCoverage = DEFAULT_MIN_TERM_COVERAGE, stats) {
|
|
113
|
+
return idfWeightedCoverage(query, text, stats) >= (minCoverage ?? DEFAULT_MIN_TERM_COVERAGE);
|
|
114
|
+
}
|
|
115
|
+
/** Semantic-side gate. Cosine is bounded, so an absolute floor is portable. */
|
|
116
|
+
export function passesVectorGate(score, minScore = DEFAULT_MIN_VECTOR_SCORE) {
|
|
117
|
+
return score >= minScore;
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=relevance.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"relevance.js","sourceRoot":"","sources":["../../src/search/relevance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE/D;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AAE5C;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAE7C;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,IAAY;IACtD,MAAM,KAAK,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEjC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IACxE,OAAO,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC;AAChC,CAAC;AASD;;wEAEwE;AACxE,SAAS,GAAG,CAAC,IAAY,EAAE,KAAkB;IAC3C,MAAM,EAAE,GAAG,KAAK,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACxD,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC/D,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,mBAAmB,CACjC,KAAa,EACb,IAAY,EACZ,KAAmB;IAEnB,IAAI,CAAC,KAAK;QAAE,OAAO,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAE7C,MAAM,KAAK,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEjC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAChC,WAAW,IAAI,MAAM,CAAC;QACtB,IAAI,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC;YAAE,aAAa,IAAI,MAAM,CAAC;IACxD,CAAC;IAED,OAAO,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,WAAW,CAAC;AAC7D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAa,EACb,IAAY,EACZ,cAAsB,yBAAyB,EAC/C,KAAmB;IAEnB,OAAO,mBAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,yBAAyB,CAAC,CAAC;AAC/F,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,gBAAgB,CAC9B,KAAa,EACb,WAAmB,wBAAwB;IAE3C,OAAO,KAAK,IAAI,QAAQ,CAAC;AAC3B,CAAC"}
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
import { DocumentStore } from "../storage/document-store.js";
|
|
14
14
|
import { VectorStore } from "../storage/vector-store.js";
|
|
15
15
|
import { EmbeddingService } from "../embeddings/embedding-service.js";
|
|
16
|
-
import type {
|
|
16
|
+
import type { SearchMode } from "../types.js";
|
|
17
17
|
import type { HighlightMode } from "./snippet.js";
|
|
18
18
|
export interface HybridResult {
|
|
19
19
|
id: string;
|
|
@@ -27,10 +27,19 @@ export interface HybridResult {
|
|
|
27
27
|
content?: string;
|
|
28
28
|
headingBreadcrumb?: string[];
|
|
29
29
|
source: "text" | "vector" | "hybrid";
|
|
30
|
+
title?: string;
|
|
31
|
+
description?: string;
|
|
32
|
+
headings?: string;
|
|
33
|
+
status?: string;
|
|
34
|
+
documentType?: string;
|
|
35
|
+
lastUpdate?: string;
|
|
36
|
+
/** Fused score before authority weighting, kept so --explain can show both. */
|
|
37
|
+
baseScore?: number;
|
|
38
|
+
/** Authority multiplier applied to `baseScore` to produce the final order. */
|
|
39
|
+
authority?: number;
|
|
30
40
|
}
|
|
31
41
|
export interface SearchOptions {
|
|
32
42
|
mode: SearchMode;
|
|
33
|
-
scope: Scope;
|
|
34
43
|
limit?: number;
|
|
35
44
|
/** How matched terms should be marked in generated snippets. Defaults to
|
|
36
45
|
* "plain" (no markup) when omitted. */
|
|
@@ -39,15 +48,48 @@ export interface SearchOptions {
|
|
|
39
48
|
* snippet generation. When omitted, snippets fall back to chunk-only
|
|
40
49
|
* generation. */
|
|
41
50
|
repoRoot?: string;
|
|
51
|
+
/** Cosine floor for semantic hits. */
|
|
52
|
+
minVectorScore?: number;
|
|
53
|
+
/** Required fraction of query terms present, for keyword hits. */
|
|
54
|
+
minCoverage?: number;
|
|
55
|
+
/** Skip status/recency weighting, so raw retrieval order can be compared. */
|
|
56
|
+
rawRank?: boolean;
|
|
57
|
+
/** Clock for recency decay; injected so tests are not time-dependent. */
|
|
58
|
+
now?: number;
|
|
42
59
|
}
|
|
43
60
|
export declare class SearchService {
|
|
44
61
|
private docStore;
|
|
45
62
|
private vectorStore;
|
|
46
63
|
private embeddingService;
|
|
47
64
|
constructor(docStore: DocumentStore, vectorStore: VectorStore, embeddingService: EmbeddingService | null);
|
|
65
|
+
/**
|
|
66
|
+
* Fetch vector chunks and collapse to the best-scoring chunk per FILE,
|
|
67
|
+
* growing the request until `fileLimit` distinct files are found (or the
|
|
68
|
+
* store has no more chunks to give).
|
|
69
|
+
*
|
|
70
|
+
* `DocumentStore.search` collapses to one result per file before slicing
|
|
71
|
+
* to a limit (see its comment); a raw HNSW search can't do the equivalent
|
|
72
|
+
* because it has to be told a chunk budget upfront, and one large document
|
|
73
|
+
* can own most of that budget -- on the real corpus,
|
|
74
|
+
* docs/architecture/engine/lessons-learned.md supplied 26 of the top 30
|
|
75
|
+
* vector hits for one query, leaving only 4 chunk-slots (not file-slots)
|
|
76
|
+
* for every other document combined. Raising the chunk budget cannot fix
|
|
77
|
+
* this in general, since a bigger document just needs a bigger multiple.
|
|
78
|
+
* Instead ask for more chunks, geometrically, until enough distinct files
|
|
79
|
+
* have shown up -- the same "collapse before slice" shape the keyword side
|
|
80
|
+
* already gets from MiniSearch returning its full match list.
|
|
81
|
+
*/
|
|
82
|
+
private fetchVectorResultsByFile;
|
|
48
83
|
search(query: string, options: SearchOptions): Promise<HybridResult[]>;
|
|
49
84
|
private textSearch;
|
|
50
85
|
private semanticSearch;
|
|
51
86
|
private hybridSearch;
|
|
87
|
+
/**
|
|
88
|
+
* Apply authority weighting and produce the final page.
|
|
89
|
+
*
|
|
90
|
+
* Every mode ends here, so `--mode keyword` cannot become a quiet bypass of
|
|
91
|
+
* the same weighting hybrid results get.
|
|
92
|
+
*/
|
|
93
|
+
private finish;
|
|
52
94
|
}
|
|
53
95
|
//# sourceMappingURL=search-service.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search-service.d.ts","sourceRoot":"","sources":["../../src/search/search-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,aAAa,
|
|
1
|
+
{"version":3,"file":"search-service.d.ts","sourceRoot":"","sources":["../../src/search/search-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,aAAa,EAAkB,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAEzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAUlD,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+EAA+E;IAC/E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;2CACuC;IACvC,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B;;qBAEiB;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kEAAkE;IAClE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,6EAA6E;IAC7E,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,yEAAyE;IACzE,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAaD,qBAAa,aAAa;IAEtB,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,gBAAgB;gBAFhB,QAAQ,EAAE,aAAa,EACvB,WAAW,EAAE,WAAW,EACxB,gBAAgB,EAAE,gBAAgB,GAAG,IAAI;IAGnD;;;;;;;;;;;;;;;;OAgBG;YACW,wBAAwB;IAwBhC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YA+B9D,UAAU;YAoCV,cAAc;YAuDd,YAAY;IAsH1B;;;;;OAKG;IACH,OAAO,CAAC,MAAM;CAyBf"}
|