@amsterdamdatalabs/enact-docs-search 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -4
- package/dist/cli.js +182 -54
- package/dist/cli.js.map +1 -1
- package/dist/dependency-errors.d.ts +3 -0
- package/dist/dependency-errors.d.ts.map +1 -0
- package/dist/dependency-errors.js +26 -0
- package/dist/dependency-errors.js.map +1 -0
- package/dist/doctor.d.ts +17 -0
- package/dist/doctor.d.ts.map +1 -0
- package/dist/doctor.js +88 -0
- package/dist/doctor.js.map +1 -0
- package/dist/embeddings/embedding-service.d.ts.map +1 -1
- package/dist/embeddings/embedding-service.js +2 -3
- package/dist/embeddings/embedding-service.js.map +1 -1
- package/dist/indexing/chunker.d.ts +6 -1
- package/dist/indexing/chunker.d.ts.map +1 -1
- package/dist/indexing/chunker.js +17 -11
- package/dist/indexing/chunker.js.map +1 -1
- package/dist/indexing/embedding-text.d.ts +22 -0
- package/dist/indexing/embedding-text.d.ts.map +1 -0
- package/dist/indexing/embedding-text.js +27 -0
- package/dist/indexing/embedding-text.js.map +1 -0
- package/dist/indexing/frontmatter.d.ts +48 -0
- package/dist/indexing/frontmatter.d.ts.map +1 -0
- package/dist/indexing/frontmatter.js +162 -0
- package/dist/indexing/frontmatter.js.map +1 -0
- package/dist/indexing/indexing-service.d.ts +1 -4
- package/dist/indexing/indexing-service.d.ts.map +1 -1
- package/dist/indexing/indexing-service.js +24 -12
- package/dist/indexing/indexing-service.js.map +1 -1
- package/dist/search/authority.d.ts +31 -0
- package/dist/search/authority.d.ts.map +1 -0
- package/dist/search/authority.js +50 -0
- package/dist/search/authority.js.map +1 -0
- package/dist/search/relevance.d.ts +80 -0
- package/dist/search/relevance.d.ts.map +1 -0
- package/dist/search/relevance.js +119 -0
- package/dist/search/relevance.js.map +1 -0
- package/dist/search/search-service.d.ts +44 -2
- package/dist/search/search-service.d.ts.map +1 -1
- package/dist/search/search-service.js +181 -23
- package/dist/search/search-service.js.map +1 -1
- package/dist/search/snippet.d.ts +11 -0
- package/dist/search/snippet.d.ts.map +1 -1
- package/dist/search/snippet.js +102 -17
- package/dist/search/snippet.js.map +1 -1
- package/dist/storage/document-store.d.ts +47 -5
- package/dist/storage/document-store.d.ts.map +1 -1
- package/dist/storage/document-store.js +94 -16
- package/dist/storage/document-store.js.map +1 -1
- package/dist/storage/persistence.d.ts +6 -10
- package/dist/storage/persistence.d.ts.map +1 -1
- package/dist/storage/persistence.js +20 -20
- package/dist/storage/persistence.js.map +1 -1
- package/dist/storage/vector-store.d.ts +2 -3
- package/dist/storage/vector-store.d.ts.map +1 -1
- package/dist/storage/vector-store.js +5 -15
- package/dist/storage/vector-store.js.map +1 -1
- package/dist/types.d.ts +2 -7
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +2 -20
- package/dist/types.js.map +1 -1
- package/package.json +9 -2
|
@@ -2,15 +2,13 @@
|
|
|
2
2
|
* Adapted from obsidian-qmd (MIT): https://github.com/thirteen37/obsidian-qmd
|
|
3
3
|
* Source: src/storage/DocumentStore.ts — the original had no Obsidian
|
|
4
4
|
* dependency at all (pure MiniSearch wrapper). Adjustments: added
|
|
5
|
-
* `startLine` to the indexed schema for path:line output
|
|
6
|
-
*
|
|
7
|
-
* `pluginBasePath` option that was already unused upstream.
|
|
5
|
+
* `startLine` to the indexed schema for path:line output and dropped the
|
|
6
|
+
* Obsidian-flavoured `pluginBasePath` option that was already unused upstream.
|
|
8
7
|
*
|
|
9
8
|
* DocumentStore provides full-text search using MiniSearch.
|
|
10
9
|
* Implements proper BM25 ranking with inverted index.
|
|
11
10
|
*/
|
|
12
11
|
import MiniSearch from "minisearch";
|
|
13
|
-
import type { Scope } from "../types.js";
|
|
14
12
|
import type { HighlightMode } from "../search/snippet.js";
|
|
15
13
|
export interface IndexedDocument {
|
|
16
14
|
/** Unique identifier for this document/chunk */
|
|
@@ -27,6 +25,18 @@ export interface IndexedDocument {
|
|
|
27
25
|
startLine: number;
|
|
28
26
|
/** Heading hierarchy at the start of this chunk */
|
|
29
27
|
headingBreadcrumb?: string[];
|
|
28
|
+
/** Document title from frontmatter, denormalised onto every chunk. */
|
|
29
|
+
title?: string;
|
|
30
|
+
/** Document description from frontmatter, denormalised onto every chunk. */
|
|
31
|
+
description?: string;
|
|
32
|
+
/** Heading breadcrumb flattened for the keyword index. */
|
|
33
|
+
headings?: string;
|
|
34
|
+
/** Frontmatter `status`, e.g. "stable" or "draft". */
|
|
35
|
+
status?: string;
|
|
36
|
+
/** Frontmatter `document_type`. */
|
|
37
|
+
documentType?: string;
|
|
38
|
+
/** Frontmatter `last_update`, ISO-8601 as written. */
|
|
39
|
+
lastUpdate?: string;
|
|
30
40
|
}
|
|
31
41
|
export interface SearchResult {
|
|
32
42
|
id: string;
|
|
@@ -37,10 +47,17 @@ export interface SearchResult {
|
|
|
37
47
|
snippet: string;
|
|
38
48
|
headingBreadcrumb?: string[];
|
|
39
49
|
content: string;
|
|
50
|
+
title?: string;
|
|
51
|
+
description?: string;
|
|
52
|
+
/** Flattened heading path. Carried so the relevance gate sees exactly the
|
|
53
|
+
* text document frequency was counted over. */
|
|
54
|
+
headings?: string;
|
|
55
|
+
status?: string;
|
|
56
|
+
documentType?: string;
|
|
57
|
+
lastUpdate?: string;
|
|
40
58
|
}
|
|
41
59
|
export interface SearchOptions {
|
|
42
60
|
limit?: number;
|
|
43
|
-
scope?: Scope;
|
|
44
61
|
/** How matched terms should be marked in generated snippets. Defaults to
|
|
45
62
|
* "plain" (no markup) when omitted. */
|
|
46
63
|
highlight?: HighlightMode;
|
|
@@ -54,6 +71,22 @@ export interface ExportedDocumentStore {
|
|
|
54
71
|
documents: IndexedDocument[];
|
|
55
72
|
index: ReturnType<MiniSearch<IndexedDocument>["toJSON"]>;
|
|
56
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* The fields a keyword hit may have matched on, concatenated.
|
|
76
|
+
*
|
|
77
|
+
* Exported and shared with the relevance gate deliberately. Document frequency
|
|
78
|
+
* and the gate must read the SAME text: when df counted `headings` and the
|
|
79
|
+
* gate only read `headingBreadcrumb`, a term could be reported as present in
|
|
80
|
+
* every document while being unmatchable in all of them -- which is round-1
|
|
81
|
+
* blocker #2 again, one level up.
|
|
82
|
+
*/
|
|
83
|
+
export declare function searchableText(doc: {
|
|
84
|
+
title?: string;
|
|
85
|
+
description?: string;
|
|
86
|
+
headings?: string;
|
|
87
|
+
headingBreadcrumb?: string[];
|
|
88
|
+
content?: string;
|
|
89
|
+
}): string;
|
|
57
90
|
export declare class DocumentStore {
|
|
58
91
|
private documents;
|
|
59
92
|
private searchIndex;
|
|
@@ -66,6 +99,15 @@ export declare class DocumentStore {
|
|
|
66
99
|
/** Search for documents using MiniSearch with BM25 scoring. */
|
|
67
100
|
search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
|
|
68
101
|
getDocumentCount(): Promise<number>;
|
|
102
|
+
/**
|
|
103
|
+
* Document frequency for each term, plus the corpus size, for IDF-weighted
|
|
104
|
+
* relevance gating. Exact matching only -- prefix and fuzzy expansion would
|
|
105
|
+
* inflate the count and make rare terms look common.
|
|
106
|
+
*/
|
|
107
|
+
corpusStats(terms: string[]): {
|
|
108
|
+
totalDocs: number;
|
|
109
|
+
documentFrequency: Map<string, number>;
|
|
110
|
+
};
|
|
69
111
|
getAllFilePaths(): Promise<string[]>;
|
|
70
112
|
export(): ExportedDocumentStore;
|
|
71
113
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-store.d.ts","sourceRoot":"","sources":["../../src/storage/document-store.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"document-store.d.ts","sourceRoot":"","sources":["../../src/storage/document-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,UAAU,MAAM,YAAY,CAAC;AAEpC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1D,MAAM,WAAW,eAAe;IAC9B,gDAAgD;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,uBAAuB;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,6DAA6D;IAC7D,UAAU,EAAE,MAAM,CAAC;IACnB,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;IAChB,qEAAqE;IACrE,SAAS,EAAE,MAAM,CAAC;IAClB,mDAAmD;IACnD,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,sEAAsE;IACtE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0DAA0D;IAC1D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sDAAsD;IACtD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sDAAsD;IACtD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;mDAC+C;IAC/C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;2CACuC;IACvC,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B;;;wCAGoC;IACpC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,eAAe,EAAE,CAAC;IAC7B,KAAK,EAAE,UAAU,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;CAC1D;AAmCD;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GAAG,MAAM,CAIT;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,SAAS,CAA2C;IAC5D,OAAO,CAAC,WAAW,CAA4C;IAC/D,OAAO,CAAC,KAAK,CAAS;IAEhB,UAAU,CAAC,YAAY,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IA0BrE,OAAO,IAAI,OAAO;IAIZ,cAAc,CAAC,IAAI,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBtD,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAIxD,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBvD,+DAA+D;IACzD,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IA4D3E,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC;IAIzC;;;;OAIG;IACH,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE;IAqBrF,eAAe,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAQ1C,MAAM,IAAI,qBAAqB;CAOhC"}
|
|
@@ -2,25 +2,59 @@
|
|
|
2
2
|
* Adapted from obsidian-qmd (MIT): https://github.com/thirteen37/obsidian-qmd
|
|
3
3
|
* Source: src/storage/DocumentStore.ts — the original had no Obsidian
|
|
4
4
|
* dependency at all (pure MiniSearch wrapper). Adjustments: added
|
|
5
|
-
* `startLine` to the indexed schema for path:line output
|
|
6
|
-
*
|
|
7
|
-
* `pluginBasePath` option that was already unused upstream.
|
|
5
|
+
* `startLine` to the indexed schema for path:line output and dropped the
|
|
6
|
+
* Obsidian-flavoured `pluginBasePath` option that was already unused upstream.
|
|
8
7
|
*
|
|
9
8
|
* DocumentStore provides full-text search using MiniSearch.
|
|
10
9
|
* Implements proper BM25 ranking with inverted index.
|
|
11
10
|
*/
|
|
12
11
|
import MiniSearch from "minisearch";
|
|
13
|
-
import {
|
|
14
|
-
|
|
12
|
+
import { extractQueryTerms, buildSnippet, containsTerm } from "../search/snippet.js";
|
|
13
|
+
/**
|
|
14
|
+
* Field weights. A document's own title is the strongest statement it makes
|
|
15
|
+
* about what it is about, so a title hit outranks a passing body mention.
|
|
16
|
+
* Keep these in one place: they are the ranking contract.
|
|
17
|
+
*/
|
|
18
|
+
const FIELD_BOOST = {
|
|
19
|
+
title: 8,
|
|
20
|
+
description: 4,
|
|
21
|
+
headings: 2,
|
|
22
|
+
content: 1,
|
|
23
|
+
};
|
|
15
24
|
const MINISEARCH_OPTIONS = {
|
|
16
|
-
fields: ["content"],
|
|
17
|
-
storeFields: [
|
|
25
|
+
fields: ["title", "description", "headings", "content"],
|
|
26
|
+
storeFields: [
|
|
27
|
+
"filePath",
|
|
28
|
+
"chunkIndex",
|
|
29
|
+
"mtimeMs",
|
|
30
|
+
"startLine",
|
|
31
|
+
"headingBreadcrumb",
|
|
32
|
+
"title",
|
|
33
|
+
"description",
|
|
34
|
+
"status",
|
|
35
|
+
"documentType",
|
|
36
|
+
"lastUpdate",
|
|
37
|
+
],
|
|
18
38
|
searchOptions: {
|
|
19
|
-
boost: {
|
|
39
|
+
boost: { ...FIELD_BOOST },
|
|
20
40
|
fuzzy: 0.2,
|
|
21
41
|
prefix: true,
|
|
22
42
|
},
|
|
23
43
|
};
|
|
44
|
+
/**
|
|
45
|
+
* The fields a keyword hit may have matched on, concatenated.
|
|
46
|
+
*
|
|
47
|
+
* Exported and shared with the relevance gate deliberately. Document frequency
|
|
48
|
+
* and the gate must read the SAME text: when df counted `headings` and the
|
|
49
|
+
* gate only read `headingBreadcrumb`, a term could be reported as present in
|
|
50
|
+
* every document while being unmatchable in all of them -- which is round-1
|
|
51
|
+
* blocker #2 again, one level up.
|
|
52
|
+
*/
|
|
53
|
+
export function searchableText(doc) {
|
|
54
|
+
return [doc.title, doc.description, doc.headings ?? (doc.headingBreadcrumb ?? []).join(" "), doc.content]
|
|
55
|
+
.filter(Boolean)
|
|
56
|
+
.join(" ");
|
|
57
|
+
}
|
|
24
58
|
export class DocumentStore {
|
|
25
59
|
documents = new Map();
|
|
26
60
|
searchIndex = null;
|
|
@@ -32,6 +66,7 @@ export class DocumentStore {
|
|
|
32
66
|
this.searchIndex = MiniSearch.loadJSON(JSON.stringify(existingData.index), {
|
|
33
67
|
fields: [...MINISEARCH_OPTIONS.fields],
|
|
34
68
|
storeFields: [...MINISEARCH_OPTIONS.storeFields],
|
|
69
|
+
searchOptions: { ...MINISEARCH_OPTIONS.searchOptions },
|
|
35
70
|
});
|
|
36
71
|
for (const doc of existingData.documents) {
|
|
37
72
|
this.documents.set(doc.id, doc);
|
|
@@ -88,26 +123,45 @@ export class DocumentStore {
|
|
|
88
123
|
if (!trimmedQuery)
|
|
89
124
|
return [];
|
|
90
125
|
const limit = options.limit ?? 20;
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
const
|
|
96
|
-
for (const result of miniSearchResults
|
|
126
|
+
const miniSearchResults = this.searchIndex.search(trimmedQuery);
|
|
127
|
+
// Collapse to the best chunk per file. Title and description are
|
|
128
|
+
// denormalised onto every chunk, so without this a long document matches
|
|
129
|
+
// once per chunk and floods the page with itself.
|
|
130
|
+
const bestByFile = new Map();
|
|
131
|
+
for (const result of miniSearchResults) {
|
|
97
132
|
const doc = this.documents.get(result.id);
|
|
98
133
|
if (!doc)
|
|
99
134
|
continue;
|
|
100
|
-
const
|
|
135
|
+
const incumbent = bestByFile.get(doc.filePath);
|
|
136
|
+
if (!incumbent || result.score > incumbent.score) {
|
|
137
|
+
bestByFile.set(doc.filePath, { id: doc.id, score: result.score });
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
const ranked = Array.from(bestByFile.values())
|
|
141
|
+
.sort((a, b) => b.score - a.score)
|
|
142
|
+
.slice(0, limit);
|
|
143
|
+
const terms = extractQueryTerms(trimmedQuery);
|
|
144
|
+
const results = [];
|
|
145
|
+
for (const entry of ranked) {
|
|
146
|
+
const doc = this.documents.get(entry.id);
|
|
147
|
+
if (!doc)
|
|
148
|
+
continue;
|
|
101
149
|
const snippet = await buildSnippet({ filePath: doc.filePath, content: doc.content, startLine: doc.startLine }, terms, options.highlight ?? "plain", options.repoRoot);
|
|
102
150
|
results.push({
|
|
103
151
|
id: doc.id,
|
|
104
152
|
filePath: doc.filePath,
|
|
105
153
|
chunkIndex: doc.chunkIndex,
|
|
106
154
|
startLine: doc.startLine,
|
|
107
|
-
score:
|
|
155
|
+
score: entry.score,
|
|
108
156
|
snippet,
|
|
109
157
|
headingBreadcrumb: doc.headingBreadcrumb,
|
|
110
158
|
content: doc.content,
|
|
159
|
+
title: doc.title,
|
|
160
|
+
description: doc.description,
|
|
161
|
+
headings: doc.headings,
|
|
162
|
+
status: doc.status,
|
|
163
|
+
documentType: doc.documentType,
|
|
164
|
+
lastUpdate: doc.lastUpdate,
|
|
111
165
|
});
|
|
112
166
|
}
|
|
113
167
|
return results;
|
|
@@ -115,6 +169,30 @@ export class DocumentStore {
|
|
|
115
169
|
async getDocumentCount() {
|
|
116
170
|
return this.documents.size;
|
|
117
171
|
}
|
|
172
|
+
/**
|
|
173
|
+
* Document frequency for each term, plus the corpus size, for IDF-weighted
|
|
174
|
+
* relevance gating. Exact matching only -- prefix and fuzzy expansion would
|
|
175
|
+
* inflate the count and make rare terms look common.
|
|
176
|
+
*/
|
|
177
|
+
corpusStats(terms) {
|
|
178
|
+
if (!this.searchIndex)
|
|
179
|
+
throw new Error("Document store not initialized");
|
|
180
|
+
// Counted with `containsTerm`, the same matcher the relevance gate uses.
|
|
181
|
+
// Asking MiniSearch instead would answer a different question -- it
|
|
182
|
+
// tokenises, so "device-tree" would be counted as device OR tree -- and a
|
|
183
|
+
// frequency that disagrees with the membership test inverts the ranking
|
|
184
|
+
// it is supposed to weight.
|
|
185
|
+
const documentFrequency = new Map(terms.map((term) => [term, 0]));
|
|
186
|
+
for (const doc of this.documents.values()) {
|
|
187
|
+
const text = searchableText(doc);
|
|
188
|
+
for (const term of terms) {
|
|
189
|
+
if (containsTerm(text, term)) {
|
|
190
|
+
documentFrequency.set(term, documentFrequency.get(term) + 1);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
return { totalDocs: this.documents.size, documentFrequency };
|
|
195
|
+
}
|
|
118
196
|
async getAllFilePaths() {
|
|
119
197
|
const filePaths = new Set();
|
|
120
198
|
for (const doc of this.documents.values()) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-store.js","sourceRoot":"","sources":["../../src/storage/document-store.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"document-store.js","sourceRoot":"","sources":["../../src/storage/document-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,UAAU,MAAM,YAAY,CAAC;AACpC,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAoErF;;;;GAIG;AACH,MAAM,WAAW,GAAG;IAClB,KAAK,EAAE,CAAC;IACR,WAAW,EAAE,CAAC;IACd,QAAQ,EAAE,CAAC;IACX,OAAO,EAAE,CAAC;CACF,CAAC;AAEX,MAAM,kBAAkB,GAAG;IACzB,MAAM,EAAE,CAAC,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,CAAC;IACvD,WAAW,EAAE;QACX,UAAU;QACV,YAAY;QACZ,SAAS;QACT,WAAW;QACX,mBAAmB;QACnB,OAAO;QACP,aAAa;QACb,QAAQ;QACR,cAAc;QACd,YAAY;KACb;IACD,aAAa,EAAE;QACb,KAAK,EAAE,EAAE,GAAG,WAAW,EAAE;QACzB,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,IAAI;KACb;CACO,CAAC;AAEX;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,GAM9B;IACC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC;SACtG,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,CAAC,GAAG,CAAC,CAAC;AACf,CAAC;AAED,MAAM,OAAO,aAAa;IAChB,SAAS,GAAiC,IAAI,GAAG,EAAE,CAAC;IACpD,WAAW,GAAuC,IAAI,CAAC;IACvD,KAAK,GAAG,KAAK,CAAC;IAEtB,KAAK,CAAC,UAAU,CAAC,YAAoC;QACnD,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,EAAE,CAAC;QAE3B,MAAM,WAAW,GACf,YAAY,IAAI,YAAY,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAE5F,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,YAAa,CAAC,KAAK,CAAC,EAAE;gBAC1E,MAAM,EAAE,CAAC,GAAG,kBAAkB,CAAC,MAAM,CAAC;gBACtC,WAAW,EAAE,CAAC,GAAG,kBAAkB,CAAC,WAAW,CAAC;gBAChD,aAAa,EAAE,EAAE,GAAG,kBAAkB,CAAC,aAAa,EAAE;aACvD,CAAC,CAAC;YACH,KAAK,MAAM,GAAG,IAAI,YAAa,CAAC,SAAS,EAAE,CAAC;gBAC1C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,WAAW,GAAG,IAAI,UAAU,CAAkB;gBACjD,MAAM,EAAE,CAAC,GAAG,kBAAkB,CAAC,MAAM,CAAC;gBACtC,WAAW,EAAE,CAAC,GAAG,kBAAkB,CAAC,WAAW,CAAC;gBAChD,aAAa,EAAE,EAAE,GAAG,kBAAkB,CAAC,aAAa,EAAE;aACvD,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,IAAuB;QAC1C,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAEzE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;gBAC7C,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAE9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,EAAU;QAC1B,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QACrC,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAEzE,MAAM,QAAQ,GAAsB,EAAE,CAAC;QACvC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBAC9B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAAyB,EAAE;QACrD,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAEzE,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAClC,IAAI,CAAC,YAAY;YAAE,OAAO,EAAE,CAAC;QAE7B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QAClC,MAAM,iBAAiB,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAEhE,iEAAiE;QACjE,yEAAyE;QACzE,kDAAkD;QAClD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAyC,CAAC;QACpE,KAAK,MAAM,MAAM,IAAI,iBAAiB,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAY,CAAC,CAAC;YACpD,IAAI,CAAC,GAAG;gBAAE,SAAS;YACnB,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC/C,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,EAAE,CAAC;gBACjD,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;aAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;aACjC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAEnB,MAAM,KAAK,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG;gBAAE,SAAS;YAEnB,MAAM,OAAO,GAAG,MAAM,YAAY,CAChC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,EAC1E,KAAK,EACL,OAAO,CAAC,SAAS,IAAI,OAAO,EAC5B,OAAO,CAAC,QAAQ,CACjB,CAAC;YAEF,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,OAAO;gBACP,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;gBACxC,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,YAAY,EAAE,GAAG,CAAC,YAAY;gBAC9B,UAAU,EAAE,GAAG,CAAC,UAAU;aAC3B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,gBAAgB;QACpB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;IAC7B,CAAC;IAED;;;;OAIG;IACH,WAAW,CAAC,KAAe;QACzB,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAEzE,yEAAyE;QACzE,oEAAoE;QACpE,0EAA0E;QAC1E,wEAAwE;QACxE,4BAA4B;QAC5B,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAiB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAClF,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;YACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;oBAC7B,iBAAiB,CAAC,GAAG,CAAC,IAAI,EAAE,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAE,GAAG,CAAC,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,iBAAiB,EAAE,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;QACpC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM;QACJ,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACzE,OAAO;YACL,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;YAC9C,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;SACjC,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { ExportedDocumentStore } from "./document-store.js";
|
|
2
2
|
import type { SidecarData } from "./vector-store.js";
|
|
3
|
-
import type { Scope } from "../types.js";
|
|
4
3
|
export interface ManifestEntry {
|
|
5
4
|
/** mtime (epoch ms) at the time this file was last text-indexed; null means removed */
|
|
6
5
|
textMtimeMs: number | null;
|
|
@@ -16,13 +15,10 @@ export interface Manifest {
|
|
|
16
15
|
embeddingModel: string;
|
|
17
16
|
/** repo-relative file path -> last-indexed mtime */
|
|
18
17
|
files: Record<string, ManifestEntry>;
|
|
19
|
-
/** Informational audit trail: scopes passed to a successful `index` run.
|
|
20
|
-
* Not used for freshness logic — see checkFreshness()'s doc comment. */
|
|
21
|
-
indexedScopes: string[];
|
|
22
18
|
/** epoch ms of the last successful index run */
|
|
23
19
|
lastIndexedAt: number;
|
|
24
20
|
}
|
|
25
|
-
export declare const MANIFEST_VERSION =
|
|
21
|
+
export declare const MANIFEST_VERSION = 5;
|
|
26
22
|
export declare const EMBEDDING_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
|
|
27
23
|
export declare class CachePermissionError extends Error {
|
|
28
24
|
}
|
|
@@ -64,19 +60,19 @@ export declare function writeVectorSidecar(cacheDir: string, data: SidecarData):
|
|
|
64
60
|
export declare function removeVectorCache(cacheDir: string): Promise<void>;
|
|
65
61
|
export declare function vectorIndexExists(cacheDir: string): Promise<boolean>;
|
|
66
62
|
export interface FreshnessReport {
|
|
67
|
-
/** Repo-relative paths that are new, changed, or removed since the last index
|
|
63
|
+
/** Repo-relative paths that are new, changed, or removed since the last index. */
|
|
68
64
|
staleCount: number;
|
|
69
65
|
/** Repo-relative paths whose vector embeddings are missing or stale. */
|
|
70
66
|
vectorStaleCount: number;
|
|
71
|
-
/** True if
|
|
67
|
+
/** True if the docs tree has never been indexed. */
|
|
72
68
|
neverIndexed: boolean;
|
|
73
69
|
}
|
|
74
70
|
/**
|
|
75
|
-
* Compare the current on-disk files
|
|
76
|
-
*
|
|
71
|
+
* Compare the current on-disk docs files against the manifest, without
|
|
72
|
+
* mutating anything. Used by `query`
|
|
77
73
|
* to refresh a stale or nonexistent index before searching (I-08).
|
|
78
74
|
*/
|
|
79
|
-
export declare function checkFreshness(manifest: Manifest | null,
|
|
75
|
+
export declare function checkFreshness(manifest: Manifest | null, currentFiles: {
|
|
80
76
|
path: string;
|
|
81
77
|
mtimeMs: number;
|
|
82
78
|
}[], requireVectors?: boolean): FreshnessReport;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"persistence.d.ts","sourceRoot":"","sources":["../../src/storage/persistence.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"persistence.d.ts","sourceRoot":"","sources":["../../src/storage/persistence.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,MAAM,WAAW,aAAa;IAC5B,uFAAuF;IACvF,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,gFAAgF;IAChF,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,6EAA6E;IAC7E,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,QAAQ;IACvB,iFAAiF;IACjF,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,cAAc,EAAE,MAAM,CAAC;IACvB,oDAAoD;IACpD,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACrC,gDAAgD;IAChD,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAClC,eAAO,MAAM,kBAAkB,4BAA4B,CAAC;AAE5D,qBAAa,oBAAqB,SAAQ,KAAK;CAAG;AAElD,2EAA2E;AAC3E,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEtD;AAED,wCAAwC;AACxC,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErD;AAED,wDAAwD;AACxD,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,0EAA0E;AAC1E,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAoBrE;AAOD;;;;;;;;;;GAUG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,GAAG,UAAU,GACxB,OAAO,CAAC,IAAI,CAAC,CASf;AAED,wBAAgB,KAAK,CAAC,QAAQ,EAAE,MAAM;;;;;EAOrC;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpE;AAED,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAS7E;AAED,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvF;AAED,wBAAgB,aAAa,IAAI,QAAQ,CAOxC;AAED,wBAAsB,uBAAuB,CAC3C,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC,CAOvC;AAED,wBAAsB,wBAAwB,CAC5C,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAEf;AAED,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAOrF;AAED,wBAAsB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAE3F;AAED,iFAAiF;AACjF,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvE;AAUD,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAI1E;AAED,MAAM,WAAW,eAAe;IAC9B,kFAAkF;IAClF,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,gBAAgB,EAAE,MAAM,CAAC;IACzB,oDAAoD;IACpD,YAAY,EAAE,OAAO,CAAC;CACvB;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,QAAQ,GAAG,IAAI,EACzB,YAAY,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAAE,EACjD,cAAc,UAAQ,GACrB,eAAe,CAmCjB"}
|
|
@@ -5,14 +5,11 @@
|
|
|
5
5
|
* localforage/IndexedDB, which has no equivalent outside a browser. This
|
|
6
6
|
* lays the same DocumentStore/VectorStore export shapes out as plain files
|
|
7
7
|
* under a per-repo cache directory, plus a manifest that tracks each
|
|
8
|
-
* indexed file's mtime
|
|
9
|
-
* ever been indexed (so a query can refresh an un-indexed scope instead of
|
|
10
|
-
* silently returning nothing).
|
|
8
|
+
* indexed file's mtime for incremental re-indexing.
|
|
11
9
|
*/
|
|
12
10
|
import { mkdir, readFile, writeFile, stat, open, unlink, rename } from "node:fs/promises";
|
|
13
11
|
import { join, resolve } from "node:path";
|
|
14
|
-
|
|
15
|
-
export const MANIFEST_VERSION = 3;
|
|
12
|
+
export const MANIFEST_VERSION = 5;
|
|
16
13
|
export const EMBEDDING_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
|
|
17
14
|
export class CachePermissionError extends Error {
|
|
18
15
|
}
|
|
@@ -106,7 +103,6 @@ export function emptyManifest() {
|
|
|
106
103
|
version: MANIFEST_VERSION,
|
|
107
104
|
embeddingModel: EMBEDDING_MODEL_ID,
|
|
108
105
|
files: {},
|
|
109
|
-
indexedScopes: [],
|
|
110
106
|
lastIndexedAt: 0,
|
|
111
107
|
};
|
|
112
108
|
}
|
|
@@ -137,26 +133,32 @@ export async function writeVectorSidecar(cacheDir, data) {
|
|
|
137
133
|
/** Remove the disposable vector cache when an incompatible manifest is found. */
|
|
138
134
|
export async function removeVectorCache(cacheDir) {
|
|
139
135
|
await Promise.all([
|
|
140
|
-
|
|
141
|
-
|
|
136
|
+
unlinkIfPresent(paths(cacheDir).vectorIndex),
|
|
137
|
+
unlinkIfPresent(paths(cacheDir).vectorSidecar),
|
|
142
138
|
]);
|
|
143
139
|
}
|
|
140
|
+
async function unlinkIfPresent(filePath) {
|
|
141
|
+
try {
|
|
142
|
+
await unlink(filePath);
|
|
143
|
+
}
|
|
144
|
+
catch (error) {
|
|
145
|
+
if (error.code !== "ENOENT")
|
|
146
|
+
throw error;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
144
149
|
export async function vectorIndexExists(cacheDir) {
|
|
145
150
|
return stat(paths(cacheDir).vectorIndex)
|
|
146
151
|
.then(() => true)
|
|
147
152
|
.catch(() => false);
|
|
148
153
|
}
|
|
149
154
|
/**
|
|
150
|
-
* Compare the current on-disk files
|
|
151
|
-
*
|
|
155
|
+
* Compare the current on-disk docs files against the manifest, without
|
|
156
|
+
* mutating anything. Used by `query`
|
|
152
157
|
* to refresh a stale or nonexistent index before searching (I-08).
|
|
153
158
|
*/
|
|
154
|
-
export function checkFreshness(manifest,
|
|
155
|
-
//
|
|
156
|
-
//
|
|
157
|
-
// (a subset) is a legitimate, fully-covered case. If there are files on
|
|
158
|
-
// disk in this scope but none of them appear in the manifest at all, no
|
|
159
|
-
// index run has ever covered this scope.
|
|
159
|
+
export function checkFreshness(manifest, currentFiles, requireVectors = false) {
|
|
160
|
+
// If there are docs on disk but none appear in the manifest, no index run
|
|
161
|
+
// has ever covered the current docs tree.
|
|
160
162
|
if (!manifest)
|
|
161
163
|
return { staleCount: 0, vectorStaleCount: 0, neverIndexed: true };
|
|
162
164
|
if (currentFiles.length > 0 && currentFiles.every((f) => {
|
|
@@ -178,11 +180,9 @@ export function checkFreshness(manifest, scope, currentFiles, requireVectors = f
|
|
|
178
180
|
vectorStaleCount++;
|
|
179
181
|
}
|
|
180
182
|
}
|
|
181
|
-
// Files present in the manifest
|
|
182
|
-
const roots = scopesToRoots(scope);
|
|
183
|
+
// Files present in the manifest but deleted on disk also count as stale.
|
|
183
184
|
for (const path of Object.keys(manifest.files)) {
|
|
184
|
-
|
|
185
|
-
if (inScope && !seen.has(path)) {
|
|
185
|
+
if (!seen.has(path)) {
|
|
186
186
|
const entry = manifest.files[path];
|
|
187
187
|
if (entry.textMtimeMs !== null)
|
|
188
188
|
staleCount++;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"persistence.js","sourceRoot":"","sources":["../../src/storage/persistence.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"persistence.js","sourceRoot":"","sources":["../../src/storage/persistence.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1F,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAwB1C,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAClC,MAAM,CAAC,MAAM,kBAAkB,GAAG,yBAAyB,CAAC;AAE5D,MAAM,OAAO,oBAAqB,SAAQ,KAAK;CAAG;AAElD,2EAA2E;AAC3E,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;AACnD,CAAC;AAED,wCAAwC;AACxC,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;AAChD,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IAChD,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;AACjD,CAAC;AAED,0EAA0E;AAC1E,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB;IACpD,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC1E,KAAK,MAAM,SAAS,IAAI,WAAW,EAAE,CAAC;YACpC,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,gBAAgB,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC3E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YACvC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,GAAI,KAA+B,CAAC,IAAI,CAAC;QACnD,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YAC9D,MAAM,IAAI,oBAAoB,CAC5B,sDAAsD,aAAa,CAAC,QAAQ,CAAC,IAAI;gBAC/E,4DAA4D,CAC/D,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,aAAa,GAAG,eAAe,CAAC;AACtC,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;AAClD,MAAM,iBAAiB,GAAG,kBAAkB,CAAC;AAC7C,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;AAElD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,QAAgB,EAChB,IAAyB;IAEzB,MAAM,OAAO,GAAG,GAAG,QAAQ,QAAQ,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC/D,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC/B,MAAM,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACtC,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,QAAgB;IACpC,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC;QACvC,aAAa,EAAE,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC;QAClD,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE,iBAAiB,CAAC;QAC9C,aAAa,EAAE,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC;KACnD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB;IACnD,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAa,CAAC;QAC3C,IAAI,MAAM,CAAC,OAAO,KAAK,gBAAgB;YAAE,OAAO,IAAI,CAAC;QACrD,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAgB,EAAE,QAAkB;IACtE,MAAM,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,OAAO,EAAE,gBAAgB;QACzB,cAAc,EAAE,kBAAkB;QAClC,KAAK,EAAE,EAAE;QACT,aAAa,EAAE,CAAC;KACjB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,QAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAA0B,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,QAAgB,EAChB,IAA2B;IAE3B,MAAM,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IACtD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAgB,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAgB,EAAE,IAAiB;IAC1E,MAAM,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IACtD,MAAM,OAAO,CAAC,GAAG,CAAC;QAChB,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC;QAC5C,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,CAAC;KAC/C,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,QAAgB;IAC7C,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAC;IACtE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IACtD,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC;SACrC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;AACxB,CAAC;AAWD;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAC5B,QAAyB,EACzB,YAAiD,EACjD,cAAc,GAAG,KAAK;IAEtB,0EAA0E;IAC1E,0CAA0C;IAC1C,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACjF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,CAAC;IAC9C,CAAC,CAAC,EAAE,CAAC;QACH,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACpE,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACjB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,CAAC,CAAC,OAAO;YAAE,UAAU,EAAE,CAAC;QAC5D,IAAI,cAAc;YAChB,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACrE,gBAAgB,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IACD,yEAAyE;IACzE,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;YACpC,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,UAAU,EAAE,CAAC;YAC7C,IAAI,cAAc,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,CAAC,EAAE,CAAC;gBAC1E,gBAAgB,EAAE,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;AAC/D,CAAC"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { Scope } from "../types.js";
|
|
2
1
|
export declare class DependencyError extends Error {
|
|
3
2
|
}
|
|
4
3
|
export interface VectorDocument {
|
|
@@ -74,8 +73,8 @@ export declare class VectorStore {
|
|
|
74
73
|
add(doc: VectorDocument): Promise<void>;
|
|
75
74
|
delete(id: string): Promise<void>;
|
|
76
75
|
deleteByFilePath(filePath: string): Promise<void>;
|
|
77
|
-
/** Search for similar vectors using HNSW
|
|
78
|
-
search(query: number[], k: number
|
|
76
|
+
/** Search for similar vectors using HNSW. */
|
|
77
|
+
search(query: number[], k: number): Promise<VectorSearchResult[]>;
|
|
79
78
|
getCount(): Promise<number>;
|
|
80
79
|
has(id: string): Promise<boolean>;
|
|
81
80
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-store.d.ts","sourceRoot":"","sources":["../../src/storage/vector-store.ts"],"names":[],"mappings":"AAiBA,
|
|
1
|
+
{"version":3,"file":"vector-store.d.ts","sourceRoot":"","sources":["../../src/storage/vector-store.ts"],"names":[],"mappings":"AAiBA,qBAAa,eAAgB,SAAQ,KAAK;CAAG;AAoB7C,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,sDAAsD;IACtD,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,oFAAoF;IACpF,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,UAAU,cAAc;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,KAAK,CAAoC;IACjD,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,KAAK,CAAS;IAEtB,OAAO,CAAC,SAAS,CAAkC;IACnD,OAAO,CAAC,SAAS,CAAkC;IACnD,OAAO,CAAC,QAAQ,CAA0C;IAE1D,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,aAAa,CAAgB;IAErC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAM;IACxB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAO;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;gBAEnB,OAAO,EAAE,kBAAkB;IAKvC,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAejC,yEAAyE;IACnE,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB1E;;;;;;;;OAQG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAmBzD,OAAO,IAAI,OAAO;IAIlB;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,cAAc;IAUhB,GAAG,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBvC,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAcjC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAUvD,6CAA6C;IACvC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAkCjE,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;IAI3B,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAGxC;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IAC9B,QAAQ,EAAE,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,CAAC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;CACpB"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { rename, unlink } from "node:fs/promises";
|
|
2
|
-
import {
|
|
2
|
+
import { semanticDependencyMessage } from "../dependency-errors.js";
|
|
3
3
|
export class DependencyError extends Error {
|
|
4
4
|
}
|
|
5
5
|
async function loadHierarchicalNSW() {
|
|
@@ -11,10 +11,7 @@ async function loadHierarchicalNSW() {
|
|
|
11
11
|
return HierarchicalNSW;
|
|
12
12
|
}
|
|
13
13
|
catch (error) {
|
|
14
|
-
throw new DependencyError("
|
|
15
|
-
"native binding failed to build for this platform).\n" +
|
|
16
|
-
"Run `npm install` in the enact-docs-search directory, then retry.\n" +
|
|
17
|
-
`Original error: ${error.message}`);
|
|
14
|
+
throw new DependencyError(semanticDependencyMessage("hnswlib-node", error));
|
|
18
15
|
}
|
|
19
16
|
}
|
|
20
17
|
export class VectorStore {
|
|
@@ -162,22 +159,15 @@ export class VectorStore {
|
|
|
162
159
|
for (const id of toDelete)
|
|
163
160
|
await this.delete(id);
|
|
164
161
|
}
|
|
165
|
-
/** Search for similar vectors using HNSW
|
|
166
|
-
async search(query, k
|
|
162
|
+
/** Search for similar vectors using HNSW. */
|
|
163
|
+
async search(query, k) {
|
|
167
164
|
if (!this.ready || !this.index)
|
|
168
165
|
throw new Error("Vector store not initialized");
|
|
169
166
|
const actualCount = this.idToLabel.size;
|
|
170
167
|
if (actualCount === 0)
|
|
171
168
|
return [];
|
|
172
169
|
const actualK = Math.min(k, actualCount);
|
|
173
|
-
const
|
|
174
|
-
? undefined
|
|
175
|
-
: (label) => {
|
|
176
|
-
const id = this.labelToId.get(label);
|
|
177
|
-
const meta = id ? this.metadata.get(id) : undefined;
|
|
178
|
-
return !!meta && scopeOfPath(meta.filePath) === scope;
|
|
179
|
-
};
|
|
180
|
-
const result = this.index.searchKnn(query, actualK, filter);
|
|
170
|
+
const result = this.index.searchKnn(query, actualK);
|
|
181
171
|
const results = [];
|
|
182
172
|
for (let i = 0; i < result.neighbors.length; i++) {
|
|
183
173
|
const label = result.neighbors[i];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-store.js","sourceRoot":"","sources":["../../src/storage/vector-store.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"vector-store.js","sourceRoot":"","sources":["../../src/storage/vector-store.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAEpE,MAAM,OAAO,eAAgB,SAAQ,KAAK;CAAG;AAS7C,KAAK,UAAU,mBAAmB;IAChC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC,CAAe,CAAC;QACzD,MAAM,eAAe,GAAG,GAAG,CAAC,eAAe,IAAI,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC;QAC5E,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC3E,OAAO,eAAe,CAAC;IACzB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,eAAe,CAAC,yBAAyB,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AA6BD,MAAM,OAAO,WAAW;IACd,KAAK,GAA+B,IAAI,CAAC;IACzC,UAAU,CAAS;IACnB,KAAK,GAAG,KAAK,CAAC;IAEd,SAAS,GAAwB,IAAI,GAAG,EAAE,CAAC;IAC3C,SAAS,GAAwB,IAAI,GAAG,EAAE,CAAC;IAC3C,QAAQ,GAAgC,IAAI,GAAG,EAAE,CAAC;IAElD,SAAS,GAAG,CAAC,CAAC;IACd,aAAa,GAAa,EAAE,CAAC;IAE7B,WAAW,CAAS;IACX,CAAC,GAAG,EAAE,CAAC;IACP,cAAc,GAAG,GAAG,CAAC;IACrB,QAAQ,GAAG,EAAE,CAAC;IAE/B,YAAY,OAA2B;QACrC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,kBAAkB,IAAI,KAAK,CAAC;IACzD,CAAC;IAED,uCAAuC;IACvC,KAAK,CAAC,UAAU;QACd,MAAM,eAAe,GAAG,MAAM,mBAAmB,EAAE,CAAC;QACpD,IAAI,CAAC,KAAK,GAAG,IAAI,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;QACzE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEhC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QAExB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,yEAAyE;IACzE,KAAK,CAAC,YAAY,CAAC,SAAiB,EAAE,OAAoB;QACxD,MAAM,eAAe,GAAG,MAAM,mBAAmB,EAAE,CAAC;QACpD,IAAI,CAAC,KAAK,GAAG,IAAI,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAE/C,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9E,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;QAEhD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,UAAU,CAAC,SAAiB;QAChC,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,GAAG,SAAS,QAAQ,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAChE,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YACrC,MAAM,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACtC,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO;YACL,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YAC/C,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,aAAa,EAAE,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC;YACtC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC;IACJ,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACK,cAAc;QACpB,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QACxB,IAAI,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACjD,OAAO,aAAa,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;YACtB,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACzC,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAmB;QAC3B,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAEhF,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,uEAAuE;QACvE,uEAAuE;QACvE,mEAAmE;QACnE,wEAAwE;QACxE,yEAAyE;QACzE,wEAAwE;QACxE,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAE/B,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAEjD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAClC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAEhF,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrC,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO;QAEhC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7B,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE/B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QACrC,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAEjE,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACvC,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;gBAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,KAAK,MAAM,EAAE,IAAI,QAAQ;YAAE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,6CAA6C;IAC7C,KAAK,CAAC,MAAM,CAAC,KAAe,EAAE,CAAS;QACrC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAEhF,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;QACxC,IAAI,WAAW,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QAEzC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAEpD,MAAM,OAAO,GAAyB,EAAE,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC;YAEtC,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,CAAC,EAAE;gBAAE,SAAS;YAElB,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,CAAC,IAAI;gBAAE,SAAS;YAEpB,MAAM,UAAU,GAAG,CAAC,GAAG,QAAQ,CAAC;YAEhC,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE;gBACF,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;aAC5C,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChC,CAAC;CACF"}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,12 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shared types for enact-docs-search.
|
|
3
3
|
*/
|
|
4
|
-
/** The
|
|
5
|
-
export
|
|
6
|
-
/** Root directory name (relative to the target repo) for each concrete scope. */
|
|
7
|
-
export declare const SCOPE_ROOTS: Record<Exclude<Scope, "all">, string>;
|
|
8
|
-
export declare function scopesToRoots(scope: Scope): string[];
|
|
9
|
-
/** Which concrete scope a repo-relative path belongs to, if any. */
|
|
10
|
-
export declare function scopeOfPath(relPath: string): Exclude<Scope, "all"> | null;
|
|
4
|
+
/** The sole searchable knowledge-tree root, relative to the target repo. */
|
|
5
|
+
export declare const DOCS_ROOT = "docs";
|
|
11
6
|
export type SearchMode = "keyword" | "semantic" | "hybrid";
|
|
12
7
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,4EAA4E;AAC5E,eAAO,MAAM,SAAS,SAAS,CAAC;AAEhC,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAC"}
|
package/dist/types.js
CHANGED
|
@@ -1,24 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shared types for enact-docs-search.
|
|
3
3
|
*/
|
|
4
|
-
/**
|
|
5
|
-
export const
|
|
6
|
-
docs: "docs",
|
|
7
|
-
plans: ".plans",
|
|
8
|
-
specs: ".specs",
|
|
9
|
-
};
|
|
10
|
-
export function scopesToRoots(scope) {
|
|
11
|
-
if (scope === "all")
|
|
12
|
-
return Object.values(SCOPE_ROOTS);
|
|
13
|
-
return [SCOPE_ROOTS[scope]];
|
|
14
|
-
}
|
|
15
|
-
/** Which concrete scope a repo-relative path belongs to, if any. */
|
|
16
|
-
export function scopeOfPath(relPath) {
|
|
17
|
-
for (const [scope, root] of Object.entries(SCOPE_ROOTS)) {
|
|
18
|
-
if (relPath === root || relPath.startsWith(root + "/")) {
|
|
19
|
-
return scope;
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
return null;
|
|
23
|
-
}
|
|
4
|
+
/** The sole searchable knowledge-tree root, relative to the target repo. */
|
|
5
|
+
export const DOCS_ROOT = "docs";
|
|
24
6
|
//# sourceMappingURL=types.js.map
|