@amsterdamdatalabs/enact-docs-search 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +2 -2
- package/README.md +55 -5
- package/dist/cli.js +75 -21
- package/dist/cli.js.map +1 -1
- package/dist/dependency-errors.d.ts +3 -0
- package/dist/dependency-errors.d.ts.map +1 -0
- package/dist/dependency-errors.js +26 -0
- package/dist/dependency-errors.js.map +1 -0
- package/dist/doctor.d.ts +17 -0
- package/dist/doctor.d.ts.map +1 -0
- package/dist/doctor.js +88 -0
- package/dist/doctor.js.map +1 -0
- package/dist/embeddings/embedding-service.d.ts +9 -0
- package/dist/embeddings/embedding-service.d.ts.map +1 -1
- package/dist/embeddings/embedding-service.js +34 -9
- package/dist/embeddings/embedding-service.js.map +1 -1
- package/dist/indexing/indexing-service.d.ts +2 -0
- package/dist/indexing/indexing-service.d.ts.map +1 -1
- package/dist/indexing/indexing-service.js +90 -32
- package/dist/indexing/indexing-service.js.map +1 -1
- package/dist/search/search-service.d.ts +1 -1
- package/dist/storage/persistence.d.ts +33 -5
- package/dist/storage/persistence.d.ts.map +1 -1
- package/dist/storage/persistence.js +88 -24
- package/dist/storage/persistence.js.map +1 -1
- package/dist/storage/vector-store.d.ts +28 -1
- package/dist/storage/vector-store.d.ts.map +1 -1
- package/dist/storage/vector-store.js +43 -9
- package/dist/storage/vector-store.js.map +1 -1
- package/package.json +10 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexing-service.d.ts","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"indexing-service.d.ts","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"AAkCA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,UAAU,GAAG,eAAe,GAAG,WAAW,GAAG,MAAM,CAAC;IAC3D,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC;AAEjE,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,KAAK,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,KAAK,CAAC;IACb,wEAAwE;IACxE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wEAAwE;IACxE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,0EAA0E;IAC1E,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC/B;AAKD,wBAAsB,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CA0M7E"}
|
|
@@ -14,7 +14,7 @@ import { VectorStore } from "../storage/vector-store.js";
|
|
|
14
14
|
import { EmbeddingService, EMBEDDING_DIMENSIONS } from "../embeddings/embedding-service.js";
|
|
15
15
|
import { Chunker } from "./chunker.js";
|
|
16
16
|
import { listMarkdownFiles, readRepoFile } from "../adapter/repo-walker.js";
|
|
17
|
-
import {
|
|
17
|
+
import { ensureRepoCache, readManifest, writeManifest, emptyManifest, readDocumentStoreExport, writeDocumentStoreExport, readVectorSidecar, writeVectorSidecar, removeVectorCache, vectorIndexExists, repoCacheDir, repoModelCacheDir, paths, EMBEDDING_MODEL_ID, } from "../storage/persistence.js";
|
|
18
18
|
import { scopesToRoots } from "../types.js";
|
|
19
19
|
const CHUNK_SIZE = 3200;
|
|
20
20
|
const CHUNK_OVERLAP = 0.15;
|
|
@@ -22,63 +22,97 @@ export async function runIndex(options) {
|
|
|
22
22
|
const start = Date.now();
|
|
23
23
|
const { repoPath, scope, onProgress } = options;
|
|
24
24
|
const cacheDir = repoCacheDir(repoPath);
|
|
25
|
-
await
|
|
26
|
-
const
|
|
25
|
+
await ensureRepoCache(repoPath);
|
|
26
|
+
const persistedManifest = await readManifest(cacheDir);
|
|
27
|
+
const manifest = persistedManifest ?? emptyManifest();
|
|
28
|
+
const cacheRequiresRebuild = persistedManifest === null;
|
|
29
|
+
if (cacheRequiresRebuild && options.skipEmbeddings) {
|
|
30
|
+
// An incompatible cache cannot be safely reconciled without loading its
|
|
31
|
+
// vector index. Drop the disposable vector files so the next semantic
|
|
32
|
+
// query rebuilds them automatically instead of mixing old metadata in.
|
|
33
|
+
await removeVectorCache(cacheDir);
|
|
34
|
+
}
|
|
27
35
|
onProgress?.({ phase: "scanning", filesProcessed: 0, totalFiles: 0 });
|
|
28
36
|
const roots = scopesToRoots(scope);
|
|
29
37
|
const files = await listMarkdownFiles(repoPath, roots, { exclude: options.exclude });
|
|
30
38
|
const docStore = new DocumentStore();
|
|
31
|
-
await docStore.initialize((await readDocumentStoreExport(cacheDir)) ?? undefined);
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
if (
|
|
35
|
-
|
|
36
|
-
|
|
39
|
+
await docStore.initialize(cacheRequiresRebuild ? undefined : (await readDocumentStoreExport(cacheDir)) ?? undefined);
|
|
40
|
+
let vectorStore = null;
|
|
41
|
+
let sidecar = null;
|
|
42
|
+
if (!options.skipEmbeddings) {
|
|
43
|
+
vectorStore = new VectorStore({ dimensions: EMBEDDING_DIMENSIONS });
|
|
44
|
+
const hasExistingVectors = await vectorIndexExists(cacheDir);
|
|
45
|
+
sidecar = !cacheRequiresRebuild && hasExistingVectors
|
|
46
|
+
? await readVectorSidecar(cacheDir)
|
|
47
|
+
: null;
|
|
48
|
+
if (sidecar && !options.force) {
|
|
37
49
|
await vectorStore.loadFromDisk(paths(cacheDir).vectorIndex, sidecar);
|
|
38
50
|
}
|
|
39
51
|
else {
|
|
40
52
|
await vectorStore.initialize();
|
|
41
53
|
}
|
|
42
54
|
}
|
|
43
|
-
else {
|
|
44
|
-
await vectorStore.initialize();
|
|
45
|
-
}
|
|
46
55
|
const chunker = new Chunker({ maxChunkSize: CHUNK_SIZE, overlapPercent: CHUNK_OVERLAP });
|
|
47
56
|
const currentPaths = new Set(files.map((f) => f.path));
|
|
48
|
-
const
|
|
57
|
+
const textChanged = [];
|
|
58
|
+
const filesToProcess = [];
|
|
49
59
|
for (const f of files) {
|
|
50
60
|
const entry = manifest.files[f.path];
|
|
51
|
-
|
|
52
|
-
|
|
61
|
+
const textIsStale = options.force || !entry || entry.textMtimeMs !== f.mtimeMs;
|
|
62
|
+
const vectorIsStale = !options.skipEmbeddings &&
|
|
63
|
+
(manifest.embeddingModel !== EMBEDDING_MODEL_ID ||
|
|
64
|
+
!sidecar || !entry || entry.vectorStale || entry.vectorMtimeMs !== f.mtimeMs);
|
|
65
|
+
if (textIsStale)
|
|
66
|
+
textChanged.push(f);
|
|
67
|
+
if (textIsStale || vectorIsStale)
|
|
68
|
+
filesToProcess.push(f);
|
|
53
69
|
}
|
|
54
70
|
// Files that were indexed under this manifest for this scope but no longer exist on disk.
|
|
55
71
|
const removed = Object.keys(manifest.files).filter((p) => {
|
|
56
72
|
const inScope = roots.some((r) => p === r || p.startsWith(r + "/"));
|
|
57
|
-
|
|
73
|
+
const entry = manifest.files[p];
|
|
74
|
+
return inScope && !currentPaths.has(p) &&
|
|
75
|
+
(entry.textMtimeMs !== null || entry.vectorMtimeMs !== null || entry.vectorStale);
|
|
58
76
|
});
|
|
59
77
|
onProgress?.({
|
|
60
78
|
phase: "text-indexing",
|
|
61
79
|
filesProcessed: 0,
|
|
62
|
-
totalFiles:
|
|
80
|
+
totalFiles: filesToProcess.length,
|
|
63
81
|
});
|
|
64
82
|
let chunksIndexed = 0;
|
|
65
83
|
const changedDocsForEmbedding = [];
|
|
66
84
|
for (const path of removed) {
|
|
67
85
|
await docStore.deleteByFilePath(path);
|
|
68
|
-
|
|
69
|
-
|
|
86
|
+
if (vectorStore)
|
|
87
|
+
await vectorStore.deleteByFilePath(path);
|
|
88
|
+
const entry = manifest.files[path];
|
|
89
|
+
entry.textMtimeMs = null;
|
|
90
|
+
if (vectorStore) {
|
|
91
|
+
entry.vectorMtimeMs = null;
|
|
92
|
+
entry.vectorStale = false;
|
|
93
|
+
}
|
|
94
|
+
else if (entry.vectorMtimeMs !== null) {
|
|
95
|
+
// Keep a tombstone until a semantic run can remove the old vector.
|
|
96
|
+
entry.vectorStale = true;
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
entry.vectorStale = false;
|
|
100
|
+
}
|
|
101
|
+
if (entry.textMtimeMs === null && entry.vectorMtimeMs === null)
|
|
102
|
+
delete manifest.files[path];
|
|
70
103
|
}
|
|
71
|
-
for (let i = 0; i <
|
|
72
|
-
const file =
|
|
104
|
+
for (let i = 0; i < filesToProcess.length; i++) {
|
|
105
|
+
const file = filesToProcess[i];
|
|
73
106
|
onProgress?.({
|
|
74
107
|
phase: "text-indexing",
|
|
75
108
|
filesProcessed: i,
|
|
76
|
-
totalFiles:
|
|
109
|
+
totalFiles: filesToProcess.length,
|
|
77
110
|
currentFile: file.path,
|
|
78
111
|
});
|
|
79
112
|
const content = await readRepoFile(file.absPath);
|
|
80
113
|
await docStore.deleteByFilePath(file.path);
|
|
81
|
-
|
|
114
|
+
if (vectorStore)
|
|
115
|
+
await vectorStore.deleteByFilePath(file.path);
|
|
82
116
|
const chunks = chunker.chunk(content, file.path);
|
|
83
117
|
const docs = chunks.map((chunk, idx) => ({
|
|
84
118
|
id: `${file.path}:${idx}`,
|
|
@@ -91,12 +125,34 @@ export async function runIndex(options) {
|
|
|
91
125
|
}));
|
|
92
126
|
await docStore.indexDocuments(docs);
|
|
93
127
|
chunksIndexed += docs.length;
|
|
94
|
-
|
|
95
|
-
|
|
128
|
+
if (vectorStore)
|
|
129
|
+
changedDocsForEmbedding.push(...docs);
|
|
130
|
+
const entry = manifest.files[file.path] ?? {
|
|
131
|
+
textMtimeMs: null,
|
|
132
|
+
vectorMtimeMs: null,
|
|
133
|
+
vectorStale: false,
|
|
134
|
+
};
|
|
135
|
+
entry.textMtimeMs = file.mtimeMs;
|
|
136
|
+
if (!vectorStore) {
|
|
137
|
+
// The text index is current, but semantic vectors are intentionally not
|
|
138
|
+
// touched by keyword-only indexing. The next semantic/hybrid query will
|
|
139
|
+
// refresh this file automatically.
|
|
140
|
+
entry.vectorStale = true;
|
|
141
|
+
}
|
|
142
|
+
else if (docs.length === 0) {
|
|
143
|
+
// An empty file is fully represented by the absence of vector points;
|
|
144
|
+
// record that fact so it does not trigger a refresh forever.
|
|
145
|
+
entry.vectorMtimeMs = file.mtimeMs;
|
|
146
|
+
entry.vectorStale = false;
|
|
147
|
+
}
|
|
148
|
+
manifest.files[file.path] = entry;
|
|
96
149
|
}
|
|
97
150
|
let embeddingsGenerated = 0;
|
|
98
|
-
if (
|
|
99
|
-
const embeddingService = new EmbeddingService(
|
|
151
|
+
if (vectorStore && changedDocsForEmbedding.length > 0) {
|
|
152
|
+
const embeddingService = new EmbeddingService({
|
|
153
|
+
modelCacheDir: repoModelCacheDir(repoPath),
|
|
154
|
+
allowRemoteModels: options.allowRemoteModels ?? true,
|
|
155
|
+
});
|
|
100
156
|
await embeddingService.initialize();
|
|
101
157
|
onProgress?.({
|
|
102
158
|
phase: "embedding",
|
|
@@ -115,6 +171,8 @@ export async function runIndex(options) {
|
|
|
115
171
|
chunkIndex: doc.chunkIndex,
|
|
116
172
|
embedding: embeddings[j],
|
|
117
173
|
});
|
|
174
|
+
manifest.files[doc.filePath].vectorMtimeMs = doc.mtimeMs;
|
|
175
|
+
manifest.files[doc.filePath].vectorStale = false;
|
|
118
176
|
embeddingsGenerated++;
|
|
119
177
|
}
|
|
120
178
|
onProgress?.({
|
|
@@ -126,13 +184,13 @@ export async function runIndex(options) {
|
|
|
126
184
|
}
|
|
127
185
|
// Persist.
|
|
128
186
|
await writeDocumentStoreExport(cacheDir, docStore.export());
|
|
129
|
-
if (
|
|
130
|
-
const
|
|
131
|
-
await writeVectorSidecar(cacheDir,
|
|
187
|
+
if (vectorStore) {
|
|
188
|
+
const updatedSidecar = await vectorStore.saveToDisk(paths(cacheDir).vectorIndex);
|
|
189
|
+
await writeVectorSidecar(cacheDir, updatedSidecar);
|
|
132
190
|
}
|
|
133
191
|
const updatedManifest = {
|
|
134
192
|
...manifest,
|
|
135
|
-
embeddingModel: EMBEDDING_MODEL_ID,
|
|
193
|
+
embeddingModel: options.skipEmbeddings ? manifest.embeddingModel : EMBEDDING_MODEL_ID,
|
|
136
194
|
indexedScopes: Array.from(new Set([...manifest.indexedScopes, scope])),
|
|
137
195
|
lastIndexedAt: Date.now(),
|
|
138
196
|
};
|
|
@@ -141,7 +199,7 @@ export async function runIndex(options) {
|
|
|
141
199
|
return {
|
|
142
200
|
scope,
|
|
143
201
|
filesScanned: files.length,
|
|
144
|
-
filesChanged:
|
|
202
|
+
filesChanged: textChanged.length,
|
|
145
203
|
filesRemoved: removed.length,
|
|
146
204
|
chunksIndexed,
|
|
147
205
|
embeddingsGenerated,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexing-service.js","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,aAAa,EAAwB,MAAM,8BAA8B,CAAC;AACnF,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,oCAAoC,CAAC;AAC5F,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAiB,MAAM,2BAA2B,CAAC;AAC3F,OAAO,EACL,
|
|
1
|
+
{"version":3,"file":"indexing-service.js","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,aAAa,EAAwB,MAAM,8BAA8B,CAAC;AACnF,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,oCAAoC,CAAC;AAC5F,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAiB,MAAM,2BAA2B,CAAC;AAC3F,OAAO,EACL,eAAe,EACf,YAAY,EACZ,aAAa,EACb,aAAa,EACb,uBAAuB,EACvB,wBAAwB,EACxB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,KAAK,EACL,kBAAkB,GAEnB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAmC5C,MAAM,UAAU,GAAG,IAAI,CAAC;AACxB,MAAM,aAAa,GAAG,IAAI,CAAC;AAE3B,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAqB;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAChD,MAAM,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAC;IAEhC,MAAM,iBAAiB,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,aAAa,EAAE,CAAC;IACtD,MAAM,oBAAoB,GAAG,iBAAiB,KAAK,IAAI,CAAC;IACxD,IAAI,oBAAoB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QACnD,wEAAwE;QACxE,sEAAsE;QACtE,uEAAuE;QACvE,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAErF,MAAM,QAAQ,GAAG,IAAI,aAAa,EAAE,CAAC;IACrC,MAAM,QAAQ,CAAC,UAAU,CACvB,oBAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,uBAAuB,CAAC,QAAQ,CAAC,CAAC,IAAI,SAAS,CAC1F,CAAC;IAEF,IAAI,WAAW,GAAuB,IAAI,CAAC;IAC3C,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QAC5B,WAAW,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACpE,MAAM,kBAAkB,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC7D,OAAO,GAAG,CAAC,oBAAoB,IAAI,kBAAkB;YACnD,CAAC,CAAC,MAAM,iBAAiB,CAAC,QAAQ,CAAC;YACnC,CAAC,CAAC,IAAI,CAAC;QACT,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC9B,MAAM,WAAW,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACvE,CAAC;aAAM,CAAC;YACN,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC;QACjC,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC;IAEzF,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,MAAM,WAAW,GAAe,EAAE,CAAC;IACnC,MAAM,cAAc,GAAe,EAAE,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,CAAC,CAAC,OAAO,CAAC;QAC/E,MAAM,aAAa,GACjB,CAAC,OAAO,CAAC,cAAc;YACvB,CAAC,QAAQ,CAAC,cAAc,KAAK,kBAAkB;gBAC7C,CAAC,OAAO,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;QAClF,IAAI,WAAW;YAAE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,WAAW,IAAI,aAAa;YAAE,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,0FAA0F;IAC1F,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACvD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;QACjC,OAAO,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YACpC,CAAC,KAAK,CAAC,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC;IACtF,CAAC,CAAC,CAAC;IAEH,UAAU,EAAE,CAAC;QACX,KAAK,EAAE,eAAe;QACtB,cAAc,EAAE,CAAC;QACjB,UAAU,EAAE,cAAc,CAAC,MAAM;KAClC,CAAC,CAAC;IAEH,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,MAAM,uBAAuB,GAAsB,EAAE,CAAC;IAEtD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,WAAW;YAAE,MAAM,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QACpC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;QACzB,IAAI,WAAW,EAAE,CAAC;YAChB,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC;YAC3B,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC;QAC5B,CAAC;aAAM,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YACxC,mEAAmE;YACnE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC;QAC5B,CAAC;QACD,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI;YAAE,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9F,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC,CAAE,CAAC;QAChC,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,eAAe;YACtB,cAAc,EAAE,CAAC;YACjB,UAAU,EAAE,cAAc,CAAC,MAAM;YACjC,WAAW,EAAE,IAAI,CAAC,IAAI;SACvB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,WAAW;YAAE,MAAM,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE/D,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,IAAI,GAAsB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YAC1D,EAAE,EAAE,GAAG,IAAI,CAAC,IAAI,IAAI,GAAG,EAAE;YACzB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,GAAG;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;SAC3C,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACpC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC;QAC7B,IAAI,WAAW;YAAE,uBAAuB,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;QAEvD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YACzC,WAAW,EAAE,IAAI;YACjB,aAAa,EAAE,IAAI;YACnB,WAAW,EAAE,KAAK;SACnB,CAAC;QACF,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC;QACjC,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,wEAAwE;YACxE,wEAAwE;YACxE,mCAAmC;YACnC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;QAC3B,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,sEAAsE;YACtE,6DAA6D;YAC7D,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC;YACnC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC;QAC5B,CAAC;QACD,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IACpC,CAAC;IAED,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,IAAI,WAAW,IAAI,uBAAuB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,CAAC;YAC5C,aAAa,EAAE,iBAAiB,CAAC,QAAQ,CAAC;YAC1C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI;SACrD,CAAC,CAAC;QACH,MAAM,gBAAgB,CAAC,UAAU,EAAE,CAAC;QAEpC,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,WAAW;YAClB,cAAc,EAAE,CAAC;YACjB,UAAU,EAAE,uBAAuB,CAAC,MAAM;SAC3C,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,uBAAuB,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;YAC1D,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YAClF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;gBACtB,MAAM,WAAW,CAAC,GAAG,CAAC;oBACpB,EAAE,EAAE,GAAG,CAAC,EAAE;oBACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,UAAU,EAAE,GAAG,CAAC,UAAU;oBAC1B,SAAS,EAAE,UAAU,CAAC,CAAC,CAAE;iBAC1B,CAAC,CAAC;gBACH,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,aAAa,GAAG,GAAG,CAAC,OAAO,CAAC;gBAC1D,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,WAAW,GAAG,KAAK,CAAC;gBAClD,mBAAmB,EAAE,CAAC;YACxB,CAAC;YACD,UAAU,EAAE,CAAC;gBACX,KAAK,EAAE,WAAW;gBAClB,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,uBAAuB,CAAC,MAAM,CAAC;gBACnE,UAAU,EAAE,uBAAuB,CAAC,MAAM;aAC3C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,WAAW;IACX,MAAM,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5D,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,cAAc,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC,CAAC;QACjF,MAAM,kBAAkB,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,eAAe,GAAa;QAChC,GAAG,QAAQ;QACX,cAAc,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,kBAAkB;QACrF,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC,CAAC;QACtE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE;KAC1B,CAAC;IACF,MAAM,aAAa,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAE/C,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAExF,OAAO;QACL,KAAK;QACL,YAAY,EAAE,KAAK,CAAC,MAAM;QAC1B,YAAY,EAAE,WAAW,CAAC,MAAM;QAChC,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,aAAa;QACb,mBAAmB;QACnB,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KACpC,CAAC;AACJ,CAAC"}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* constant as upstream). Dropped: `QueryExpansionService` and
|
|
7
7
|
* `RerankingService` integration — both call out to an LLM, which is out of
|
|
8
8
|
* scope for this offline CLI (the brief's three dependencies are minisearch,
|
|
9
|
-
* @
|
|
9
|
+
* @huggingface/transformers, and hnswlib-node; no LLM client). Also dropped:
|
|
10
10
|
* `groupByFile`/section dedup UI conveniences not needed by a line-oriented
|
|
11
11
|
* CLI, since callers get raw scored chunks that already carry a line number.
|
|
12
12
|
*/
|
|
@@ -2,8 +2,12 @@ import type { ExportedDocumentStore } from "./document-store.js";
|
|
|
2
2
|
import type { SidecarData } from "./vector-store.js";
|
|
3
3
|
import type { Scope } from "../types.js";
|
|
4
4
|
export interface ManifestEntry {
|
|
5
|
-
/** mtime (epoch ms) at the time this file was last indexed */
|
|
6
|
-
|
|
5
|
+
/** mtime (epoch ms) at the time this file was last text-indexed; null means removed */
|
|
6
|
+
textMtimeMs: number | null;
|
|
7
|
+
/** mtime (epoch ms) represented by the vector index; null means not embedded */
|
|
8
|
+
vectorMtimeMs: number | null;
|
|
9
|
+
/** True when the vector cache must be reconciled by a semantic index run. */
|
|
10
|
+
vectorStale: boolean;
|
|
7
11
|
}
|
|
8
12
|
export interface Manifest {
|
|
9
13
|
/** Schema/model version — bump to force a full rebuild on incompatible change */
|
|
@@ -18,10 +22,30 @@ export interface Manifest {
|
|
|
18
22
|
/** epoch ms of the last successful index run */
|
|
19
23
|
lastIndexedAt: number;
|
|
20
24
|
}
|
|
21
|
-
export declare const MANIFEST_VERSION =
|
|
25
|
+
export declare const MANIFEST_VERSION = 3;
|
|
22
26
|
export declare const EMBEDDING_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
|
|
23
|
-
|
|
27
|
+
export declare class CachePermissionError extends Error {
|
|
28
|
+
}
|
|
29
|
+
/** Fixed repo-local cache root. This is intentionally not configurable. */
|
|
30
|
+
export declare function repoCacheRoot(repoPath: string): string;
|
|
31
|
+
/** Repo-local index cache directory. */
|
|
24
32
|
export declare function repoCacheDir(repoPath: string): string;
|
|
33
|
+
/** Repo-local Transformers.js model cache directory. */
|
|
34
|
+
export declare function repoModelCacheDir(repoPath: string): string;
|
|
35
|
+
/** Ensure both cache trees are available before a command starts work. */
|
|
36
|
+
export declare function ensureRepoCache(repoPath: string): Promise<void>;
|
|
37
|
+
/**
|
|
38
|
+
* Write `data` to `destPath` without ever leaving a partially-written file
|
|
39
|
+
* at that path. A crash (OOM kill, SIGKILL, power loss) mid-`writeFile`
|
|
40
|
+
* would otherwise truncate the destination in place; the next run would
|
|
41
|
+
* then hit a corrupt manifest/sidecar/index and, for the binary vector
|
|
42
|
+
* index, fail hard on load with no path back to a working cache. Writing to
|
|
43
|
+
* a sibling temp file first and `rename`-ing over the destination is safe
|
|
44
|
+
* because POSIX rename within the same directory (same filesystem) is
|
|
45
|
+
* atomic: readers only ever observe the old complete file or the new
|
|
46
|
+
* complete file, never a partial one.
|
|
47
|
+
*/
|
|
48
|
+
export declare function atomicWriteFile(destPath: string, data: string | Uint8Array): Promise<void>;
|
|
25
49
|
export declare function paths(cacheDir: string): {
|
|
26
50
|
manifest: string;
|
|
27
51
|
documentStore: string;
|
|
@@ -36,10 +60,14 @@ export declare function readDocumentStoreExport(cacheDir: string): Promise<Expor
|
|
|
36
60
|
export declare function writeDocumentStoreExport(cacheDir: string, data: ExportedDocumentStore): Promise<void>;
|
|
37
61
|
export declare function readVectorSidecar(cacheDir: string): Promise<SidecarData | null>;
|
|
38
62
|
export declare function writeVectorSidecar(cacheDir: string, data: SidecarData): Promise<void>;
|
|
63
|
+
/** Remove the disposable vector cache when an incompatible manifest is found. */
|
|
64
|
+
export declare function removeVectorCache(cacheDir: string): Promise<void>;
|
|
39
65
|
export declare function vectorIndexExists(cacheDir: string): Promise<boolean>;
|
|
40
66
|
export interface FreshnessReport {
|
|
41
67
|
/** Repo-relative paths that are new, changed, or removed since the last index for this scope. */
|
|
42
68
|
staleCount: number;
|
|
69
|
+
/** Repo-relative paths whose vector embeddings are missing or stale. */
|
|
70
|
+
vectorStaleCount: number;
|
|
43
71
|
/** True if this scope has never been indexed at all. */
|
|
44
72
|
neverIndexed: boolean;
|
|
45
73
|
}
|
|
@@ -51,5 +79,5 @@ export interface FreshnessReport {
|
|
|
51
79
|
export declare function checkFreshness(manifest: Manifest | null, scope: Scope, currentFiles: {
|
|
52
80
|
path: string;
|
|
53
81
|
mtimeMs: number;
|
|
54
|
-
}[]): FreshnessReport;
|
|
82
|
+
}[], requireVectors?: boolean): FreshnessReport;
|
|
55
83
|
//# sourceMappingURL=persistence.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"persistence.d.ts","sourceRoot":"","sources":["../../src/storage/persistence.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"persistence.d.ts","sourceRoot":"","sources":["../../src/storage/persistence.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,WAAW,aAAa;IAC5B,uFAAuF;IACvF,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,gFAAgF;IAChF,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,6EAA6E;IAC7E,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,QAAQ;IACvB,iFAAiF;IACjF,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,cAAc,EAAE,MAAM,CAAC;IACvB,oDAAoD;IACpD,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACrC;6EACyE;IACzE,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,gDAAgD;IAChD,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAClC,eAAO,MAAM,kBAAkB,4BAA4B,CAAC;AAE5D,qBAAa,oBAAqB,SAAQ,KAAK;CAAG;AAElD,2EAA2E;AAC3E,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEtD;AAED,wCAAwC;AACxC,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErD;AAED,wDAAwD;AACxD,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,0EAA0E;AAC1E,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAoBrE;AAOD;;;;;;;;;;GAUG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,GAAG,UAAU,GACxB,OAAO,CAAC,IAAI,CAAC,CASf;AAED,wBAAgB,KAAK,CAAC,QAAQ,EAAE,MAAM;;;;;EAOrC;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpE;AAED,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAS7E;AAED,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvF;AAED,wBAAgB,aAAa,IAAI,QAAQ,CAQxC;AAED,wBAAsB,uBAAuB,CAC3C,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC,CAOvC;AAED,wBAAsB,wBAAwB,CAC5C,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAEf;AAED,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAOrF;AAED,wBAAsB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAE3F;AAED,iFAAiF;AACjF,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvE;AAED,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAI1E;AAED,MAAM,WAAW,eAAe;IAC9B,iGAAiG;IACjG,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,gBAAgB,EAAE,MAAM,CAAC;IACzB,wDAAwD;IACxD,YAAY,EAAE,OAAO,CAAC;CACvB;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,QAAQ,GAAG,IAAI,EACzB,KAAK,EAAE,KAAK,EACZ,YAAY,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAAE,EACjD,cAAc,UAAQ,GACrB,eAAe,CAwCjB"}
|
|
@@ -9,29 +9,72 @@
|
|
|
9
9
|
* ever been indexed (so a query can refresh an un-indexed scope instead of
|
|
10
10
|
* silently returning nothing).
|
|
11
11
|
*/
|
|
12
|
-
import { mkdir, readFile, writeFile, stat } from "node:fs/promises";
|
|
13
|
-
import { createHash } from "node:crypto";
|
|
14
|
-
import { homedir } from "node:os";
|
|
12
|
+
import { mkdir, readFile, writeFile, stat, open, unlink, rename } from "node:fs/promises";
|
|
15
13
|
import { join, resolve } from "node:path";
|
|
16
14
|
import { scopesToRoots } from "../types.js";
|
|
17
|
-
export const MANIFEST_VERSION =
|
|
15
|
+
export const MANIFEST_VERSION = 3;
|
|
18
16
|
export const EMBEDDING_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
|
|
19
|
-
|
|
20
|
-
const xdg = process.env.XDG_CACHE_HOME;
|
|
21
|
-
const base = xdg && xdg.trim().length > 0 ? xdg : join(homedir(), ".cache");
|
|
22
|
-
return join(base, "enact-docs-search");
|
|
17
|
+
export class CachePermissionError extends Error {
|
|
23
18
|
}
|
|
24
|
-
/**
|
|
19
|
+
/** Fixed repo-local cache root. This is intentionally not configurable. */
|
|
20
|
+
export function repoCacheRoot(repoPath) {
|
|
21
|
+
return join(resolve(repoPath), ".cache", "docs");
|
|
22
|
+
}
|
|
23
|
+
/** Repo-local index cache directory. */
|
|
25
24
|
export function repoCacheDir(repoPath) {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
return join(repoCacheRoot(repoPath), "index");
|
|
26
|
+
}
|
|
27
|
+
/** Repo-local Transformers.js model cache directory. */
|
|
28
|
+
export function repoModelCacheDir(repoPath) {
|
|
29
|
+
return join(repoCacheRoot(repoPath), "models");
|
|
30
|
+
}
|
|
31
|
+
/** Ensure both cache trees are available before a command starts work. */
|
|
32
|
+
export async function ensureRepoCache(repoPath) {
|
|
33
|
+
try {
|
|
34
|
+
const directories = [repoCacheDir(repoPath), repoModelCacheDir(repoPath)];
|
|
35
|
+
for (const directory of directories) {
|
|
36
|
+
await mkdir(directory, { recursive: true });
|
|
37
|
+
const probe = join(directory, `.write-probe-${process.pid}-${Date.now()}`);
|
|
38
|
+
const handle = await open(probe, "wx");
|
|
39
|
+
await handle.close();
|
|
40
|
+
await unlink(probe);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
const code = error.code;
|
|
45
|
+
if (code === "EACCES" || code === "EPERM" || code === "EROFS") {
|
|
46
|
+
throw new CachePermissionError(`The repository-local search cache is not writable: ${repoCacheRoot(repoPath)}. ` +
|
|
47
|
+
"The target repository must allow writes under .cache/docs.");
|
|
48
|
+
}
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
30
51
|
}
|
|
31
52
|
const MANIFEST_FILE = "manifest.json";
|
|
32
53
|
const DOCUMENT_STORE_FILE = "document-store.json";
|
|
33
54
|
const VECTOR_INDEX_FILE = "vector-index.bin";
|
|
34
55
|
const VECTOR_SIDECAR_FILE = "vector-sidecar.json";
|
|
56
|
+
/**
|
|
57
|
+
* Write `data` to `destPath` without ever leaving a partially-written file
|
|
58
|
+
* at that path. A crash (OOM kill, SIGKILL, power loss) mid-`writeFile`
|
|
59
|
+
* would otherwise truncate the destination in place; the next run would
|
|
60
|
+
* then hit a corrupt manifest/sidecar/index and, for the binary vector
|
|
61
|
+
* index, fail hard on load with no path back to a working cache. Writing to
|
|
62
|
+
* a sibling temp file first and `rename`-ing over the destination is safe
|
|
63
|
+
* because POSIX rename within the same directory (same filesystem) is
|
|
64
|
+
* atomic: readers only ever observe the old complete file or the new
|
|
65
|
+
* complete file, never a partial one.
|
|
66
|
+
*/
|
|
67
|
+
export async function atomicWriteFile(destPath, data) {
|
|
68
|
+
const tmpPath = `${destPath}.tmp-${process.pid}-${Date.now()}`;
|
|
69
|
+
try {
|
|
70
|
+
await writeFile(tmpPath, data);
|
|
71
|
+
await rename(tmpPath, destPath);
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
await unlink(tmpPath).catch(() => { });
|
|
75
|
+
throw error;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
35
78
|
export function paths(cacheDir) {
|
|
36
79
|
return {
|
|
37
80
|
manifest: join(cacheDir, MANIFEST_FILE),
|
|
@@ -56,7 +99,7 @@ export async function readManifest(cacheDir) {
|
|
|
56
99
|
}
|
|
57
100
|
}
|
|
58
101
|
export async function writeManifest(cacheDir, manifest) {
|
|
59
|
-
await
|
|
102
|
+
await atomicWriteFile(paths(cacheDir).manifest, JSON.stringify(manifest, null, 2));
|
|
60
103
|
}
|
|
61
104
|
export function emptyManifest() {
|
|
62
105
|
return {
|
|
@@ -77,7 +120,7 @@ export async function readDocumentStoreExport(cacheDir) {
|
|
|
77
120
|
}
|
|
78
121
|
}
|
|
79
122
|
export async function writeDocumentStoreExport(cacheDir, data) {
|
|
80
|
-
await
|
|
123
|
+
await atomicWriteFile(paths(cacheDir).documentStore, JSON.stringify(data));
|
|
81
124
|
}
|
|
82
125
|
export async function readVectorSidecar(cacheDir) {
|
|
83
126
|
try {
|
|
@@ -89,7 +132,14 @@ export async function readVectorSidecar(cacheDir) {
|
|
|
89
132
|
}
|
|
90
133
|
}
|
|
91
134
|
export async function writeVectorSidecar(cacheDir, data) {
|
|
92
|
-
await
|
|
135
|
+
await atomicWriteFile(paths(cacheDir).vectorSidecar, JSON.stringify(data));
|
|
136
|
+
}
|
|
137
|
+
/** Remove the disposable vector cache when an incompatible manifest is found. */
|
|
138
|
+
export async function removeVectorCache(cacheDir) {
|
|
139
|
+
await Promise.all([
|
|
140
|
+
unlink(paths(cacheDir).vectorIndex).catch(() => { }),
|
|
141
|
+
unlink(paths(cacheDir).vectorSidecar).catch(() => { }),
|
|
142
|
+
]);
|
|
93
143
|
}
|
|
94
144
|
export async function vectorIndexExists(cacheDir) {
|
|
95
145
|
return stat(paths(cacheDir).vectorIndex)
|
|
@@ -101,32 +151,46 @@ export async function vectorIndexExists(cacheDir) {
|
|
|
101
151
|
* caller) against the manifest, without mutating anything. Used by `query`
|
|
102
152
|
* to refresh a stale or nonexistent index before searching (I-08).
|
|
103
153
|
*/
|
|
104
|
-
export function checkFreshness(manifest, scope, currentFiles) {
|
|
154
|
+
export function checkFreshness(manifest, scope, currentFiles, requireVectors = false) {
|
|
105
155
|
// "Never indexed" is judged by overlap with the manifest, not by a scope
|
|
106
156
|
// label match: indexing with --scope all and then querying --scope docs
|
|
107
157
|
// (a subset) is a legitimate, fully-covered case. If there are files on
|
|
108
158
|
// disk in this scope but none of them appear in the manifest at all, no
|
|
109
159
|
// index run has ever covered this scope.
|
|
110
160
|
if (!manifest)
|
|
111
|
-
return { staleCount: 0, neverIndexed: true };
|
|
112
|
-
if (currentFiles.length > 0 && currentFiles.every((f) =>
|
|
113
|
-
|
|
161
|
+
return { staleCount: 0, vectorStaleCount: 0, neverIndexed: true };
|
|
162
|
+
if (currentFiles.length > 0 && currentFiles.every((f) => {
|
|
163
|
+
const entry = manifest.files[f.path];
|
|
164
|
+
return !entry || entry.textMtimeMs === null;
|
|
165
|
+
})) {
|
|
166
|
+
return { staleCount: 0, vectorStaleCount: 0, neverIndexed: true };
|
|
114
167
|
}
|
|
115
168
|
let staleCount = 0;
|
|
169
|
+
let vectorStaleCount = 0;
|
|
116
170
|
const seen = new Set();
|
|
117
171
|
for (const f of currentFiles) {
|
|
118
172
|
seen.add(f.path);
|
|
119
173
|
const entry = manifest.files[f.path];
|
|
120
|
-
if (!entry || entry.
|
|
174
|
+
if (!entry || entry.textMtimeMs !== f.mtimeMs)
|
|
121
175
|
staleCount++;
|
|
176
|
+
if (requireVectors &&
|
|
177
|
+
(!entry || entry.vectorStale || entry.vectorMtimeMs !== f.mtimeMs)) {
|
|
178
|
+
vectorStaleCount++;
|
|
179
|
+
}
|
|
122
180
|
}
|
|
123
181
|
// Files present in the manifest for this scope but deleted on disk also count as stale.
|
|
124
182
|
const roots = scopesToRoots(scope);
|
|
125
183
|
for (const path of Object.keys(manifest.files)) {
|
|
126
184
|
const inScope = roots.some((r) => path === r || path.startsWith(r + "/"));
|
|
127
|
-
if (inScope && !seen.has(path))
|
|
128
|
-
|
|
185
|
+
if (inScope && !seen.has(path)) {
|
|
186
|
+
const entry = manifest.files[path];
|
|
187
|
+
if (entry.textMtimeMs !== null)
|
|
188
|
+
staleCount++;
|
|
189
|
+
if (requireVectors && (entry.vectorStale || entry.vectorMtimeMs !== null)) {
|
|
190
|
+
vectorStaleCount++;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
129
193
|
}
|
|
130
|
-
return { staleCount, neverIndexed: false };
|
|
194
|
+
return { staleCount, vectorStaleCount, neverIndexed: false };
|
|
131
195
|
}
|
|
132
196
|
//# sourceMappingURL=persistence.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"persistence.js","sourceRoot":"","sources":["../../src/storage/persistence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,
|
|
1
|
+
{"version":3,"file":"persistence.js","sourceRoot":"","sources":["../../src/storage/persistence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1F,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAG1C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AA0B5C,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAClC,MAAM,CAAC,MAAM,kBAAkB,GAAG,yBAAyB,CAAC;AAE5D,MAAM,OAAO,oBAAqB,SAAQ,KAAK;CAAG;AAElD,2EAA2E;AAC3E,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;AACnD,CAAC;AAED,wCAAwC;AACxC,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;AAChD,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IAChD,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;AACjD,CAAC;AAED,0EAA0E;AAC1E,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB;IACpD,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC1E,KAAK,MAAM,SAAS,IAAI,WAAW,EAAE,CAAC;YACpC,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,gBAAgB,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC3E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YACvC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,GAAI,KAA+B,CAAC,IAAI,CAAC;QACnD,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YAC9D,MAAM,IAAI,oBAAoB,CAC5B,sDAAsD,aAAa,CAAC,QAAQ,CAAC,IAAI;gBAC/E,4DAA4D,CAC/D,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,aAAa,GAAG,eAAe,CAAC;AACtC,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;AAClD,MAAM,iBAAiB,GAAG,kBAAkB,CAAC;AAC7C,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;AAElD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,QAAgB,EAChB,IAAyB;IAEzB,MAAM,OAAO,GAAG,GAAG,QAAQ,QAAQ,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC/D,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC/B,MAAM,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACtC,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,QAAgB;IACpC,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC;QACvC,aAAa,EAAE,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC;QAClD,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE,iBAAiB,CAAC;QAC9C,aAAa,EAAE,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC;KACnD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB;IACnD,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAa,CAAC;QAC3C,IAAI,MAAM,CAAC,OAAO,KAAK,gBAAgB;YAAE,OAAO,IAAI,CAAC;QACrD,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAgB,EAAE,QAAkB;IACtE,MAAM,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,OAAO,EAAE,gBAAgB;QACzB,cAAc,EAAE,kBAAkB;QAClC,KAAK,EAAE,EAAE;QACT,aAAa,EAAE,EAAE;QACjB,aAAa,EAAE,CAAC;KACjB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,QAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAA0B,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,QAAgB,EAChB,IAA2B;IAE3B,MAAM,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IACtD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAgB,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAgB,EAAE,IAAiB;IAC1E,MAAM,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IACtD,MAAM,OAAO,CAAC,GAAG,CAAC;QAChB,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;QACnD,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;KACtD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IACtD,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC;SACrC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;AACxB,CAAC;AAWD;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAC5B,QAAyB,EACzB,KAAY,EACZ,YAAiD,EACjD,cAAc,GAAG,KAAK;IAEtB,yEAAyE;IACzE,wEAAwE;IACxE,wEAAwE;IACxE,wEAAwE;IACxE,yCAAyC;IACzC,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACjF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,CAAC;IAC9C,CAAC,CAAC,EAAE,CAAC;QACH,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACpE,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACjB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,CAAC,CAAC,OAAO;YAAE,UAAU,EAAE,CAAC;QAC5D,IAAI,cAAc;YAChB,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACrE,gBAAgB,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IACD,wFAAwF;IACxF,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QAC1E,IAAI,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;YACpC,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,UAAU,EAAE,CAAC;YAC7C,IAAI,cAAc,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,CAAC,EAAE,CAAC;gBAC1E,gBAAgB,EAAE,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;AAC/D,CAAC"}
|
|
@@ -16,6 +16,8 @@ export interface VectorSearchResult {
|
|
|
16
16
|
}
|
|
17
17
|
export interface VectorStoreOptions {
|
|
18
18
|
dimensions: number;
|
|
19
|
+
/** Initial HNSW capacity. Grows automatically past this; override in tests only. */
|
|
20
|
+
initialMaxElements?: number;
|
|
19
21
|
}
|
|
20
22
|
interface VectorMetadata {
|
|
21
23
|
id: string;
|
|
@@ -40,9 +42,34 @@ export declare class VectorStore {
|
|
|
40
42
|
initialize(): Promise<void>;
|
|
41
43
|
/** Load a previously persisted index plus its label/metadata sidecar. */
|
|
42
44
|
loadFromDisk(indexPath: string, sidecar: SidecarData): Promise<void>;
|
|
43
|
-
/**
|
|
45
|
+
/**
|
|
46
|
+
* Persist the raw HNSW index plus the sidecar label/metadata mapping.
|
|
47
|
+
*
|
|
48
|
+
* Writes to a sibling temp path and renames over `indexPath` so a crash
|
|
49
|
+
* mid-write (this binary file is large and the native write isn't
|
|
50
|
+
* instantaneous) can never leave a truncated `indexPath` in place —
|
|
51
|
+
* `loadFromDisk`'s `readIndex` has no way to recover from a corrupt file,
|
|
52
|
+
* and that failure would otherwise repeat on every subsequent run.
|
|
53
|
+
*/
|
|
44
54
|
saveToDisk(indexPath: string): Promise<SidecarData>;
|
|
45
55
|
isReady(): boolean;
|
|
56
|
+
/**
|
|
57
|
+
* hnswlib's internal `cur_element_count` counts every label ever added,
|
|
58
|
+
* including ones later `markDelete`d — deletion only tombstones a slot, it
|
|
59
|
+
* never frees capacity (see hnswalg.h's addPoint: capacity is checked
|
|
60
|
+
* against `cur_element_count`, not the live point count). `add()` never
|
|
61
|
+
* reuses a deleted label (see the comment in `add()` below), so capacity
|
|
62
|
+
* is consumed monotonically by every add call, live or not.
|
|
63
|
+
*
|
|
64
|
+
* This must NOT be approximated with `this.idToLabel.size` (the live
|
|
65
|
+
* count): after any delete+re-add cycle — which is exactly what every
|
|
66
|
+
* incremental re-index does per changed file — live count stays flat
|
|
67
|
+
* while consumed capacity keeps climbing, so that check would never fire
|
|
68
|
+
* and `addPoint` would eventually throw "The number of elements exceeds
|
|
69
|
+
* the specified limit" past the initial capacity. `getCurrentCount()`
|
|
70
|
+
* reads the native counter directly, so it can't drift out of sync with
|
|
71
|
+
* our own bookkeeping.
|
|
72
|
+
*/
|
|
46
73
|
private resizeIfNeeded;
|
|
47
74
|
add(doc: VectorDocument): Promise<void>;
|
|
48
75
|
delete(id: string): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-store.d.ts","sourceRoot":"","sources":["../../src/storage/vector-store.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"vector-store.d.ts","sourceRoot":"","sources":["../../src/storage/vector-store.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAGzC,qBAAa,eAAgB,SAAQ,KAAK;CAAG;AAoB7C,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,sDAAsD;IACtD,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,oFAAoF;IACpF,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,UAAU,cAAc;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,KAAK,CAAoC;IACjD,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,KAAK,CAAS;IAEtB,OAAO,CAAC,SAAS,CAAkC;IACnD,OAAO,CAAC,SAAS,CAAkC;IACnD,OAAO,CAAC,QAAQ,CAA0C;IAE1D,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,aAAa,CAAgB;IAErC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAM;IACxB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAO;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;gBAEnB,OAAO,EAAE,kBAAkB;IAKvC,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAejC,yEAAyE;IACnE,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB1E;;;;;;;;OAQG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAmBzD,OAAO,IAAI,OAAO;IAIlB;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,cAAc;IAUhB,GAAG,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBvC,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAcjC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAUvD,+EAA+E;IACzE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,GAAE,KAAa,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IA2CvF,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;IAI3B,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAGxC;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IAC9B,QAAQ,EAAE,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,CAAC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;CACpB"}
|