@amsterdamdatalabs/enact-docs-search 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- {"version":3,"file":"indexing-service.d.ts","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"AAgCA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,UAAU,GAAG,eAAe,GAAG,WAAW,GAAG,MAAM,CAAC;IAC3D,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC;AAEjE,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,KAAK,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,KAAK,CAAC;IACb,wEAAwE;IACxE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wEAAwE;IACxE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC/B;AAKD,wBAAsB,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CAoJ7E"}
1
+ {"version":3,"file":"indexing-service.d.ts","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"AAkCA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,UAAU,GAAG,eAAe,GAAG,WAAW,GAAG,MAAM,CAAC;IAC3D,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC;AAEjE,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,KAAK,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,KAAK,CAAC;IACb,wEAAwE;IACxE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wEAAwE;IACxE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,0EAA0E;IAC1E,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC/B;AAKD,wBAAsB,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CA0M7E"}
@@ -14,7 +14,7 @@ import { VectorStore } from "../storage/vector-store.js";
14
14
  import { EmbeddingService, EMBEDDING_DIMENSIONS } from "../embeddings/embedding-service.js";
15
15
  import { Chunker } from "./chunker.js";
16
16
  import { listMarkdownFiles, readRepoFile } from "../adapter/repo-walker.js";
17
- import { ensureCacheDir, readManifest, writeManifest, emptyManifest, readDocumentStoreExport, writeDocumentStoreExport, readVectorSidecar, writeVectorSidecar, vectorIndexExists, repoCacheDir, paths, EMBEDDING_MODEL_ID, } from "../storage/persistence.js";
17
+ import { ensureRepoCache, readManifest, writeManifest, emptyManifest, readDocumentStoreExport, writeDocumentStoreExport, readVectorSidecar, writeVectorSidecar, removeVectorCache, vectorIndexExists, repoCacheDir, repoModelCacheDir, paths, EMBEDDING_MODEL_ID, } from "../storage/persistence.js";
18
18
  import { scopesToRoots } from "../types.js";
19
19
  const CHUNK_SIZE = 3200;
20
20
  const CHUNK_OVERLAP = 0.15;
@@ -22,63 +22,97 @@ export async function runIndex(options) {
22
22
  const start = Date.now();
23
23
  const { repoPath, scope, onProgress } = options;
24
24
  const cacheDir = repoCacheDir(repoPath);
25
- await ensureCacheDir(cacheDir);
26
- const manifest = (options.force ? null : await readManifest(cacheDir)) ?? emptyManifest();
25
+ await ensureRepoCache(repoPath);
26
+ const persistedManifest = await readManifest(cacheDir);
27
+ const manifest = persistedManifest ?? emptyManifest();
28
+ const cacheRequiresRebuild = persistedManifest === null;
29
+ if (cacheRequiresRebuild && options.skipEmbeddings) {
30
+ // An incompatible cache cannot be safely reconciled without loading its
31
+ // vector index. Drop the disposable vector files so the next semantic
32
+ // query rebuilds them automatically instead of mixing old metadata in.
33
+ await removeVectorCache(cacheDir);
34
+ }
27
35
  onProgress?.({ phase: "scanning", filesProcessed: 0, totalFiles: 0 });
28
36
  const roots = scopesToRoots(scope);
29
37
  const files = await listMarkdownFiles(repoPath, roots, { exclude: options.exclude });
30
38
  const docStore = new DocumentStore();
31
- await docStore.initialize((await readDocumentStoreExport(cacheDir)) ?? undefined);
32
- const vectorStore = new VectorStore({ dimensions: EMBEDDING_DIMENSIONS });
33
- const hasExistingVectors = await vectorIndexExists(cacheDir);
34
- if (hasExistingVectors && !options.force) {
35
- const sidecar = await readVectorSidecar(cacheDir);
36
- if (sidecar) {
39
+ await docStore.initialize(cacheRequiresRebuild ? undefined : (await readDocumentStoreExport(cacheDir)) ?? undefined);
40
+ let vectorStore = null;
41
+ let sidecar = null;
42
+ if (!options.skipEmbeddings) {
43
+ vectorStore = new VectorStore({ dimensions: EMBEDDING_DIMENSIONS });
44
+ const hasExistingVectors = await vectorIndexExists(cacheDir);
45
+ sidecar = !cacheRequiresRebuild && hasExistingVectors
46
+ ? await readVectorSidecar(cacheDir)
47
+ : null;
48
+ if (sidecar && !options.force) {
37
49
  await vectorStore.loadFromDisk(paths(cacheDir).vectorIndex, sidecar);
38
50
  }
39
51
  else {
40
52
  await vectorStore.initialize();
41
53
  }
42
54
  }
43
- else {
44
- await vectorStore.initialize();
45
- }
46
55
  const chunker = new Chunker({ maxChunkSize: CHUNK_SIZE, overlapPercent: CHUNK_OVERLAP });
47
56
  const currentPaths = new Set(files.map((f) => f.path));
48
- const changed = [];
57
+ const textChanged = [];
58
+ const filesToProcess = [];
49
59
  for (const f of files) {
50
60
  const entry = manifest.files[f.path];
51
- if (!entry || entry.mtimeMs !== f.mtimeMs)
52
- changed.push(f);
61
+ const textIsStale = options.force || !entry || entry.textMtimeMs !== f.mtimeMs;
62
+ const vectorIsStale = !options.skipEmbeddings &&
63
+ (manifest.embeddingModel !== EMBEDDING_MODEL_ID ||
64
+ !sidecar || !entry || entry.vectorStale || entry.vectorMtimeMs !== f.mtimeMs);
65
+ if (textIsStale)
66
+ textChanged.push(f);
67
+ if (textIsStale || vectorIsStale)
68
+ filesToProcess.push(f);
53
69
  }
54
70
  // Files that were indexed under this manifest for this scope but no longer exist on disk.
55
71
  const removed = Object.keys(manifest.files).filter((p) => {
56
72
  const inScope = roots.some((r) => p === r || p.startsWith(r + "/"));
57
- return inScope && !currentPaths.has(p);
73
+ const entry = manifest.files[p];
74
+ return inScope && !currentPaths.has(p) &&
75
+ (entry.textMtimeMs !== null || entry.vectorMtimeMs !== null || entry.vectorStale);
58
76
  });
59
77
  onProgress?.({
60
78
  phase: "text-indexing",
61
79
  filesProcessed: 0,
62
- totalFiles: changed.length,
80
+ totalFiles: filesToProcess.length,
63
81
  });
64
82
  let chunksIndexed = 0;
65
83
  const changedDocsForEmbedding = [];
66
84
  for (const path of removed) {
67
85
  await docStore.deleteByFilePath(path);
68
- await vectorStore.deleteByFilePath(path);
69
- delete manifest.files[path];
86
+ if (vectorStore)
87
+ await vectorStore.deleteByFilePath(path);
88
+ const entry = manifest.files[path];
89
+ entry.textMtimeMs = null;
90
+ if (vectorStore) {
91
+ entry.vectorMtimeMs = null;
92
+ entry.vectorStale = false;
93
+ }
94
+ else if (entry.vectorMtimeMs !== null) {
95
+ // Keep a tombstone until a semantic run can remove the old vector.
96
+ entry.vectorStale = true;
97
+ }
98
+ else {
99
+ entry.vectorStale = false;
100
+ }
101
+ if (entry.textMtimeMs === null && entry.vectorMtimeMs === null)
102
+ delete manifest.files[path];
70
103
  }
71
- for (let i = 0; i < changed.length; i++) {
72
- const file = changed[i];
104
+ for (let i = 0; i < filesToProcess.length; i++) {
105
+ const file = filesToProcess[i];
73
106
  onProgress?.({
74
107
  phase: "text-indexing",
75
108
  filesProcessed: i,
76
- totalFiles: changed.length,
109
+ totalFiles: filesToProcess.length,
77
110
  currentFile: file.path,
78
111
  });
79
112
  const content = await readRepoFile(file.absPath);
80
113
  await docStore.deleteByFilePath(file.path);
81
- await vectorStore.deleteByFilePath(file.path);
114
+ if (vectorStore)
115
+ await vectorStore.deleteByFilePath(file.path);
82
116
  const chunks = chunker.chunk(content, file.path);
83
117
  const docs = chunks.map((chunk, idx) => ({
84
118
  id: `${file.path}:${idx}`,
@@ -91,12 +125,34 @@ export async function runIndex(options) {
91
125
  }));
92
126
  await docStore.indexDocuments(docs);
93
127
  chunksIndexed += docs.length;
94
- changedDocsForEmbedding.push(...docs);
95
- manifest.files[file.path] = { mtimeMs: file.mtimeMs };
128
+ if (vectorStore)
129
+ changedDocsForEmbedding.push(...docs);
130
+ const entry = manifest.files[file.path] ?? {
131
+ textMtimeMs: null,
132
+ vectorMtimeMs: null,
133
+ vectorStale: false,
134
+ };
135
+ entry.textMtimeMs = file.mtimeMs;
136
+ if (!vectorStore) {
137
+ // The text index is current, but semantic vectors are intentionally not
138
+ // touched by keyword-only indexing. The next semantic/hybrid query will
139
+ // refresh this file automatically.
140
+ entry.vectorStale = true;
141
+ }
142
+ else if (docs.length === 0) {
143
+ // An empty file is fully represented by the absence of vector points;
144
+ // record that fact so it does not trigger a refresh forever.
145
+ entry.vectorMtimeMs = file.mtimeMs;
146
+ entry.vectorStale = false;
147
+ }
148
+ manifest.files[file.path] = entry;
96
149
  }
97
150
  let embeddingsGenerated = 0;
98
- if (!options.skipEmbeddings && changedDocsForEmbedding.length > 0) {
99
- const embeddingService = new EmbeddingService();
151
+ if (vectorStore && changedDocsForEmbedding.length > 0) {
152
+ const embeddingService = new EmbeddingService({
153
+ modelCacheDir: repoModelCacheDir(repoPath),
154
+ allowRemoteModels: options.allowRemoteModels ?? true,
155
+ });
100
156
  await embeddingService.initialize();
101
157
  onProgress?.({
102
158
  phase: "embedding",
@@ -115,6 +171,8 @@ export async function runIndex(options) {
115
171
  chunkIndex: doc.chunkIndex,
116
172
  embedding: embeddings[j],
117
173
  });
174
+ manifest.files[doc.filePath].vectorMtimeMs = doc.mtimeMs;
175
+ manifest.files[doc.filePath].vectorStale = false;
118
176
  embeddingsGenerated++;
119
177
  }
120
178
  onProgress?.({
@@ -126,13 +184,13 @@ export async function runIndex(options) {
126
184
  }
127
185
  // Persist.
128
186
  await writeDocumentStoreExport(cacheDir, docStore.export());
129
- if (!options.skipEmbeddings) {
130
- const sidecar = await vectorStore.saveToDisk(paths(cacheDir).vectorIndex);
131
- await writeVectorSidecar(cacheDir, sidecar);
187
+ if (vectorStore) {
188
+ const updatedSidecar = await vectorStore.saveToDisk(paths(cacheDir).vectorIndex);
189
+ await writeVectorSidecar(cacheDir, updatedSidecar);
132
190
  }
133
191
  const updatedManifest = {
134
192
  ...manifest,
135
- embeddingModel: EMBEDDING_MODEL_ID,
193
+ embeddingModel: options.skipEmbeddings ? manifest.embeddingModel : EMBEDDING_MODEL_ID,
136
194
  indexedScopes: Array.from(new Set([...manifest.indexedScopes, scope])),
137
195
  lastIndexedAt: Date.now(),
138
196
  };
@@ -141,7 +199,7 @@ export async function runIndex(options) {
141
199
  return {
142
200
  scope,
143
201
  filesScanned: files.length,
144
- filesChanged: changed.length,
202
+ filesChanged: textChanged.length,
145
203
  filesRemoved: removed.length,
146
204
  chunksIndexed,
147
205
  embeddingsGenerated,
@@ -1 +1 @@
1
- {"version":3,"file":"indexing-service.js","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,aAAa,EAAwB,MAAM,8BAA8B,CAAC;AACnF,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,oCAAoC,CAAC;AAC5F,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAiB,MAAM,2BAA2B,CAAC;AAC3F,OAAO,EACL,cAAc,EACd,YAAY,EACZ,aAAa,EACb,aAAa,EACb,uBAAuB,EACvB,wBAAwB,EACxB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,YAAY,EACZ,KAAK,EACL,kBAAkB,GAEnB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAiC5C,MAAM,UAAU,GAAG,IAAI,CAAC;AACxB,MAAM,aAAa,GAAG,IAAI,CAAC;AAE3B,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAqB;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAChD,MAAM,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;IAE/B,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI,aAAa,EAAE,CAAC;IAE1F,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAErF,MAAM,QAAQ,GAAG,IAAI,aAAa,EAAE,CAAC;IACrC,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC,MAAM,uBAAuB,CAAC,QAAQ,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC;IAElF,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,oBAAoB,EAAE,CAAC,CAAC;IAC1E,MAAM,kBAAkB,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC7D,IAAI,kBAAkB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,WAAW,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACvE,CAAC;aAAM,CAAC;YACN,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC;QACjC,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC;IACjC,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC;IAEzF,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,0FAA0F;IAC1F,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACvD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QACpE,OAAO,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,UAAU,EAAE,CAAC;QACX,KAAK,EAAE,eAAe;QACtB,cAAc,EAAE,CAAC;QACjB,UAAU,EAAE,OAAO,CAAC,MAAM;KAC3B,CAAC,CAAC;IAEH,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,MAAM,uBAAuB,GAAsB,EAAE,CAAC;IAEtD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACzC,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;QACzB,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,eAAe;YACtB,cAAc,EAAE,CAAC;YACjB,UAAU,EAAE,OAAO,CAAC,MAAM;YAC1B,WAAW,EAAE,IAAI,CAAC,IAAI;SACvB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,IAAI,GAAsB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YAC1D,EAAE,EAAE,GAAG,IAAI,CAAC,IAAI,IAAI,GAAG,EAAE;YACzB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,GAAG;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;SAC3C,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACpC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC;QAC7B,uBAAuB,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;QAEtC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;IACxD,CAAC;IAED,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,uBAAuB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClE,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,EAAE,CAAC;QAChD,MAAM,gBAAgB,CAAC,UAAU,EAAE,CAAC;QAEpC,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,WAAW;YAClB,cAAc,EAAE,CAAC;YACjB,UAAU,EAAE,uBAAuB,CAAC,MAAM;SAC3C,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,uBAAuB,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;YAC1D,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YAClF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;gBACtB,MAAM,WAAW,CAAC,GAAG,CAAC;oBACpB,EAAE,EAAE,GAAG,CAAC,EAAE;oBACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,UAAU,EAAE,GAAG,CAAC,UAAU;oBAC1B,SAAS,EAAE,UAAU,CAAC,CAAC,CAAE;iBAC1B,CAAC,CAAC;gBACH,mBAAmB,EAAE,CAAC;YACxB,CAAC;YACD,UAAU,EAAE,CAAC;gBACX,KAAK,EAAE,WAAW;gBAClB,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,uBAAuB,CAAC,MAAM,CAAC;gBACnE,UAAU,EAAE,uBAAuB,CAAC,MAAM;aAC3C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,WAAW;IACX,MAAM,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5D,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC,CAAC;QAC1E,MAAM,kBAAkB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,eAAe,GAAa;QAChC,GAAG,QAAQ;QACX,cAAc,EAAE,kBAAkB;QAClC,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC,CAAC;QACtE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE;KAC1B,CAAC;IACF,MAAM,aAAa,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAE/C,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAExF,OAAO;QACL,KAAK;QACL,YAAY,EAAE,KAAK,CAAC,MAAM;QAC1B,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,aAAa;QACb,mBAAmB;QACnB,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KACpC,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"indexing-service.js","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,aAAa,EAAwB,MAAM,8BAA8B,CAAC;AACnF,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,oCAAoC,CAAC;AAC5F,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAiB,MAAM,2BAA2B,CAAC;AAC3F,OAAO,EACL,eAAe,EACf,YAAY,EACZ,aAAa,EACb,aAAa,EACb,uBAAuB,EACvB,wBAAwB,EACxB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,KAAK,EACL,kBAAkB,GAEnB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAmC5C,MAAM,UAAU,GAAG,IAAI,CAAC;AACxB,MAAM,aAAa,GAAG,IAAI,CAAC;AAE3B,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAqB;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAChD,MAAM,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAC;IAEhC,MAAM,iBAAiB,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,aAAa,EAAE,CAAC;IACtD,MAAM,oBAAoB,GAAG,iBAAiB,KAAK,IAAI,CAAC;IACxD,IAAI,oBAAoB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QACnD,wEAAwE;QACxE,sEAAsE;QACtE,uEAAuE;QACvE,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAErF,MAAM,QAAQ,GAAG,IAAI,aAAa,EAAE,CAAC;IACrC,MAAM,QAAQ,CAAC,UAAU,CACvB,oBAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,uBAAuB,CAAC,QAAQ,CAAC,CAAC,IAAI,SAAS,CAC1F,CAAC;IAEF,IAAI,WAAW,GAAuB,IAAI,CAAC;IAC3C,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QAC5B,WAAW,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACpE,MAAM,kBAAkB,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC7D,OAAO,GAAG,CAAC,oBAAoB,IAAI,kBAAkB;YACnD,CAAC,CAAC,MAAM,iBAAiB,CAAC,QAAQ,CAAC;YACnC,CAAC,CAAC,IAAI,CAAC;QACT,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC9B,MAAM,WAAW,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACvE,CAAC;aAAM,CAAC;YACN,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC;QACjC,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC;IAEzF,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,MAAM,WAAW,GAAe,EAAE,CAAC;IACnC,MAAM,cAAc,GAAe,EAAE,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,CAAC,CAAC,OAAO,CAAC;QAC/E,MAAM,aAAa,GACjB,CAAC,OAAO,CAAC,cAAc;YACvB,CAAC,QAAQ,CAAC,cAAc,KAAK,kBAAkB;gBAC7C,CAAC,OAAO,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;QAClF,IAAI,WAAW;YAAE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,WAAW,IAAI,aAAa;YAAE,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,0FAA0F;IAC1F,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACvD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;QACjC,OAAO,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YACpC,CAAC,KAAK,CAAC,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC;IACtF,CAAC,CAAC,CAAC;IAEH,UAAU,EAAE,CAAC;QACX,KAAK,EAAE,eAAe;QACtB,cAAc,EAAE,CAAC;QACjB,UAAU,EAAE,cAAc,CAAC,MAAM;KAClC,CAAC,CAAC;IAEH,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,MAAM,uBAAuB,GAAsB,EAAE,CAAC;IAEtD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,WAAW;YAAE,MAAM,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QACpC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;QACzB,IAAI,WAAW,EAAE,CAAC;YAChB,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC;YAC3B,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC;QAC5B,CAAC;aAAM,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YACxC,mEAAmE;YACnE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC;QAC5B,CAAC;QACD,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI;YAAE,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9F,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC,CAAE,CAAC;QAChC,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,eAAe;YACtB,cAAc,EAAE,CAAC;YACjB,UAAU,EAAE,cAAc,CAAC,MAAM;YACjC,WAAW,EAAE,IAAI,CAAC,IAAI;SACvB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,WAAW;YAAE,MAAM,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE/D,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,IAAI,GAAsB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YAC1D,EAAE,EAAE,GAAG,IAAI,CAAC,IAAI,IAAI,GAAG,EAAE;YACzB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,GAAG;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;SAC3C,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACpC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC;QAC7B,IAAI,WAAW;YAAE,uBAAuB,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;QAEvD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YACzC,WAAW,EAAE,IAAI;YACjB,aAAa,EAAE,IAAI;YACnB,WAAW,EAAE,KAAK;SACnB,CAAC;QACF,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC;QACjC,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,wEAAwE;YACxE,wEAAwE;YACxE,mCAAmC;YACnC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;QAC3B,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,sEAAsE;YACtE,6DAA6D;YAC7D,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC;YACnC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC;QAC5B,CAAC;QACD,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IACpC,CAAC;IAED,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,IAAI,WAAW,IAAI,uBAAuB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,CAAC;YAC5C,aAAa,EAAE,iBAAiB,CAAC,QAAQ,CAAC;YAC1C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI;SACrD,CAAC,CAAC;QACH,MAAM,gBAAgB,CAAC,UAAU,EAAE,CAAC;QAEpC,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,WAAW;YAClB,cAAc,EAAE,CAAC;YACjB,UAAU,EAAE,uBAAuB,CAAC,MAAM;SAC3C,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,uBAAuB,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;YAC1D,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YAClF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;gBACtB,MAAM,WAAW,CAAC,GAAG,CAAC;oBACpB,EAAE,EAAE,GAAG,CAAC,EAAE;oBACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,UAAU,EAAE,GAAG,CAAC,UAAU;oBAC1B,SAAS,EAAE,UAAU,CAAC,CAAC,CAAE;iBAC1B,CAAC,CAAC;gBACH,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,aAAa,GAAG,GAAG,CAAC,OAAO,CAAC;gBAC1D,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,WAAW,GAAG,KAAK,CAAC;gBAClD,mBAAmB,EAAE,CAAC;YACxB,CAAC;YACD,UAAU,EAAE,CAAC;gBACX,KAAK,EAAE,WAAW;gBAClB,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,uBAAuB,CAAC,MAAM,CAAC;gBACnE,UAAU,EAAE,uBAAuB,CAAC,MAAM;aAC3C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,WAAW;IACX,MAAM,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5D,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,cAAc,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC,CAAC;QACjF,MAAM,kBAAkB,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,eAAe,GAAa;QAChC,GAAG,QAAQ;QACX,cAAc,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,kBAAkB;QACrF,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC,CAAC;QACtE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE;KAC1B,CAAC;IACF,MAAM,aAAa,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAE/C,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAExF,OAAO;QACL,KAAK;QACL,YAAY,EAAE,KAAK,CAAC,MAAM;QAC1B,YAAY,EAAE,WAAW,CAAC,MAAM;QAChC,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,aAAa;QACb,mBAAmB;QACnB,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KACpC,CAAC;AACJ,CAAC"}
@@ -6,7 +6,7 @@
6
6
  * constant as upstream). Dropped: `QueryExpansionService` and
7
7
  * `RerankingService` integration — both call out to an LLM, which is out of
8
8
  * scope for this offline CLI (the brief's three dependencies are minisearch,
9
- * @xenova/transformers, and hnswlib-node; no LLM client). Also dropped:
9
+ * @huggingface/transformers, and hnswlib-node; no LLM client). Also dropped:
10
10
  * `groupByFile`/section dedup UI conveniences not needed by a line-oriented
11
11
  * CLI, since callers get raw scored chunks that already carry a line number.
12
12
  */
@@ -2,8 +2,12 @@ import type { ExportedDocumentStore } from "./document-store.js";
2
2
  import type { SidecarData } from "./vector-store.js";
3
3
  import type { Scope } from "../types.js";
4
4
  export interface ManifestEntry {
5
- /** mtime (epoch ms) at the time this file was last indexed */
6
- mtimeMs: number;
5
+ /** mtime (epoch ms) at the time this file was last text-indexed; null means removed */
6
+ textMtimeMs: number | null;
7
+ /** mtime (epoch ms) represented by the vector index; null means not embedded */
8
+ vectorMtimeMs: number | null;
9
+ /** True when the vector cache must be reconciled by a semantic index run. */
10
+ vectorStale: boolean;
7
11
  }
8
12
  export interface Manifest {
9
13
  /** Schema/model version — bump to force a full rebuild on incompatible change */
@@ -18,10 +22,30 @@ export interface Manifest {
18
22
  /** epoch ms of the last successful index run */
19
23
  lastIndexedAt: number;
20
24
  }
21
- export declare const MANIFEST_VERSION = 1;
25
+ export declare const MANIFEST_VERSION = 3;
22
26
  export declare const EMBEDDING_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
23
- /** Deterministic, collision-resistant per-repo cache directory. */
27
+ export declare class CachePermissionError extends Error {
28
+ }
29
+ /** Fixed repo-local cache root. This is intentionally not configurable. */
30
+ export declare function repoCacheRoot(repoPath: string): string;
31
+ /** Repo-local index cache directory. */
24
32
  export declare function repoCacheDir(repoPath: string): string;
33
+ /** Repo-local Transformers.js model cache directory. */
34
+ export declare function repoModelCacheDir(repoPath: string): string;
35
+ /** Ensure both cache trees are available before a command starts work. */
36
+ export declare function ensureRepoCache(repoPath: string): Promise<void>;
37
+ /**
38
+ * Write `data` to `destPath` without ever leaving a partially-written file
39
+ * at that path. A crash (OOM kill, SIGKILL, power loss) mid-`writeFile`
40
+ * would otherwise truncate the destination in place; the next run would
41
+ * then hit a corrupt manifest/sidecar/index and, for the binary vector
42
+ * index, fail hard on load with no path back to a working cache. Writing to
43
+ * a sibling temp file first and `rename`-ing over the destination is safe
44
+ * because POSIX rename within the same directory (same filesystem) is
45
+ * atomic: readers only ever observe the old complete file or the new
46
+ * complete file, never a partial one.
47
+ */
48
+ export declare function atomicWriteFile(destPath: string, data: string | Uint8Array): Promise<void>;
25
49
  export declare function paths(cacheDir: string): {
26
50
  manifest: string;
27
51
  documentStore: string;
@@ -36,10 +60,14 @@ export declare function readDocumentStoreExport(cacheDir: string): Promise<Expor
36
60
  export declare function writeDocumentStoreExport(cacheDir: string, data: ExportedDocumentStore): Promise<void>;
37
61
  export declare function readVectorSidecar(cacheDir: string): Promise<SidecarData | null>;
38
62
  export declare function writeVectorSidecar(cacheDir: string, data: SidecarData): Promise<void>;
63
+ /** Remove the disposable vector cache when an incompatible manifest is found. */
64
+ export declare function removeVectorCache(cacheDir: string): Promise<void>;
39
65
  export declare function vectorIndexExists(cacheDir: string): Promise<boolean>;
40
66
  export interface FreshnessReport {
41
67
  /** Repo-relative paths that are new, changed, or removed since the last index for this scope. */
42
68
  staleCount: number;
69
+ /** Repo-relative paths whose vector embeddings are missing or stale. */
70
+ vectorStaleCount: number;
43
71
  /** True if this scope has never been indexed at all. */
44
72
  neverIndexed: boolean;
45
73
  }
@@ -51,5 +79,5 @@ export interface FreshnessReport {
51
79
  export declare function checkFreshness(manifest: Manifest | null, scope: Scope, currentFiles: {
52
80
  path: string;
53
81
  mtimeMs: number;
54
- }[]): FreshnessReport;
82
+ }[], requireVectors?: boolean): FreshnessReport;
55
83
  //# sourceMappingURL=persistence.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"persistence.d.ts","sourceRoot":"","sources":["../../src/storage/persistence.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,WAAW,aAAa;IAC5B,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,QAAQ;IACvB,iFAAiF;IACjF,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,cAAc,EAAE,MAAM,CAAC;IACvB,oDAAoD;IACpD,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACrC;6EACyE;IACzE,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,gDAAgD;IAChD,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAClC,eAAO,MAAM,kBAAkB,4BAA4B,CAAC;AAQ5D,mEAAmE;AACnE,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAKrD;AAOD,wBAAgB,KAAK,CAAC,QAAQ,EAAE,MAAM;;;;;EAOrC;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpE;AAED,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAS7E;AAED,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvF;AAED,wBAAgB,aAAa,IAAI,QAAQ,CAQxC;AAED,wBAAsB,uBAAuB,CAC3C,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC,CAOvC;AAED,wBAAsB,wBAAwB,CAC5C,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAEf;AAED,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAOrF;AAED,wBAAsB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAE3F;AAED,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAI1E;AAED,MAAM,WAAW,eAAe;IAC9B,iGAAiG;IACjG,UAAU,EAAE,MAAM,CAAC;IACnB,wDAAwD;IACxD,YAAY,EAAE,OAAO,CAAC;CACvB;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,QAAQ,GAAG,IAAI,EACzB,KAAK,EAAE,KAAK,EACZ,YAAY,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAAE,GAChD,eAAe,CA0BjB"}
1
+ {"version":3,"file":"persistence.d.ts","sourceRoot":"","sources":["../../src/storage/persistence.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,WAAW,aAAa;IAC5B,uFAAuF;IACvF,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,gFAAgF;IAChF,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,6EAA6E;IAC7E,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,QAAQ;IACvB,iFAAiF;IACjF,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,cAAc,EAAE,MAAM,CAAC;IACvB,oDAAoD;IACpD,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACrC;6EACyE;IACzE,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,gDAAgD;IAChD,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAClC,eAAO,MAAM,kBAAkB,4BAA4B,CAAC;AAE5D,qBAAa,oBAAqB,SAAQ,KAAK;CAAG;AAElD,2EAA2E;AAC3E,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEtD;AAED,wCAAwC;AACxC,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErD;AAED,wDAAwD;AACxD,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,0EAA0E;AAC1E,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAoBrE;AAOD;;;;;;;;;;GAUG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,GAAG,UAAU,GACxB,OAAO,CAAC,IAAI,CAAC,CASf;AAED,wBAAgB,KAAK,CAAC,QAAQ,EAAE,MAAM;;;;;EAOrC;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpE;AAED,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAS7E;AAED,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAEvF;AAED,wBAAgB,aAAa,IAAI,QAAQ,CAQxC;AAED,wBAAsB,uBAAuB,CAC3C,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC,CAOvC;AAED,wBAAsB,wBAAwB,CAC5C,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAEf;AAED,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAOrF;AAED,wBAAsB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAE3F;AAED,iFAAiF;AACjF,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKvE;AAED,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAI1E;AAED,MAAM,WAAW,eAAe;IAC9B,iGAAiG;IACjG,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,gBAAgB,EAAE,MAAM,CAAC;IACzB,wDAAwD;IACxD,YAAY,EAAE,OAAO,CAAC;CACvB;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,QAAQ,GAAG,IAAI,EACzB,KAAK,EAAE,KAAK,EACZ,YAAY,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAAE,EACjD,cAAc,UAAQ,GACrB,eAAe,CAwCjB"}
@@ -9,29 +9,72 @@
9
9
  * ever been indexed (so a query can refresh an un-indexed scope instead of
10
10
  * silently returning nothing).
11
11
  */
12
- import { mkdir, readFile, writeFile, stat } from "node:fs/promises";
13
- import { createHash } from "node:crypto";
14
- import { homedir } from "node:os";
12
+ import { mkdir, readFile, writeFile, stat, open, unlink, rename } from "node:fs/promises";
15
13
  import { join, resolve } from "node:path";
16
14
  import { scopesToRoots } from "../types.js";
17
- export const MANIFEST_VERSION = 1;
15
+ export const MANIFEST_VERSION = 3;
18
16
  export const EMBEDDING_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
19
- function cacheRoot() {
20
- const xdg = process.env.XDG_CACHE_HOME;
21
- const base = xdg && xdg.trim().length > 0 ? xdg : join(homedir(), ".cache");
22
- return join(base, "enact-docs-search");
17
+ export class CachePermissionError extends Error {
23
18
  }
24
- /** Deterministic, collision-resistant per-repo cache directory. */
19
+ /** Fixed repo-local cache root. This is intentionally not configurable. */
20
+ export function repoCacheRoot(repoPath) {
21
+ return join(resolve(repoPath), ".cache", "docs");
22
+ }
23
+ /** Repo-local index cache directory. */
25
24
  export function repoCacheDir(repoPath) {
26
- const abs = resolve(repoPath);
27
- const hash = createHash("sha256").update(abs).digest("hex").slice(0, 16);
28
- const base = abs.split("/").filter(Boolean).pop() ?? "repo";
29
- return join(cacheRoot(), `${base}-${hash}`);
25
+ return join(repoCacheRoot(repoPath), "index");
26
+ }
27
+ /** Repo-local Transformers.js model cache directory. */
28
+ export function repoModelCacheDir(repoPath) {
29
+ return join(repoCacheRoot(repoPath), "models");
30
+ }
31
+ /** Ensure both cache trees are available before a command starts work. */
32
+ export async function ensureRepoCache(repoPath) {
33
+ try {
34
+ const directories = [repoCacheDir(repoPath), repoModelCacheDir(repoPath)];
35
+ for (const directory of directories) {
36
+ await mkdir(directory, { recursive: true });
37
+ const probe = join(directory, `.write-probe-${process.pid}-${Date.now()}`);
38
+ const handle = await open(probe, "wx");
39
+ await handle.close();
40
+ await unlink(probe);
41
+ }
42
+ }
43
+ catch (error) {
44
+ const code = error.code;
45
+ if (code === "EACCES" || code === "EPERM" || code === "EROFS") {
46
+ throw new CachePermissionError(`The repository-local search cache is not writable: ${repoCacheRoot(repoPath)}. ` +
47
+ "The target repository must allow writes under .cache/docs.");
48
+ }
49
+ throw error;
50
+ }
30
51
  }
31
52
  const MANIFEST_FILE = "manifest.json";
32
53
  const DOCUMENT_STORE_FILE = "document-store.json";
33
54
  const VECTOR_INDEX_FILE = "vector-index.bin";
34
55
  const VECTOR_SIDECAR_FILE = "vector-sidecar.json";
56
+ /**
57
+ * Write `data` to `destPath` without ever leaving a partially-written file
58
+ * at that path. A crash (OOM kill, SIGKILL, power loss) mid-`writeFile`
59
+ * would otherwise truncate the destination in place; the next run would
60
+ * then hit a corrupt manifest/sidecar/index and, for the binary vector
61
+ * index, fail hard on load with no path back to a working cache. Writing to
62
+ * a sibling temp file first and `rename`-ing over the destination is safe
63
+ * because POSIX rename within the same directory (same filesystem) is
64
+ * atomic: readers only ever observe the old complete file or the new
65
+ * complete file, never a partial one.
66
+ */
67
+ export async function atomicWriteFile(destPath, data) {
68
+ const tmpPath = `${destPath}.tmp-${process.pid}-${Date.now()}`;
69
+ try {
70
+ await writeFile(tmpPath, data);
71
+ await rename(tmpPath, destPath);
72
+ }
73
+ catch (error) {
74
+ await unlink(tmpPath).catch(() => { });
75
+ throw error;
76
+ }
77
+ }
35
78
  export function paths(cacheDir) {
36
79
  return {
37
80
  manifest: join(cacheDir, MANIFEST_FILE),
@@ -56,7 +99,7 @@ export async function readManifest(cacheDir) {
56
99
  }
57
100
  }
58
101
  export async function writeManifest(cacheDir, manifest) {
59
- await writeFile(paths(cacheDir).manifest, JSON.stringify(manifest, null, 2), "utf-8");
102
+ await atomicWriteFile(paths(cacheDir).manifest, JSON.stringify(manifest, null, 2));
60
103
  }
61
104
  export function emptyManifest() {
62
105
  return {
@@ -77,7 +120,7 @@ export async function readDocumentStoreExport(cacheDir) {
77
120
  }
78
121
  }
79
122
  export async function writeDocumentStoreExport(cacheDir, data) {
80
- await writeFile(paths(cacheDir).documentStore, JSON.stringify(data), "utf-8");
123
+ await atomicWriteFile(paths(cacheDir).documentStore, JSON.stringify(data));
81
124
  }
82
125
  export async function readVectorSidecar(cacheDir) {
83
126
  try {
@@ -89,7 +132,14 @@ export async function readVectorSidecar(cacheDir) {
89
132
  }
90
133
  }
91
134
  export async function writeVectorSidecar(cacheDir, data) {
92
- await writeFile(paths(cacheDir).vectorSidecar, JSON.stringify(data), "utf-8");
135
+ await atomicWriteFile(paths(cacheDir).vectorSidecar, JSON.stringify(data));
136
+ }
137
+ /** Remove the disposable vector cache when an incompatible manifest is found. */
138
+ export async function removeVectorCache(cacheDir) {
139
+ await Promise.all([
140
+ unlink(paths(cacheDir).vectorIndex).catch(() => { }),
141
+ unlink(paths(cacheDir).vectorSidecar).catch(() => { }),
142
+ ]);
93
143
  }
94
144
  export async function vectorIndexExists(cacheDir) {
95
145
  return stat(paths(cacheDir).vectorIndex)
@@ -101,32 +151,46 @@ export async function vectorIndexExists(cacheDir) {
101
151
  * caller) against the manifest, without mutating anything. Used by `query`
102
152
  * to refresh a stale or nonexistent index before searching (I-08).
103
153
  */
104
- export function checkFreshness(manifest, scope, currentFiles) {
154
+ export function checkFreshness(manifest, scope, currentFiles, requireVectors = false) {
105
155
  // "Never indexed" is judged by overlap with the manifest, not by a scope
106
156
  // label match: indexing with --scope all and then querying --scope docs
107
157
  // (a subset) is a legitimate, fully-covered case. If there are files on
108
158
  // disk in this scope but none of them appear in the manifest at all, no
109
159
  // index run has ever covered this scope.
110
160
  if (!manifest)
111
- return { staleCount: 0, neverIndexed: true };
112
- if (currentFiles.length > 0 && currentFiles.every((f) => !(f.path in manifest.files))) {
113
- return { staleCount: 0, neverIndexed: true };
161
+ return { staleCount: 0, vectorStaleCount: 0, neverIndexed: true };
162
+ if (currentFiles.length > 0 && currentFiles.every((f) => {
163
+ const entry = manifest.files[f.path];
164
+ return !entry || entry.textMtimeMs === null;
165
+ })) {
166
+ return { staleCount: 0, vectorStaleCount: 0, neverIndexed: true };
114
167
  }
115
168
  let staleCount = 0;
169
+ let vectorStaleCount = 0;
116
170
  const seen = new Set();
117
171
  for (const f of currentFiles) {
118
172
  seen.add(f.path);
119
173
  const entry = manifest.files[f.path];
120
- if (!entry || entry.mtimeMs !== f.mtimeMs)
174
+ if (!entry || entry.textMtimeMs !== f.mtimeMs)
121
175
  staleCount++;
176
+ if (requireVectors &&
177
+ (!entry || entry.vectorStale || entry.vectorMtimeMs !== f.mtimeMs)) {
178
+ vectorStaleCount++;
179
+ }
122
180
  }
123
181
  // Files present in the manifest for this scope but deleted on disk also count as stale.
124
182
  const roots = scopesToRoots(scope);
125
183
  for (const path of Object.keys(manifest.files)) {
126
184
  const inScope = roots.some((r) => path === r || path.startsWith(r + "/"));
127
- if (inScope && !seen.has(path))
128
- staleCount++;
185
+ if (inScope && !seen.has(path)) {
186
+ const entry = manifest.files[path];
187
+ if (entry.textMtimeMs !== null)
188
+ staleCount++;
189
+ if (requireVectors && (entry.vectorStale || entry.vectorMtimeMs !== null)) {
190
+ vectorStaleCount++;
191
+ }
192
+ }
129
193
  }
130
- return { staleCount, neverIndexed: false };
194
+ return { staleCount, vectorStaleCount, neverIndexed: false };
131
195
  }
132
196
  //# sourceMappingURL=persistence.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"persistence.js","sourceRoot":"","sources":["../../src/storage/persistence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACpE,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAG1C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAsB5C,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAClC,MAAM,CAAC,MAAM,kBAAkB,GAAG,yBAAyB,CAAC;AAE5D,SAAS,SAAS;IAChB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IACvC,MAAM,IAAI,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC5E,OAAO,IAAI,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;AACzC,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzE,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC;IAC5D,OAAO,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,aAAa,GAAG,eAAe,CAAC;AACtC,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;AAClD,MAAM,iBAAiB,GAAG,kBAAkB,CAAC;AAC7C,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;AAElD,MAAM,UAAU,KAAK,CAAC,QAAgB;IACpC,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC;QACvC,aAAa,EAAE,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC;QAClD,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE,iBAAiB,CAAC;QAC9C,aAAa,EAAE,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC;KACnD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB;IACnD,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAa,CAAC;QAC3C,IAAI,MAAM,CAAC,OAAO,KAAK,gBAAgB;YAAE,OAAO,IAAI,CAAC;QACrD,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAgB,EAAE,QAAkB;IACtE,MAAM,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AACxF,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,OAAO,EAAE,gBAAgB;QACzB,cAAc,EAAE,kBAAkB;QAClC,KAAK,EAAE,EAAE;QACT,aAAa,EAAE,EAAE;QACjB,aAAa,EAAE,CAAC;KACjB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,QAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAA0B,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,QAAgB,EAChB,IAA2B;IAE3B,MAAM,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IACtD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAgB,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAgB,EAAE,IAAiB;IAC1E,MAAM,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IACtD,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC;SACrC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;AACxB,CAAC;AASD;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAC5B,QAAyB,EACzB,KAAY,EACZ,YAAiD;IAEjD,yEAAyE;IACzE,wEAAwE;IACxE,wEAAwE;IACxE,wEAAwE;IACxE,yCAAyC;IACzC,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IAC5D,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACtF,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IAC/C,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACjB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO;YAAE,UAAU,EAAE,CAAC;IAC1D,CAAC;IACD,wFAAwF;IACxF,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QAC1E,IAAI,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,UAAU,EAAE,CAAC;IAC/C,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;AAC7C,CAAC"}
1
+ {"version":3,"file":"persistence.js","sourceRoot":"","sources":["../../src/storage/persistence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1F,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAG1C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AA0B5C,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAClC,MAAM,CAAC,MAAM,kBAAkB,GAAG,yBAAyB,CAAC;AAE5D,MAAM,OAAO,oBAAqB,SAAQ,KAAK;CAAG;AAElD,2EAA2E;AAC3E,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;AACnD,CAAC;AAED,wCAAwC;AACxC,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;AAChD,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IAChD,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;AACjD,CAAC;AAED,0EAA0E;AAC1E,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB;IACpD,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC1E,KAAK,MAAM,SAAS,IAAI,WAAW,EAAE,CAAC;YACpC,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,gBAAgB,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC3E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YACvC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACrB,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,GAAI,KAA+B,CAAC,IAAI,CAAC;QACnD,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YAC9D,MAAM,IAAI,oBAAoB,CAC5B,sDAAsD,aAAa,CAAC,QAAQ,CAAC,IAAI;gBAC/E,4DAA4D,CAC/D,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,aAAa,GAAG,eAAe,CAAC;AACtC,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;AAClD,MAAM,iBAAiB,GAAG,kBAAkB,CAAC;AAC7C,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;AAElD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,QAAgB,EAChB,IAAyB;IAEzB,MAAM,OAAO,GAAG,GAAG,QAAQ,QAAQ,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC/D,IAAI,CAAC;QACH,MAAM,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC/B,MAAM,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACtC,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,QAAgB;IACpC,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC;QACvC,aAAa,EAAE,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC;QAClD,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE,iBAAiB,CAAC;QAC9C,aAAa,EAAE,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC;KACnD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB;IACnD,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAa,CAAC;QAC3C,IAAI,MAAM,CAAC,OAAO,KAAK,gBAAgB;YAAE,OAAO,IAAI,CAAC;QACrD,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAgB,EAAE,QAAkB;IACtE,MAAM,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,OAAO,EAAE,gBAAgB;QACzB,cAAc,EAAE,kBAAkB;QAClC,KAAK,EAAE,EAAE;QACT,aAAa,EAAE,EAAE;QACjB,aAAa,EAAE,CAAC;KACjB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,QAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAA0B,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,QAAgB,EAChB,IAA2B;IAE3B,MAAM,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IACtD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAgB,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAgB,EAAE,IAAiB;IAC1E,MAAM,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IACtD,MAAM,OAAO,CAAC,GAAG,CAAC;QAChB,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;QACnD,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;KACtD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IACtD,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC;SACrC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;AACxB,CAAC;AAWD;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAC5B,QAAyB,EACzB,KAAY,EACZ,YAAiD,EACjD,cAAc,GAAG,KAAK;IAEtB,yEAAyE;IACzE,wEAAwE;IACxE,wEAAwE;IACxE,wEAAwE;IACxE,yCAAyC;IACzC,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACjF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,CAAC;IAC9C,CAAC,CAAC,EAAE,CAAC;QACH,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACpE,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACjB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,CAAC,CAAC,OAAO;YAAE,UAAU,EAAE,CAAC;QAC5D,IAAI,cAAc;YAChB,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACrE,gBAAgB,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IACD,wFAAwF;IACxF,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QAC1E,IAAI,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;YACpC,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,UAAU,EAAE,CAAC;YAC7C,IAAI,cAAc,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,CAAC,EAAE,CAAC;gBAC1E,gBAAgB,EAAE,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;AAC/D,CAAC"}
@@ -16,6 +16,8 @@ export interface VectorSearchResult {
16
16
  }
17
17
  export interface VectorStoreOptions {
18
18
  dimensions: number;
19
+ /** Initial HNSW capacity. Grows automatically past this; override in tests only. */
20
+ initialMaxElements?: number;
19
21
  }
20
22
  interface VectorMetadata {
21
23
  id: string;
@@ -40,9 +42,34 @@ export declare class VectorStore {
40
42
  initialize(): Promise<void>;
41
43
  /** Load a previously persisted index plus its label/metadata sidecar. */
42
44
  loadFromDisk(indexPath: string, sidecar: SidecarData): Promise<void>;
43
- /** Persist the raw HNSW index plus the sidecar label/metadata mapping. */
45
+ /**
46
+ * Persist the raw HNSW index plus the sidecar label/metadata mapping.
47
+ *
48
+ * Writes to a sibling temp path and renames over `indexPath` so a crash
49
+ * mid-write (this binary file is large and the native write isn't
50
+ * instantaneous) can never leave a truncated `indexPath` in place —
51
+ * `loadFromDisk`'s `readIndex` has no way to recover from a corrupt file,
52
+ * and that failure would otherwise repeat on every subsequent run.
53
+ */
44
54
  saveToDisk(indexPath: string): Promise<SidecarData>;
45
55
  isReady(): boolean;
56
+ /**
57
+ * hnswlib's internal `cur_element_count` counts every label ever added,
58
+ * including ones later `markDelete`d — deletion only tombstones a slot, it
59
+ * never frees capacity (see hnswalg.h's addPoint: capacity is checked
60
+ * against `cur_element_count`, not the live point count). `add()` never
61
+ * reuses a deleted label (see the comment in `add()` below), so capacity
62
+ * is consumed monotonically by every add call, live or not.
63
+ *
64
+ * This must NOT be approximated with `this.idToLabel.size` (the live
65
+ * count): after any delete+re-add cycle — which is exactly what every
66
+ * incremental re-index does per changed file — live count stays flat
67
+ * while consumed capacity keeps climbing, so that check would never fire
68
+ * and `addPoint` would eventually throw "The number of elements exceeds
69
+ * the specified limit" past the initial capacity. `getCurrentCount()`
70
+ * reads the native counter directly, so it can't drift out of sync with
71
+ * our own bookkeeping.
72
+ */
46
73
  private resizeIfNeeded;
47
74
  add(doc: VectorDocument): Promise<void>;
48
75
  delete(id: string): Promise<void>;
@@ -1 +1 @@
1
- {"version":3,"file":"vector-store.d.ts","sourceRoot":"","sources":["../../src/storage/vector-store.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEzC,qBAAa,eAAgB,SAAQ,KAAK;CAAG;AAyB7C,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,sDAAsD;IACtD,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,cAAc;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,KAAK,CAAoC;IACjD,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,KAAK,CAAS;IAEtB,OAAO,CAAC,SAAS,CAAkC;IACnD,OAAO,CAAC,SAAS,CAAkC;IACnD,OAAO,CAAC,QAAQ,CAA0C;IAE1D,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,aAAa,CAAgB;IAErC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAM;IACxB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAO;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;gBAEnB,OAAO,EAAE,kBAAkB;IAIvC,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAejC,yEAAyE;IACnE,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB1E,0EAA0E;IACpE,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAYzD,OAAO,IAAI,OAAO;IAIlB,OAAO,CAAC,cAAc;IAShB,GAAG,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBvC,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAcjC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAUvD,+EAA+E;IACzE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,GAAE,KAAa,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IA2CvF,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;IAI3B,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAGxC;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IAC9B,QAAQ,EAAE,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,CAAC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;CACpB"}
1
+ {"version":3,"file":"vector-store.d.ts","sourceRoot":"","sources":["../../src/storage/vector-store.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAGzC,qBAAa,eAAgB,SAAQ,KAAK;CAAG;AAoB7C,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,sDAAsD;IACtD,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,oFAAoF;IACpF,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,UAAU,cAAc;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,KAAK,CAAoC;IACjD,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,KAAK,CAAS;IAEtB,OAAO,CAAC,SAAS,CAAkC;IACnD,OAAO,CAAC,SAAS,CAAkC;IACnD,OAAO,CAAC,QAAQ,CAA0C;IAE1D,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,aAAa,CAAgB;IAErC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAM;IACxB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAO;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAM;gBAEnB,OAAO,EAAE,kBAAkB;IAKvC,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAejC,yEAAyE;IACnE,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB1E;;;;;;;;OAQG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAmBzD,OAAO,IAAI,OAAO;IAIlB;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,cAAc;IAUhB,GAAG,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBvC,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAcjC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAUvD,+EAA+E;IACzE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,GAAE,KAAa,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IA2CvF,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;IAI3B,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAGxC;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IAC9B,QAAQ,EAAE,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,CAAC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;CACpB"}