opencode-rag-plugin 2.10.1 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/ReadMe.md +40 -28
  2. package/dist/api.js +1 -0
  3. package/dist/cli/commands/clickhouse.d.ts +61 -0
  4. package/dist/cli/commands/clickhouse.js +335 -0
  5. package/dist/cli/commands/eval-tuner.js +1 -1
  6. package/dist/cli/commands/eval.js +1 -1
  7. package/dist/cli/commands/index-command.js +2 -1
  8. package/dist/cli/commands/index.d.ts +1 -0
  9. package/dist/cli/commands/index.js +1 -0
  10. package/dist/cli/commands/init-helpers.js +2 -2
  11. package/dist/cli/commands/init.js +49 -11
  12. package/dist/cli/commands/quirk.js +6 -6
  13. package/dist/cli/commands/status.js +70 -6
  14. package/dist/cli/commands/ui.js +2 -2
  15. package/dist/cli/index.js +2 -1
  16. package/dist/cli/types.d.ts +14 -0
  17. package/dist/core/bootstrap.d.ts +7 -0
  18. package/dist/core/bootstrap.js +40 -9
  19. package/dist/core/config.d.ts +128 -5
  20. package/dist/core/config.js +205 -8
  21. package/dist/core/interfaces.d.ts +54 -0
  22. package/dist/core/runtime-overrides.d.ts +9 -0
  23. package/dist/core/runtime-overrides.js +44 -0
  24. package/dist/core/state-sync.d.ts +91 -0
  25. package/dist/core/state-sync.js +273 -0
  26. package/dist/eval/fast-index.js +2 -2
  27. package/dist/eval/rerank-eval.js +2 -2
  28. package/dist/eval/run-branch-compare.js +2 -2
  29. package/dist/eval/run-token-test.js +2 -2
  30. package/dist/index.d.ts +5 -2
  31. package/dist/index.js +2 -1
  32. package/dist/indexer/pipeline.d.ts +6 -0
  33. package/dist/indexer/pipeline.js +173 -56
  34. package/dist/indexer/progress.d.ts +75 -0
  35. package/dist/indexer/progress.js +114 -0
  36. package/dist/indexer/stats.d.ts +14 -0
  37. package/dist/plugin.d.ts +6 -3
  38. package/dist/plugin.js +39 -20
  39. package/dist/quirks/quirk-store.d.ts +10 -0
  40. package/dist/quirks/quirk-store.js +110 -15
  41. package/dist/quirks/types.d.ts +7 -0
  42. package/dist/tui.js +128 -111
  43. package/dist/vectorstore/clickhouse.d.ts +150 -0
  44. package/dist/vectorstore/clickhouse.js +931 -0
  45. package/dist/vectorstore/dimension.d.ts +19 -0
  46. package/dist/vectorstore/dimension.js +36 -0
  47. package/dist/vectorstore/factory.d.ts +9 -3
  48. package/dist/vectorstore/factory.js +11 -2
  49. package/dist/vectorstore/lancedb.d.ts +2 -2
  50. package/dist/watcher.d.ts +29 -0
  51. package/dist/watcher.js +76 -2
  52. package/dist/web/api.d.ts +3 -4
  53. package/dist/web/api.js +18 -3
  54. package/dist/web/server.js +28 -2
  55. package/dist/web/ui/assets/index-B-RH6fko.css +2 -0
  56. package/dist/web/ui/index.html +2 -2
  57. package/package.json +2 -1
  58. package/dist/web/ui/assets/index-CWcp0jNf.css +0 -2
  59. /package/dist/web/ui/assets/{index-DbXFrn53.js → index-ChGi2bSp.js} +0 -0
@@ -2,6 +2,7 @@
2
2
  * @fileoverview Orchestrates the full indexing pipeline: scan, chunk, embed, and store.
3
3
  */
4
4
  import fs from "node:fs/promises";
5
+ import os from "node:os";
5
6
  import path from "node:path";
6
7
  import pLimit from "p-limit";
7
8
  import { scanWorkspaceFiles } from "../content/reader.js";
@@ -12,6 +13,7 @@ import { createVectorStore } from "../vectorstore/factory.js";
12
13
  import { swapStoreDirectories } from "../vectorstore/lancedb.js";
13
14
  import { reconcileQuirks, QUIRK_FILE_PREFIX } from "../quirks/quirk-store.js";
14
15
  import { rehydrateKeywordIndex } from "./keyword-rehydration.js";
16
+ import { PassProgress } from "./progress.js";
15
17
  import { createIndexStats } from "./stats.js";
16
18
  import { prepareFile, buildTextsToEmbed } from "./worker.js";
17
19
  import { tagChunksRole } from "../core/chunk-role.js";
@@ -131,10 +133,48 @@ export async function runIndexPass(options) {
131
133
  // concurrent-pass protection above silently stops working.
132
134
  logger.debug(`Could not write index lock file: ${err.message} — concurrent-pass protection disabled`);
133
135
  }
136
+ // Cross-machine lease: central stores have no local directory in common, so
137
+ // the file lock above only protects this machine. The lease is advisory
138
+ // (ClickHouse has no CAS) but stops a second machine from re-embedding the
139
+ // same workspace concurrently.
140
+ const leaseOwner = `${os.hostname()}:${process.pid}`;
141
+ const leaseKind = "index";
142
+ const leaseTtlMs = Math.max(10, options.config.vectorStore.clickhouse?.lease?.ttlSeconds ?? 90) * 1000;
143
+ let leaseHeld = false;
144
+ let leaseTimer;
145
+ if (typeof options.store.acquireLease === "function") {
146
+ try {
147
+ leaseHeld = await options.store.acquireLease(leaseOwner, leaseKind, leaseTtlMs);
148
+ }
149
+ catch (err) {
150
+ // Do not leak the local lock file when the central store is unreachable.
151
+ try {
152
+ await fs.unlink(lockPath);
153
+ }
154
+ catch { /* best effort */ }
155
+ throw err;
156
+ }
157
+ if (!leaseHeld) {
158
+ logger.warn("Another machine holds the central store's index lease. Skipping.");
159
+ try {
160
+ await fs.unlink(lockPath);
161
+ }
162
+ catch { /* best effort */ }
163
+ return { ...createIndexStats(0, "missing"), skipped: true };
164
+ }
165
+ leaseTimer = setInterval(() => {
166
+ void options.store.renewLease?.(leaseOwner, leaseKind, leaseTtlMs).catch(() => { });
167
+ }, Math.max(5_000, Math.floor(leaseTtlMs / 3)));
168
+ leaseTimer.unref();
169
+ }
134
170
  try {
135
171
  return await runIndexPassInner(options, logger);
136
172
  }
137
173
  finally {
174
+ if (leaseTimer)
175
+ clearInterval(leaseTimer);
176
+ if (leaseHeld)
177
+ await options.store.releaseLease?.(leaseOwner, leaseKind).catch(() => { });
138
178
  try {
139
179
  await fs.unlink(lockPath);
140
180
  }
@@ -306,6 +346,8 @@ async function runIndexPassInner(options, logger) {
306
346
  // Effective store used throughout the pass — may be a temp store for atomic rebuild.
307
347
  let effectiveStore = options.store;
308
348
  let tempStorePath;
349
+ /** True when the rebuild writes into a private epoch of a central store. */
350
+ let rebuildTarget = false;
309
351
  /** Set when the temporary rebuild store actually received rows. */
310
352
  let tempStoreWroteChunks = false;
311
353
  if (options.force || (manifestStatus !== "ok" && (existingCount > 0 || dimensionMismatch))) {
@@ -321,13 +363,21 @@ async function runIndexPassInner(options, logger) {
321
363
  // Build the new index into a temporary store first, then atomically
322
364
  // swap on completion. Original data stays untouched if the process
323
365
  // is aborted (Ctrl+C, crash) before the swap completes.
324
- if (options.dimension) {
366
+ if (typeof options.store.beginRebuild === "function") {
367
+ // Central stores (ClickHouse) rebuild into a private epoch and promote
368
+ // atomically; no local temp directory and no directory swap.
369
+ effectiveStore = await options.store.beginRebuild();
370
+ rebuildTarget = true;
371
+ logger.debug("Rebuilding index in a private store epoch");
372
+ options.keywordIndex?.clear();
373
+ }
374
+ else if (options.dimension) {
325
375
  tempStorePath = options.storePath + "_tmp";
326
376
  try {
327
377
  await fs.rm(tempStorePath, { recursive: true, force: true });
328
378
  }
329
379
  catch { /* may not exist */ }
330
- effectiveStore = createVectorStore(options.config, tempStorePath, options.dimension);
380
+ effectiveStore = createVectorStore(options.config, tempStorePath, options.dimension, options.cwd);
331
381
  logger.debug(`Rebuilding index in temporary store at ${tempStorePath}`);
332
382
  // Only clear the shared in-memory keyword index once the rebuild is
333
383
  // guaranteed to proceed — the guard below may still bail out, and
@@ -495,17 +545,20 @@ async function runIndexPassInner(options, logger) {
495
545
  logger.debug(`Processing ${workspaceFiles.length} files (${newCount} new, ${modifiedCount} modified, ${unchangedCount} unchanged)...`);
496
546
  const limit = pLimit(options.config.indexing.concurrency);
497
547
  const deferDescriptions = !!options.descriptionProvider;
498
- const activeCount = workspaceFiles.filter((f) => !f.isEmpty && !f.isTooSmall && (!manifest.files[f.normalizedPath] || manifest.files[f.normalizedPath].hash !== f.hash)).length;
548
+ const isActiveFile = (f) => !f.isEmpty && !f.isTooSmall && (!manifest.files[f.normalizedPath] || manifest.files[f.normalizedPath].hash !== f.hash);
549
+ const activeCount = workspaceFiles.filter(isActiveFile).length;
499
550
  logger.info(`Chunking ${activeCount} file(s)...`);
551
+ // Whole-pass progress in file units: every progress line reports this
552
+ // tracker's percentage, never a per-chunk or per-stage fraction.
553
+ const passProgress = new PassProgress(activeCount);
500
554
  let chunkedDone = 0;
501
555
  const chunkProgressInterval = Math.max(1, Math.floor(activeCount / 20));
502
556
  const chunkPhaseStart = Date.now();
503
557
  /** Chunk one window of files (used by the windowed pipeline loop below). */
504
- async function prepareWindow(windowFiles) {
558
+ async function prepareWindow(windowFiles, winProgress) {
505
559
  const prepared = await Promise.all(windowFiles.map((file) => limit(async () => {
506
560
  const fileLabel = path.relative(options.cwd, file.normalizedPath).replace(/\\/g, "/");
507
- const isActive = !file.isEmpty && !file.isTooSmall &&
508
- (!manifest.files[file.normalizedPath] || manifest.files[file.normalizedPath].hash !== file.hash);
561
+ const isActive = isActiveFile(file);
509
562
  if (isActive) {
510
563
  options.progress?.startFile(fileLabel);
511
564
  }
@@ -517,9 +570,9 @@ async function runIndexPassInner(options, logger) {
517
570
  }
518
571
  if (isActive) {
519
572
  chunkedDone++;
573
+ winProgress.chunkFile();
520
574
  if (chunkedDone % chunkProgressInterval === 0 || chunkedDone === activeCount) {
521
- const pct = ((chunkedDone / activeCount) * 100).toFixed(1);
522
- logger.info(` Chunking: ${chunkedDone}/${activeCount} files (${pct}%)`);
575
+ logProgress("Chunking", "", chunkedDone, activeCount, "files");
523
576
  }
524
577
  }
525
578
  if (prep.earlyResult && isActive) {
@@ -530,11 +583,14 @@ async function runIndexPassInner(options, logger) {
530
583
  return prepared;
531
584
  }
532
585
  const aborted = () => options.abortSignal?.aborted ?? false;
533
- // Shared progress logger: "stage <file> (chunk i/n) — X/total remaining (P%)".
534
- const logChunkProgress = (stage, fileLabel, index, count, completed, total) => {
535
- const remaining = total - completed;
536
- const pct = total > 0 ? ((remaining / total) * 100).toFixed(1) : "0.0";
537
- logger.info(`${stage} ${fileLabel} (chunk ${index}/${count}) — ${remaining}/${total} remaining (${pct}%)`);
586
+ // Progress line shared by all four stages: "<Stage> [file] — NN.N% overall (done/total unit)".
587
+ // The percentage is ALWAYS the fraction of the WHOLE pass completed
588
+ // (PassProgress across all windows and stages) — never a per-chunk or
589
+ // per-stage fraction. The stage-local counters ride along as a plain
590
+ // done/total count in parentheses.
591
+ const logProgress = (stage, fileLabel, done, total, unit) => {
592
+ const detail = total > 0 ? ` (${done}/${total} ${unit})` : "";
593
+ logger.info(`${stage}${fileLabel ? ` ${fileLabel}` : ""} — ${passProgress.label} overall${detail}`);
538
594
  };
539
595
  // File metadata look-up for manifest entries
540
596
  const fileMeta = new Map(workspaceFiles.map((f) => [f.normalizedPath, { mtime: f.mtime, size: f.size }]));
@@ -568,10 +624,14 @@ async function runIndexPassInner(options, logger) {
568
624
  * version-manifest accumulation (K+2 versions per file) that made the
569
625
  * store phase degrade quadratically as the index grew.
570
626
  */
571
- async function storeWindow(prepared, windowEarlyResults) {
627
+ async function storeWindow(prepared, windowEarlyResults, winProgress) {
572
628
  const filesToStore = prepared.filter((p) => !windowEarlyResults.has(prepared.indexOf(p)) && p.chunks && (p.textToEmbed?.length ?? 0) > 0).length;
573
629
  const storePhaseStart = Date.now();
574
630
  logger.info(`Store phase: storing ${filesToStore} file(s) into vector database...`);
631
+ // Nothing to store (every file in this window had an early result or no
632
+ // chunks) — the store stage is trivially done, credit it immediately.
633
+ if (filesToStore === 0)
634
+ winProgress.fill("store");
575
635
  let storedFiles = 0;
576
636
  // Files whose chunks ALL failed to embed while no manifest entry exists.
577
637
  // Rows an earlier pass wrote for them would otherwise never be removed:
@@ -703,7 +763,8 @@ async function runIndexPassInner(options, logger) {
703
763
  }
704
764
  options.progress?.finishFile(prep.fileLabel);
705
765
  storedFiles++;
706
- logChunkProgress("Storing", prep.fileLabel, storedFiles, filesToStore, storedFiles, filesToStore);
766
+ winProgress.stage("store", storedFiles, filesToStore);
767
+ logProgress("Storing", prep.fileLabel, storedFiles, filesToStore, "files");
707
768
  storeResults.push(result);
708
769
  }
709
770
  const workerResults = storeResults;
@@ -753,7 +814,11 @@ async function runIndexPassInner(options, logger) {
753
814
  }
754
815
  windowCount++;
755
816
  const windowFiles = workspaceFiles.slice(windowStart, windowStart + WINDOW_SIZE);
756
- const prepared = await prepareWindow(windowFiles);
817
+ // This window's credit handle. The active-file count is taken here, up
818
+ // front: manifest entries for THIS window's files only appear once its
819
+ // store phase runs, so the count stays stable across the window.
820
+ const winProgress = passProgress.beginWindow(windowFiles.filter(isActiveFile).length);
821
+ const prepared = await prepareWindow(windowFiles, winProgress);
757
822
  if (windowStart === 0) {
758
823
  const chunkPhaseSec = ((Date.now() - chunkPhaseStart) / 1000).toFixed(1);
759
824
  logger.info(`Chunking complete: ${activeCount} files in ${chunkPhaseSec}s`);
@@ -780,31 +845,16 @@ async function runIndexPassInner(options, logger) {
780
845
  for (const chunk of oversizedChunks) {
781
846
  chunk.description = buildFallbackDescription(chunk);
782
847
  }
783
- // Per-file chunk index + label for clear progress reporting, plus a shared
784
- // progress logger emitting "stage <file> (chunk i/n) — X/total remaining (P%)".
848
+ // chunk.id → file label for the progress lines. The overall
849
+ // percentage comes from PassProgress; the per-chunk index inside the
850
+ // file is deliberately NOT shown.
785
851
  const chunkToFileLabel = new Map();
786
852
  for (const prep of deferredPreps) {
787
853
  for (const chunk of prep.chunks ?? []) {
788
854
  chunkToFileLabel.set(chunk.id, prep.fileLabel);
789
855
  }
790
856
  }
791
- const chunkMeta = new Map();
792
- {
793
- const perFileCount = new Map();
794
- for (const chunk of allChunks) {
795
- perFileCount.set(chunk.metadata.filePath, (perFileCount.get(chunk.metadata.filePath) ?? 0) + 1);
796
- }
797
- const perFileSeen = new Map();
798
- for (const chunk of allChunks) {
799
- const seen = perFileSeen.get(chunk.metadata.filePath) ?? 0;
800
- perFileSeen.set(chunk.metadata.filePath, seen + 1);
801
- chunkMeta.set(chunk.id, {
802
- fileLabel: chunkToFileLabel.get(chunk.id) ?? chunk.metadata.filePath,
803
- index: seen + 1,
804
- count: perFileCount.get(chunk.metadata.filePath) ?? 1,
805
- });
806
- }
807
- }
857
+ const describeFileLabel = (chunk) => chunkToFileLabel.get(chunk.id) ?? chunk.metadata.filePath;
808
858
  // Advance progress to Description stage before descriptions start
809
859
  for (const prep of deferredPreps) {
810
860
  options.progress?.finishStage(prep.fileLabel);
@@ -831,6 +881,8 @@ async function runIndexPassInner(options, logger) {
831
881
  for (const { chunk, desc } of cacheHits) {
832
882
  chunk.description = desc;
833
883
  }
884
+ // Cache hits are already-described chunks of this window.
885
+ winProgress.stage("describe", cacheHits.length, allChunks.length);
834
886
  }
835
887
  // Build chunkId → prep map for tracking description failures per-file
836
888
  const chunkToPrep = new Map();
@@ -862,9 +914,8 @@ async function runIndexPassInner(options, logger) {
862
914
  total: totalMisses,
863
915
  onProgress: (chunk) => {
864
916
  describedDone++;
865
- const meta = chunkMeta.get(chunk.id);
866
- if (meta)
867
- logChunkProgress("Describing", meta.fileLabel, meta.index, meta.count, describedDone, totalMisses);
917
+ winProgress.stage("describe", cacheHits.length + describedDone, allChunks.length);
918
+ logProgress("Describing", describeFileLabel(chunk), describedDone, totalMisses, "texts");
868
919
  },
869
920
  });
870
921
  const newCacheEntries = [];
@@ -911,9 +962,8 @@ async function runIndexPassInner(options, logger) {
911
962
  total: allChunks.length,
912
963
  onProgress: (chunk) => {
913
964
  describedDone++;
914
- const meta = chunkMeta.get(chunk.id);
915
- if (meta)
916
- logChunkProgress("Describing", meta.fileLabel, meta.index, meta.count, describedDone, allChunks.length);
965
+ winProgress.stage("describe", describedDone, allChunks.length);
966
+ logProgress("Describing", describeFileLabel(chunk), describedDone, allChunks.length, "texts");
917
967
  },
918
968
  });
919
969
  for (const chunk of allChunks) {
@@ -939,6 +989,11 @@ async function runIndexPassInner(options, logger) {
939
989
  logger.info(`Description phase complete: ${totalDescribedChunks} chunks across ${deferredPreps.length} files`);
940
990
  }
941
991
  }
992
+ // The describe stage of this window is finished — or was skipped entirely
993
+ // (no description provider, nothing to describe, or a global description
994
+ // failure). Credit the remaining fraction so the overall percentage keeps
995
+ // moving toward 100.
996
+ winProgress.fill("describe");
942
997
  // Cross-file embedding batch: collect this window's texts into a single queue,
943
998
  // embed in one batched call (with concurrency), then distribute back.
944
999
  const isOllama = options.embedder.name === "ollama";
@@ -946,7 +1001,6 @@ async function runIndexPassInner(options, logger) {
946
1001
  const defaultConcurrency = options.config.indexing.embedConcurrency ?? 1;
947
1002
  // ── Phase 1: Collect embed queue + handle early results ────────────────
948
1003
  const embedQueue = [];
949
- let totalEmbedChunks = 0;
950
1004
  earlyWorkerResults.clear();
951
1005
  for (let fi = 0; fi < prepared.length; fi++) {
952
1006
  const prep = prepared[fi];
@@ -975,13 +1029,11 @@ async function runIndexPassInner(options, logger) {
975
1029
  for (let ci = 0; ci < prep.textToEmbed.length; ci++) {
976
1030
  embedQueue.push({ fileIdx: fi, chunkIdx: ci, text: prep.textToEmbed[ci] });
977
1031
  }
978
- totalEmbedChunks += prep.textToEmbed.length;
979
1032
  }
980
1033
  // ── Phase 2: Embed all texts in a single batched call ──────────────────
981
1034
  const batchSize = isOllama
982
1035
  ? Math.min(options.config.indexing.ollamaMaxBatchSize ?? 500, defaultBatchSize)
983
1036
  : defaultBatchSize;
984
- let embeddedDone = 0;
985
1037
  const allTexts = embedQueue.map(item => item.text);
986
1038
  let allEmbeddings = [];
987
1039
  const embedPhaseStart = Date.now();
@@ -989,8 +1041,8 @@ async function runIndexPassInner(options, logger) {
989
1041
  logger.info(`Embedding phase: ${allTexts.length} texts in batches of ${batchSize}...`);
990
1042
  try {
991
1043
  allEmbeddings = await embedBatch(options.embedder, allTexts, batchSize, "document", defaultConcurrency, (completed, total) => {
992
- embeddedDone = completed;
993
- logChunkProgress("Embedding", "", completed, total, embeddedDone, totalEmbedChunks);
1044
+ winProgress.stage("embed", completed, total);
1045
+ logProgress("Embedding", "", completed, total, "texts");
994
1046
  });
995
1047
  logger.info(`Embedding complete: ${allTexts.length} texts in ${((Date.now() - embedPhaseStart) / 1000).toFixed(1)}s`);
996
1048
  }
@@ -1027,6 +1079,10 @@ async function runIndexPassInner(options, logger) {
1027
1079
  }
1028
1080
  }
1029
1081
  }
1082
+ // Embed stage done for this window — worked, skipped (nothing to embed),
1083
+ // or globally failed (a failed embed leaves nothing to store, so holding
1084
+ // the credit back would stall the percentage without helping anyone).
1085
+ winProgress.fill("embed");
1030
1086
  // ── Distribute embeddings back to per-file chunks ─────────────────────
1031
1087
  for (let i = 0; i < embedQueue.length; i++) {
1032
1088
  const { fileIdx, chunkIdx } = embedQueue[i];
@@ -1048,7 +1104,7 @@ async function runIndexPassInner(options, logger) {
1048
1104
  // asynchronously (overlapped with the next window's prepare/embed), so a
1049
1105
  // shared mutable map would make its per-file loop read the WRONG window's
1050
1106
  // entries and skip every file ("Store phase: storing N → 0 files stored").
1051
- prevStore = storeWindow(prepared, new Map(earlyWorkerResults));
1107
+ prevStore = storeWindow(prepared, new Map(earlyWorkerResults), winProgress);
1052
1108
  // Periodically compact fragments and prune old versions so the store
1053
1109
  // phase doesn't slow down as the index grows during long runs. Skipped on
1054
1110
  // windows that wrote nothing — an idle pass (no changed files) adds no
@@ -1128,9 +1184,52 @@ async function runIndexPassInner(options, logger) {
1128
1184
  if (!aborted()) {
1129
1185
  tryUpdateLastGitCommit(options.cwd, manifest);
1130
1186
  }
1131
- // ── Atomically promote temp store if a full rebuild was performed ──
1187
+ // ── Atomically promote a rebuilt epoch / temp store ──
1132
1188
  let tempStorePromoted = false;
1133
- if (tempStorePath) {
1189
+ if (rebuildTarget) {
1190
+ if (!aborted()) {
1191
+ if (!tempStoreWroteChunks) {
1192
+ // Nothing was written (e.g. every embedding request failed). Keep the
1193
+ // existing epoch rather than promoting an empty one over good data.
1194
+ logger.warn("Rebuild produced no chunks — keeping the existing index and manifest unchanged.");
1195
+ try {
1196
+ await options.store.discardRebuild?.();
1197
+ }
1198
+ catch { /* best effort */ }
1199
+ }
1200
+ else {
1201
+ try {
1202
+ await options.store.promoteRebuild?.();
1203
+ tempStorePromoted = true;
1204
+ logger.debug("Promoted rebuilt store epoch");
1205
+ }
1206
+ catch (err) {
1207
+ logger.warn(`Could not promote the rebuilt epoch: ${err.message}`);
1208
+ try {
1209
+ await options.store.discardRebuild?.();
1210
+ }
1211
+ catch { /* best effort */ }
1212
+ }
1213
+ }
1214
+ try {
1215
+ await effectiveStore.close();
1216
+ }
1217
+ catch { /* separate client handle */ }
1218
+ }
1219
+ else {
1220
+ // Aborted — discard the epoch, keep the active data intact.
1221
+ try {
1222
+ await options.store.discardRebuild?.();
1223
+ }
1224
+ catch { /* best effort */ }
1225
+ effectiveStore.close().catch(() => { });
1226
+ logger.debug("Index pass cancelled; discarded the rebuild epoch.");
1227
+ // Skip the final manifest/keyword save: the old state is still valid.
1228
+ await descCache.save();
1229
+ return stats;
1230
+ }
1231
+ }
1232
+ else if (tempStorePath) {
1134
1233
  if (!aborted()) {
1135
1234
  if (!tempStoreWroteChunks) {
1136
1235
  // Nothing was written (e.g. every embedding request failed). Keep the
@@ -1188,7 +1287,7 @@ async function runIndexPassInner(options, logger) {
1188
1287
  // for the new store. When the rebuild did not promote (empty temp store or
1189
1288
  // failed swap), keep the previous manifest/keyword index on disk — saving
1190
1289
  // the cleared state would orphan the existing store's data.
1191
- const keepPreviousIndexState = tempStorePath !== undefined && !tempStorePromoted;
1290
+ const keepPreviousIndexState = (tempStorePath !== undefined || rebuildTarget) && !tempStorePromoted;
1192
1291
  if (keepPreviousIndexState) {
1193
1292
  logger.warn("Rebuild did not complete — keeping the previous manifest and keyword index.");
1194
1293
  }
@@ -1223,6 +1322,14 @@ async function runIndexPassInner(options, logger) {
1223
1322
  // a successful swap this points to the new data).
1224
1323
  await saveManifest(options.storePath, manifest);
1225
1324
  await options.keywordIndex?.save(options.storePath);
1325
+ // ClickHouse: upload the durable state bundle (manifest, keyword index,
1326
+ // quirk log) so a fresh clone on another machine needs no re-embedding.
1327
+ try {
1328
+ await options.stateSync?.flush(["manifest", "keyword-index", "quirks", "runtime-overrides", "doc-progress", ".desc-cache"]);
1329
+ }
1330
+ catch (err) {
1331
+ logger.warn(`State upload failed: ${err.message}`);
1332
+ }
1226
1333
  }
1227
1334
  // Compact fragments and prune old version manifests so countRows() can't
1228
1335
  // hang on accumulated versions from many add/delete cycles. After a temp
@@ -1234,14 +1341,14 @@ async function runIndexPassInner(options, logger) {
1234
1341
  // Every optimize rebuilds the ANN index (a ~190 MB vector copy at 4096
1235
1342
  // dims), so passes within the throttle window skip it; the next due pass
1236
1343
  // compacts and prunes, which also reclaims the skipped builds.
1237
- if (!optimizeDue({ force: Boolean(tempStorePath) })) {
1344
+ if (!optimizeDue({ force: Boolean(tempStorePath || rebuildTarget) })) {
1238
1345
  logger.debug(`Skipping vector store optimize (throttled: < ${MIN_OPTIMIZE_INTERVAL_MS / 60000} min since the last run)`);
1239
1346
  }
1240
1347
  else {
1241
1348
  logger.info("Optimizing vector store (compacting fragments, pruning old versions)...");
1242
1349
  const optimizeStart = Date.now();
1243
1350
  try {
1244
- const optimizeTarget = tempStorePath ? options.store : effectiveStore;
1351
+ const optimizeTarget = (tempStorePath || rebuildTarget) ? options.store : effectiveStore;
1245
1352
  await optimizeTarget.optimize?.(tempStorePath ? { aggressive: true, logger: logger.warn } : { logger: logger.warn });
1246
1353
  logger.info(`Vector store optimized in ${((Date.now() - optimizeStart) / 1000).toFixed(1)}s`);
1247
1354
  }
@@ -1326,6 +1433,7 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
1326
1433
  storeChunkCount: storeCount,
1327
1434
  manifestExpectedChunks: 0,
1328
1435
  orphanChunks: 0,
1436
+ scanned: false,
1329
1437
  };
1330
1438
  }
1331
1439
  let upToDateFiles = 0;
@@ -1359,24 +1467,31 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
1359
1467
  manifestExpectedChunks = Object.values(manifest.files).reduce((sum, entry) => sum + entry.chunkCount, 0);
1360
1468
  }
1361
1469
  // Rows without a manifest entry (partial-embedding leftovers). Quirk chunks
1362
- // live in quirks.jsonl and are deliberately not counted as orphans. A store
1363
- // that cannot enumerate files simply reports 0.
1470
+ // live in quirks.jsonl and are deliberately not counted as orphans — they
1471
+ // are counted separately so `Indexed = Expected + Orphan (+ Quirk)` actually
1472
+ // reconciles. A store that cannot enumerate files reports 0 orphans and
1473
+ // leaves quirkChunks undefined (unknown, not zero).
1364
1474
  let orphanChunks = 0;
1475
+ let quirkChunks = 0;
1365
1476
  try {
1366
1477
  const manifestKeysLower = new Set(Object.keys(manifest.files).map((k) => k.toLowerCase()));
1478
+ quirkChunks = 0;
1367
1479
  for (const file of await store.listFiles()) {
1368
1480
  const key = normalizeFilePath(file.filePath);
1481
+ if (path.basename(key).startsWith(QUIRK_FILE_PREFIX)) {
1482
+ quirkChunks += file.chunkCount;
1483
+ continue;
1484
+ }
1369
1485
  if (manifest.files[key])
1370
1486
  continue;
1371
1487
  if (manifestKeysLower.has(key.toLowerCase()))
1372
1488
  continue;
1373
- if (path.basename(key).startsWith(QUIRK_FILE_PREFIX))
1374
- continue;
1375
1489
  orphanChunks += file.chunkCount;
1376
1490
  }
1377
1491
  }
1378
1492
  catch {
1379
1493
  orphanChunks = 0;
1494
+ quirkChunks = undefined;
1380
1495
  }
1381
1496
  return {
1382
1497
  manifestStatus: loadResult.status,
@@ -1388,6 +1503,8 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
1388
1503
  storeChunkCount: storeCount,
1389
1504
  manifestExpectedChunks,
1390
1505
  orphanChunks,
1506
+ scanned: !skipScan,
1507
+ quirkChunks,
1391
1508
  };
1392
1509
  }
1393
1510
  //# sourceMappingURL=pipeline.js.map
@@ -0,0 +1,75 @@
1
+ /**
2
+ * @fileoverview Whole-pass progress accounting for a single index run.
3
+ *
4
+ * Every progress line the `index` command prints reports ONE percentage:
5
+ * the fraction of the WHOLE pass completed — never a per-chunk or
6
+ * per-stage fraction. This module owns that accounting; the pipeline
7
+ * threads a `WindowProgress` handle through the chunk/describe/embed/store
8
+ * stages of each window and emits lines via the shared `logProgress`
9
+ * helper in pipeline.ts.
10
+ */
11
+ /** Stage names that report a done/total fraction (chunking counts files directly). */
12
+ export type ProgressStage = "describe" | "embed" | "store";
13
+ /**
14
+ * Tracks the progress of one full index pass in file units.
15
+ *
16
+ * Every active (new or modified, non-empty, large-enough) file contributes
17
+ * one unit, split into four equal credits: chunked, described, embedded,
18
+ * stored. Credits are pushed by `WindowProgress` handles as their window
19
+ * advances, so `percent` rises monotonically from 0 toward 100 across ALL
20
+ * windows and stages — unlike the old per-stage "remaining" percentage
21
+ * that counted down and restarted inside every window.
22
+ *
23
+ * Files are weighted equally regardless of chunk count: the percentage is
24
+ * an honest estimate of "how far through the run are we", not a precise
25
+ * work-meter.
26
+ */
27
+ export declare class PassProgress {
28
+ readonly totalFiles: number;
29
+ private credited;
30
+ /** @param totalFiles - Total number of active files the pass will process. */
31
+ constructor(totalFiles: number);
32
+ /**
33
+ * Start accounting for one processing window.
34
+ * @param activeFiles - How many of the window's files are active (the same
35
+ * predicate that produced `totalFiles`). Windows with 0 active files
36
+ * contribute nothing to the percentage.
37
+ */
38
+ beginWindow(activeFiles: number): WindowProgress;
39
+ /** @internal Called by {@link WindowProgress} to push credit into the pass total. */
40
+ push(credit: number): void;
41
+ /** Whole-pass percentage, 0–100, clamped. */
42
+ get percent(): number;
43
+ /** Formatted percentage ("42.3%") for progress lines. */
44
+ get label(): string;
45
+ }
46
+ /**
47
+ * Credit handle for one window of the windowed pipeline. The pipeline
48
+ * reports each stage's done/total fraction here; the handle converts it to
49
+ * whole-pass credit by scaling with the window's active-file count.
50
+ *
51
+ * All updates are clamped to move FORWARD only, so repeated, out-of-order,
52
+ * or overlapping callbacks (concurrent embed sub-batches, the store phase
53
+ * still running while the next window prepares) can never make the overall
54
+ * percentage go backwards.
55
+ */
56
+ export declare class WindowProgress {
57
+ private readonly owner;
58
+ readonly files: number;
59
+ private done;
60
+ private chunkedFiles;
61
+ constructor(owner: PassProgress, files: number);
62
+ /** Fold the stage-fraction delta into the pass total (forward-only). */
63
+ private advance;
64
+ /** One of this window's active files finished chunking. */
65
+ chunkFile(): void;
66
+ /**
67
+ * Report live progress of a per-stage work queue (described texts,
68
+ * embedded texts, stored files). `total` 0 means the stage had no work for
69
+ * this window (early-result files, full description-cache hits, a skipped
70
+ * stage) and fills the stage's credit immediately.
71
+ */
72
+ stage(stage: ProgressStage, done: number, total: number): void;
73
+ /** Mark a stage complete regardless of its counters (stage done, skipped, or failed). */
74
+ fill(stage: ProgressStage): void;
75
+ }
@@ -0,0 +1,114 @@
1
+ /**
2
+ * @fileoverview Whole-pass progress accounting for a single index run.
3
+ *
4
+ * Every progress line the `index` command prints reports ONE percentage:
5
+ * the fraction of the WHOLE pass completed — never a per-chunk or
6
+ * per-stage fraction. This module owns that accounting; the pipeline
7
+ * threads a `WindowProgress` handle through the chunk/describe/embed/store
8
+ * stages of each window and emits lines via the shared `logProgress`
9
+ * helper in pipeline.ts.
10
+ */
11
+ /** One of the four equal stages every active file passes through. */
12
+ const QUARTER = 0.25;
13
+ /**
14
+ * Tracks the progress of one full index pass in file units.
15
+ *
16
+ * Every active (new or modified, non-empty, large-enough) file contributes
17
+ * one unit, split into four equal credits: chunked, described, embedded,
18
+ * stored. Credits are pushed by `WindowProgress` handles as their window
19
+ * advances, so `percent` rises monotonically from 0 toward 100 across ALL
20
+ * windows and stages — unlike the old per-stage "remaining" percentage
21
+ * that counted down and restarted inside every window.
22
+ *
23
+ * Files are weighted equally regardless of chunk count: the percentage is
24
+ * an honest estimate of "how far through the run are we", not a precise
25
+ * work-meter.
26
+ */
27
+ export class PassProgress {
28
+ totalFiles;
29
+ credited = 0;
30
+ /** @param totalFiles - Total number of active files the pass will process. */
31
+ constructor(totalFiles) {
32
+ this.totalFiles = totalFiles;
33
+ }
34
+ /**
35
+ * Start accounting for one processing window.
36
+ * @param activeFiles - How many of the window's files are active (the same
37
+ * predicate that produced `totalFiles`). Windows with 0 active files
38
+ * contribute nothing to the percentage.
39
+ */
40
+ beginWindow(activeFiles) {
41
+ return new WindowProgress(this, activeFiles);
42
+ }
43
+ /** @internal Called by {@link WindowProgress} to push credit into the pass total. */
44
+ push(credit) {
45
+ this.credited += credit;
46
+ }
47
+ /** Whole-pass percentage, 0–100, clamped. */
48
+ get percent() {
49
+ if (this.totalFiles <= 0)
50
+ return 100;
51
+ return Math.min(100, (this.credited / this.totalFiles) * 100);
52
+ }
53
+ /** Formatted percentage ("42.3%") for progress lines. */
54
+ get label() {
55
+ return `${this.percent.toFixed(1)}%`;
56
+ }
57
+ }
58
+ /**
59
+ * Credit handle for one window of the windowed pipeline. The pipeline
60
+ * reports each stage's done/total fraction here; the handle converts it to
61
+ * whole-pass credit by scaling with the window's active-file count.
62
+ *
63
+ * All updates are clamped to move FORWARD only, so repeated, out-of-order,
64
+ * or overlapping callbacks (concurrent embed sub-batches, the store phase
65
+ * still running while the next window prepares) can never make the overall
66
+ * percentage go backwards.
67
+ */
68
+ export class WindowProgress {
69
+ owner;
70
+ files;
71
+ done = {
72
+ chunk: 0,
73
+ describe: 0,
74
+ embed: 0,
75
+ store: 0,
76
+ };
77
+ chunkedFiles = 0;
78
+ constructor(owner, files) {
79
+ this.owner = owner;
80
+ this.files = files;
81
+ }
82
+ /** Fold the stage-fraction delta into the pass total (forward-only). */
83
+ advance(stage, fraction) {
84
+ if (this.files <= 0)
85
+ return;
86
+ let target = fraction;
87
+ if (!Number.isFinite(target))
88
+ target = 1;
89
+ target = Math.min(1, Math.max(0, target));
90
+ if (target <= this.done[stage])
91
+ return;
92
+ this.owner.push((target - this.done[stage]) * QUARTER * this.files);
93
+ this.done[stage] = target;
94
+ }
95
+ /** One of this window's active files finished chunking. */
96
+ chunkFile() {
97
+ this.chunkedFiles = Math.min(this.files, this.chunkedFiles + 1);
98
+ this.advance("chunk", this.chunkedFiles / this.files);
99
+ }
100
+ /**
101
+ * Report live progress of a per-stage work queue (described texts,
102
+ * embedded texts, stored files). `total` 0 means the stage had no work for
103
+ * this window (early-result files, full description-cache hits, a skipped
104
+ * stage) and fills the stage's credit immediately.
105
+ */
106
+ stage(stage, done, total) {
107
+ this.advance(stage, total > 0 ? done / total : 1);
108
+ }
109
+ /** Mark a stage complete regardless of its counters (stage done, skipped, or failed). */
110
+ fill(stage) {
111
+ this.advance(stage, 1);
112
+ }
113
+ }
114
+ //# sourceMappingURL=progress.js.map