opencode-rag-plugin 2.10.0 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/ReadMe.md +40 -28
  2. package/dist/api.js +1 -0
  3. package/dist/cli/commands/clickhouse.d.ts +61 -0
  4. package/dist/cli/commands/clickhouse.js +335 -0
  5. package/dist/cli/commands/eval-tuner.js +1 -1
  6. package/dist/cli/commands/eval.js +1 -1
  7. package/dist/cli/commands/index-command.js +2 -1
  8. package/dist/cli/commands/index.d.ts +1 -0
  9. package/dist/cli/commands/index.js +1 -0
  10. package/dist/cli/commands/init-helpers.js +2 -2
  11. package/dist/cli/commands/init.js +49 -11
  12. package/dist/cli/commands/quirk.js +6 -6
  13. package/dist/cli/commands/status.js +70 -6
  14. package/dist/cli/commands/ui.js +2 -2
  15. package/dist/cli/index.js +2 -1
  16. package/dist/cli/types.d.ts +14 -0
  17. package/dist/core/bootstrap.d.ts +7 -0
  18. package/dist/core/bootstrap.js +40 -9
  19. package/dist/core/config.d.ts +128 -5
  20. package/dist/core/config.js +205 -8
  21. package/dist/core/interfaces.d.ts +54 -0
  22. package/dist/core/runtime-overrides.d.ts +9 -0
  23. package/dist/core/runtime-overrides.js +44 -0
  24. package/dist/core/state-sync.d.ts +91 -0
  25. package/dist/core/state-sync.js +273 -0
  26. package/dist/eval/fast-index.js +2 -2
  27. package/dist/eval/rerank-eval.js +2 -2
  28. package/dist/eval/run-branch-compare.js +2 -2
  29. package/dist/eval/run-token-test.js +2 -2
  30. package/dist/index.d.ts +5 -2
  31. package/dist/index.js +2 -1
  32. package/dist/indexer/pipeline.d.ts +32 -0
  33. package/dist/indexer/pipeline.js +230 -65
  34. package/dist/indexer/progress.d.ts +75 -0
  35. package/dist/indexer/progress.js +114 -0
  36. package/dist/indexer/stats.d.ts +14 -0
  37. package/dist/plugin.d.ts +6 -3
  38. package/dist/plugin.js +39 -20
  39. package/dist/quirks/quirk-store.d.ts +10 -0
  40. package/dist/quirks/quirk-store.js +110 -15
  41. package/dist/quirks/types.d.ts +7 -0
  42. package/dist/tui.js +134 -113
  43. package/dist/vectorstore/clickhouse.d.ts +150 -0
  44. package/dist/vectorstore/clickhouse.js +931 -0
  45. package/dist/vectorstore/dimension.d.ts +19 -0
  46. package/dist/vectorstore/dimension.js +36 -0
  47. package/dist/vectorstore/factory.d.ts +9 -3
  48. package/dist/vectorstore/factory.js +11 -2
  49. package/dist/vectorstore/lancedb.d.ts +27 -4
  50. package/dist/vectorstore/lancedb.js +180 -15
  51. package/dist/watcher.d.ts +29 -0
  52. package/dist/watcher.js +76 -2
  53. package/dist/web/api.d.ts +3 -4
  54. package/dist/web/api.js +18 -3
  55. package/dist/web/server.js +28 -2
  56. package/dist/web/ui/assets/index-B-RH6fko.css +2 -0
  57. package/dist/web/ui/index.html +2 -2
  58. package/package.json +2 -1
  59. package/dist/web/ui/assets/index-CWcp0jNf.css +0 -2
  60. /package/dist/web/ui/assets/{index-DbXFrn53.js → index-ChGi2bSp.js} +0 -0
@@ -2,6 +2,7 @@
2
2
  * @fileoverview Orchestrates the full indexing pipeline: scan, chunk, embed, and store.
3
3
  */
4
4
  import fs from "node:fs/promises";
5
+ import os from "node:os";
5
6
  import path from "node:path";
6
7
  import pLimit from "p-limit";
7
8
  import { scanWorkspaceFiles } from "../content/reader.js";
@@ -12,6 +13,7 @@ import { createVectorStore } from "../vectorstore/factory.js";
12
13
  import { swapStoreDirectories } from "../vectorstore/lancedb.js";
13
14
  import { reconcileQuirks, QUIRK_FILE_PREFIX } from "../quirks/quirk-store.js";
14
15
  import { rehydrateKeywordIndex } from "./keyword-rehydration.js";
16
+ import { PassProgress } from "./progress.js";
15
17
  import { createIndexStats } from "./stats.js";
16
18
  import { prepareFile, buildTextsToEmbed } from "./worker.js";
17
19
  import { tagChunksRole } from "../core/chunk-role.js";
@@ -59,6 +61,43 @@ function isPidAlive(pid) {
59
61
  * @param options - Configuration for the index pass.
60
62
  * @returns Aggregate statistics for the pass.
61
63
  */
64
+ /** Minimum gap between two regular (non-aggressive) store optimize runs. */
65
+ export const MIN_OPTIMIZE_INTERVAL_MS = 15 * 60 * 1000;
66
+ /** Last time a regular optimize actually ran in this process. */
67
+ let lastOptimizeAt = 0;
68
+ /**
69
+ * Decide whether a regular store optimize may run now, and stamp the attempt.
70
+ *
71
+ * LanceDB's `Table.optimize()` always rewrites the ANN index (it decides the
72
+ * index is stale and rebuilds it), and one IVF_FLAT build is a full,
73
+ * uncompressed copy of the vector column - 12k chunks x 4096 float32 is about
74
+ * 190 MB. The watch loop optimized at the end of EVERY pass, so a busy session
75
+ * left one dead build per pass behind: on 2026-10-26 the store carried 15 builds
76
+ * (2.5 GB) against 380 MB of live data. Optimizing at most every
77
+ * {@link MIN_OPTIMIZE_INTERVAL_MS} bounds that to the builds of the last
78
+ * interval, and the version prune inside the same call reclaims the rest.
79
+ *
80
+ * Aggressive runs (a private temp store during a full rebuild) always return
81
+ * true.
82
+ *
83
+ * @param options - `force` bypasses the throttle; `now` overrides the clock (tests).
84
+ * @returns True when the caller should run the optimize.
85
+ */
86
+ export function optimizeDue(options) {
87
+ const now = options?.now ?? Date.now();
88
+ if (options?.force) {
89
+ lastOptimizeAt = now;
90
+ return true;
91
+ }
92
+ if (now - lastOptimizeAt < MIN_OPTIMIZE_INTERVAL_MS)
93
+ return false;
94
+ lastOptimizeAt = now;
95
+ return true;
96
+ }
97
+ /** Reset the optimize throttle (tests only). */
98
+ export function resetOptimizeThrottle() {
99
+ lastOptimizeAt = 0;
100
+ }
62
101
  export async function runIndexPass(options) {
63
102
  const logger = createLogger(options.logger);
64
103
  const lockPath = path.join(options.storePath, LOCK_FILE);
@@ -94,10 +133,48 @@ export async function runIndexPass(options) {
94
133
  // concurrent-pass protection above silently stops working.
95
134
  logger.debug(`Could not write index lock file: ${err.message} — concurrent-pass protection disabled`);
96
135
  }
136
+ // Cross-machine lease: central stores have no local directory in common, so
137
+ // the file lock above only protects this machine. The lease is advisory
138
+ // (ClickHouse has no CAS) but stops a second machine from re-embedding the
139
+ // same workspace concurrently.
140
+ const leaseOwner = `${os.hostname()}:${process.pid}`;
141
+ const leaseKind = "index";
142
+ const leaseTtlMs = Math.max(10, options.config.vectorStore.clickhouse?.lease?.ttlSeconds ?? 90) * 1000;
143
+ let leaseHeld = false;
144
+ let leaseTimer;
145
+ if (typeof options.store.acquireLease === "function") {
146
+ try {
147
+ leaseHeld = await options.store.acquireLease(leaseOwner, leaseKind, leaseTtlMs);
148
+ }
149
+ catch (err) {
150
+ // Do not leak the local lock file when the central store is unreachable.
151
+ try {
152
+ await fs.unlink(lockPath);
153
+ }
154
+ catch { /* best effort */ }
155
+ throw err;
156
+ }
157
+ if (!leaseHeld) {
158
+ logger.warn("Another machine holds the central store's index lease. Skipping.");
159
+ try {
160
+ await fs.unlink(lockPath);
161
+ }
162
+ catch { /* best effort */ }
163
+ return { ...createIndexStats(0, "missing"), skipped: true };
164
+ }
165
+ leaseTimer = setInterval(() => {
166
+ void options.store.renewLease?.(leaseOwner, leaseKind, leaseTtlMs).catch(() => { });
167
+ }, Math.max(5_000, Math.floor(leaseTtlMs / 3)));
168
+ leaseTimer.unref();
169
+ }
97
170
  try {
98
171
  return await runIndexPassInner(options, logger);
99
172
  }
100
173
  finally {
174
+ if (leaseTimer)
175
+ clearInterval(leaseTimer);
176
+ if (leaseHeld)
177
+ await options.store.releaseLease?.(leaseOwner, leaseKind).catch(() => { });
101
178
  try {
102
179
  await fs.unlink(lockPath);
103
180
  }
@@ -170,11 +247,14 @@ async function runIndexPassInner(options, logger) {
170
247
  if (!options.force && manifestStatus === "ok" && Object.keys(manifest.files).length > 0) {
171
248
  try {
172
249
  const storedPaths = await options.store.getFilePaths();
173
- const storedSet = new Set(storedPaths);
250
+ // Case-insensitive on Windows: the store may hold `c:/...` while the
251
+ // manifest keys `C:/...` (path.resolve keeps the drive-letter case).
252
+ // A case-sensitive compare re-indexes every such file on every pass.
253
+ const storedSet = new Set(storedPaths.map((p) => p.toLowerCase()));
174
254
  let removedCount = 0;
175
255
  for (const [p, entry] of Object.entries(manifest.files)) {
176
256
  // Path missing from store entirely (aborted before store finished)
177
- if (entry.chunkCount > 0 && !storedSet.has(p)) {
257
+ if (entry.chunkCount > 0 && !storedSet.has(p.toLowerCase())) {
178
258
  delete manifest.files[p];
179
259
  removedCount++;
180
260
  }
@@ -266,6 +346,8 @@ async function runIndexPassInner(options, logger) {
266
346
  // Effective store used throughout the pass — may be a temp store for atomic rebuild.
267
347
  let effectiveStore = options.store;
268
348
  let tempStorePath;
349
+ /** True when the rebuild writes into a private epoch of a central store. */
350
+ let rebuildTarget = false;
269
351
  /** Set when the temporary rebuild store actually received rows. */
270
352
  let tempStoreWroteChunks = false;
271
353
  if (options.force || (manifestStatus !== "ok" && (existingCount > 0 || dimensionMismatch))) {
@@ -281,13 +363,21 @@ async function runIndexPassInner(options, logger) {
281
363
  // Build the new index into a temporary store first, then atomically
282
364
  // swap on completion. Original data stays untouched if the process
283
365
  // is aborted (Ctrl+C, crash) before the swap completes.
284
- if (options.dimension) {
366
+ if (typeof options.store.beginRebuild === "function") {
367
+ // Central stores (ClickHouse) rebuild into a private epoch and promote
368
+ // atomically; no local temp directory and no directory swap.
369
+ effectiveStore = await options.store.beginRebuild();
370
+ rebuildTarget = true;
371
+ logger.debug("Rebuilding index in a private store epoch");
372
+ options.keywordIndex?.clear();
373
+ }
374
+ else if (options.dimension) {
285
375
  tempStorePath = options.storePath + "_tmp";
286
376
  try {
287
377
  await fs.rm(tempStorePath, { recursive: true, force: true });
288
378
  }
289
379
  catch { /* may not exist */ }
290
- effectiveStore = createVectorStore(options.config, tempStorePath, options.dimension);
380
+ effectiveStore = createVectorStore(options.config, tempStorePath, options.dimension, options.cwd);
291
381
  logger.debug(`Rebuilding index in temporary store at ${tempStorePath}`);
292
382
  // Only clear the shared in-memory keyword index once the rebuild is
293
383
  // guaranteed to proceed — the guard below may still bail out, and
@@ -455,17 +545,20 @@ async function runIndexPassInner(options, logger) {
455
545
  logger.debug(`Processing ${workspaceFiles.length} files (${newCount} new, ${modifiedCount} modified, ${unchangedCount} unchanged)...`);
456
546
  const limit = pLimit(options.config.indexing.concurrency);
457
547
  const deferDescriptions = !!options.descriptionProvider;
458
- const activeCount = workspaceFiles.filter((f) => !f.isEmpty && !f.isTooSmall && (!manifest.files[f.normalizedPath] || manifest.files[f.normalizedPath].hash !== f.hash)).length;
548
+ const isActiveFile = (f) => !f.isEmpty && !f.isTooSmall && (!manifest.files[f.normalizedPath] || manifest.files[f.normalizedPath].hash !== f.hash);
549
+ const activeCount = workspaceFiles.filter(isActiveFile).length;
459
550
  logger.info(`Chunking ${activeCount} file(s)...`);
551
+ // Whole-pass progress in file units: every progress line reports this
552
+ // tracker's percentage, never a per-chunk or per-stage fraction.
553
+ const passProgress = new PassProgress(activeCount);
460
554
  let chunkedDone = 0;
461
555
  const chunkProgressInterval = Math.max(1, Math.floor(activeCount / 20));
462
556
  const chunkPhaseStart = Date.now();
463
557
  /** Chunk one window of files (used by the windowed pipeline loop below). */
464
- async function prepareWindow(windowFiles) {
558
+ async function prepareWindow(windowFiles, winProgress) {
465
559
  const prepared = await Promise.all(windowFiles.map((file) => limit(async () => {
466
560
  const fileLabel = path.relative(options.cwd, file.normalizedPath).replace(/\\/g, "/");
467
- const isActive = !file.isEmpty && !file.isTooSmall &&
468
- (!manifest.files[file.normalizedPath] || manifest.files[file.normalizedPath].hash !== file.hash);
561
+ const isActive = isActiveFile(file);
469
562
  if (isActive) {
470
563
  options.progress?.startFile(fileLabel);
471
564
  }
@@ -477,9 +570,9 @@ async function runIndexPassInner(options, logger) {
477
570
  }
478
571
  if (isActive) {
479
572
  chunkedDone++;
573
+ winProgress.chunkFile();
480
574
  if (chunkedDone % chunkProgressInterval === 0 || chunkedDone === activeCount) {
481
- const pct = ((chunkedDone / activeCount) * 100).toFixed(1);
482
- logger.info(` Chunking: ${chunkedDone}/${activeCount} files (${pct}%)`);
575
+ logProgress("Chunking", "", chunkedDone, activeCount, "files");
483
576
  }
484
577
  }
485
578
  if (prep.earlyResult && isActive) {
@@ -490,11 +583,14 @@ async function runIndexPassInner(options, logger) {
490
583
  return prepared;
491
584
  }
492
585
  const aborted = () => options.abortSignal?.aborted ?? false;
493
- // Shared progress logger: "stage <file> (chunk i/n) — X/total remaining (P%)".
494
- const logChunkProgress = (stage, fileLabel, index, count, completed, total) => {
495
- const remaining = total - completed;
496
- const pct = total > 0 ? ((remaining / total) * 100).toFixed(1) : "0.0";
497
- logger.info(`${stage} ${fileLabel} (chunk ${index}/${count}) — ${remaining}/${total} remaining (${pct}%)`);
586
+ // Progress line shared by all four stages: "<Stage> [file] — NN.N% overall (done/total unit)".
587
+ // The percentage is ALWAYS the fraction of the WHOLE pass completed
588
+ // (PassProgress across all windows and stages) — never a per-chunk or
589
+ // per-stage fraction. The stage-local counters ride along as a plain
590
+ // done/total count in parentheses.
591
+ const logProgress = (stage, fileLabel, done, total, unit) => {
592
+ const detail = total > 0 ? ` (${done}/${total} ${unit})` : "";
593
+ logger.info(`${stage}${fileLabel ? ` ${fileLabel}` : ""} — ${passProgress.label} overall${detail}`);
498
594
  };
499
595
  // File metadata look-up for manifest entries
500
596
  const fileMeta = new Map(workspaceFiles.map((f) => [f.normalizedPath, { mtime: f.mtime, size: f.size }]));
@@ -528,10 +624,14 @@ async function runIndexPassInner(options, logger) {
528
624
  * version-manifest accumulation (K+2 versions per file) that made the
529
625
  * store phase degrade quadratically as the index grew.
530
626
  */
531
- async function storeWindow(prepared, windowEarlyResults) {
627
+ async function storeWindow(prepared, windowEarlyResults, winProgress) {
532
628
  const filesToStore = prepared.filter((p) => !windowEarlyResults.has(prepared.indexOf(p)) && p.chunks && (p.textToEmbed?.length ?? 0) > 0).length;
533
629
  const storePhaseStart = Date.now();
534
630
  logger.info(`Store phase: storing ${filesToStore} file(s) into vector database...`);
631
+ // Nothing to store (every file in this window had an early result or no
632
+ // chunks) — the store stage is trivially done, credit it immediately.
633
+ if (filesToStore === 0)
634
+ winProgress.fill("store");
535
635
  let storedFiles = 0;
536
636
  // Files whose chunks ALL failed to embed while no manifest entry exists.
537
637
  // Rows an earlier pass wrote for them would otherwise never be removed:
@@ -663,7 +763,8 @@ async function runIndexPassInner(options, logger) {
663
763
  }
664
764
  options.progress?.finishFile(prep.fileLabel);
665
765
  storedFiles++;
666
- logChunkProgress("Storing", prep.fileLabel, storedFiles, filesToStore, storedFiles, filesToStore);
766
+ winProgress.stage("store", storedFiles, filesToStore);
767
+ logProgress("Storing", prep.fileLabel, storedFiles, filesToStore, "files");
667
768
  storeResults.push(result);
668
769
  }
669
770
  const workerResults = storeResults;
@@ -713,7 +814,11 @@ async function runIndexPassInner(options, logger) {
713
814
  }
714
815
  windowCount++;
715
816
  const windowFiles = workspaceFiles.slice(windowStart, windowStart + WINDOW_SIZE);
716
- const prepared = await prepareWindow(windowFiles);
817
+ // This window's credit handle. The active-file count is taken here, up
818
+ // front: manifest entries for THIS window's files only appear once its
819
+ // store phase runs, so the count stays stable across the window.
820
+ const winProgress = passProgress.beginWindow(windowFiles.filter(isActiveFile).length);
821
+ const prepared = await prepareWindow(windowFiles, winProgress);
717
822
  if (windowStart === 0) {
718
823
  const chunkPhaseSec = ((Date.now() - chunkPhaseStart) / 1000).toFixed(1);
719
824
  logger.info(`Chunking complete: ${activeCount} files in ${chunkPhaseSec}s`);
@@ -740,31 +845,16 @@ async function runIndexPassInner(options, logger) {
740
845
  for (const chunk of oversizedChunks) {
741
846
  chunk.description = buildFallbackDescription(chunk);
742
847
  }
743
- // Per-file chunk index + label for clear progress reporting, plus a shared
744
- // progress logger emitting "stage <file> (chunk i/n) — X/total remaining (P%)".
848
+ // chunk.id → file label for the progress lines. The overall
849
+ // percentage comes from PassProgress; the per-chunk index inside the
850
+ // file is deliberately NOT shown.
745
851
  const chunkToFileLabel = new Map();
746
852
  for (const prep of deferredPreps) {
747
853
  for (const chunk of prep.chunks ?? []) {
748
854
  chunkToFileLabel.set(chunk.id, prep.fileLabel);
749
855
  }
750
856
  }
751
- const chunkMeta = new Map();
752
- {
753
- const perFileCount = new Map();
754
- for (const chunk of allChunks) {
755
- perFileCount.set(chunk.metadata.filePath, (perFileCount.get(chunk.metadata.filePath) ?? 0) + 1);
756
- }
757
- const perFileSeen = new Map();
758
- for (const chunk of allChunks) {
759
- const seen = perFileSeen.get(chunk.metadata.filePath) ?? 0;
760
- perFileSeen.set(chunk.metadata.filePath, seen + 1);
761
- chunkMeta.set(chunk.id, {
762
- fileLabel: chunkToFileLabel.get(chunk.id) ?? chunk.metadata.filePath,
763
- index: seen + 1,
764
- count: perFileCount.get(chunk.metadata.filePath) ?? 1,
765
- });
766
- }
767
- }
857
+ const describeFileLabel = (chunk) => chunkToFileLabel.get(chunk.id) ?? chunk.metadata.filePath;
768
858
  // Advance progress to Description stage before descriptions start
769
859
  for (const prep of deferredPreps) {
770
860
  options.progress?.finishStage(prep.fileLabel);
@@ -791,6 +881,8 @@ async function runIndexPassInner(options, logger) {
791
881
  for (const { chunk, desc } of cacheHits) {
792
882
  chunk.description = desc;
793
883
  }
884
+ // Cache hits are already-described chunks of this window.
885
+ winProgress.stage("describe", cacheHits.length, allChunks.length);
794
886
  }
795
887
  // Build chunkId → prep map for tracking description failures per-file
796
888
  const chunkToPrep = new Map();
@@ -822,9 +914,8 @@ async function runIndexPassInner(options, logger) {
822
914
  total: totalMisses,
823
915
  onProgress: (chunk) => {
824
916
  describedDone++;
825
- const meta = chunkMeta.get(chunk.id);
826
- if (meta)
827
- logChunkProgress("Describing", meta.fileLabel, meta.index, meta.count, describedDone, totalMisses);
917
+ winProgress.stage("describe", cacheHits.length + describedDone, allChunks.length);
918
+ logProgress("Describing", describeFileLabel(chunk), describedDone, totalMisses, "texts");
828
919
  },
829
920
  });
830
921
  const newCacheEntries = [];
@@ -871,9 +962,8 @@ async function runIndexPassInner(options, logger) {
871
962
  total: allChunks.length,
872
963
  onProgress: (chunk) => {
873
964
  describedDone++;
874
- const meta = chunkMeta.get(chunk.id);
875
- if (meta)
876
- logChunkProgress("Describing", meta.fileLabel, meta.index, meta.count, describedDone, allChunks.length);
965
+ winProgress.stage("describe", describedDone, allChunks.length);
966
+ logProgress("Describing", describeFileLabel(chunk), describedDone, allChunks.length, "texts");
877
967
  },
878
968
  });
879
969
  for (const chunk of allChunks) {
@@ -899,6 +989,11 @@ async function runIndexPassInner(options, logger) {
899
989
  logger.info(`Description phase complete: ${totalDescribedChunks} chunks across ${deferredPreps.length} files`);
900
990
  }
901
991
  }
992
+ // The describe stage of this window is finished — or was skipped entirely
993
+ // (no description provider, nothing to describe, or a global description
994
+ // failure). Credit the remaining fraction so the overall percentage keeps
995
+ // moving toward 100.
996
+ winProgress.fill("describe");
902
997
  // Cross-file embedding batch: collect this window's texts into a single queue,
903
998
  // embed in one batched call (with concurrency), then distribute back.
904
999
  const isOllama = options.embedder.name === "ollama";
@@ -906,7 +1001,6 @@ async function runIndexPassInner(options, logger) {
906
1001
  const defaultConcurrency = options.config.indexing.embedConcurrency ?? 1;
907
1002
  // ── Phase 1: Collect embed queue + handle early results ────────────────
908
1003
  const embedQueue = [];
909
- let totalEmbedChunks = 0;
910
1004
  earlyWorkerResults.clear();
911
1005
  for (let fi = 0; fi < prepared.length; fi++) {
912
1006
  const prep = prepared[fi];
@@ -935,13 +1029,11 @@ async function runIndexPassInner(options, logger) {
935
1029
  for (let ci = 0; ci < prep.textToEmbed.length; ci++) {
936
1030
  embedQueue.push({ fileIdx: fi, chunkIdx: ci, text: prep.textToEmbed[ci] });
937
1031
  }
938
- totalEmbedChunks += prep.textToEmbed.length;
939
1032
  }
940
1033
  // ── Phase 2: Embed all texts in a single batched call ──────────────────
941
1034
  const batchSize = isOllama
942
1035
  ? Math.min(options.config.indexing.ollamaMaxBatchSize ?? 500, defaultBatchSize)
943
1036
  : defaultBatchSize;
944
- let embeddedDone = 0;
945
1037
  const allTexts = embedQueue.map(item => item.text);
946
1038
  let allEmbeddings = [];
947
1039
  const embedPhaseStart = Date.now();
@@ -949,8 +1041,8 @@ async function runIndexPassInner(options, logger) {
949
1041
  logger.info(`Embedding phase: ${allTexts.length} texts in batches of ${batchSize}...`);
950
1042
  try {
951
1043
  allEmbeddings = await embedBatch(options.embedder, allTexts, batchSize, "document", defaultConcurrency, (completed, total) => {
952
- embeddedDone = completed;
953
- logChunkProgress("Embedding", "", completed, total, embeddedDone, totalEmbedChunks);
1044
+ winProgress.stage("embed", completed, total);
1045
+ logProgress("Embedding", "", completed, total, "texts");
954
1046
  });
955
1047
  logger.info(`Embedding complete: ${allTexts.length} texts in ${((Date.now() - embedPhaseStart) / 1000).toFixed(1)}s`);
956
1048
  }
@@ -987,6 +1079,10 @@ async function runIndexPassInner(options, logger) {
987
1079
  }
988
1080
  }
989
1081
  }
1082
+ // Embed stage done for this window — worked, skipped (nothing to embed),
1083
+ // or globally failed (a failed embed leaves nothing to store, so holding
1084
+ // the credit back would stall the percentage without helping anyone).
1085
+ winProgress.fill("embed");
990
1086
  // ── Distribute embeddings back to per-file chunks ─────────────────────
991
1087
  for (let i = 0; i < embedQueue.length; i++) {
992
1088
  const { fileIdx, chunkIdx } = embedQueue[i];
@@ -1008,7 +1104,7 @@ async function runIndexPassInner(options, logger) {
1008
1104
  // asynchronously (overlapped with the next window's prepare/embed), so a
1009
1105
  // shared mutable map would make its per-file loop read the WRONG window's
1010
1106
  // entries and skip every file ("Store phase: storing N → 0 files stored").
1011
- prevStore = storeWindow(prepared, new Map(earlyWorkerResults));
1107
+ prevStore = storeWindow(prepared, new Map(earlyWorkerResults), winProgress);
1012
1108
  // Periodically compact fragments and prune old versions so the store
1013
1109
  // phase doesn't slow down as the index grows during long runs. Skipped on
1014
1110
  // windows that wrote nothing — an idle pass (no changed files) adds no
@@ -1016,7 +1112,7 @@ async function runIndexPassInner(options, logger) {
1016
1112
  if (optimizeInterval > 0 && windowCount % optimizeInterval === 0) {
1017
1113
  const storedInWindow = await prevStore;
1018
1114
  prevStore = null;
1019
- if (storedInWindow > 0) {
1115
+ if (storedInWindow > 0 && optimizeDue({ force: Boolean(tempStorePath) })) {
1020
1116
  logger.info("Optimizing vector store (mid-run compaction, pruning old versions)...");
1021
1117
  await effectiveStore.optimize?.(tempStorePath
1022
1118
  ? { aggressive: true, skipIndex: true, logger: logger.warn }
@@ -1088,9 +1184,52 @@ async function runIndexPassInner(options, logger) {
1088
1184
  if (!aborted()) {
1089
1185
  tryUpdateLastGitCommit(options.cwd, manifest);
1090
1186
  }
1091
- // ── Atomically promote temp store if a full rebuild was performed ──
1187
+ // ── Atomically promote a rebuilt epoch / temp store ──
1092
1188
  let tempStorePromoted = false;
1093
- if (tempStorePath) {
1189
+ if (rebuildTarget) {
1190
+ if (!aborted()) {
1191
+ if (!tempStoreWroteChunks) {
1192
+ // Nothing was written (e.g. every embedding request failed). Keep the
1193
+ // existing epoch rather than promoting an empty one over good data.
1194
+ logger.warn("Rebuild produced no chunks — keeping the existing index and manifest unchanged.");
1195
+ try {
1196
+ await options.store.discardRebuild?.();
1197
+ }
1198
+ catch { /* best effort */ }
1199
+ }
1200
+ else {
1201
+ try {
1202
+ await options.store.promoteRebuild?.();
1203
+ tempStorePromoted = true;
1204
+ logger.debug("Promoted rebuilt store epoch");
1205
+ }
1206
+ catch (err) {
1207
+ logger.warn(`Could not promote the rebuilt epoch: ${err.message}`);
1208
+ try {
1209
+ await options.store.discardRebuild?.();
1210
+ }
1211
+ catch { /* best effort */ }
1212
+ }
1213
+ }
1214
+ try {
1215
+ await effectiveStore.close();
1216
+ }
1217
+ catch { /* separate client handle */ }
1218
+ }
1219
+ else {
1220
+ // Aborted — discard the epoch, keep the active data intact.
1221
+ try {
1222
+ await options.store.discardRebuild?.();
1223
+ }
1224
+ catch { /* best effort */ }
1225
+ effectiveStore.close().catch(() => { });
1226
+ logger.debug("Index pass cancelled; discarded the rebuild epoch.");
1227
+ // Skip the final manifest/keyword save: the old state is still valid.
1228
+ await descCache.save();
1229
+ return stats;
1230
+ }
1231
+ }
1232
+ else if (tempStorePath) {
1094
1233
  if (!aborted()) {
1095
1234
  if (!tempStoreWroteChunks) {
1096
1235
  // Nothing was written (e.g. every embedding request failed). Keep the
@@ -1148,7 +1287,7 @@ async function runIndexPassInner(options, logger) {
1148
1287
  // for the new store. When the rebuild did not promote (empty temp store or
1149
1288
  // failed swap), keep the previous manifest/keyword index on disk — saving
1150
1289
  // the cleared state would orphan the existing store's data.
1151
- const keepPreviousIndexState = tempStorePath !== undefined && !tempStorePromoted;
1290
+ const keepPreviousIndexState = (tempStorePath !== undefined || rebuildTarget) && !tempStorePromoted;
1152
1291
  if (keepPreviousIndexState) {
1153
1292
  logger.warn("Rebuild did not complete — keeping the previous manifest and keyword index.");
1154
1293
  }
@@ -1183,6 +1322,14 @@ async function runIndexPassInner(options, logger) {
1183
1322
  // a successful swap this points to the new data).
1184
1323
  await saveManifest(options.storePath, manifest);
1185
1324
  await options.keywordIndex?.save(options.storePath);
1325
+ // ClickHouse: upload the durable state bundle (manifest, keyword index,
1326
+ // quirk log) so a fresh clone on another machine needs no re-embedding.
1327
+ try {
1328
+ await options.stateSync?.flush(["manifest", "keyword-index", "quirks", "runtime-overrides", "doc-progress", ".desc-cache"]);
1329
+ }
1330
+ catch (err) {
1331
+ logger.warn(`State upload failed: ${err.message}`);
1332
+ }
1186
1333
  }
1187
1334
  // Compact fragments and prune old version manifests so countRows() can't
1188
1335
  // hang on accumulated versions from many add/delete cycles. After a temp
@@ -1191,15 +1338,23 @@ async function runIndexPassInner(options, logger) {
1191
1338
  // swapped-in store is private to this process. Skipped when the rebuild did
1192
1339
  // not promote (the old store is untouched and its handle may be closed).
1193
1340
  if (!aborted() && !keepPreviousIndexState) {
1194
- logger.info("Optimizing vector store (compacting fragments, pruning old versions)...");
1195
- const optimizeStart = Date.now();
1196
- try {
1197
- const optimizeTarget = tempStorePath ? options.store : effectiveStore;
1198
- await optimizeTarget.optimize?.(tempStorePath ? { aggressive: true, logger: logger.warn } : { logger: logger.warn });
1199
- logger.info(`Vector store optimized in ${((Date.now() - optimizeStart) / 1000).toFixed(1)}s`);
1341
+ // Every optimize rebuilds the ANN index (a ~190 MB vector copy at 4096
1342
+ // dims), so passes within the throttle window skip it; the next due pass
1343
+ // compacts and prunes, which also reclaims the skipped builds.
1344
+ if (!optimizeDue({ force: Boolean(tempStorePath || rebuildTarget) })) {
1345
+ logger.debug(`Skipping vector store optimize (throttled: < ${MIN_OPTIMIZE_INTERVAL_MS / 60000} min since the last run)`);
1200
1346
  }
1201
- catch (err) {
1202
- logger.warn(`Vector store optimization failed: ${err.message}`);
1347
+ else {
1348
+ logger.info("Optimizing vector store (compacting fragments, pruning old versions)...");
1349
+ const optimizeStart = Date.now();
1350
+ try {
1351
+ const optimizeTarget = (tempStorePath || rebuildTarget) ? options.store : effectiveStore;
1352
+ await optimizeTarget.optimize?.(tempStorePath ? { aggressive: true, logger: logger.warn } : { logger: logger.warn });
1353
+ logger.info(`Vector store optimized in ${((Date.now() - optimizeStart) / 1000).toFixed(1)}s`);
1354
+ }
1355
+ catch (err) {
1356
+ logger.warn(`Vector store optimization failed: ${err.message}`);
1357
+ }
1203
1358
  }
1204
1359
  }
1205
1360
  // Persist description cache
@@ -1278,6 +1433,7 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
1278
1433
  storeChunkCount: storeCount,
1279
1434
  manifestExpectedChunks: 0,
1280
1435
  orphanChunks: 0,
1436
+ scanned: false,
1281
1437
  };
1282
1438
  }
1283
1439
  let upToDateFiles = 0;
@@ -1311,24 +1467,31 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
1311
1467
  manifestExpectedChunks = Object.values(manifest.files).reduce((sum, entry) => sum + entry.chunkCount, 0);
1312
1468
  }
1313
1469
  // Rows without a manifest entry (partial-embedding leftovers). Quirk chunks
1314
- // live in quirks.jsonl and are deliberately not counted as orphans. A store
1315
- // that cannot enumerate files simply reports 0.
1470
+ // live in quirks.jsonl and are deliberately not counted as orphans — they
1471
+ // are counted separately so `Indexed = Expected + Orphan (+ Quirk)` actually
1472
+ // reconciles. A store that cannot enumerate files reports 0 orphans and
1473
+ // leaves quirkChunks undefined (unknown, not zero).
1316
1474
  let orphanChunks = 0;
1475
+ let quirkChunks = 0;
1317
1476
  try {
1318
1477
  const manifestKeysLower = new Set(Object.keys(manifest.files).map((k) => k.toLowerCase()));
1478
+ quirkChunks = 0;
1319
1479
  for (const file of await store.listFiles()) {
1320
1480
  const key = normalizeFilePath(file.filePath);
1481
+ if (path.basename(key).startsWith(QUIRK_FILE_PREFIX)) {
1482
+ quirkChunks += file.chunkCount;
1483
+ continue;
1484
+ }
1321
1485
  if (manifest.files[key])
1322
1486
  continue;
1323
1487
  if (manifestKeysLower.has(key.toLowerCase()))
1324
1488
  continue;
1325
- if (path.basename(key).startsWith(QUIRK_FILE_PREFIX))
1326
- continue;
1327
1489
  orphanChunks += file.chunkCount;
1328
1490
  }
1329
1491
  }
1330
1492
  catch {
1331
1493
  orphanChunks = 0;
1494
+ quirkChunks = undefined;
1332
1495
  }
1333
1496
  return {
1334
1497
  manifestStatus: loadResult.status,
@@ -1340,6 +1503,8 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
1340
1503
  storeChunkCount: storeCount,
1341
1504
  manifestExpectedChunks,
1342
1505
  orphanChunks,
1506
+ scanned: !skipScan,
1507
+ quirkChunks,
1343
1508
  };
1344
1509
  }
1345
1510
  //# sourceMappingURL=pipeline.js.map
@@ -0,0 +1,75 @@
1
+ /**
2
+ * @fileoverview Whole-pass progress accounting for a single index run.
3
+ *
4
+ * Every progress line the `index` command prints reports ONE percentage:
5
+ * the fraction of the WHOLE pass completed — never a per-chunk or
6
+ * per-stage fraction. This module owns that accounting; the pipeline
7
+ * threads a `WindowProgress` handle through the chunk/describe/embed/store
8
+ * stages of each window and emits lines via the shared `logProgress`
9
+ * helper in pipeline.ts.
10
+ */
11
+ /** Stage names that report a done/total fraction (chunking counts files directly). */
12
+ export type ProgressStage = "describe" | "embed" | "store";
13
+ /**
14
+ * Tracks the progress of one full index pass in file units.
15
+ *
16
+ * Every active (new or modified, non-empty, large-enough) file contributes
17
+ * one unit, split into four equal credits: chunked, described, embedded,
18
+ * stored. Credits are pushed by `WindowProgress` handles as their window
19
+ * advances, so `percent` rises monotonically from 0 toward 100 across ALL
20
+ * windows and stages — unlike the old per-stage "remaining" percentage
21
+ * that counted down and restarted inside every window.
22
+ *
23
+ * Files are weighted equally regardless of chunk count: the percentage is
24
+ * an honest estimate of "how far through the run are we", not a precise
25
+ * work-meter.
26
+ */
27
+ export declare class PassProgress {
28
+ readonly totalFiles: number;
29
+ private credited;
30
+ /** @param totalFiles - Total number of active files the pass will process. */
31
+ constructor(totalFiles: number);
32
+ /**
33
+ * Start accounting for one processing window.
34
+ * @param activeFiles - How many of the window's files are active (the same
35
+ * predicate that produced `totalFiles`). Windows with 0 active files
36
+ * contribute nothing to the percentage.
37
+ */
38
+ beginWindow(activeFiles: number): WindowProgress;
39
+ /** @internal Called by {@link WindowProgress} to push credit into the pass total. */
40
+ push(credit: number): void;
41
+ /** Whole-pass percentage, 0–100, clamped. */
42
+ get percent(): number;
43
+ /** Formatted percentage ("42.3%") for progress lines. */
44
+ get label(): string;
45
+ }
46
+ /**
47
+ * Credit handle for one window of the windowed pipeline. The pipeline
48
+ * reports each stage's done/total fraction here; the handle converts it to
49
+ * whole-pass credit by scaling with the window's active-file count.
50
+ *
51
+ * All updates are clamped to move FORWARD only, so repeated, out-of-order,
52
+ * or overlapping callbacks (concurrent embed sub-batches, the store phase
53
+ * still running while the next window prepares) can never make the overall
54
+ * percentage go backwards.
55
+ */
56
+ export declare class WindowProgress {
57
+ private readonly owner;
58
+ readonly files: number;
59
+ private done;
60
+ private chunkedFiles;
61
+ constructor(owner: PassProgress, files: number);
62
+ /** Fold the stage-fraction delta into the pass total (forward-only). */
63
+ private advance;
64
+ /** One of this window's active files finished chunking. */
65
+ chunkFile(): void;
66
+ /**
67
+ * Report live progress of a per-stage work queue (described texts,
68
+ * embedded texts, stored files). `total` 0 means the stage had no work for
69
+ * this window (early-result files, full description-cache hits, a skipped
70
+ * stage) and fills the stage's credit immediately.
71
+ */
72
+ stage(stage: ProgressStage, done: number, total: number): void;
73
+ /** Mark a stage complete regardless of its counters (stage done, skipped, or failed). */
74
+ fill(stage: ProgressStage): void;
75
+ }