opencode-rag-plugin 2.10.0 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +40 -28
- package/dist/api.js +1 -0
- package/dist/cli/commands/clickhouse.d.ts +61 -0
- package/dist/cli/commands/clickhouse.js +335 -0
- package/dist/cli/commands/eval-tuner.js +1 -1
- package/dist/cli/commands/eval.js +1 -1
- package/dist/cli/commands/index-command.js +2 -1
- package/dist/cli/commands/index.d.ts +1 -0
- package/dist/cli/commands/index.js +1 -0
- package/dist/cli/commands/init-helpers.js +2 -2
- package/dist/cli/commands/init.js +49 -11
- package/dist/cli/commands/quirk.js +6 -6
- package/dist/cli/commands/status.js +70 -6
- package/dist/cli/commands/ui.js +2 -2
- package/dist/cli/index.js +2 -1
- package/dist/cli/types.d.ts +14 -0
- package/dist/core/bootstrap.d.ts +7 -0
- package/dist/core/bootstrap.js +40 -9
- package/dist/core/config.d.ts +128 -5
- package/dist/core/config.js +205 -8
- package/dist/core/interfaces.d.ts +54 -0
- package/dist/core/runtime-overrides.d.ts +9 -0
- package/dist/core/runtime-overrides.js +44 -0
- package/dist/core/state-sync.d.ts +91 -0
- package/dist/core/state-sync.js +273 -0
- package/dist/eval/fast-index.js +2 -2
- package/dist/eval/rerank-eval.js +2 -2
- package/dist/eval/run-branch-compare.js +2 -2
- package/dist/eval/run-token-test.js +2 -2
- package/dist/index.d.ts +5 -2
- package/dist/index.js +2 -1
- package/dist/indexer/pipeline.d.ts +32 -0
- package/dist/indexer/pipeline.js +230 -65
- package/dist/indexer/progress.d.ts +75 -0
- package/dist/indexer/progress.js +114 -0
- package/dist/indexer/stats.d.ts +14 -0
- package/dist/plugin.d.ts +6 -3
- package/dist/plugin.js +39 -20
- package/dist/quirks/quirk-store.d.ts +10 -0
- package/dist/quirks/quirk-store.js +110 -15
- package/dist/quirks/types.d.ts +7 -0
- package/dist/tui.js +134 -113
- package/dist/vectorstore/clickhouse.d.ts +150 -0
- package/dist/vectorstore/clickhouse.js +931 -0
- package/dist/vectorstore/dimension.d.ts +19 -0
- package/dist/vectorstore/dimension.js +36 -0
- package/dist/vectorstore/factory.d.ts +9 -3
- package/dist/vectorstore/factory.js +11 -2
- package/dist/vectorstore/lancedb.d.ts +27 -4
- package/dist/vectorstore/lancedb.js +180 -15
- package/dist/watcher.d.ts +29 -0
- package/dist/watcher.js +76 -2
- package/dist/web/api.d.ts +3 -4
- package/dist/web/api.js +18 -3
- package/dist/web/server.js +28 -2
- package/dist/web/ui/assets/index-B-RH6fko.css +2 -0
- package/dist/web/ui/index.html +2 -2
- package/package.json +2 -1
- package/dist/web/ui/assets/index-CWcp0jNf.css +0 -2
- /package/dist/web/ui/assets/{index-DbXFrn53.js → index-ChGi2bSp.js} +0 -0
package/dist/indexer/pipeline.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* @fileoverview Orchestrates the full indexing pipeline: scan, chunk, embed, and store.
|
|
3
3
|
*/
|
|
4
4
|
import fs from "node:fs/promises";
|
|
5
|
+
import os from "node:os";
|
|
5
6
|
import path from "node:path";
|
|
6
7
|
import pLimit from "p-limit";
|
|
7
8
|
import { scanWorkspaceFiles } from "../content/reader.js";
|
|
@@ -12,6 +13,7 @@ import { createVectorStore } from "../vectorstore/factory.js";
|
|
|
12
13
|
import { swapStoreDirectories } from "../vectorstore/lancedb.js";
|
|
13
14
|
import { reconcileQuirks, QUIRK_FILE_PREFIX } from "../quirks/quirk-store.js";
|
|
14
15
|
import { rehydrateKeywordIndex } from "./keyword-rehydration.js";
|
|
16
|
+
import { PassProgress } from "./progress.js";
|
|
15
17
|
import { createIndexStats } from "./stats.js";
|
|
16
18
|
import { prepareFile, buildTextsToEmbed } from "./worker.js";
|
|
17
19
|
import { tagChunksRole } from "../core/chunk-role.js";
|
|
@@ -59,6 +61,43 @@ function isPidAlive(pid) {
|
|
|
59
61
|
* @param options - Configuration for the index pass.
|
|
60
62
|
* @returns Aggregate statistics for the pass.
|
|
61
63
|
*/
|
|
64
|
+
/** Minimum gap between two regular (non-aggressive) store optimize runs. */
|
|
65
|
+
export const MIN_OPTIMIZE_INTERVAL_MS = 15 * 60 * 1000;
|
|
66
|
+
/** Last time a regular optimize actually ran in this process. */
|
|
67
|
+
let lastOptimizeAt = 0;
|
|
68
|
+
/**
|
|
69
|
+
* Decide whether a regular store optimize may run now, and stamp the attempt.
|
|
70
|
+
*
|
|
71
|
+
* LanceDB's `Table.optimize()` always rewrites the ANN index (it decides the
|
|
72
|
+
* index is stale and rebuilds it), and one IVF_FLAT build is a full,
|
|
73
|
+
* uncompressed copy of the vector column - 12k chunks x 4096 float32 is about
|
|
74
|
+
* 190 MB. The watch loop optimized at the end of EVERY pass, so a busy session
|
|
75
|
+
* left one dead build per pass behind: on 2026-10-26 the store carried 15 builds
|
|
76
|
+
* (2.5 GB) against 380 MB of live data. Optimizing at most every
|
|
77
|
+
* {@link MIN_OPTIMIZE_INTERVAL_MS} bounds that to the builds of the last
|
|
78
|
+
* interval, and the version prune inside the same call reclaims the rest.
|
|
79
|
+
*
|
|
80
|
+
* Aggressive runs (a private temp store during a full rebuild) always return
|
|
81
|
+
* true.
|
|
82
|
+
*
|
|
83
|
+
* @param options - `force` bypasses the throttle; `now` overrides the clock (tests).
|
|
84
|
+
* @returns True when the caller should run the optimize.
|
|
85
|
+
*/
|
|
86
|
+
export function optimizeDue(options) {
|
|
87
|
+
const now = options?.now ?? Date.now();
|
|
88
|
+
if (options?.force) {
|
|
89
|
+
lastOptimizeAt = now;
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
92
|
+
if (now - lastOptimizeAt < MIN_OPTIMIZE_INTERVAL_MS)
|
|
93
|
+
return false;
|
|
94
|
+
lastOptimizeAt = now;
|
|
95
|
+
return true;
|
|
96
|
+
}
|
|
97
|
+
/** Reset the optimize throttle (tests only). */
|
|
98
|
+
export function resetOptimizeThrottle() {
|
|
99
|
+
lastOptimizeAt = 0;
|
|
100
|
+
}
|
|
62
101
|
export async function runIndexPass(options) {
|
|
63
102
|
const logger = createLogger(options.logger);
|
|
64
103
|
const lockPath = path.join(options.storePath, LOCK_FILE);
|
|
@@ -94,10 +133,48 @@ export async function runIndexPass(options) {
|
|
|
94
133
|
// concurrent-pass protection above silently stops working.
|
|
95
134
|
logger.debug(`Could not write index lock file: ${err.message} — concurrent-pass protection disabled`);
|
|
96
135
|
}
|
|
136
|
+
// Cross-machine lease: central stores have no local directory in common, so
|
|
137
|
+
// the file lock above only protects this machine. The lease is advisory
|
|
138
|
+
// (ClickHouse has no CAS) but stops a second machine from re-embedding the
|
|
139
|
+
// same workspace concurrently.
|
|
140
|
+
const leaseOwner = `${os.hostname()}:${process.pid}`;
|
|
141
|
+
const leaseKind = "index";
|
|
142
|
+
const leaseTtlMs = Math.max(10, options.config.vectorStore.clickhouse?.lease?.ttlSeconds ?? 90) * 1000;
|
|
143
|
+
let leaseHeld = false;
|
|
144
|
+
let leaseTimer;
|
|
145
|
+
if (typeof options.store.acquireLease === "function") {
|
|
146
|
+
try {
|
|
147
|
+
leaseHeld = await options.store.acquireLease(leaseOwner, leaseKind, leaseTtlMs);
|
|
148
|
+
}
|
|
149
|
+
catch (err) {
|
|
150
|
+
// Do not leak the local lock file when the central store is unreachable.
|
|
151
|
+
try {
|
|
152
|
+
await fs.unlink(lockPath);
|
|
153
|
+
}
|
|
154
|
+
catch { /* best effort */ }
|
|
155
|
+
throw err;
|
|
156
|
+
}
|
|
157
|
+
if (!leaseHeld) {
|
|
158
|
+
logger.warn("Another machine holds the central store's index lease. Skipping.");
|
|
159
|
+
try {
|
|
160
|
+
await fs.unlink(lockPath);
|
|
161
|
+
}
|
|
162
|
+
catch { /* best effort */ }
|
|
163
|
+
return { ...createIndexStats(0, "missing"), skipped: true };
|
|
164
|
+
}
|
|
165
|
+
leaseTimer = setInterval(() => {
|
|
166
|
+
void options.store.renewLease?.(leaseOwner, leaseKind, leaseTtlMs).catch(() => { });
|
|
167
|
+
}, Math.max(5_000, Math.floor(leaseTtlMs / 3)));
|
|
168
|
+
leaseTimer.unref();
|
|
169
|
+
}
|
|
97
170
|
try {
|
|
98
171
|
return await runIndexPassInner(options, logger);
|
|
99
172
|
}
|
|
100
173
|
finally {
|
|
174
|
+
if (leaseTimer)
|
|
175
|
+
clearInterval(leaseTimer);
|
|
176
|
+
if (leaseHeld)
|
|
177
|
+
await options.store.releaseLease?.(leaseOwner, leaseKind).catch(() => { });
|
|
101
178
|
try {
|
|
102
179
|
await fs.unlink(lockPath);
|
|
103
180
|
}
|
|
@@ -170,11 +247,14 @@ async function runIndexPassInner(options, logger) {
|
|
|
170
247
|
if (!options.force && manifestStatus === "ok" && Object.keys(manifest.files).length > 0) {
|
|
171
248
|
try {
|
|
172
249
|
const storedPaths = await options.store.getFilePaths();
|
|
173
|
-
|
|
250
|
+
// Case-insensitive on Windows: the store may hold `c:/...` while the
|
|
251
|
+
// manifest keys `C:/...` (path.resolve keeps the drive-letter case).
|
|
252
|
+
// A case-sensitive compare re-indexes every such file on every pass.
|
|
253
|
+
const storedSet = new Set(storedPaths.map((p) => p.toLowerCase()));
|
|
174
254
|
let removedCount = 0;
|
|
175
255
|
for (const [p, entry] of Object.entries(manifest.files)) {
|
|
176
256
|
// Path missing from store entirely (aborted before store finished)
|
|
177
|
-
if (entry.chunkCount > 0 && !storedSet.has(p)) {
|
|
257
|
+
if (entry.chunkCount > 0 && !storedSet.has(p.toLowerCase())) {
|
|
178
258
|
delete manifest.files[p];
|
|
179
259
|
removedCount++;
|
|
180
260
|
}
|
|
@@ -266,6 +346,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
266
346
|
// Effective store used throughout the pass — may be a temp store for atomic rebuild.
|
|
267
347
|
let effectiveStore = options.store;
|
|
268
348
|
let tempStorePath;
|
|
349
|
+
/** True when the rebuild writes into a private epoch of a central store. */
|
|
350
|
+
let rebuildTarget = false;
|
|
269
351
|
/** Set when the temporary rebuild store actually received rows. */
|
|
270
352
|
let tempStoreWroteChunks = false;
|
|
271
353
|
if (options.force || (manifestStatus !== "ok" && (existingCount > 0 || dimensionMismatch))) {
|
|
@@ -281,13 +363,21 @@ async function runIndexPassInner(options, logger) {
|
|
|
281
363
|
// Build the new index into a temporary store first, then atomically
|
|
282
364
|
// swap on completion. Original data stays untouched if the process
|
|
283
365
|
// is aborted (Ctrl+C, crash) before the swap completes.
|
|
284
|
-
if (options.
|
|
366
|
+
if (typeof options.store.beginRebuild === "function") {
|
|
367
|
+
// Central stores (ClickHouse) rebuild into a private epoch and promote
|
|
368
|
+
// atomically; no local temp directory and no directory swap.
|
|
369
|
+
effectiveStore = await options.store.beginRebuild();
|
|
370
|
+
rebuildTarget = true;
|
|
371
|
+
logger.debug("Rebuilding index in a private store epoch");
|
|
372
|
+
options.keywordIndex?.clear();
|
|
373
|
+
}
|
|
374
|
+
else if (options.dimension) {
|
|
285
375
|
tempStorePath = options.storePath + "_tmp";
|
|
286
376
|
try {
|
|
287
377
|
await fs.rm(tempStorePath, { recursive: true, force: true });
|
|
288
378
|
}
|
|
289
379
|
catch { /* may not exist */ }
|
|
290
|
-
effectiveStore = createVectorStore(options.config, tempStorePath, options.dimension);
|
|
380
|
+
effectiveStore = createVectorStore(options.config, tempStorePath, options.dimension, options.cwd);
|
|
291
381
|
logger.debug(`Rebuilding index in temporary store at ${tempStorePath}`);
|
|
292
382
|
// Only clear the shared in-memory keyword index once the rebuild is
|
|
293
383
|
// guaranteed to proceed — the guard below may still bail out, and
|
|
@@ -455,17 +545,20 @@ async function runIndexPassInner(options, logger) {
|
|
|
455
545
|
logger.debug(`Processing ${workspaceFiles.length} files (${newCount} new, ${modifiedCount} modified, ${unchangedCount} unchanged)...`);
|
|
456
546
|
const limit = pLimit(options.config.indexing.concurrency);
|
|
457
547
|
const deferDescriptions = !!options.descriptionProvider;
|
|
458
|
-
const
|
|
548
|
+
const isActiveFile = (f) => !f.isEmpty && !f.isTooSmall && (!manifest.files[f.normalizedPath] || manifest.files[f.normalizedPath].hash !== f.hash);
|
|
549
|
+
const activeCount = workspaceFiles.filter(isActiveFile).length;
|
|
459
550
|
logger.info(`Chunking ${activeCount} file(s)...`);
|
|
551
|
+
// Whole-pass progress in file units: every progress line reports this
|
|
552
|
+
// tracker's percentage, never a per-chunk or per-stage fraction.
|
|
553
|
+
const passProgress = new PassProgress(activeCount);
|
|
460
554
|
let chunkedDone = 0;
|
|
461
555
|
const chunkProgressInterval = Math.max(1, Math.floor(activeCount / 20));
|
|
462
556
|
const chunkPhaseStart = Date.now();
|
|
463
557
|
/** Chunk one window of files (used by the windowed pipeline loop below). */
|
|
464
|
-
async function prepareWindow(windowFiles) {
|
|
558
|
+
async function prepareWindow(windowFiles, winProgress) {
|
|
465
559
|
const prepared = await Promise.all(windowFiles.map((file) => limit(async () => {
|
|
466
560
|
const fileLabel = path.relative(options.cwd, file.normalizedPath).replace(/\\/g, "/");
|
|
467
|
-
const isActive =
|
|
468
|
-
(!manifest.files[file.normalizedPath] || manifest.files[file.normalizedPath].hash !== file.hash);
|
|
561
|
+
const isActive = isActiveFile(file);
|
|
469
562
|
if (isActive) {
|
|
470
563
|
options.progress?.startFile(fileLabel);
|
|
471
564
|
}
|
|
@@ -477,9 +570,9 @@ async function runIndexPassInner(options, logger) {
|
|
|
477
570
|
}
|
|
478
571
|
if (isActive) {
|
|
479
572
|
chunkedDone++;
|
|
573
|
+
winProgress.chunkFile();
|
|
480
574
|
if (chunkedDone % chunkProgressInterval === 0 || chunkedDone === activeCount) {
|
|
481
|
-
|
|
482
|
-
logger.info(` Chunking: ${chunkedDone}/${activeCount} files (${pct}%)`);
|
|
575
|
+
logProgress("Chunking", "", chunkedDone, activeCount, "files");
|
|
483
576
|
}
|
|
484
577
|
}
|
|
485
578
|
if (prep.earlyResult && isActive) {
|
|
@@ -490,11 +583,14 @@ async function runIndexPassInner(options, logger) {
|
|
|
490
583
|
return prepared;
|
|
491
584
|
}
|
|
492
585
|
const aborted = () => options.abortSignal?.aborted ?? false;
|
|
493
|
-
//
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
586
|
+
// Progress line shared by all four stages: "<Stage> [file] — NN.N% overall (done/total unit)".
|
|
587
|
+
// The percentage is ALWAYS the fraction of the WHOLE pass completed
|
|
588
|
+
// (PassProgress across all windows and stages) — never a per-chunk or
|
|
589
|
+
// per-stage fraction. The stage-local counters ride along as a plain
|
|
590
|
+
// done/total count in parentheses.
|
|
591
|
+
const logProgress = (stage, fileLabel, done, total, unit) => {
|
|
592
|
+
const detail = total > 0 ? ` (${done}/${total} ${unit})` : "";
|
|
593
|
+
logger.info(`${stage}${fileLabel ? ` ${fileLabel}` : ""} — ${passProgress.label} overall${detail}`);
|
|
498
594
|
};
|
|
499
595
|
// File metadata look-up for manifest entries
|
|
500
596
|
const fileMeta = new Map(workspaceFiles.map((f) => [f.normalizedPath, { mtime: f.mtime, size: f.size }]));
|
|
@@ -528,10 +624,14 @@ async function runIndexPassInner(options, logger) {
|
|
|
528
624
|
* version-manifest accumulation (K+2 versions per file) that made the
|
|
529
625
|
* store phase degrade quadratically as the index grew.
|
|
530
626
|
*/
|
|
531
|
-
async function storeWindow(prepared, windowEarlyResults) {
|
|
627
|
+
async function storeWindow(prepared, windowEarlyResults, winProgress) {
|
|
532
628
|
const filesToStore = prepared.filter((p) => !windowEarlyResults.has(prepared.indexOf(p)) && p.chunks && (p.textToEmbed?.length ?? 0) > 0).length;
|
|
533
629
|
const storePhaseStart = Date.now();
|
|
534
630
|
logger.info(`Store phase: storing ${filesToStore} file(s) into vector database...`);
|
|
631
|
+
// Nothing to store (every file in this window had an early result or no
|
|
632
|
+
// chunks) — the store stage is trivially done, credit it immediately.
|
|
633
|
+
if (filesToStore === 0)
|
|
634
|
+
winProgress.fill("store");
|
|
535
635
|
let storedFiles = 0;
|
|
536
636
|
// Files whose chunks ALL failed to embed while no manifest entry exists.
|
|
537
637
|
// Rows an earlier pass wrote for them would otherwise never be removed:
|
|
@@ -663,7 +763,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
663
763
|
}
|
|
664
764
|
options.progress?.finishFile(prep.fileLabel);
|
|
665
765
|
storedFiles++;
|
|
666
|
-
|
|
766
|
+
winProgress.stage("store", storedFiles, filesToStore);
|
|
767
|
+
logProgress("Storing", prep.fileLabel, storedFiles, filesToStore, "files");
|
|
667
768
|
storeResults.push(result);
|
|
668
769
|
}
|
|
669
770
|
const workerResults = storeResults;
|
|
@@ -713,7 +814,11 @@ async function runIndexPassInner(options, logger) {
|
|
|
713
814
|
}
|
|
714
815
|
windowCount++;
|
|
715
816
|
const windowFiles = workspaceFiles.slice(windowStart, windowStart + WINDOW_SIZE);
|
|
716
|
-
|
|
817
|
+
// This window's credit handle. The active-file count is taken here, up
|
|
818
|
+
// front: manifest entries for THIS window's files only appear once its
|
|
819
|
+
// store phase runs, so the count stays stable across the window.
|
|
820
|
+
const winProgress = passProgress.beginWindow(windowFiles.filter(isActiveFile).length);
|
|
821
|
+
const prepared = await prepareWindow(windowFiles, winProgress);
|
|
717
822
|
if (windowStart === 0) {
|
|
718
823
|
const chunkPhaseSec = ((Date.now() - chunkPhaseStart) / 1000).toFixed(1);
|
|
719
824
|
logger.info(`Chunking complete: ${activeCount} files in ${chunkPhaseSec}s`);
|
|
@@ -740,31 +845,16 @@ async function runIndexPassInner(options, logger) {
|
|
|
740
845
|
for (const chunk of oversizedChunks) {
|
|
741
846
|
chunk.description = buildFallbackDescription(chunk);
|
|
742
847
|
}
|
|
743
|
-
//
|
|
744
|
-
//
|
|
848
|
+
// chunk.id → file label for the progress lines. The overall
|
|
849
|
+
// percentage comes from PassProgress; the per-chunk index inside the
|
|
850
|
+
// file is deliberately NOT shown.
|
|
745
851
|
const chunkToFileLabel = new Map();
|
|
746
852
|
for (const prep of deferredPreps) {
|
|
747
853
|
for (const chunk of prep.chunks ?? []) {
|
|
748
854
|
chunkToFileLabel.set(chunk.id, prep.fileLabel);
|
|
749
855
|
}
|
|
750
856
|
}
|
|
751
|
-
const
|
|
752
|
-
{
|
|
753
|
-
const perFileCount = new Map();
|
|
754
|
-
for (const chunk of allChunks) {
|
|
755
|
-
perFileCount.set(chunk.metadata.filePath, (perFileCount.get(chunk.metadata.filePath) ?? 0) + 1);
|
|
756
|
-
}
|
|
757
|
-
const perFileSeen = new Map();
|
|
758
|
-
for (const chunk of allChunks) {
|
|
759
|
-
const seen = perFileSeen.get(chunk.metadata.filePath) ?? 0;
|
|
760
|
-
perFileSeen.set(chunk.metadata.filePath, seen + 1);
|
|
761
|
-
chunkMeta.set(chunk.id, {
|
|
762
|
-
fileLabel: chunkToFileLabel.get(chunk.id) ?? chunk.metadata.filePath,
|
|
763
|
-
index: seen + 1,
|
|
764
|
-
count: perFileCount.get(chunk.metadata.filePath) ?? 1,
|
|
765
|
-
});
|
|
766
|
-
}
|
|
767
|
-
}
|
|
857
|
+
const describeFileLabel = (chunk) => chunkToFileLabel.get(chunk.id) ?? chunk.metadata.filePath;
|
|
768
858
|
// Advance progress to Description stage before descriptions start
|
|
769
859
|
for (const prep of deferredPreps) {
|
|
770
860
|
options.progress?.finishStage(prep.fileLabel);
|
|
@@ -791,6 +881,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
791
881
|
for (const { chunk, desc } of cacheHits) {
|
|
792
882
|
chunk.description = desc;
|
|
793
883
|
}
|
|
884
|
+
// Cache hits are already-described chunks of this window.
|
|
885
|
+
winProgress.stage("describe", cacheHits.length, allChunks.length);
|
|
794
886
|
}
|
|
795
887
|
// Build chunkId → prep map for tracking description failures per-file
|
|
796
888
|
const chunkToPrep = new Map();
|
|
@@ -822,9 +914,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
822
914
|
total: totalMisses,
|
|
823
915
|
onProgress: (chunk) => {
|
|
824
916
|
describedDone++;
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
logChunkProgress("Describing", meta.fileLabel, meta.index, meta.count, describedDone, totalMisses);
|
|
917
|
+
winProgress.stage("describe", cacheHits.length + describedDone, allChunks.length);
|
|
918
|
+
logProgress("Describing", describeFileLabel(chunk), describedDone, totalMisses, "texts");
|
|
828
919
|
},
|
|
829
920
|
});
|
|
830
921
|
const newCacheEntries = [];
|
|
@@ -871,9 +962,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
871
962
|
total: allChunks.length,
|
|
872
963
|
onProgress: (chunk) => {
|
|
873
964
|
describedDone++;
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
logChunkProgress("Describing", meta.fileLabel, meta.index, meta.count, describedDone, allChunks.length);
|
|
965
|
+
winProgress.stage("describe", describedDone, allChunks.length);
|
|
966
|
+
logProgress("Describing", describeFileLabel(chunk), describedDone, allChunks.length, "texts");
|
|
877
967
|
},
|
|
878
968
|
});
|
|
879
969
|
for (const chunk of allChunks) {
|
|
@@ -899,6 +989,11 @@ async function runIndexPassInner(options, logger) {
|
|
|
899
989
|
logger.info(`Description phase complete: ${totalDescribedChunks} chunks across ${deferredPreps.length} files`);
|
|
900
990
|
}
|
|
901
991
|
}
|
|
992
|
+
// The describe stage of this window is finished — or was skipped entirely
|
|
993
|
+
// (no description provider, nothing to describe, or a global description
|
|
994
|
+
// failure). Credit the remaining fraction so the overall percentage keeps
|
|
995
|
+
// moving toward 100.
|
|
996
|
+
winProgress.fill("describe");
|
|
902
997
|
// Cross-file embedding batch: collect this window's texts into a single queue,
|
|
903
998
|
// embed in one batched call (with concurrency), then distribute back.
|
|
904
999
|
const isOllama = options.embedder.name === "ollama";
|
|
@@ -906,7 +1001,6 @@ async function runIndexPassInner(options, logger) {
|
|
|
906
1001
|
const defaultConcurrency = options.config.indexing.embedConcurrency ?? 1;
|
|
907
1002
|
// ── Phase 1: Collect embed queue + handle early results ────────────────
|
|
908
1003
|
const embedQueue = [];
|
|
909
|
-
let totalEmbedChunks = 0;
|
|
910
1004
|
earlyWorkerResults.clear();
|
|
911
1005
|
for (let fi = 0; fi < prepared.length; fi++) {
|
|
912
1006
|
const prep = prepared[fi];
|
|
@@ -935,13 +1029,11 @@ async function runIndexPassInner(options, logger) {
|
|
|
935
1029
|
for (let ci = 0; ci < prep.textToEmbed.length; ci++) {
|
|
936
1030
|
embedQueue.push({ fileIdx: fi, chunkIdx: ci, text: prep.textToEmbed[ci] });
|
|
937
1031
|
}
|
|
938
|
-
totalEmbedChunks += prep.textToEmbed.length;
|
|
939
1032
|
}
|
|
940
1033
|
// ── Phase 2: Embed all texts in a single batched call ──────────────────
|
|
941
1034
|
const batchSize = isOllama
|
|
942
1035
|
? Math.min(options.config.indexing.ollamaMaxBatchSize ?? 500, defaultBatchSize)
|
|
943
1036
|
: defaultBatchSize;
|
|
944
|
-
let embeddedDone = 0;
|
|
945
1037
|
const allTexts = embedQueue.map(item => item.text);
|
|
946
1038
|
let allEmbeddings = [];
|
|
947
1039
|
const embedPhaseStart = Date.now();
|
|
@@ -949,8 +1041,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
949
1041
|
logger.info(`Embedding phase: ${allTexts.length} texts in batches of ${batchSize}...`);
|
|
950
1042
|
try {
|
|
951
1043
|
allEmbeddings = await embedBatch(options.embedder, allTexts, batchSize, "document", defaultConcurrency, (completed, total) => {
|
|
952
|
-
|
|
953
|
-
|
|
1044
|
+
winProgress.stage("embed", completed, total);
|
|
1045
|
+
logProgress("Embedding", "", completed, total, "texts");
|
|
954
1046
|
});
|
|
955
1047
|
logger.info(`Embedding complete: ${allTexts.length} texts in ${((Date.now() - embedPhaseStart) / 1000).toFixed(1)}s`);
|
|
956
1048
|
}
|
|
@@ -987,6 +1079,10 @@ async function runIndexPassInner(options, logger) {
|
|
|
987
1079
|
}
|
|
988
1080
|
}
|
|
989
1081
|
}
|
|
1082
|
+
// Embed stage done for this window — worked, skipped (nothing to embed),
|
|
1083
|
+
// or globally failed (a failed embed leaves nothing to store, so holding
|
|
1084
|
+
// the credit back would stall the percentage without helping anyone).
|
|
1085
|
+
winProgress.fill("embed");
|
|
990
1086
|
// ── Distribute embeddings back to per-file chunks ─────────────────────
|
|
991
1087
|
for (let i = 0; i < embedQueue.length; i++) {
|
|
992
1088
|
const { fileIdx, chunkIdx } = embedQueue[i];
|
|
@@ -1008,7 +1104,7 @@ async function runIndexPassInner(options, logger) {
|
|
|
1008
1104
|
// asynchronously (overlapped with the next window's prepare/embed), so a
|
|
1009
1105
|
// shared mutable map would make its per-file loop read the WRONG window's
|
|
1010
1106
|
// entries and skip every file ("Store phase: storing N → 0 files stored").
|
|
1011
|
-
prevStore = storeWindow(prepared, new Map(earlyWorkerResults));
|
|
1107
|
+
prevStore = storeWindow(prepared, new Map(earlyWorkerResults), winProgress);
|
|
1012
1108
|
// Periodically compact fragments and prune old versions so the store
|
|
1013
1109
|
// phase doesn't slow down as the index grows during long runs. Skipped on
|
|
1014
1110
|
// windows that wrote nothing — an idle pass (no changed files) adds no
|
|
@@ -1016,7 +1112,7 @@ async function runIndexPassInner(options, logger) {
|
|
|
1016
1112
|
if (optimizeInterval > 0 && windowCount % optimizeInterval === 0) {
|
|
1017
1113
|
const storedInWindow = await prevStore;
|
|
1018
1114
|
prevStore = null;
|
|
1019
|
-
if (storedInWindow > 0) {
|
|
1115
|
+
if (storedInWindow > 0 && optimizeDue({ force: Boolean(tempStorePath) })) {
|
|
1020
1116
|
logger.info("Optimizing vector store (mid-run compaction, pruning old versions)...");
|
|
1021
1117
|
await effectiveStore.optimize?.(tempStorePath
|
|
1022
1118
|
? { aggressive: true, skipIndex: true, logger: logger.warn }
|
|
@@ -1088,9 +1184,52 @@ async function runIndexPassInner(options, logger) {
|
|
|
1088
1184
|
if (!aborted()) {
|
|
1089
1185
|
tryUpdateLastGitCommit(options.cwd, manifest);
|
|
1090
1186
|
}
|
|
1091
|
-
// ── Atomically promote
|
|
1187
|
+
// ── Atomically promote a rebuilt epoch / temp store ──
|
|
1092
1188
|
let tempStorePromoted = false;
|
|
1093
|
-
if (
|
|
1189
|
+
if (rebuildTarget) {
|
|
1190
|
+
if (!aborted()) {
|
|
1191
|
+
if (!tempStoreWroteChunks) {
|
|
1192
|
+
// Nothing was written (e.g. every embedding request failed). Keep the
|
|
1193
|
+
// existing epoch rather than promoting an empty one over good data.
|
|
1194
|
+
logger.warn("Rebuild produced no chunks — keeping the existing index and manifest unchanged.");
|
|
1195
|
+
try {
|
|
1196
|
+
await options.store.discardRebuild?.();
|
|
1197
|
+
}
|
|
1198
|
+
catch { /* best effort */ }
|
|
1199
|
+
}
|
|
1200
|
+
else {
|
|
1201
|
+
try {
|
|
1202
|
+
await options.store.promoteRebuild?.();
|
|
1203
|
+
tempStorePromoted = true;
|
|
1204
|
+
logger.debug("Promoted rebuilt store epoch");
|
|
1205
|
+
}
|
|
1206
|
+
catch (err) {
|
|
1207
|
+
logger.warn(`Could not promote the rebuilt epoch: ${err.message}`);
|
|
1208
|
+
try {
|
|
1209
|
+
await options.store.discardRebuild?.();
|
|
1210
|
+
}
|
|
1211
|
+
catch { /* best effort */ }
|
|
1212
|
+
}
|
|
1213
|
+
}
|
|
1214
|
+
try {
|
|
1215
|
+
await effectiveStore.close();
|
|
1216
|
+
}
|
|
1217
|
+
catch { /* separate client handle */ }
|
|
1218
|
+
}
|
|
1219
|
+
else {
|
|
1220
|
+
// Aborted — discard the epoch, keep the active data intact.
|
|
1221
|
+
try {
|
|
1222
|
+
await options.store.discardRebuild?.();
|
|
1223
|
+
}
|
|
1224
|
+
catch { /* best effort */ }
|
|
1225
|
+
effectiveStore.close().catch(() => { });
|
|
1226
|
+
logger.debug("Index pass cancelled; discarded the rebuild epoch.");
|
|
1227
|
+
// Skip the final manifest/keyword save: the old state is still valid.
|
|
1228
|
+
await descCache.save();
|
|
1229
|
+
return stats;
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
else if (tempStorePath) {
|
|
1094
1233
|
if (!aborted()) {
|
|
1095
1234
|
if (!tempStoreWroteChunks) {
|
|
1096
1235
|
// Nothing was written (e.g. every embedding request failed). Keep the
|
|
@@ -1148,7 +1287,7 @@ async function runIndexPassInner(options, logger) {
|
|
|
1148
1287
|
// for the new store. When the rebuild did not promote (empty temp store or
|
|
1149
1288
|
// failed swap), keep the previous manifest/keyword index on disk — saving
|
|
1150
1289
|
// the cleared state would orphan the existing store's data.
|
|
1151
|
-
const keepPreviousIndexState = tempStorePath !== undefined && !tempStorePromoted;
|
|
1290
|
+
const keepPreviousIndexState = (tempStorePath !== undefined || rebuildTarget) && !tempStorePromoted;
|
|
1152
1291
|
if (keepPreviousIndexState) {
|
|
1153
1292
|
logger.warn("Rebuild did not complete — keeping the previous manifest and keyword index.");
|
|
1154
1293
|
}
|
|
@@ -1183,6 +1322,14 @@ async function runIndexPassInner(options, logger) {
|
|
|
1183
1322
|
// a successful swap this points to the new data).
|
|
1184
1323
|
await saveManifest(options.storePath, manifest);
|
|
1185
1324
|
await options.keywordIndex?.save(options.storePath);
|
|
1325
|
+
// ClickHouse: upload the durable state bundle (manifest, keyword index,
|
|
1326
|
+
// quirk log) so a fresh clone on another machine needs no re-embedding.
|
|
1327
|
+
try {
|
|
1328
|
+
await options.stateSync?.flush(["manifest", "keyword-index", "quirks", "runtime-overrides", "doc-progress", ".desc-cache"]);
|
|
1329
|
+
}
|
|
1330
|
+
catch (err) {
|
|
1331
|
+
logger.warn(`State upload failed: ${err.message}`);
|
|
1332
|
+
}
|
|
1186
1333
|
}
|
|
1187
1334
|
// Compact fragments and prune old version manifests so countRows() can't
|
|
1188
1335
|
// hang on accumulated versions from many add/delete cycles. After a temp
|
|
@@ -1191,15 +1338,23 @@ async function runIndexPassInner(options, logger) {
|
|
|
1191
1338
|
// swapped-in store is private to this process. Skipped when the rebuild did
|
|
1192
1339
|
// not promote (the old store is untouched and its handle may be closed).
|
|
1193
1340
|
if (!aborted() && !keepPreviousIndexState) {
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
logger.info(`Vector store optimized in ${((Date.now() - optimizeStart) / 1000).toFixed(1)}s`);
|
|
1341
|
+
// Every optimize rebuilds the ANN index (a ~190 MB vector copy at 4096
|
|
1342
|
+
// dims), so passes within the throttle window skip it; the next due pass
|
|
1343
|
+
// compacts and prunes, which also reclaims the skipped builds.
|
|
1344
|
+
if (!optimizeDue({ force: Boolean(tempStorePath || rebuildTarget) })) {
|
|
1345
|
+
logger.debug(`Skipping vector store optimize (throttled: < ${MIN_OPTIMIZE_INTERVAL_MS / 60000} min since the last run)`);
|
|
1200
1346
|
}
|
|
1201
|
-
|
|
1202
|
-
logger.
|
|
1347
|
+
else {
|
|
1348
|
+
logger.info("Optimizing vector store (compacting fragments, pruning old versions)...");
|
|
1349
|
+
const optimizeStart = Date.now();
|
|
1350
|
+
try {
|
|
1351
|
+
const optimizeTarget = (tempStorePath || rebuildTarget) ? options.store : effectiveStore;
|
|
1352
|
+
await optimizeTarget.optimize?.(tempStorePath ? { aggressive: true, logger: logger.warn } : { logger: logger.warn });
|
|
1353
|
+
logger.info(`Vector store optimized in ${((Date.now() - optimizeStart) / 1000).toFixed(1)}s`);
|
|
1354
|
+
}
|
|
1355
|
+
catch (err) {
|
|
1356
|
+
logger.warn(`Vector store optimization failed: ${err.message}`);
|
|
1357
|
+
}
|
|
1203
1358
|
}
|
|
1204
1359
|
}
|
|
1205
1360
|
// Persist description cache
|
|
@@ -1278,6 +1433,7 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
|
|
|
1278
1433
|
storeChunkCount: storeCount,
|
|
1279
1434
|
manifestExpectedChunks: 0,
|
|
1280
1435
|
orphanChunks: 0,
|
|
1436
|
+
scanned: false,
|
|
1281
1437
|
};
|
|
1282
1438
|
}
|
|
1283
1439
|
let upToDateFiles = 0;
|
|
@@ -1311,24 +1467,31 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
|
|
|
1311
1467
|
manifestExpectedChunks = Object.values(manifest.files).reduce((sum, entry) => sum + entry.chunkCount, 0);
|
|
1312
1468
|
}
|
|
1313
1469
|
// Rows without a manifest entry (partial-embedding leftovers). Quirk chunks
|
|
1314
|
-
// live in quirks.jsonl and are deliberately not counted as orphans
|
|
1315
|
-
//
|
|
1470
|
+
// live in quirks.jsonl and are deliberately not counted as orphans — they
|
|
1471
|
+
// are counted separately so `Indexed = Expected + Orphan (+ Quirk)` actually
|
|
1472
|
+
// reconciles. A store that cannot enumerate files reports 0 orphans and
|
|
1473
|
+
// leaves quirkChunks undefined (unknown, not zero).
|
|
1316
1474
|
let orphanChunks = 0;
|
|
1475
|
+
let quirkChunks = 0;
|
|
1317
1476
|
try {
|
|
1318
1477
|
const manifestKeysLower = new Set(Object.keys(manifest.files).map((k) => k.toLowerCase()));
|
|
1478
|
+
quirkChunks = 0;
|
|
1319
1479
|
for (const file of await store.listFiles()) {
|
|
1320
1480
|
const key = normalizeFilePath(file.filePath);
|
|
1481
|
+
if (path.basename(key).startsWith(QUIRK_FILE_PREFIX)) {
|
|
1482
|
+
quirkChunks += file.chunkCount;
|
|
1483
|
+
continue;
|
|
1484
|
+
}
|
|
1321
1485
|
if (manifest.files[key])
|
|
1322
1486
|
continue;
|
|
1323
1487
|
if (manifestKeysLower.has(key.toLowerCase()))
|
|
1324
1488
|
continue;
|
|
1325
|
-
if (path.basename(key).startsWith(QUIRK_FILE_PREFIX))
|
|
1326
|
-
continue;
|
|
1327
1489
|
orphanChunks += file.chunkCount;
|
|
1328
1490
|
}
|
|
1329
1491
|
}
|
|
1330
1492
|
catch {
|
|
1331
1493
|
orphanChunks = 0;
|
|
1494
|
+
quirkChunks = undefined;
|
|
1332
1495
|
}
|
|
1333
1496
|
return {
|
|
1334
1497
|
manifestStatus: loadResult.status,
|
|
@@ -1340,6 +1503,8 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
|
|
|
1340
1503
|
storeChunkCount: storeCount,
|
|
1341
1504
|
manifestExpectedChunks,
|
|
1342
1505
|
orphanChunks,
|
|
1506
|
+
scanned: !skipScan,
|
|
1507
|
+
quirkChunks,
|
|
1343
1508
|
};
|
|
1344
1509
|
}
|
|
1345
1510
|
//# sourceMappingURL=pipeline.js.map
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Whole-pass progress accounting for a single index run.
|
|
3
|
+
*
|
|
4
|
+
* Every progress line the `index` command prints reports ONE percentage:
|
|
5
|
+
* the fraction of the WHOLE pass completed — never a per-chunk or
|
|
6
|
+
* per-stage fraction. This module owns that accounting; the pipeline
|
|
7
|
+
* threads a `WindowProgress` handle through the chunk/describe/embed/store
|
|
8
|
+
* stages of each window and emits lines via the shared `logProgress`
|
|
9
|
+
* helper in pipeline.ts.
|
|
10
|
+
*/
|
|
11
|
+
/** Stage names that report a done/total fraction (chunking counts files directly). */
|
|
12
|
+
export type ProgressStage = "describe" | "embed" | "store";
|
|
13
|
+
/**
|
|
14
|
+
* Tracks the progress of one full index pass in file units.
|
|
15
|
+
*
|
|
16
|
+
* Every active (new or modified, non-empty, large-enough) file contributes
|
|
17
|
+
* one unit, split into four equal credits: chunked, described, embedded,
|
|
18
|
+
* stored. Credits are pushed by `WindowProgress` handles as their window
|
|
19
|
+
* advances, so `percent` rises monotonically from 0 toward 100 across ALL
|
|
20
|
+
* windows and stages — unlike the old per-stage "remaining" percentage
|
|
21
|
+
* that counted down and restarted inside every window.
|
|
22
|
+
*
|
|
23
|
+
* Files are weighted equally regardless of chunk count: the percentage is
|
|
24
|
+
* an honest estimate of "how far through the run are we", not a precise
|
|
25
|
+
* work-meter.
|
|
26
|
+
*/
|
|
27
|
+
export declare class PassProgress {
|
|
28
|
+
readonly totalFiles: number;
|
|
29
|
+
private credited;
|
|
30
|
+
/** @param totalFiles - Total number of active files the pass will process. */
|
|
31
|
+
constructor(totalFiles: number);
|
|
32
|
+
/**
|
|
33
|
+
* Start accounting for one processing window.
|
|
34
|
+
* @param activeFiles - How many of the window's files are active (the same
|
|
35
|
+
* predicate that produced `totalFiles`). Windows with 0 active files
|
|
36
|
+
* contribute nothing to the percentage.
|
|
37
|
+
*/
|
|
38
|
+
beginWindow(activeFiles: number): WindowProgress;
|
|
39
|
+
/** @internal Called by {@link WindowProgress} to push credit into the pass total. */
|
|
40
|
+
push(credit: number): void;
|
|
41
|
+
/** Whole-pass percentage, 0–100, clamped. */
|
|
42
|
+
get percent(): number;
|
|
43
|
+
/** Formatted percentage ("42.3%") for progress lines. */
|
|
44
|
+
get label(): string;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Credit handle for one window of the windowed pipeline. The pipeline
|
|
48
|
+
* reports each stage's done/total fraction here; the handle converts it to
|
|
49
|
+
* whole-pass credit by scaling with the window's active-file count.
|
|
50
|
+
*
|
|
51
|
+
* All updates are clamped to move FORWARD only, so repeated, out-of-order,
|
|
52
|
+
* or overlapping callbacks (concurrent embed sub-batches, the store phase
|
|
53
|
+
* still running while the next window prepares) can never make the overall
|
|
54
|
+
* percentage go backwards.
|
|
55
|
+
*/
|
|
56
|
+
export declare class WindowProgress {
|
|
57
|
+
private readonly owner;
|
|
58
|
+
readonly files: number;
|
|
59
|
+
private done;
|
|
60
|
+
private chunkedFiles;
|
|
61
|
+
constructor(owner: PassProgress, files: number);
|
|
62
|
+
/** Fold the stage-fraction delta into the pass total (forward-only). */
|
|
63
|
+
private advance;
|
|
64
|
+
/** One of this window's active files finished chunking. */
|
|
65
|
+
chunkFile(): void;
|
|
66
|
+
/**
|
|
67
|
+
* Report live progress of a per-stage work queue (described texts,
|
|
68
|
+
* embedded texts, stored files). `total` 0 means the stage had no work for
|
|
69
|
+
* this window (early-result files, full description-cache hits, a skipped
|
|
70
|
+
* stage) and fills the stage's credit immediately.
|
|
71
|
+
*/
|
|
72
|
+
stage(stage: ProgressStage, done: number, total: number): void;
|
|
73
|
+
/** Mark a stage complete regardless of its counters (stage done, skipped, or failed). */
|
|
74
|
+
fill(stage: ProgressStage): void;
|
|
75
|
+
}
|