opencode-rag-plugin 2.10.1 → 2.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +40 -28
- package/dist/api.js +1 -0
- package/dist/chunker/grammar.d.ts +4 -1
- package/dist/chunker/grammar.js +9 -3
- package/dist/chunker/skeleton.js +61 -4
- package/dist/cli/commands/clickhouse.d.ts +61 -0
- package/dist/cli/commands/clickhouse.js +335 -0
- package/dist/cli/commands/eval-tuner.js +1 -1
- package/dist/cli/commands/eval.js +1 -1
- package/dist/cli/commands/index-command.js +2 -1
- package/dist/cli/commands/index.d.ts +1 -0
- package/dist/cli/commands/index.js +1 -0
- package/dist/cli/commands/init-helpers.js +2 -2
- package/dist/cli/commands/init.js +49 -11
- package/dist/cli/commands/quirk.js +6 -6
- package/dist/cli/commands/status.js +70 -6
- package/dist/cli/commands/ui.js +2 -2
- package/dist/cli/index.js +2 -1
- package/dist/cli/types.d.ts +14 -0
- package/dist/core/bootstrap.d.ts +7 -0
- package/dist/core/bootstrap.js +40 -9
- package/dist/core/config.d.ts +128 -5
- package/dist/core/config.js +205 -8
- package/dist/core/interfaces.d.ts +54 -0
- package/dist/core/runtime-overrides.d.ts +9 -0
- package/dist/core/runtime-overrides.js +44 -0
- package/dist/core/state-sync.d.ts +91 -0
- package/dist/core/state-sync.js +273 -0
- package/dist/eval/fast-index.js +2 -2
- package/dist/eval/rerank-eval.js +2 -2
- package/dist/eval/run-branch-compare.js +2 -2
- package/dist/eval/run-token-test.js +2 -2
- package/dist/index.d.ts +5 -2
- package/dist/index.js +2 -1
- package/dist/indexer/pipeline.d.ts +6 -0
- package/dist/indexer/pipeline.js +173 -56
- package/dist/indexer/progress.d.ts +75 -0
- package/dist/indexer/progress.js +114 -0
- package/dist/indexer/stats.d.ts +14 -0
- package/dist/plugin.d.ts +6 -3
- package/dist/plugin.js +39 -20
- package/dist/quirks/quirk-store.d.ts +10 -0
- package/dist/quirks/quirk-store.js +110 -15
- package/dist/quirks/types.d.ts +7 -0
- package/dist/tui.js +128 -111
- package/dist/vectorstore/clickhouse.d.ts +150 -0
- package/dist/vectorstore/clickhouse.js +931 -0
- package/dist/vectorstore/dimension.d.ts +19 -0
- package/dist/vectorstore/dimension.js +36 -0
- package/dist/vectorstore/factory.d.ts +9 -3
- package/dist/vectorstore/factory.js +11 -2
- package/dist/vectorstore/lancedb.d.ts +2 -2
- package/dist/watcher.d.ts +29 -0
- package/dist/watcher.js +76 -2
- package/dist/web/api.d.ts +3 -4
- package/dist/web/api.js +18 -3
- package/dist/web/server.js +28 -2
- package/dist/web/ui/assets/{ScatterPlot3D-CKwyh1Ec.js → ScatterPlot3D-XjXIwavM.js} +5 -5
- package/dist/web/ui/assets/index-B-RH6fko.css +2 -0
- package/dist/web/ui/assets/index-FFPRbgLx.js +4 -0
- package/dist/web/ui/index.html +2 -2
- package/package.json +7 -2
- package/dist/web/ui/assets/index-CWcp0jNf.css +0 -2
- package/dist/web/ui/assets/index-DbXFrn53.js +0 -4
package/dist/indexer/pipeline.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* @fileoverview Orchestrates the full indexing pipeline: scan, chunk, embed, and store.
|
|
3
3
|
*/
|
|
4
4
|
import fs from "node:fs/promises";
|
|
5
|
+
import os from "node:os";
|
|
5
6
|
import path from "node:path";
|
|
6
7
|
import pLimit from "p-limit";
|
|
7
8
|
import { scanWorkspaceFiles } from "../content/reader.js";
|
|
@@ -12,6 +13,7 @@ import { createVectorStore } from "../vectorstore/factory.js";
|
|
|
12
13
|
import { swapStoreDirectories } from "../vectorstore/lancedb.js";
|
|
13
14
|
import { reconcileQuirks, QUIRK_FILE_PREFIX } from "../quirks/quirk-store.js";
|
|
14
15
|
import { rehydrateKeywordIndex } from "./keyword-rehydration.js";
|
|
16
|
+
import { PassProgress } from "./progress.js";
|
|
15
17
|
import { createIndexStats } from "./stats.js";
|
|
16
18
|
import { prepareFile, buildTextsToEmbed } from "./worker.js";
|
|
17
19
|
import { tagChunksRole } from "../core/chunk-role.js";
|
|
@@ -131,10 +133,48 @@ export async function runIndexPass(options) {
|
|
|
131
133
|
// concurrent-pass protection above silently stops working.
|
|
132
134
|
logger.debug(`Could not write index lock file: ${err.message} — concurrent-pass protection disabled`);
|
|
133
135
|
}
|
|
136
|
+
// Cross-machine lease: central stores have no local directory in common, so
|
|
137
|
+
// the file lock above only protects this machine. The lease is advisory
|
|
138
|
+
// (ClickHouse has no CAS) but stops a second machine from re-embedding the
|
|
139
|
+
// same workspace concurrently.
|
|
140
|
+
const leaseOwner = `${os.hostname()}:${process.pid}`;
|
|
141
|
+
const leaseKind = "index";
|
|
142
|
+
const leaseTtlMs = Math.max(10, options.config.vectorStore.clickhouse?.lease?.ttlSeconds ?? 90) * 1000;
|
|
143
|
+
let leaseHeld = false;
|
|
144
|
+
let leaseTimer;
|
|
145
|
+
if (typeof options.store.acquireLease === "function") {
|
|
146
|
+
try {
|
|
147
|
+
leaseHeld = await options.store.acquireLease(leaseOwner, leaseKind, leaseTtlMs);
|
|
148
|
+
}
|
|
149
|
+
catch (err) {
|
|
150
|
+
// Do not leak the local lock file when the central store is unreachable.
|
|
151
|
+
try {
|
|
152
|
+
await fs.unlink(lockPath);
|
|
153
|
+
}
|
|
154
|
+
catch { /* best effort */ }
|
|
155
|
+
throw err;
|
|
156
|
+
}
|
|
157
|
+
if (!leaseHeld) {
|
|
158
|
+
logger.warn("Another machine holds the central store's index lease. Skipping.");
|
|
159
|
+
try {
|
|
160
|
+
await fs.unlink(lockPath);
|
|
161
|
+
}
|
|
162
|
+
catch { /* best effort */ }
|
|
163
|
+
return { ...createIndexStats(0, "missing"), skipped: true };
|
|
164
|
+
}
|
|
165
|
+
leaseTimer = setInterval(() => {
|
|
166
|
+
void options.store.renewLease?.(leaseOwner, leaseKind, leaseTtlMs).catch(() => { });
|
|
167
|
+
}, Math.max(5_000, Math.floor(leaseTtlMs / 3)));
|
|
168
|
+
leaseTimer.unref();
|
|
169
|
+
}
|
|
134
170
|
try {
|
|
135
171
|
return await runIndexPassInner(options, logger);
|
|
136
172
|
}
|
|
137
173
|
finally {
|
|
174
|
+
if (leaseTimer)
|
|
175
|
+
clearInterval(leaseTimer);
|
|
176
|
+
if (leaseHeld)
|
|
177
|
+
await options.store.releaseLease?.(leaseOwner, leaseKind).catch(() => { });
|
|
138
178
|
try {
|
|
139
179
|
await fs.unlink(lockPath);
|
|
140
180
|
}
|
|
@@ -306,6 +346,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
306
346
|
// Effective store used throughout the pass — may be a temp store for atomic rebuild.
|
|
307
347
|
let effectiveStore = options.store;
|
|
308
348
|
let tempStorePath;
|
|
349
|
+
/** True when the rebuild writes into a private epoch of a central store. */
|
|
350
|
+
let rebuildTarget = false;
|
|
309
351
|
/** Set when the temporary rebuild store actually received rows. */
|
|
310
352
|
let tempStoreWroteChunks = false;
|
|
311
353
|
if (options.force || (manifestStatus !== "ok" && (existingCount > 0 || dimensionMismatch))) {
|
|
@@ -321,13 +363,21 @@ async function runIndexPassInner(options, logger) {
|
|
|
321
363
|
// Build the new index into a temporary store first, then atomically
|
|
322
364
|
// swap on completion. Original data stays untouched if the process
|
|
323
365
|
// is aborted (Ctrl+C, crash) before the swap completes.
|
|
324
|
-
if (options.
|
|
366
|
+
if (typeof options.store.beginRebuild === "function") {
|
|
367
|
+
// Central stores (ClickHouse) rebuild into a private epoch and promote
|
|
368
|
+
// atomically; no local temp directory and no directory swap.
|
|
369
|
+
effectiveStore = await options.store.beginRebuild();
|
|
370
|
+
rebuildTarget = true;
|
|
371
|
+
logger.debug("Rebuilding index in a private store epoch");
|
|
372
|
+
options.keywordIndex?.clear();
|
|
373
|
+
}
|
|
374
|
+
else if (options.dimension) {
|
|
325
375
|
tempStorePath = options.storePath + "_tmp";
|
|
326
376
|
try {
|
|
327
377
|
await fs.rm(tempStorePath, { recursive: true, force: true });
|
|
328
378
|
}
|
|
329
379
|
catch { /* may not exist */ }
|
|
330
|
-
effectiveStore = createVectorStore(options.config, tempStorePath, options.dimension);
|
|
380
|
+
effectiveStore = createVectorStore(options.config, tempStorePath, options.dimension, options.cwd);
|
|
331
381
|
logger.debug(`Rebuilding index in temporary store at ${tempStorePath}`);
|
|
332
382
|
// Only clear the shared in-memory keyword index once the rebuild is
|
|
333
383
|
// guaranteed to proceed — the guard below may still bail out, and
|
|
@@ -495,17 +545,20 @@ async function runIndexPassInner(options, logger) {
|
|
|
495
545
|
logger.debug(`Processing ${workspaceFiles.length} files (${newCount} new, ${modifiedCount} modified, ${unchangedCount} unchanged)...`);
|
|
496
546
|
const limit = pLimit(options.config.indexing.concurrency);
|
|
497
547
|
const deferDescriptions = !!options.descriptionProvider;
|
|
498
|
-
const
|
|
548
|
+
const isActiveFile = (f) => !f.isEmpty && !f.isTooSmall && (!manifest.files[f.normalizedPath] || manifest.files[f.normalizedPath].hash !== f.hash);
|
|
549
|
+
const activeCount = workspaceFiles.filter(isActiveFile).length;
|
|
499
550
|
logger.info(`Chunking ${activeCount} file(s)...`);
|
|
551
|
+
// Whole-pass progress in file units: every progress line reports this
|
|
552
|
+
// tracker's percentage, never a per-chunk or per-stage fraction.
|
|
553
|
+
const passProgress = new PassProgress(activeCount);
|
|
500
554
|
let chunkedDone = 0;
|
|
501
555
|
const chunkProgressInterval = Math.max(1, Math.floor(activeCount / 20));
|
|
502
556
|
const chunkPhaseStart = Date.now();
|
|
503
557
|
/** Chunk one window of files (used by the windowed pipeline loop below). */
|
|
504
|
-
async function prepareWindow(windowFiles) {
|
|
558
|
+
async function prepareWindow(windowFiles, winProgress) {
|
|
505
559
|
const prepared = await Promise.all(windowFiles.map((file) => limit(async () => {
|
|
506
560
|
const fileLabel = path.relative(options.cwd, file.normalizedPath).replace(/\\/g, "/");
|
|
507
|
-
const isActive =
|
|
508
|
-
(!manifest.files[file.normalizedPath] || manifest.files[file.normalizedPath].hash !== file.hash);
|
|
561
|
+
const isActive = isActiveFile(file);
|
|
509
562
|
if (isActive) {
|
|
510
563
|
options.progress?.startFile(fileLabel);
|
|
511
564
|
}
|
|
@@ -517,9 +570,9 @@ async function runIndexPassInner(options, logger) {
|
|
|
517
570
|
}
|
|
518
571
|
if (isActive) {
|
|
519
572
|
chunkedDone++;
|
|
573
|
+
winProgress.chunkFile();
|
|
520
574
|
if (chunkedDone % chunkProgressInterval === 0 || chunkedDone === activeCount) {
|
|
521
|
-
|
|
522
|
-
logger.info(` Chunking: ${chunkedDone}/${activeCount} files (${pct}%)`);
|
|
575
|
+
logProgress("Chunking", "", chunkedDone, activeCount, "files");
|
|
523
576
|
}
|
|
524
577
|
}
|
|
525
578
|
if (prep.earlyResult && isActive) {
|
|
@@ -530,11 +583,14 @@ async function runIndexPassInner(options, logger) {
|
|
|
530
583
|
return prepared;
|
|
531
584
|
}
|
|
532
585
|
const aborted = () => options.abortSignal?.aborted ?? false;
|
|
533
|
-
//
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
586
|
+
// Progress line shared by all four stages: "<Stage> [file] — NN.N% overall (done/total unit)".
|
|
587
|
+
// The percentage is ALWAYS the fraction of the WHOLE pass completed
|
|
588
|
+
// (PassProgress across all windows and stages) — never a per-chunk or
|
|
589
|
+
// per-stage fraction. The stage-local counters ride along as a plain
|
|
590
|
+
// done/total count in parentheses.
|
|
591
|
+
const logProgress = (stage, fileLabel, done, total, unit) => {
|
|
592
|
+
const detail = total > 0 ? ` (${done}/${total} ${unit})` : "";
|
|
593
|
+
logger.info(`${stage}${fileLabel ? ` ${fileLabel}` : ""} — ${passProgress.label} overall${detail}`);
|
|
538
594
|
};
|
|
539
595
|
// File metadata look-up for manifest entries
|
|
540
596
|
const fileMeta = new Map(workspaceFiles.map((f) => [f.normalizedPath, { mtime: f.mtime, size: f.size }]));
|
|
@@ -568,10 +624,14 @@ async function runIndexPassInner(options, logger) {
|
|
|
568
624
|
* version-manifest accumulation (K+2 versions per file) that made the
|
|
569
625
|
* store phase degrade quadratically as the index grew.
|
|
570
626
|
*/
|
|
571
|
-
async function storeWindow(prepared, windowEarlyResults) {
|
|
627
|
+
async function storeWindow(prepared, windowEarlyResults, winProgress) {
|
|
572
628
|
const filesToStore = prepared.filter((p) => !windowEarlyResults.has(prepared.indexOf(p)) && p.chunks && (p.textToEmbed?.length ?? 0) > 0).length;
|
|
573
629
|
const storePhaseStart = Date.now();
|
|
574
630
|
logger.info(`Store phase: storing ${filesToStore} file(s) into vector database...`);
|
|
631
|
+
// Nothing to store (every file in this window had an early result or no
|
|
632
|
+
// chunks) — the store stage is trivially done, credit it immediately.
|
|
633
|
+
if (filesToStore === 0)
|
|
634
|
+
winProgress.fill("store");
|
|
575
635
|
let storedFiles = 0;
|
|
576
636
|
// Files whose chunks ALL failed to embed while no manifest entry exists.
|
|
577
637
|
// Rows an earlier pass wrote for them would otherwise never be removed:
|
|
@@ -703,7 +763,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
703
763
|
}
|
|
704
764
|
options.progress?.finishFile(prep.fileLabel);
|
|
705
765
|
storedFiles++;
|
|
706
|
-
|
|
766
|
+
winProgress.stage("store", storedFiles, filesToStore);
|
|
767
|
+
logProgress("Storing", prep.fileLabel, storedFiles, filesToStore, "files");
|
|
707
768
|
storeResults.push(result);
|
|
708
769
|
}
|
|
709
770
|
const workerResults = storeResults;
|
|
@@ -753,7 +814,11 @@ async function runIndexPassInner(options, logger) {
|
|
|
753
814
|
}
|
|
754
815
|
windowCount++;
|
|
755
816
|
const windowFiles = workspaceFiles.slice(windowStart, windowStart + WINDOW_SIZE);
|
|
756
|
-
|
|
817
|
+
// This window's credit handle. The active-file count is taken here, up
|
|
818
|
+
// front: manifest entries for THIS window's files only appear once its
|
|
819
|
+
// store phase runs, so the count stays stable across the window.
|
|
820
|
+
const winProgress = passProgress.beginWindow(windowFiles.filter(isActiveFile).length);
|
|
821
|
+
const prepared = await prepareWindow(windowFiles, winProgress);
|
|
757
822
|
if (windowStart === 0) {
|
|
758
823
|
const chunkPhaseSec = ((Date.now() - chunkPhaseStart) / 1000).toFixed(1);
|
|
759
824
|
logger.info(`Chunking complete: ${activeCount} files in ${chunkPhaseSec}s`);
|
|
@@ -780,31 +845,16 @@ async function runIndexPassInner(options, logger) {
|
|
|
780
845
|
for (const chunk of oversizedChunks) {
|
|
781
846
|
chunk.description = buildFallbackDescription(chunk);
|
|
782
847
|
}
|
|
783
|
-
//
|
|
784
|
-
//
|
|
848
|
+
// chunk.id → file label for the progress lines. The overall
|
|
849
|
+
// percentage comes from PassProgress; the per-chunk index inside the
|
|
850
|
+
// file is deliberately NOT shown.
|
|
785
851
|
const chunkToFileLabel = new Map();
|
|
786
852
|
for (const prep of deferredPreps) {
|
|
787
853
|
for (const chunk of prep.chunks ?? []) {
|
|
788
854
|
chunkToFileLabel.set(chunk.id, prep.fileLabel);
|
|
789
855
|
}
|
|
790
856
|
}
|
|
791
|
-
const
|
|
792
|
-
{
|
|
793
|
-
const perFileCount = new Map();
|
|
794
|
-
for (const chunk of allChunks) {
|
|
795
|
-
perFileCount.set(chunk.metadata.filePath, (perFileCount.get(chunk.metadata.filePath) ?? 0) + 1);
|
|
796
|
-
}
|
|
797
|
-
const perFileSeen = new Map();
|
|
798
|
-
for (const chunk of allChunks) {
|
|
799
|
-
const seen = perFileSeen.get(chunk.metadata.filePath) ?? 0;
|
|
800
|
-
perFileSeen.set(chunk.metadata.filePath, seen + 1);
|
|
801
|
-
chunkMeta.set(chunk.id, {
|
|
802
|
-
fileLabel: chunkToFileLabel.get(chunk.id) ?? chunk.metadata.filePath,
|
|
803
|
-
index: seen + 1,
|
|
804
|
-
count: perFileCount.get(chunk.metadata.filePath) ?? 1,
|
|
805
|
-
});
|
|
806
|
-
}
|
|
807
|
-
}
|
|
857
|
+
const describeFileLabel = (chunk) => chunkToFileLabel.get(chunk.id) ?? chunk.metadata.filePath;
|
|
808
858
|
// Advance progress to Description stage before descriptions start
|
|
809
859
|
for (const prep of deferredPreps) {
|
|
810
860
|
options.progress?.finishStage(prep.fileLabel);
|
|
@@ -831,6 +881,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
831
881
|
for (const { chunk, desc } of cacheHits) {
|
|
832
882
|
chunk.description = desc;
|
|
833
883
|
}
|
|
884
|
+
// Cache hits are already-described chunks of this window.
|
|
885
|
+
winProgress.stage("describe", cacheHits.length, allChunks.length);
|
|
834
886
|
}
|
|
835
887
|
// Build chunkId → prep map for tracking description failures per-file
|
|
836
888
|
const chunkToPrep = new Map();
|
|
@@ -862,9 +914,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
862
914
|
total: totalMisses,
|
|
863
915
|
onProgress: (chunk) => {
|
|
864
916
|
describedDone++;
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
logChunkProgress("Describing", meta.fileLabel, meta.index, meta.count, describedDone, totalMisses);
|
|
917
|
+
winProgress.stage("describe", cacheHits.length + describedDone, allChunks.length);
|
|
918
|
+
logProgress("Describing", describeFileLabel(chunk), describedDone, totalMisses, "texts");
|
|
868
919
|
},
|
|
869
920
|
});
|
|
870
921
|
const newCacheEntries = [];
|
|
@@ -911,9 +962,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
911
962
|
total: allChunks.length,
|
|
912
963
|
onProgress: (chunk) => {
|
|
913
964
|
describedDone++;
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
logChunkProgress("Describing", meta.fileLabel, meta.index, meta.count, describedDone, allChunks.length);
|
|
965
|
+
winProgress.stage("describe", describedDone, allChunks.length);
|
|
966
|
+
logProgress("Describing", describeFileLabel(chunk), describedDone, allChunks.length, "texts");
|
|
917
967
|
},
|
|
918
968
|
});
|
|
919
969
|
for (const chunk of allChunks) {
|
|
@@ -939,6 +989,11 @@ async function runIndexPassInner(options, logger) {
|
|
|
939
989
|
logger.info(`Description phase complete: ${totalDescribedChunks} chunks across ${deferredPreps.length} files`);
|
|
940
990
|
}
|
|
941
991
|
}
|
|
992
|
+
// The describe stage of this window is finished — or was skipped entirely
|
|
993
|
+
// (no description provider, nothing to describe, or a global description
|
|
994
|
+
// failure). Credit the remaining fraction so the overall percentage keeps
|
|
995
|
+
// moving toward 100.
|
|
996
|
+
winProgress.fill("describe");
|
|
942
997
|
// Cross-file embedding batch: collect this window's texts into a single queue,
|
|
943
998
|
// embed in one batched call (with concurrency), then distribute back.
|
|
944
999
|
const isOllama = options.embedder.name === "ollama";
|
|
@@ -946,7 +1001,6 @@ async function runIndexPassInner(options, logger) {
|
|
|
946
1001
|
const defaultConcurrency = options.config.indexing.embedConcurrency ?? 1;
|
|
947
1002
|
// ── Phase 1: Collect embed queue + handle early results ────────────────
|
|
948
1003
|
const embedQueue = [];
|
|
949
|
-
let totalEmbedChunks = 0;
|
|
950
1004
|
earlyWorkerResults.clear();
|
|
951
1005
|
for (let fi = 0; fi < prepared.length; fi++) {
|
|
952
1006
|
const prep = prepared[fi];
|
|
@@ -975,13 +1029,11 @@ async function runIndexPassInner(options, logger) {
|
|
|
975
1029
|
for (let ci = 0; ci < prep.textToEmbed.length; ci++) {
|
|
976
1030
|
embedQueue.push({ fileIdx: fi, chunkIdx: ci, text: prep.textToEmbed[ci] });
|
|
977
1031
|
}
|
|
978
|
-
totalEmbedChunks += prep.textToEmbed.length;
|
|
979
1032
|
}
|
|
980
1033
|
// ── Phase 2: Embed all texts in a single batched call ──────────────────
|
|
981
1034
|
const batchSize = isOllama
|
|
982
1035
|
? Math.min(options.config.indexing.ollamaMaxBatchSize ?? 500, defaultBatchSize)
|
|
983
1036
|
: defaultBatchSize;
|
|
984
|
-
let embeddedDone = 0;
|
|
985
1037
|
const allTexts = embedQueue.map(item => item.text);
|
|
986
1038
|
let allEmbeddings = [];
|
|
987
1039
|
const embedPhaseStart = Date.now();
|
|
@@ -989,8 +1041,8 @@ async function runIndexPassInner(options, logger) {
|
|
|
989
1041
|
logger.info(`Embedding phase: ${allTexts.length} texts in batches of ${batchSize}...`);
|
|
990
1042
|
try {
|
|
991
1043
|
allEmbeddings = await embedBatch(options.embedder, allTexts, batchSize, "document", defaultConcurrency, (completed, total) => {
|
|
992
|
-
|
|
993
|
-
|
|
1044
|
+
winProgress.stage("embed", completed, total);
|
|
1045
|
+
logProgress("Embedding", "", completed, total, "texts");
|
|
994
1046
|
});
|
|
995
1047
|
logger.info(`Embedding complete: ${allTexts.length} texts in ${((Date.now() - embedPhaseStart) / 1000).toFixed(1)}s`);
|
|
996
1048
|
}
|
|
@@ -1027,6 +1079,10 @@ async function runIndexPassInner(options, logger) {
|
|
|
1027
1079
|
}
|
|
1028
1080
|
}
|
|
1029
1081
|
}
|
|
1082
|
+
// Embed stage done for this window — worked, skipped (nothing to embed),
|
|
1083
|
+
// or globally failed (a failed embed leaves nothing to store, so holding
|
|
1084
|
+
// the credit back would stall the percentage without helping anyone).
|
|
1085
|
+
winProgress.fill("embed");
|
|
1030
1086
|
// ── Distribute embeddings back to per-file chunks ─────────────────────
|
|
1031
1087
|
for (let i = 0; i < embedQueue.length; i++) {
|
|
1032
1088
|
const { fileIdx, chunkIdx } = embedQueue[i];
|
|
@@ -1048,7 +1104,7 @@ async function runIndexPassInner(options, logger) {
|
|
|
1048
1104
|
// asynchronously (overlapped with the next window's prepare/embed), so a
|
|
1049
1105
|
// shared mutable map would make its per-file loop read the WRONG window's
|
|
1050
1106
|
// entries and skip every file ("Store phase: storing N → 0 files stored").
|
|
1051
|
-
prevStore = storeWindow(prepared, new Map(earlyWorkerResults));
|
|
1107
|
+
prevStore = storeWindow(prepared, new Map(earlyWorkerResults), winProgress);
|
|
1052
1108
|
// Periodically compact fragments and prune old versions so the store
|
|
1053
1109
|
// phase doesn't slow down as the index grows during long runs. Skipped on
|
|
1054
1110
|
// windows that wrote nothing — an idle pass (no changed files) adds no
|
|
@@ -1128,9 +1184,52 @@ async function runIndexPassInner(options, logger) {
|
|
|
1128
1184
|
if (!aborted()) {
|
|
1129
1185
|
tryUpdateLastGitCommit(options.cwd, manifest);
|
|
1130
1186
|
}
|
|
1131
|
-
// ── Atomically promote
|
|
1187
|
+
// ── Atomically promote a rebuilt epoch / temp store ──
|
|
1132
1188
|
let tempStorePromoted = false;
|
|
1133
|
-
if (
|
|
1189
|
+
if (rebuildTarget) {
|
|
1190
|
+
if (!aborted()) {
|
|
1191
|
+
if (!tempStoreWroteChunks) {
|
|
1192
|
+
// Nothing was written (e.g. every embedding request failed). Keep the
|
|
1193
|
+
// existing epoch rather than promoting an empty one over good data.
|
|
1194
|
+
logger.warn("Rebuild produced no chunks — keeping the existing index and manifest unchanged.");
|
|
1195
|
+
try {
|
|
1196
|
+
await options.store.discardRebuild?.();
|
|
1197
|
+
}
|
|
1198
|
+
catch { /* best effort */ }
|
|
1199
|
+
}
|
|
1200
|
+
else {
|
|
1201
|
+
try {
|
|
1202
|
+
await options.store.promoteRebuild?.();
|
|
1203
|
+
tempStorePromoted = true;
|
|
1204
|
+
logger.debug("Promoted rebuilt store epoch");
|
|
1205
|
+
}
|
|
1206
|
+
catch (err) {
|
|
1207
|
+
logger.warn(`Could not promote the rebuilt epoch: ${err.message}`);
|
|
1208
|
+
try {
|
|
1209
|
+
await options.store.discardRebuild?.();
|
|
1210
|
+
}
|
|
1211
|
+
catch { /* best effort */ }
|
|
1212
|
+
}
|
|
1213
|
+
}
|
|
1214
|
+
try {
|
|
1215
|
+
await effectiveStore.close();
|
|
1216
|
+
}
|
|
1217
|
+
catch { /* separate client handle */ }
|
|
1218
|
+
}
|
|
1219
|
+
else {
|
|
1220
|
+
// Aborted — discard the epoch, keep the active data intact.
|
|
1221
|
+
try {
|
|
1222
|
+
await options.store.discardRebuild?.();
|
|
1223
|
+
}
|
|
1224
|
+
catch { /* best effort */ }
|
|
1225
|
+
effectiveStore.close().catch(() => { });
|
|
1226
|
+
logger.debug("Index pass cancelled; discarded the rebuild epoch.");
|
|
1227
|
+
// Skip the final manifest/keyword save: the old state is still valid.
|
|
1228
|
+
await descCache.save();
|
|
1229
|
+
return stats;
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
else if (tempStorePath) {
|
|
1134
1233
|
if (!aborted()) {
|
|
1135
1234
|
if (!tempStoreWroteChunks) {
|
|
1136
1235
|
// Nothing was written (e.g. every embedding request failed). Keep the
|
|
@@ -1188,7 +1287,7 @@ async function runIndexPassInner(options, logger) {
|
|
|
1188
1287
|
// for the new store. When the rebuild did not promote (empty temp store or
|
|
1189
1288
|
// failed swap), keep the previous manifest/keyword index on disk — saving
|
|
1190
1289
|
// the cleared state would orphan the existing store's data.
|
|
1191
|
-
const keepPreviousIndexState = tempStorePath !== undefined && !tempStorePromoted;
|
|
1290
|
+
const keepPreviousIndexState = (tempStorePath !== undefined || rebuildTarget) && !tempStorePromoted;
|
|
1192
1291
|
if (keepPreviousIndexState) {
|
|
1193
1292
|
logger.warn("Rebuild did not complete — keeping the previous manifest and keyword index.");
|
|
1194
1293
|
}
|
|
@@ -1223,6 +1322,14 @@ async function runIndexPassInner(options, logger) {
|
|
|
1223
1322
|
// a successful swap this points to the new data).
|
|
1224
1323
|
await saveManifest(options.storePath, manifest);
|
|
1225
1324
|
await options.keywordIndex?.save(options.storePath);
|
|
1325
|
+
// ClickHouse: upload the durable state bundle (manifest, keyword index,
|
|
1326
|
+
// quirk log) so a fresh clone on another machine needs no re-embedding.
|
|
1327
|
+
try {
|
|
1328
|
+
await options.stateSync?.flush(["manifest", "keyword-index", "quirks", "runtime-overrides", "doc-progress", ".desc-cache"]);
|
|
1329
|
+
}
|
|
1330
|
+
catch (err) {
|
|
1331
|
+
logger.warn(`State upload failed: ${err.message}`);
|
|
1332
|
+
}
|
|
1226
1333
|
}
|
|
1227
1334
|
// Compact fragments and prune old version manifests so countRows() can't
|
|
1228
1335
|
// hang on accumulated versions from many add/delete cycles. After a temp
|
|
@@ -1234,14 +1341,14 @@ async function runIndexPassInner(options, logger) {
|
|
|
1234
1341
|
// Every optimize rebuilds the ANN index (a ~190 MB vector copy at 4096
|
|
1235
1342
|
// dims), so passes within the throttle window skip it; the next due pass
|
|
1236
1343
|
// compacts and prunes, which also reclaims the skipped builds.
|
|
1237
|
-
if (!optimizeDue({ force: Boolean(tempStorePath) })) {
|
|
1344
|
+
if (!optimizeDue({ force: Boolean(tempStorePath || rebuildTarget) })) {
|
|
1238
1345
|
logger.debug(`Skipping vector store optimize (throttled: < ${MIN_OPTIMIZE_INTERVAL_MS / 60000} min since the last run)`);
|
|
1239
1346
|
}
|
|
1240
1347
|
else {
|
|
1241
1348
|
logger.info("Optimizing vector store (compacting fragments, pruning old versions)...");
|
|
1242
1349
|
const optimizeStart = Date.now();
|
|
1243
1350
|
try {
|
|
1244
|
-
const optimizeTarget = tempStorePath ? options.store : effectiveStore;
|
|
1351
|
+
const optimizeTarget = (tempStorePath || rebuildTarget) ? options.store : effectiveStore;
|
|
1245
1352
|
await optimizeTarget.optimize?.(tempStorePath ? { aggressive: true, logger: logger.warn } : { logger: logger.warn });
|
|
1246
1353
|
logger.info(`Vector store optimized in ${((Date.now() - optimizeStart) / 1000).toFixed(1)}s`);
|
|
1247
1354
|
}
|
|
@@ -1326,6 +1433,7 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
|
|
|
1326
1433
|
storeChunkCount: storeCount,
|
|
1327
1434
|
manifestExpectedChunks: 0,
|
|
1328
1435
|
orphanChunks: 0,
|
|
1436
|
+
scanned: false,
|
|
1329
1437
|
};
|
|
1330
1438
|
}
|
|
1331
1439
|
let upToDateFiles = 0;
|
|
@@ -1359,24 +1467,31 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
|
|
|
1359
1467
|
manifestExpectedChunks = Object.values(manifest.files).reduce((sum, entry) => sum + entry.chunkCount, 0);
|
|
1360
1468
|
}
|
|
1361
1469
|
// Rows without a manifest entry (partial-embedding leftovers). Quirk chunks
|
|
1362
|
-
// live in quirks.jsonl and are deliberately not counted as orphans
|
|
1363
|
-
//
|
|
1470
|
+
// live in quirks.jsonl and are deliberately not counted as orphans — they
|
|
1471
|
+
// are counted separately so `Indexed = Expected + Orphan (+ Quirk)` actually
|
|
1472
|
+
// reconciles. A store that cannot enumerate files reports 0 orphans and
|
|
1473
|
+
// leaves quirkChunks undefined (unknown, not zero).
|
|
1364
1474
|
let orphanChunks = 0;
|
|
1475
|
+
let quirkChunks = 0;
|
|
1365
1476
|
try {
|
|
1366
1477
|
const manifestKeysLower = new Set(Object.keys(manifest.files).map((k) => k.toLowerCase()));
|
|
1478
|
+
quirkChunks = 0;
|
|
1367
1479
|
for (const file of await store.listFiles()) {
|
|
1368
1480
|
const key = normalizeFilePath(file.filePath);
|
|
1481
|
+
if (path.basename(key).startsWith(QUIRK_FILE_PREFIX)) {
|
|
1482
|
+
quirkChunks += file.chunkCount;
|
|
1483
|
+
continue;
|
|
1484
|
+
}
|
|
1369
1485
|
if (manifest.files[key])
|
|
1370
1486
|
continue;
|
|
1371
1487
|
if (manifestKeysLower.has(key.toLowerCase()))
|
|
1372
1488
|
continue;
|
|
1373
|
-
if (path.basename(key).startsWith(QUIRK_FILE_PREFIX))
|
|
1374
|
-
continue;
|
|
1375
1489
|
orphanChunks += file.chunkCount;
|
|
1376
1490
|
}
|
|
1377
1491
|
}
|
|
1378
1492
|
catch {
|
|
1379
1493
|
orphanChunks = 0;
|
|
1494
|
+
quirkChunks = undefined;
|
|
1380
1495
|
}
|
|
1381
1496
|
return {
|
|
1382
1497
|
manifestStatus: loadResult.status,
|
|
@@ -1388,6 +1503,8 @@ export async function getIndexStatusSummary(cwd, storePath, config, store, skipS
|
|
|
1388
1503
|
storeChunkCount: storeCount,
|
|
1389
1504
|
manifestExpectedChunks,
|
|
1390
1505
|
orphanChunks,
|
|
1506
|
+
scanned: !skipScan,
|
|
1507
|
+
quirkChunks,
|
|
1391
1508
|
};
|
|
1392
1509
|
}
|
|
1393
1510
|
//# sourceMappingURL=pipeline.js.map
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Whole-pass progress accounting for a single index run.
|
|
3
|
+
*
|
|
4
|
+
* Every progress line the `index` command prints reports ONE percentage:
|
|
5
|
+
* the fraction of the WHOLE pass completed — never a per-chunk or
|
|
6
|
+
* per-stage fraction. This module owns that accounting; the pipeline
|
|
7
|
+
* threads a `WindowProgress` handle through the chunk/describe/embed/store
|
|
8
|
+
* stages of each window and emits lines via the shared `logProgress`
|
|
9
|
+
* helper in pipeline.ts.
|
|
10
|
+
*/
|
|
11
|
+
/** Stage names that report a done/total fraction (chunking counts files directly). */
|
|
12
|
+
export type ProgressStage = "describe" | "embed" | "store";
|
|
13
|
+
/**
|
|
14
|
+
* Tracks the progress of one full index pass in file units.
|
|
15
|
+
*
|
|
16
|
+
* Every active (new or modified, non-empty, large-enough) file contributes
|
|
17
|
+
* one unit, split into four equal credits: chunked, described, embedded,
|
|
18
|
+
* stored. Credits are pushed by `WindowProgress` handles as their window
|
|
19
|
+
* advances, so `percent` rises monotonically from 0 toward 100 across ALL
|
|
20
|
+
* windows and stages — unlike the old per-stage "remaining" percentage
|
|
21
|
+
* that counted down and restarted inside every window.
|
|
22
|
+
*
|
|
23
|
+
* Files are weighted equally regardless of chunk count: the percentage is
|
|
24
|
+
* an honest estimate of "how far through the run are we", not a precise
|
|
25
|
+
* work-meter.
|
|
26
|
+
*/
|
|
27
|
+
export declare class PassProgress {
|
|
28
|
+
readonly totalFiles: number;
|
|
29
|
+
private credited;
|
|
30
|
+
/** @param totalFiles - Total number of active files the pass will process. */
|
|
31
|
+
constructor(totalFiles: number);
|
|
32
|
+
/**
|
|
33
|
+
* Start accounting for one processing window.
|
|
34
|
+
* @param activeFiles - How many of the window's files are active (the same
|
|
35
|
+
* predicate that produced `totalFiles`). Windows with 0 active files
|
|
36
|
+
* contribute nothing to the percentage.
|
|
37
|
+
*/
|
|
38
|
+
beginWindow(activeFiles: number): WindowProgress;
|
|
39
|
+
/** @internal Called by {@link WindowProgress} to push credit into the pass total. */
|
|
40
|
+
push(credit: number): void;
|
|
41
|
+
/** Whole-pass percentage, 0–100, clamped. */
|
|
42
|
+
get percent(): number;
|
|
43
|
+
/** Formatted percentage ("42.3%") for progress lines. */
|
|
44
|
+
get label(): string;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Credit handle for one window of the windowed pipeline. The pipeline
|
|
48
|
+
* reports each stage's done/total fraction here; the handle converts it to
|
|
49
|
+
* whole-pass credit by scaling with the window's active-file count.
|
|
50
|
+
*
|
|
51
|
+
* All updates are clamped to move FORWARD only, so repeated, out-of-order,
|
|
52
|
+
* or overlapping callbacks (concurrent embed sub-batches, the store phase
|
|
53
|
+
* still running while the next window prepares) can never make the overall
|
|
54
|
+
* percentage go backwards.
|
|
55
|
+
*/
|
|
56
|
+
export declare class WindowProgress {
|
|
57
|
+
private readonly owner;
|
|
58
|
+
readonly files: number;
|
|
59
|
+
private done;
|
|
60
|
+
private chunkedFiles;
|
|
61
|
+
constructor(owner: PassProgress, files: number);
|
|
62
|
+
/** Fold the stage-fraction delta into the pass total (forward-only). */
|
|
63
|
+
private advance;
|
|
64
|
+
/** One of this window's active files finished chunking. */
|
|
65
|
+
chunkFile(): void;
|
|
66
|
+
/**
|
|
67
|
+
* Report live progress of a per-stage work queue (described texts,
|
|
68
|
+
* embedded texts, stored files). `total` 0 means the stage had no work for
|
|
69
|
+
* this window (early-result files, full description-cache hits, a skipped
|
|
70
|
+
* stage) and fills the stage's credit immediately.
|
|
71
|
+
*/
|
|
72
|
+
stage(stage: ProgressStage, done: number, total: number): void;
|
|
73
|
+
/** Mark a stage complete regardless of its counters (stage done, skipped, or failed). */
|
|
74
|
+
fill(stage: ProgressStage): void;
|
|
75
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Whole-pass progress accounting for a single index run.
|
|
3
|
+
*
|
|
4
|
+
* Every progress line the `index` command prints reports ONE percentage:
|
|
5
|
+
* the fraction of the WHOLE pass completed — never a per-chunk or
|
|
6
|
+
* per-stage fraction. This module owns that accounting; the pipeline
|
|
7
|
+
* threads a `WindowProgress` handle through the chunk/describe/embed/store
|
|
8
|
+
* stages of each window and emits lines via the shared `logProgress`
|
|
9
|
+
* helper in pipeline.ts.
|
|
10
|
+
*/
|
|
11
|
+
/** One of the four equal stages every active file passes through. */
|
|
12
|
+
const QUARTER = 0.25;
|
|
13
|
+
/**
|
|
14
|
+
* Tracks the progress of one full index pass in file units.
|
|
15
|
+
*
|
|
16
|
+
* Every active (new or modified, non-empty, large-enough) file contributes
|
|
17
|
+
* one unit, split into four equal credits: chunked, described, embedded,
|
|
18
|
+
* stored. Credits are pushed by `WindowProgress` handles as their window
|
|
19
|
+
* advances, so `percent` rises monotonically from 0 toward 100 across ALL
|
|
20
|
+
* windows and stages — unlike the old per-stage "remaining" percentage
|
|
21
|
+
* that counted down and restarted inside every window.
|
|
22
|
+
*
|
|
23
|
+
* Files are weighted equally regardless of chunk count: the percentage is
|
|
24
|
+
* an honest estimate of "how far through the run are we", not a precise
|
|
25
|
+
* work-meter.
|
|
26
|
+
*/
|
|
27
|
+
export class PassProgress {
|
|
28
|
+
totalFiles;
|
|
29
|
+
credited = 0;
|
|
30
|
+
/** @param totalFiles - Total number of active files the pass will process. */
|
|
31
|
+
constructor(totalFiles) {
|
|
32
|
+
this.totalFiles = totalFiles;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Start accounting for one processing window.
|
|
36
|
+
* @param activeFiles - How many of the window's files are active (the same
|
|
37
|
+
* predicate that produced `totalFiles`). Windows with 0 active files
|
|
38
|
+
* contribute nothing to the percentage.
|
|
39
|
+
*/
|
|
40
|
+
beginWindow(activeFiles) {
|
|
41
|
+
return new WindowProgress(this, activeFiles);
|
|
42
|
+
}
|
|
43
|
+
/** @internal Called by {@link WindowProgress} to push credit into the pass total. */
|
|
44
|
+
push(credit) {
|
|
45
|
+
this.credited += credit;
|
|
46
|
+
}
|
|
47
|
+
/** Whole-pass percentage, 0–100, clamped. */
|
|
48
|
+
get percent() {
|
|
49
|
+
if (this.totalFiles <= 0)
|
|
50
|
+
return 100;
|
|
51
|
+
return Math.min(100, (this.credited / this.totalFiles) * 100);
|
|
52
|
+
}
|
|
53
|
+
/** Formatted percentage ("42.3%") for progress lines. */
|
|
54
|
+
get label() {
|
|
55
|
+
return `${this.percent.toFixed(1)}%`;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Credit handle for one window of the windowed pipeline. The pipeline
|
|
60
|
+
* reports each stage's done/total fraction here; the handle converts it to
|
|
61
|
+
* whole-pass credit by scaling with the window's active-file count.
|
|
62
|
+
*
|
|
63
|
+
* All updates are clamped to move FORWARD only, so repeated, out-of-order,
|
|
64
|
+
* or overlapping callbacks (concurrent embed sub-batches, the store phase
|
|
65
|
+
* still running while the next window prepares) can never make the overall
|
|
66
|
+
* percentage go backwards.
|
|
67
|
+
*/
|
|
68
|
+
export class WindowProgress {
|
|
69
|
+
owner;
|
|
70
|
+
files;
|
|
71
|
+
done = {
|
|
72
|
+
chunk: 0,
|
|
73
|
+
describe: 0,
|
|
74
|
+
embed: 0,
|
|
75
|
+
store: 0,
|
|
76
|
+
};
|
|
77
|
+
chunkedFiles = 0;
|
|
78
|
+
constructor(owner, files) {
|
|
79
|
+
this.owner = owner;
|
|
80
|
+
this.files = files;
|
|
81
|
+
}
|
|
82
|
+
/** Fold the stage-fraction delta into the pass total (forward-only). */
|
|
83
|
+
advance(stage, fraction) {
|
|
84
|
+
if (this.files <= 0)
|
|
85
|
+
return;
|
|
86
|
+
let target = fraction;
|
|
87
|
+
if (!Number.isFinite(target))
|
|
88
|
+
target = 1;
|
|
89
|
+
target = Math.min(1, Math.max(0, target));
|
|
90
|
+
if (target <= this.done[stage])
|
|
91
|
+
return;
|
|
92
|
+
this.owner.push((target - this.done[stage]) * QUARTER * this.files);
|
|
93
|
+
this.done[stage] = target;
|
|
94
|
+
}
|
|
95
|
+
/** One of this window's active files finished chunking. */
|
|
96
|
+
chunkFile() {
|
|
97
|
+
this.chunkedFiles = Math.min(this.files, this.chunkedFiles + 1);
|
|
98
|
+
this.advance("chunk", this.chunkedFiles / this.files);
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Report live progress of a per-stage work queue (described texts,
|
|
102
|
+
* embedded texts, stored files). `total` 0 means the stage had no work for
|
|
103
|
+
* this window (early-result files, full description-cache hits, a skipped
|
|
104
|
+
* stage) and fills the stage's credit immediately.
|
|
105
|
+
*/
|
|
106
|
+
stage(stage, done, total) {
|
|
107
|
+
this.advance(stage, total > 0 ? done / total : 1);
|
|
108
|
+
}
|
|
109
|
+
/** Mark a stage complete regardless of its counters (stage done, skipped, or failed). */
|
|
110
|
+
fill(stage) {
|
|
111
|
+
this.advance(stage, 1);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
//# sourceMappingURL=progress.js.map
|