@angri450/zvec-grep 0.2.2

package/dist/engine/pipeline/indexing/index.js

@@ -0,0 +1,1219 @@
+import { lstat, readFile } from "node:fs/promises";
+import { workspaceIndexDetail, detail, EngineError, errorDetails, isEngineError, } from "../../errors.js";
+import { sha256Bytes } from "../../utils/hash.js";
+import { normalizePath } from "../../utils/path.js";
+import { ConcurrentTiming, TimingCollector } from "../../utils/timing.js";
+import { extractForIndexing, vectorContentForFragment, } from "../../extraction/index.js";
+import { scanDirectoryPath, scanFilePath, scanRootPaths, } from "./scanner/index.js";
+import { indexChunkOptions } from "./input-budget.js";
+const EMBEDDING_TRANSIENT_MAX_RETRIES = 3;
+const EMBEDDING_RATE_LIMIT_MAX_RETRIES = 6;
+const EMBEDDING_TRANSIENT_RETRY_BASE_DELAY_MS = 500;
+const EMBEDDING_RATE_LIMIT_RETRY_BASE_DELAY_MS = 2000;
+const EMBEDDING_TRANSIENT_RETRY_MAX_DELAY_MS = 8000;
+const EMBEDDING_RATE_LIMIT_RETRY_MAX_DELAY_MS = 30000;
+const EMBEDDING_RETRY_JITTER_MS = 500;
+const EMBEDDING_SUCCESS_STREAK_MIN = 4;
+const PERMANENT_REMOTE_MODEL_PROVIDER_CODES = new Set([
+    "invalid_model",
+    "model_not_found",
+    "unsupported_model",
+    "invalid_dimension",
+    "invalid_dimensions",
+    "unsupported_dimension",
+    "unsupported_dimensions",
+    "dimension_out_of_range",
+    "invalid_embedding_dimension",
+    "unsupported_embedding_dimension",
+]);
+export async function indexWorkspace(ctx) {
+    try {
+        return await indexWorkspaceUnchecked(ctx);
+    }
+    catch (error) {
+        throw toEngineError(error, "Indexing workspace failed", {
+            code: "ZVEC_GREP.ENGINE.INDEXING.WORKSPACE_FAILED",
+            context: workspaceIndexContext(ctx.workspaceIndex),
+        });
+    }
+}
+export async function indexWorkspacePaths(ctx, changedPaths) {
+    try {
+        return await indexWorkspacePathsUnchecked(ctx, changedPaths);
+    }
+    catch (error) {
+        throw toEngineError(error, "Indexing changed paths failed", {
+            code: "ZVEC_GREP.ENGINE.INDEXING.WORKSPACE_FAILED",
+            context: workspaceIndexContext(ctx.workspaceIndex),
+        });
+    }
+}
+export async function getWorkspaceIndexStatus(workspaceIndex, storedFiles) {
+    try {
+        const scan = await scanRootPaths(workspaceIndex.id, workspaceIndex.rootPaths, { knownFiles: storedFiles });
+        const diff = await computeDiffFromFiles(scan.files, storedFiles);
+        const pendingFiles = storedFiles.filter((file) => file.indexStatus?.indexedTime === null);
+        const failedFiles = pendingFiles.filter((file) => file.indexStatus?.error !== undefined);
+        const indexedFiles = storedFiles.filter((file) => file.indexStatus?.indexedTime !== undefined &&
+            file.indexStatus.indexedTime !== null);
+        const entitiesIndexed = indexedFiles.reduce((count, file) => count + (file.indexStatus?.entityCount ?? 0), 0);
+        const fragmentsTruncated = indexedFiles.reduce((count, file) => count + (file.indexStatus?.truncatedFragmentCount ?? 0), 0);
+        return {
+            filesScanned: scan.files.length,
+            filesStored: storedFiles.length,
+            filesIndexed: indexedFiles.length,
+            entitiesIndexed,
+            fragmentsTruncated,
+            filesPending: pendingFiles.length,
+            filesFailed: failedFiles.length,
+            filesAdded: diff.added.length,
+            filesModified: diff.modified.length,
+            filesDeleted: diff.deleted.length,
+            filesUnchanged: diff.unchanged.length,
+            pendingFiles,
+            failedFiles,
+            addedFiles: diff.added,
+            modifiedFiles: diff.modified,
+            deletedFiles: diff.deleted,
+        };
+    }
+    catch (error) {
+        throw toEngineError(error, "Inspecting workspace index status failed", {
+            code: "ZVEC_GREP.ENGINE.INDEXING.STATUS_FAILED",
+            context: workspaceIndexContext(workspaceIndex),
+        });
+    }
+}
+async function indexWorkspaceUnchecked(ctx) {
+    const start = Date.now();
+    const report = ctx.onProgress ?? (() => undefined);
+    const timings = new TimingCollector();
+    throwIfIndexCancelled(ctx);
+    const firstPass = await runIndexPass(ctx, report, "Scanning files...", timings);
+    const passes = [firstPass];
+    let progressBase;
+    if (firstPass.stats.filesFailed > 0) {
+        const failed = firstPass.stats.filesFailed;
+        progressBase = retryProgressBase(firstPass);
+        report({
+            phase: "scanning",
+            filesTotal: progressBase.filesTotal,
+            filesIndexed: progressBase.filesSucceeded,
+            detail: `Retrying ${failed} failed ${failed === 1 ? "file" : "files"}...`,
+        });
+        passes.push(await runIndexPass(ctx, report, "Scanning retry candidates...", timings, progressBase));
+    }
+    const finalPass = passes[passes.length - 1];
+    throwIfIndexCancelled(ctx);
+    reportIndexFinalizing(ctx, report, finalPass, progressBase);
+    await timings.time("index_optimize", () => optimizeStorage(ctx));
+    const result = buildIndexResult(ctx, passes, Date.now() - start, timings);
+    if (result.filesFailed > 0) {
+        report({ phase: "done", detail: "Indexing completed with failed files" });
+        throw filesFailedError(ctx, result, finalPass, passes.length);
+    }
+    report({ phase: "done", detail: "Indexing complete" });
+    return result;
+}
+async function indexWorkspacePathsUnchecked(ctx, changedPaths) {
+    const start = Date.now();
+    const report = ctx.onProgress ?? (() => undefined);
+    const timings = new TimingCollector();
+    const normalizedPaths = [...new Set(changedPaths.map(normalizePath))];
+    throwIfIndexCancelled(ctx);
+    const firstPass = await runPathIndexPass(ctx, report, normalizedPaths, timings);
+    const passes = [firstPass];
+    let progressBase;
+    if (firstPass.stats.filesFailed > 0) {
+        progressBase = retryProgressBase(firstPass);
+        passes.push(await runPathIndexPass(ctx, report, normalizedPaths, timings, progressBase));
+    }
+    const finalPass = passes[passes.length - 1];
+    throwIfIndexCancelled(ctx);
+    reportIndexFinalizing(ctx, report, finalPass, progressBase);
+    await timings.time("index_optimize", () => optimizeStorage(ctx));
+    const result = buildIndexResult(ctx, passes, Date.now() - start, timings);
+    if (result.filesFailed > 0) {
+        throw filesFailedError(ctx, result, finalPass, passes.length);
+    }
+    report({ phase: "done", detail: "Indexing complete" });
+    return result;
+}
+function filesFailedError(ctx, result, finalPass, passCount) {
+    return new EngineError(`Indexing completed with ${result.filesFailed} failed ${result.filesFailed === 1 ? "file" : "files"}`, {
+        code: "ZVEC_GREP.ENGINE.INDEXING.FILES_FAILED",
+        context: errorDetails([
+            workspaceIndexDetail(ctx.workspaceIndex.name),
+            detail("filesFailed", result.filesFailed),
+            detail("filesScanned", result.filesScanned),
+            detail("failedFiles", summarizeFailedFiles(finalPass.stats.failedFiles)),
+            detail("failedReasons", summarizeFailedFiles(finalPass.stats.failedFileReasons)),
+            detail("hint", passCount > 1
+                ? "Retried failed files once automatically; if failures persist, fix the failed files or embedding configuration."
+                : "If the failure was transient, rerun the same indexing command; if it persists, fix the failed files or embedding configuration."),
+        ]),
+    });
+}
+async function runPathIndexPass(ctx, report, changedPaths, timings, progressBase) {
+    reportScanning(report, "Scanning changed paths...", progressBase);
+    const scanned = await timings.time("index_scan_paths", async () => {
+        const files = [];
+        const diagnostics = emptyScanDiagnostics();
+        for (const path of changedPaths) {
+            throwIfIndexCancelled(ctx);
+            const info = await lstat(path).catch(() => null);
+            const scan = info?.isDirectory()
+                ? await scanDirectoryPath(ctx.workspaceIndex.id, ctx.workspaceIndex.rootPaths, path, { signal: ctx.signal })
+                : await scanFilePath(ctx.workspaceIndex.id, ctx.workspaceIndex.rootPaths, path, {
+                    signal: ctx.signal,
+                });
+            files.push(...scan.files);
+            mergeScanDiagnostics(diagnostics, scan.diagnostics);
+        }
+        return {
+            files: [...new Map(files.map((file) => [file.id, file])).values()],
+            diagnostics,
+        };
+    });
+    throwIfIndexCancelled(ctx);
+    const exactExisting = [];
+    const prefixPaths = [];
+    for (const path of changedPaths) {
+        const exact = ctx.storage.getFileByPath(path);
+        if (exact) {
+            exactExisting.push(exact);
+        }
+        else {
+            prefixPaths.push(path);
+        }
+    }
+    const existing = [
+        ...new Map([
+            ...exactExisting,
+            ...ctx.storage.listFilesByPathPrefixes(prefixPaths),
+        ].map((file) => [file.id, file])).values(),
+    ];
+    return runDiffPass(ctx, report, scanned.files, existing, timings, progressBase, scanned.diagnostics);
+}
+async function runIndexPass(ctx, report, scanningDetail, timings, progressBase) {
+    reportScanning(report, scanningDetail, progressBase);
+    const existing = ctx.storage.listFiles();
+    const scan = await timings.time("index_scan", () => scanRootPaths(ctx.workspaceIndex.id, ctx.workspaceIndex.rootPaths, {
+        signal: ctx.signal,
+        knownFiles: existing,
+    }));
+    throwIfIndexCancelled(ctx);
+    return runDiffPass(ctx, report, scan.files, existing, timings, progressBase, scan.diagnostics);
+}
+async function runDiffPass(ctx, report, scannedFiles, existingFiles, timings, progressBase, scanDiagnostics = emptyScanDiagnostics()) {
+    const diff = await timings.time("index_diff", () => computeDiffFromFiles(scannedFiles, existingFiles));
+    throwIfIndexCancelled(ctx);
+    const pending = [...diff.added, ...diff.modified, ...diff.pending];
+    report({
+        phase: "scanning",
+        filesTotal: progressBase?.filesTotal ?? scannedFiles.length,
+        filesIndexed: progressBase?.filesSucceeded ?? 0,
+        detail: `${diff.added.length} added, ${diff.modified.length} modified, ${diff.pending.length} pending, ${diff.deleted.length} deleted, ${diff.unchanged.length} unchanged`,
+    });
+    timings.timeSync("index_delete_stale", () => {
+        for (const file of diff.deleted) {
+            throwIfIndexCancelled(ctx);
+            try {
+                ctx.storage.deleteFile(file.id);
+            }
+            catch (error) {
+                throw toEngineError(error, "Indexing failed to delete stale file records", {
+                    code: "ZVEC_GREP.ENGINE.INDEXING.DELETE_FILE_FAILED",
+                    context: fileContext(file),
+                });
+            }
+        }
+    });
+    const reportIndexing = (currentStats, detail, embedding) => {
+        report({
+            phase: "indexing",
+            filesTotal: progressBase?.filesTotal ?? pending.length,
+            filesIndexed: (progressBase?.filesSucceeded ?? 0) +
+                currentStats.filesIndexed +
+                currentStats.filesFailed,
+            filesFailed: currentStats.filesFailed,
+            detail,
+            embedding,
+        });
+    };
+    reportIndexing({
+        filesIndexed: 0,
+        filesFailed: 0,
+        failedFiles: [],
+        failedFileReasons: [],
+        entitiesCreated: 0,
+    });
+    const stats = await indexFiles(pending, ctx, reportIndexing, timings);
+    throwIfIndexCancelled(ctx);
+    return {
+        filesScanned: scannedFiles.length,
+        scanDiagnostics,
+        diff,
+        stats,
+    };
+}
+function reportIndexFinalizing(ctx, report, pass, progressBase) {
+    report({
+        phase: "indexing",
+        filesTotal: progressBase?.filesTotal ??
+            pass.diff.added.length +
+                pass.diff.modified.length +
+                pass.diff.pending.length,
+        filesIndexed: (progressBase?.filesSucceeded ?? 0) +
+            pass.stats.filesIndexed +
+            pass.stats.filesFailed,
+        filesFailed: pass.stats.filesFailed,
+        detail: "finalizing index",
+    });
+}
+function retryProgressBase(pass) {
+    return {
+        filesSucceeded: pass.stats.filesIndexed,
+        filesTotal: pass.diff.added.length +
+            pass.diff.modified.length +
+            pass.diff.pending.length,
+    };
+}
+function reportScanning(report, detail, progressBase) {
+    report({
+        phase: "scanning",
+        ...(progressBase
+            ? {
+                filesTotal: progressBase.filesTotal,
+                filesIndexed: progressBase.filesSucceeded,
+            }
+            : {}),
+        detail,
+    });
+}
+function buildIndexResult(ctx, passes, durationMs, timings) {
+    const firstPass = passes[0];
+    const retryPasses = passes.slice(1);
+    const finalPass = passes[passes.length - 1];
+    return {
+        filesScanned: finalPass.filesScanned,
+        filesAdded: firstPass.diff.added.length +
+            retryPasses.reduce((count, pass) => count + pass.diff.added.length, 0),
+        filesModified: firstPass.diff.modified.length +
+            retryPasses.reduce((count, pass) => count + pass.diff.modified.length, 0),
+        filesPending: firstPass.diff.pending.length +
+            retryPasses.reduce((count, pass) => count + pass.diff.pending.length, 0),
+        filesDeleted: firstPass.diff.deleted.length +
+            retryPasses.reduce((count, pass) => count + pass.diff.deleted.length, 0),
+        filesUnchanged: firstPass.diff.unchanged.length,
+        filesFailed: finalPass.stats.filesFailed,
+        entitiesCreated: passes.reduce((count, pass) => count + pass.stats.entitiesCreated, 0),
+        durationMs,
+        timings: timings.entries(),
+        ...(finalPass.scanDiagnostics.skippedFiles > 0
+            ? { scanDiagnostics: finalPass.scanDiagnostics }
+            : {}),
+    };
+}
+const MAX_SKIPPED_FILE_SAMPLES = 20;
+function emptyScanDiagnostics() {
+    return {
+        skippedFiles: 0,
+        skippedByReason: {
+            empty: 0,
+            too_large: 0,
+            unsupported: 0,
+            binary: 0,
+        },
+        skippedSamples: [],
+    };
+}
+function mergeScanDiagnostics(target, source) {
+    target.skippedFiles += source.skippedFiles;
+    for (const reason of [
+        "empty",
+        "too_large",
+        "unsupported",
+        "binary",
+    ]) {
+        target.skippedByReason[reason] += source.skippedByReason[reason];
+    }
+    const remaining = MAX_SKIPPED_FILE_SAMPLES - target.skippedSamples.length;
+    if (remaining > 0) {
+        target.skippedSamples.push(...source.skippedSamples.slice(0, remaining));
+    }
+}
+async function optimizeStorage(ctx) {
+    try {
+        await ctx.storage.finalizeWrites();
+    }
+    catch (error) {
+        throw toEngineError(error, "Indexing failed to finalize storage", {
+            code: "ZVEC_GREP.ENGINE.INDEXING.OPTIMIZE_FAILED",
+            context: workspaceIndexContext(ctx.workspaceIndex),
+        });
+    }
+}
+async function computeDiffFromFiles(scannedFiles, existingFiles) {
+    const existingById = new Map(existingFiles.map((file) => [file.id, file]));
+    const seen = new Set();
+    const added = [];
+    const modified = [];
+    const pending = [];
+    const unchanged = [];
+    for (const file of scannedFiles) {
+        seen.add(file.id);
+        const existing = existingById.get(file.id);
+        if (!existing) {
+            added.push(await withContentHash(file));
+            continue;
+        }
+        if (existing.indexStatus?.indexedTime === null) {
+            pending.push(await withContentHash(file));
+            continue;
+        }
+        if (existing.sizeBytes === file.sizeBytes &&
+            existing.lastModifiedTime === file.lastModifiedTime &&
+            existing.contentHash) {
+            unchanged.push(existing);
+            continue;
+        }
+        const hashed = await withContentHash(file);
+        if (existing.sizeBytes === hashed.sizeBytes &&
+            existing.contentHash === hashed.contentHash) {
+            unchanged.push(existing);
+            continue;
+        }
+        modified.push(hashed);
+    }
+    const deleted = [...existingById.values()].filter((file) => !seen.has(file.id));
+    return { added, modified, pending, deleted, unchanged };
+}
+async function indexFiles(files, ctx, onProgress, timings) {
+    const stats = {
+        filesIndexed: 0,
+        filesFailed: 0,
+        failedFiles: [],
+        failedFileReasons: [],
+        entitiesCreated: 0,
+    };
+    const batchFiles = [];
+    let batchFragmentCount = 0;
+    const embeddingScheduler = createEmbeddingScheduler(ctx.embeddingConcurrency, ctx.embeddingModel);
+    const embeddingTiming = new ConcurrentTiming(timings, "index_embedding");
+    const runningEmbeddings = new Set();
+    let firstEmbeddingError;
+    const prepareModel = ctx.embeddingModel.info.provider === "local"
+        ? ctx.embeddingModel.prepare?.bind(ctx.embeddingModel)
+        : undefined;
+    let modelPrepared = false;
+    const reportEmbeddingProgress = (currentStats, detail) => {
+        onProgress(currentStats, detail, embeddingScheduler.snapshot());
+    };
+    const throwFirstEmbeddingError = () => {
+        const failure = firstEmbeddingError;
+        if (failure) {
+            throw failure.error;
+        }
+    };
+    const flushBatch = async () => {
+        throwFirstEmbeddingError();
+        throwIfIndexCancelled(ctx);
+        if (batchFiles.length === 0) {
+            return;
+        }
+        const filesToEmbed = batchFiles.splice(0);
+        batchFragmentCount = 0;
+        await scheduleEmbeddingTask(() => embedAndCommitBatch(filesToEmbed, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming));
+    };
+    const scheduleEmbeddingTask = async (task) => {
+        throwFirstEmbeddingError();
+        throwIfIndexCancelled(ctx);
+        if (!modelPrepared && prepareModel) {
+            // Finish shared initialization before any file or fragment batch is queued.
+            try {
+                await embeddingTiming.time(() => prepareModel({
+                    signal: ctx.signal,
+                    onProgress: (progress) => reportModelDownloadProgress(stats, progress, onProgress, embeddingScheduler),
+                }));
+            }
+            catch (error) {
+                throwIfIndexCancelled(ctx);
+                throw error;
+            }
+            throwIfIndexCancelled(ctx);
+        }
+        modelPrepared = true;
+        const cleanup = () => {
+            runningEmbeddings.delete(promise);
+        };
+        const promise = Promise.resolve()
+            .then(task)
+            .then(cleanup, (error) => {
+            if (!firstEmbeddingError) {
+                firstEmbeddingError = { error };
+            }
+            cleanup();
+        });
+        runningEmbeddings.add(promise);
+        if (runningEmbeddings.size >= embeddingScheduler.taskConcurrency) {
+            await Promise.race(runningEmbeddings);
+        }
+        throwFirstEmbeddingError();
+        throwIfIndexCancelled(ctx);
+    };
+    try {
+        for (const file of files) {
+            throwFirstEmbeddingError();
+            throwIfIndexCancelled(ctx);
+            onProgress(stats, `reading ${file.relativePath}`);
+            const prepared = await timings.time("index_prepare", () => prepareFile(file, ctx));
+            throwFirstEmbeddingError();
+            throwIfIndexCancelled(ctx);
+            if ("failedReason" in prepared) {
+                recordFileFailed(stats, file, prepared.failedReason);
+                onProgress(stats, `failed ${file.relativePath}`);
+                continue;
+            }
+            if (prepared.fragments.length === 0) {
+                const committed = commitFile(prepared, [], ctx, stats);
+                onProgress(stats, finishedFileDetail(committed, file.relativePath));
+                continue;
+            }
+            if (prepared.fragments.length > ctx.embeddingModel.info.limits.maxBatchSize) {
+                await flushBatch();
+                await scheduleEmbeddingTask(async () => {
+                    throwIfIndexCancelled(ctx);
+                    reportEmbeddingProgress(stats, `embedding ${file.relativePath}`);
+                    await embedAndCommitFile(prepared, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming);
+                });
+                continue;
+            }
+            if (batchFragmentCount > 0 &&
+                batchFragmentCount + prepared.fragments.length >
+                    ctx.embeddingModel.info.limits.maxBatchSize) {
+                await flushBatch();
+            }
+            batchFiles.push(prepared);
+            batchFragmentCount += prepared.fragments.length;
+            if (batchFragmentCount === ctx.embeddingModel.info.limits.maxBatchSize) {
+                await flushBatch();
+            }
+        }
+        await flushBatch();
+        await Promise.all(runningEmbeddings);
+        throwFirstEmbeddingError();
+        throwIfIndexCancelled(ctx);
+    }
+    catch (error) {
+        await Promise.allSettled(runningEmbeddings);
+        throw error;
+    }
+    throwIfIndexCancelled(ctx);
+    return stats;
+}
+async function prepareFile(file, ctx) {
+    try {
+        throwIfIndexCancelled(ctx);
+        const source = await readSource(file);
+        const chunkOptions = indexChunkOptions(ctx.embeddingModel.info.limits.maxInputTokens, source.kind === "text" ? source.text : undefined);
+        const extracted = await extractForIndexing(source, chunkOptions);
+        throwIfIndexCancelled(ctx);
+        const fragments = extracted
+            .filter(({ fragment }) => ctx.embeddingModel.info.inputKinds.includes(fragment.content.kind))
+            .map(({ fragment, embeddingSource }) => ({
+            fragment,
+            embeddingContent: vectorContentForFragment(fragment, embeddingSource, chunkOptions.maxChunkChars),
+        }));
+        return { file, fragments };
+    }
+    catch (error) {
+        if (indexIsCancelled(ctx)) {
+            throw indexCancellationError(ctx);
+        }
+        return {
+            file,
+            failedReason: markFileFailed(ctx, file, error, "prepare"),
+        };
+    }
+}
+async function embedAndCommitBatch(files, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming) {
+    try {
+        throwIfIndexCancelled(ctx);
+        const contents = files.flatMap((file) => file.fragments.map((fragment) => fragment.embeddingContent));
+        onProgress(stats, `embedding ${describePreparedFiles(files)}`, embeddingScheduler.snapshot());
+        const embedding = await embeddingTiming.time(() => embedContentsWithRetry(contents, ctx.embeddingModel, embeddingScheduler, ctx.signal, (progress) => reportModelDownloadProgress(stats, progress, onProgress, embeddingScheduler)));
+        const truncatedInputIndexes = new Set(embedding.truncated);
+        throwIfIndexCancelled(ctx);
+        let offset = 0;
+        for (const file of files) {
+            throwIfIndexCancelled(ctx);
+            const end = offset + file.fragments.length;
+            const fileVectors = embedding.vectors.slice(offset, end);
+            let truncatedFragmentCount = 0;
+            for (let index = offset; index < end; index++) {
+                if (truncatedInputIndexes.has(index)) {
+                    truncatedFragmentCount++;
+                }
+            }
+            offset += file.fragments.length;
+            const committed = timings.timeSync("index_commit", () => commitFile(file, fileVectors, ctx, stats, truncatedFragmentCount));
+            onProgress(stats, finishedFileDetail(committed, file.file.relativePath));
+        }
+    }
+    catch (error) {
+        if (indexIsCancelled(ctx)) {
+            throw indexCancellationError(ctx);
+        }
+        if (shouldFailFastEmbeddingError(error, ctx.embeddingModel)) {
+            throw error;
+        }
+        for (const file of files) {
+            onProgress(stats, `embedding ${file.file.relativePath}`, embeddingScheduler.snapshot());
+            await embedAndCommitFile(file, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming);
+        }
+    }
+}
+async function embedAndCommitFile(file, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming) {
+    try {
+        throwIfIndexCancelled(ctx);
+        const embedding = await embeddingTiming.time(() => embedFragments(file.fragments, ctx.embeddingModel, embeddingScheduler, ctx.signal, (progress) => reportModelDownloadProgress(stats, progress, onProgress, embeddingScheduler)));
+        throwIfIndexCancelled(ctx);
+        const committed = timings.timeSync("index_commit", () => commitFile(file, embedding.vectors, ctx, stats, embedding.truncated.length));
+        onProgress(stats, finishedFileDetail(committed, file.file.relativePath));
+    }
+    catch (error) {
+        if (indexIsCancelled(ctx)) {
+            throw indexCancellationError(ctx);
+        }
+        if (shouldFailFastEmbeddingError(error, ctx.embeddingModel)) {
+            throw error;
+        }
+        const reason = markFileFailed(ctx, file.file, error, "embed");
+        recordFileFailed(stats, file.file, reason);
+        onProgress(stats, finishedFileDetail(false, file.file.relativePath));
+    }
+}
+function commitFile(file, vectors, ctx, stats, truncatedFragmentCount = 0) {
+    try {
+        throwIfIndexCancelled(ctx);
+        if (file.fragments.length !== vectors.length) {
+            throw new EngineError("Embedding returned mismatched entity/vector counts", {
+                code: "ZVEC_GREP.ENGINE.STORAGE.ENTITY_VECTOR_COUNT_MISMATCH",
+                context: `fileId=${file.file.id} fragmentCount=${file.fragments.length} vectorCount=${vectors.length}`,
+            });
+        }
+        ctx.storage.replaceFile(file.file, file.fragments.map(({ fragment }, index) => ({
+            fragment,
+            vector: vectors[index],
+        })), {
+            truncatedFragmentCount,
+        });
+        stats.filesIndexed++;
+        stats.entitiesCreated += countPublicEntities(file.fragments.map(({ fragment }) => fragment));
+        return true;
+    }
+    catch (error) {
+        if (indexIsCancelled(ctx)) {
+            throw indexCancellationError(ctx);
+        }
+        const reason = markFileFailed(ctx, file.file, error, "commit");
+        recordFileFailed(stats, file.file, reason);
+        return false;
+    }
+}
+function throwIfIndexCancelled(ctx) {
+    if (!indexIsCancelled(ctx)) {
+        return;
+    }
+    throw indexCancellationError(ctx);
+}
+function indexIsCancelled(ctx) {
+    return ctx.signal?.aborted === true;
+}
+function indexCancellationError(ctx) {
+    return ctx.signal?.reason instanceof Error
+        ? ctx.signal.reason
+        : new EngineError("Indexing was cancelled.", {
+            code: "ZVEC_GREP.ENGINE.INDEXING.CANCELLED",
+            context: workspaceIndexContext(ctx.workspaceIndex),
+        });
+}
+function recordFileFailed(stats, file, reason) {
+    stats.filesFailed++;
+    stats.failedFiles.push(file.relativePath);
+    if (reason) {
+        stats.failedFileReasons.push(`${file.relativePath}: ${reason}`);
+    }
+}
+function summarizeFailedFiles(files) {
+    const shown = files.slice(0, 5);
+    const remaining = files.length - shown.length;
+    return remaining > 0
+        ? `${shown.join(", ")} and ${remaining} more`
+        : shown.join(", ");
+}
+function describePreparedFiles(files) {
+    if (files.length === 0) {
+        return "0 files";
+    }
+    if (files.length === 1) {
+        return files[0].file.relativePath;
+    }
+    return `${files.length} files, starting with ${files[0].file.relativePath}`;
+}
+function finishedFileDetail(succeeded, relativePath) {
+    return succeeded ? `indexed ${relativePath}` : `failed ${relativePath}`;
+}
+async function readSource(file) {
+    let bytes;
+    try {
+        bytes = await readFile(file.absolutePath);
+    }
+    catch (error) {
+        throw toEngineError(error, "Indexing failed to read source file", {
+            code: "ZVEC_GREP.ENGINE.INDEXING.READ_SOURCE_FAILED",
+            context: fileContext(file),
+        });
+    }
+    if (file.kind === "image") {
+        return {
+            kind: "image",
+            file,
+            data: bytes,
+            format: file.format,
+        };
+    }
+    return {
+        kind: "text",
+        file,
+        text: bytes.toString("utf8"),
+    };
+}
+async function embedFragments(fragments, model, embeddingScheduler, signal, onModelProgress) {
+    const batches = [];
+    for (let start = 0; start < fragments.length; start += model.info.limits.maxBatchSize) {
+        const batch = fragments.slice(start, start + model.info.limits.maxBatchSize);
+        batches.push({ start, fragments: batch });
+    }
+    const vectors = new Array(fragments.length);
+    const truncatedInputIndexes = [];
+    const batchAbortController = new AbortController();
+    const batchSignal = signal
+        ? AbortSignal.any([signal, batchAbortController.signal])
+        : batchAbortController.signal;
+    const results = await Promise.allSettled(batches.map((batch) => embedFragmentBatch(batch.fragments, model, batch.start, embeddingScheduler, batchSignal, onModelProgress, (error) => batchAbortController.abort(error))));
+    let firstError;
+    for (const [index, result] of results.entries()) {
+        if (result.status === "rejected") {
+            firstError ??= result.reason;
+            continue;
+        }
+        const start = batches[index].start;
+        for (const [offset, vector] of result.value.vectors.entries()) {
+            vectors[start + offset] = vector;
+        }
+        truncatedInputIndexes.push(...result.value.truncated.map((inputIndex) => start + inputIndex));
+    }
+    if (firstError) {
+        throw firstError;
+    }
+    return {
+        vectors,
+        truncated: truncatedInputIndexes,
+    };
+}
+async function embedFragmentBatch(fragments, model, startIndex, embeddingScheduler, signal, onModelProgress, onTerminalFailure) {
+    const contents = fragments.map((fragment) => fragment.embeddingContent);
+    try {
+        return await embedContentsWithRetry(contents, model, embeddingScheduler, signal, onModelProgress, onTerminalFailure);
+    }
+    catch (error) {
+        if (fragments.length === 1 || shouldFailFastEmbeddingError(error, model)) {
+            throw error;
+        }
+        return await embedFragmentBatchOneByOne(fragments, model, startIndex, embeddingScheduler, signal, onModelProgress, onTerminalFailure);
+    }
+}
+async function embedFragmentBatchOneByOne(fragments, model, startIndex, embeddingScheduler, signal, onModelProgress, onTerminalFailure) {
+    const vectors = [];
+    const truncatedInputIndexes = [];
+    for (const [index, fragment] of fragments.entries()) {
+        try {
+            const result = await embedContentsWithRetry([fragment.embeddingContent], model, embeddingScheduler, signal, onModelProgress, onTerminalFailure);
+            vectors.push(result.vectors[0]);
+            if (result.truncated.length > 0) {
+                truncatedInputIndexes.push(index);
+            }
+        }
+        catch (error) {
+            if (shouldFailFastEmbeddingError(error, model)) {
+                throw error;
+            }
+            throw new EngineError("Embedding entity fragment failed after one-by-one fallback", {
+                code: "ZVEC_GREP.ENGINE.INDEXING.EMBEDDING_FRAGMENT_FAILED",
+                context: errorDetails([
+                    detail("model", model.info.reference),
+                    detail("fragmentId", fragment.fragment.id),
+                    detail("fragmentIndex", startIndex + index),
+                    isEngineError(error) ? detail("causeCode", error.code) : null,
+                    error instanceof Error ? detail("cause", error.message) : null,
+                    isEngineError(error) ? error.context : null,
+                ]),
+                cause: error,
+            });
+        }
+    }
+    return {
+        vectors,
+        truncated: truncatedInputIndexes,
+    };
+}
+async function embedContentsWithRetry(contents, model, embeddingScheduler, signal, onModelProgress, onTerminalFailure) {
+    let attempt = 0;
+    while (true) {
+        let retry = nonRetryableEmbeddingError();
+        let delayMs = 0;
+        try {
+            throwIfAborted(signal);
+            const result = await embeddingScheduler.run(() => model.embed(contents, {
+                purpose: "document",
+                signal,
+                onProgress: onModelProgress,
+            }), signal, (error) => {
+                retry = classifyEmbeddingRetry(error, model);
+                delayMs = retryDelayMs(attempt, retry);
+                if (retry.retryable) {
+                    embeddingScheduler.recordRetryableFailure({
+                        rateLimited: retry.rateLimited,
+                        delayMs,
+                    });
+                }
+                if (retry.failFast &&
+                    (!retry.retryable || attempt >= maxRetryAttempts(retry))) {
+                    onTerminalFailure?.(error);
+                }
+            });
+            embeddingScheduler.recordSuccess();
+            return result;
+        }
+        catch (error) {
+            if (signal?.aborted) {
+                throw signal.reason instanceof Error
+                    ? signal.reason
+                    : new Error("Embedding was cancelled.");
+            }
+            if (!retry.retryable) {
+                retry = classifyEmbeddingRetry(error, model);
+                delayMs = retryDelayMs(attempt, retry);
+            }
+            if (attempt >= maxRetryAttempts(retry) || !retry.retryable) {
+                throw error;
+            }
+            await abortableDelay(delayMs, signal);
+            attempt++;
+        }
+    }
+}
+function reportModelDownloadProgress(stats, progress, onProgress, embeddingScheduler) {
+    onProgress(stats, `downloading ${progress.model}`, {
+        ...embeddingScheduler.snapshot(),
+        ...progress,
+    });
+}
+function createEmbeddingScheduler(requestedConcurrency, model) {
+    return new AdaptiveEmbeddingScheduler(resolveEmbeddingConcurrencyPolicy(requestedConcurrency, model));
+}
+function resolveEmbeddingConcurrencyPolicy(requestedConcurrency, model) {
+    if (requestedConcurrency !== undefined &&
+        Number.isInteger(requestedConcurrency) &&
+        requestedConcurrency > 0) {
+        const concurrency = Math.min(requestedConcurrency, model.info.limits.maxConcurrentBatches ?? requestedConcurrency);
+        return {
+            initial: concurrency,
+            min: 1,
+            max: concurrency,
+            adaptive: concurrency > 1,
+        };
+    }
+    const remote = model.info.provider === "qwen";
+    const multimodal = model.info.inputKinds.includes("image");
+    const configuredLocalDefault = model.info.defaultConcurrency;
+    const localDefault = configuredLocalDefault !== undefined &&
+        Number.isInteger(configuredLocalDefault) &&
+        configuredLocalDefault > 0
+        ? configuredLocalDefault
+        : 1;
+    const initial = remote ? (multimodal ? 4 : 8) : localDefault;
+    const max = remote ? (multimodal ? 8 : 12) : localDefault;
+    const min = Math.min(initial, 4);
+    return {
+        initial,
+        min,
+        max: Math.max(initial, max),
+        adaptive: max > 1,
+    };
+}
+class AdaptiveEmbeddingScheduler {
+    policy;
+    active = 0;
+    cooldownUntil = 0;
+    currentConcurrency;
+    retryableFailures = 0;
+    successStreak = 0;
+    queue = [];
+    constructor(policy) {
+        this.policy = policy;
+        this.currentConcurrency = policy.initial;
+    }
+    get taskConcurrency() {
+        return this.policy.max;
+    }
+    async run(task, signal, onError) {
+        throwIfAborted(signal);
+        await this.waitForCooldown(signal);
+        await this.acquire(signal);
+        try {
+            await this.waitForCooldown(signal);
+            throwIfAborted(signal);
+            return await task();
+        }
+        catch (error) {
+            onError?.(error);
+            throw error;
+        }
+        finally {
+            this.release();
+        }
+    }
+    recordSuccess() {
+        if (!this.policy.adaptive || this.currentConcurrency >= this.policy.max) {
+            return;
+        }
+        this.successStreak++;
+        if (this.successStreak <
+            Math.max(EMBEDDING_SUCCESS_STREAK_MIN, this.currentConcurrency * 2)) {
+            return;
+        }
+        this.currentConcurrency++;
+        this.successStreak = 0;
+        this.drainQueue();
+    }
+    recordRetryableFailure(retry) {
+        this.retryableFailures++;
+        if (retry.rateLimited && retry.delayMs > 0) {
+            this.cooldownUntil = Math.max(this.cooldownUntil, Date.now() + retry.delayMs);
+        }
+        if (!this.policy.adaptive) {
+            return;
+        }
+        this.currentConcurrency = Math.max(this.policy.min, Math.floor(this.currentConcurrency / 2));
+        this.successStreak = 0;
+    }
+    snapshot() {
+        return {
+            concurrency: this.currentConcurrency,
+            maxConcurrency: this.policy.max,
+            retryableFailures: this.retryableFailures,
+        };
+    }
+    async waitForCooldown(signal) {
+        const remaining = this.cooldownUntil - Date.now();
+        if (remaining > 0) {
+            await abortableDelay(remaining, signal);
+        }
+    }
+    async acquire(signal) {
+        throwIfAborted(signal);
+        if (this.active < this.currentConcurrency) {
+            this.active++;
+            return;
+        }
+        await new Promise((resolve, reject) => {
+            const queued = () => {
+                signal?.removeEventListener("abort", abort);
+                resolve();
+            };
+            const abort = () => {
+                const index = this.queue.indexOf(queued);
+                if (index >= 0) {
+                    this.queue.splice(index, 1);
+                }
+                reject(abortError(signal));
+            };
+            if (signal?.aborted) {
+                reject(abortError(signal));
+                return;
+            }
+            signal?.addEventListener("abort", abort, { once: true });
+            this.queue.push(queued);
+        });
+    }
+    release() {
+        this.active--;
+        this.drainQueue();
+    }
+    drainQueue() {
+        while (this.active < this.currentConcurrency) {
+            const resolve = this.queue.shift();
+            if (!resolve) {
+                return;
+            }
+            this.active++;
+            resolve();
+        }
+    }
+}
+function oneLine(value) {
+    return value.replace(/\s+/g, " ").trim();
+}
+function shouldFailFastEmbeddingError(error, model) {
+    return classifyEmbeddingRetry(error, model).failFast;
+}
+function classifyEmbeddingRetry(error, model) {
+    const codes = errorCodes(error);
+    const text = errorChainText(error);
+    const status = httpStatusFromText(text);
+    const remoteEmbedding = model.info.provider !== "local";
+    const rateLimited = remoteEmbedding &&
+        (status === 429 ||
+            /rate limit|quota exceeded|too many requests|request rate increased too quickly/i.test(text));
+    const serverError = remoteEmbedding &&
+        typeof status === "number" &&
+        status >= 500 &&
+        status <= 599;
+    const requestTimeout = remoteEmbedding && status === 408;
+    const requestFailure = codes.some((code) => code.endsWith("_REQUEST_FAILED"));
+    const transientNetworkFailure = remoteEmbedding && requestFailure && isTransientNetworkFailure(text);
+    const sharedLocalModelFailure = codes.some((code) => code === "ZVEC_GREP.ENGINE.MODELS.MODEL2VEC_DOWNLOAD_FAILED" ||
+        code === "ZVEC_GREP.ENGINE.MODELS.MODEL2VEC_LOAD_FAILED" ||
+        code === "ZVEC_GREP.ENGINE.MODELS.MODEL2VEC_DISPOSED" ||
+        code === "ZVEC_GREP.ENGINE.MODELS.TRANSFORMERS_JS_LOAD_FAILED");
+    const retryable = !sharedLocalModelFailure &&
+        (rateLimited || serverError || requestTimeout || transientNetworkFailure);
+    const remoteConfigurationFailure = remoteEmbedding &&
+        (codes.some((code) => code.endsWith("_MISSING_API_KEY") || code.endsWith("_MISSING_ENDPOINT")) ||
+            status === 401 ||
+            status === 403 ||
+            status === 404 ||
+            /\b(?:invalid|missing|unauthorized|forbidden)[ _-]?(?:api[ _-]?)?key\b/i.test(text));
+    const permanentRemoteModelFailure = remoteEmbedding &&
+        (codes.includes("ZVEC_GREP.ENGINE.MODELS.EMBEDDING_DIMENSION_MISMATCH") ||
+            (status === 400 && isPermanentRemoteModelBadRequest(text)));
+    return {
+        retryable,
+        rateLimited,
+        failFast: retryable ||
+            sharedLocalModelFailure ||
+            remoteConfigurationFailure ||
+            permanentRemoteModelFailure ||
+            (remoteEmbedding && requestFailure),
+        retryAfterMs: retryAfterMsFromText(text),
+    };
+}
+function isPermanentRemoteModelBadRequest(text) {
+    const providerCode = /\bproviderCode=([^\s]+)/i.exec(text)?.[1];
+    const normalizedCode = providerCode
+        ?.replace(/[^a-z0-9]+/gi, "_")
+        .replace(/^_+|_+$/g, "")
+        .toLowerCase();
+    if (normalizedCode &&
+        PERMANENT_REMOTE_MODEL_PROVIDER_CODES.has(normalizedCode)) {
+        return true;
+    }
+    const providerMessage = /\bproviderMessage=(.*?)(?=\s+ZVEC_GREP\.|$)/i.exec(text)?.[1];
+    return (providerMessage !== undefined &&
+        /(?:\b(?:invalid|unsupported|unknown)\b.{0,48}\bmodel\b|\bmodel\b.{0,48}\b(?:invalid|unsupported|unknown|not found|does not exist)\b|\b(?:invalid|unsupported|out of range)\b.{0,48}\bdimensions?\b|\bdimensions?\b.{0,48}\b(?:invalid|unsupported|not supported|out of range|must|should|expected|between|only supports?)\b)/i.test(providerMessage));
+}
+function nonRetryableEmbeddingError() {
+    return {
+        retryable: false,
+        rateLimited: false,
+        failFast: false,
+    };
+}
+function errorCodes(error) {
+    const codes = [];
+    for (const current of errorChain(error)) {
+        if (typeof current === "object" &&
+            current !== null &&
+            "code" in current &&
+            typeof current.code === "string") {
+            codes.push(current.code);
+        }
+    }
+    return codes;
+}
+function errorChainText(error) {
+    const parts = [];
+    for (const current of errorChain(error)) {
+        if (current instanceof Error) {
+            parts.push(current.name, current.message);
+        }
+        else {
+            parts.push(String(current));
+        }
+        if (isEngineError(current) && current.context) {
+            parts.push(current.context);
+        }
+        if (typeof current === "object" &&
+            current !== null &&
+            "code" in current &&
+            typeof current.code === "string") {
+            parts.push(current.code);
+        }
+    }
+    return parts.join(" ");
+}
+function errorChain(error) {
+    const chain = [];
+    const seen = new Set();
+    let current = error;
+    while (current !== undefined && current !== null && !seen.has(current)) {
+        seen.add(current);
+        chain.push(current);
+        if (typeof current !== "object" ||
+            !("cause" in current) ||
+            current.cause === undefined) {
+            break;
+        }
+        current = current.cause;
+    }
+    return chain;
+}
+function isTransientNetworkFailure(text) {
+    return /\b(?:EAI_AGAIN|ECONNREFUSED|ECONNRESET|EHOSTUNREACH|ENETDOWN|ENETUNREACH|ENOTFOUND|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_SOCKET|TimeoutError)\b|connection (?:reset|timed out)|network connection (?:failed|was lost)|socket hang up|temporary failure/i.test(text);
+}
+function maxRetryAttempts(retry) {
+    return retry.rateLimited
+        ? EMBEDDING_RATE_LIMIT_MAX_RETRIES
+        : EMBEDDING_TRANSIENT_MAX_RETRIES;
+}
+function retryDelayMs(attempt, retry) {
+    if (typeof retry.retryAfterMs === "number") {
+        return retry.retryAfterMs;
+    }
+    const base = retry.rateLimited
+        ? EMBEDDING_RATE_LIMIT_RETRY_BASE_DELAY_MS
+        : EMBEDDING_TRANSIENT_RETRY_BASE_DELAY_MS;
+    const max = retry.rateLimited
+        ? EMBEDDING_RATE_LIMIT_RETRY_MAX_DELAY_MS
+        : EMBEDDING_TRANSIENT_RETRY_MAX_DELAY_MS;
+    const exponential = base * 2 ** attempt;
+    const jitter = Math.floor(Math.random() * EMBEDDING_RETRY_JITTER_MS);
+    return Math.min(exponential + jitter, max);
+}
+function httpStatusFromText(text) {
+    const match = /\bstatus=(\d{3})\b/i.exec(text);
+    if (!match) {
+        return undefined;
+    }
+    return Number(match[1]);
+}
+function retryAfterMsFromText(text) {
+    const milliseconds = /\bretryAfterMs=(\d+)\b/i.exec(text);
+    if (milliseconds) {
+        return Number(milliseconds[1]);
+    }
+    const seconds = /\bretryAfter=(\d+(?:\.\d+)?)\b/i.exec(text);
+    if (seconds) {
+        return Math.round(Number(seconds[1]) * 1000);
+    }
+    return undefined;
+}
+function abortableDelay(ms, signal) {
+    if (ms <= 0) {
+        return Promise.resolve();
+    }
+    throwIfAborted(signal);
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            signal?.removeEventListener("abort", abort);
+            resolve();
+        }, ms);
+        const abort = () => {
+            clearTimeout(timer);
+            reject(abortError(signal));
+        };
+        signal?.addEventListener("abort", abort, { once: true });
+    });
+}
+function throwIfAborted(signal) {
+    if (signal?.aborted) {
+        throw abortError(signal);
+    }
+}
+function abortError(signal) {
+    return signal?.reason instanceof Error
+        ? signal.reason
+        : new Error("Embedding was cancelled.");
+}
+function countPublicEntities(fragments) {
+    let count = 0;
+    for (const fragment of fragments) {
+        if (!fragment.group || fragment.group === fragment.id) {
+            count++;
+        }
+    }
+    return count;
+}
+async function withContentHash(file) {
+    let bytes;
+    try {
+        bytes = await readFile(file.absolutePath);
+    }
+    catch (error) {
+        throw toEngineError(error, "Indexing failed to compute file content hash", {
+            code: "ZVEC_GREP.ENGINE.INDEXING.CONTENT_HASH_FAILED",
+            context: fileContext(file),
+        });
+    }
+    return {
+        ...file,
+        contentHash: sha256Bytes(bytes),
+    };
+}
+function markFileFailed(ctx, file, error, stage) {
+    const reason = fileFailureReason(stage, error);
+    try {
+        ctx.storage.markFileFailed(file, reason);
+        return reason;
+    }
+    catch (markError) {
+        throw toEngineError(markError, "Indexing failed to record file failure", {
+            code: "ZVEC_GREP.ENGINE.INDEXING.MARK_FILE_FAILED",
+            context: `${fileContext(file)} stage=${stage} original=${reason}`,
+        });
+    }
+}
+function fileFailureReason(stage, error) {
+    return oneLine(`${stage}: ${errorToMessage(error)}`);
+}
+function toEngineError(error, message, options) {
+    if (isEngineError(error)) {
+        return error;
+    }
+    return new EngineError(message, {
+        ...options,
+        cause: error,
+    });
+}
+function workspaceIndexContext(workspaceIndex) {
+    return (errorDetails([
+        detail("workspace_index_id", workspaceIndex.id),
+        workspaceIndexDetail(workspaceIndex.name),
+    ]) ?? "");
+}
+function fileContext(file) {
+    return `fileId=${file.id} path=${file.relativePath}`;
+}
+function errorToMessage(error) {
+    if (isEngineError(error) && error.context) {
+        return `${error.code}: ${error.message} (${error.context})`;
+    }
+    if (isEngineError(error)) {
+        return `${error.code}: ${error.message}`;
+    }
+    return error instanceof Error ? error.message : String(error);
+}
+//# sourceMappingURL=index.js.map