@zvec/zvec-grep 0.2.2 → 0.3.0-rc.2

package/dist/engine/pipeline/indexing/index.js

@@ -1,1217 +0,0 @@
-import { lstat, readFile } from "node:fs/promises";
-import { workspaceIndexDetail, detail, EngineError, errorDetails, isEngineError, } from "../../errors.js";
-import { sha256Bytes } from "../../utils/hash.js";
-import { normalizePath } from "../../utils/path.js";
-import { ConcurrentTiming, TimingCollector } from "../../utils/timing.js";
-import { extractForIndexing, vectorContentForFragment, } from "../../extraction/index.js";
-import { scanDirectoryPath, scanFilePath, scanRootPaths, } from "./scanner/index.js";
-import { indexChunkOptions } from "./input-budget.js";
-const EMBEDDING_TRANSIENT_MAX_RETRIES = 3;
-const EMBEDDING_RATE_LIMIT_MAX_RETRIES = 6;
-const EMBEDDING_TRANSIENT_RETRY_BASE_DELAY_MS = 500;
-const EMBEDDING_RATE_LIMIT_RETRY_BASE_DELAY_MS = 2000;
-const EMBEDDING_TRANSIENT_RETRY_MAX_DELAY_MS = 8000;
-const EMBEDDING_RATE_LIMIT_RETRY_MAX_DELAY_MS = 30000;
-const EMBEDDING_RETRY_JITTER_MS = 500;
-const EMBEDDING_SUCCESS_STREAK_MIN = 4;
-const PERMANENT_REMOTE_MODEL_PROVIDER_CODES = new Set([
-    "invalid_model",
-    "model_not_found",
-    "unsupported_model",
-    "invalid_dimension",
-    "invalid_dimensions",
-    "unsupported_dimension",
-    "unsupported_dimensions",
-    "dimension_out_of_range",
-    "invalid_embedding_dimension",
-    "unsupported_embedding_dimension",
-]);
-export async function indexWorkspace(ctx) {
-    try {
-        return await indexWorkspaceUnchecked(ctx);
-    }
-    catch (error) {
-        throw toEngineError(error, "Indexing workspace failed", {
-            code: "ZVEC_GREP.ENGINE.INDEXING.WORKSPACE_FAILED",
-            context: workspaceIndexContext(ctx.workspaceIndex),
-        });
-    }
-}
-export async function indexWorkspacePaths(ctx, changedPaths) {
-    try {
-        return await indexWorkspacePathsUnchecked(ctx, changedPaths);
-    }
-    catch (error) {
-        throw toEngineError(error, "Indexing changed paths failed", {
-            code: "ZVEC_GREP.ENGINE.INDEXING.WORKSPACE_FAILED",
-            context: workspaceIndexContext(ctx.workspaceIndex),
-        });
-    }
-}
-export async function getWorkspaceIndexStatus(workspaceIndex, storedFiles) {
-    try {
-        const scan = await scanRootPaths(workspaceIndex.id, workspaceIndex.rootPaths, { knownFiles: storedFiles });
-        const diff = await computeDiffFromFiles(scan.files, storedFiles);
-        const pendingFiles = storedFiles.filter((file) => file.indexStatus?.indexedTime === null);
-        const failedFiles = pendingFiles.filter((file) => file.indexStatus?.error !== undefined);
-        const indexedFiles = storedFiles.filter((file) => file.indexStatus?.indexedTime !== undefined &&
-            file.indexStatus.indexedTime !== null);
-        const entitiesIndexed = indexedFiles.reduce((count, file) => count + (file.indexStatus?.entityCount ?? 0), 0);
-        const fragmentsTruncated = indexedFiles.reduce((count, file) => count + (file.indexStatus?.truncatedFragmentCount ?? 0), 0);
-        return {
-            filesScanned: scan.files.length,
-            filesStored: storedFiles.length,
-            filesIndexed: indexedFiles.length,
-            entitiesIndexed,
-            fragmentsTruncated,
-            filesPending: pendingFiles.length,
-            filesFailed: failedFiles.length,
-            filesAdded: diff.added.length,
-            filesModified: diff.modified.length,
-            filesDeleted: diff.deleted.length,
-            filesUnchanged: diff.unchanged.length,
-            pendingFiles,
-            failedFiles,
-            addedFiles: diff.added,
-            modifiedFiles: diff.modified,
-            deletedFiles: diff.deleted,
-        };
-    }
-    catch (error) {
-        throw toEngineError(error, "Inspecting workspace index status failed", {
-            code: "ZVEC_GREP.ENGINE.INDEXING.STATUS_FAILED",
-            context: workspaceIndexContext(workspaceIndex),
-        });
-    }
-}
-async function indexWorkspaceUnchecked(ctx) {
-    const start = Date.now();
-    const report = ctx.onProgress ?? (() => undefined);
-    const timings = new TimingCollector();
-    throwIfIndexCancelled(ctx);
-    const firstPass = await runIndexPass(ctx, report, "Scanning files...", timings);
-    const passes = [firstPass];
-    let progressBase;
-    if (firstPass.stats.filesFailed > 0) {
-        const failed = firstPass.stats.filesFailed;
-        progressBase = retryProgressBase(firstPass);
-        report({
-            phase: "scanning",
-            filesTotal: progressBase.filesTotal,
-            filesIndexed: progressBase.filesSucceeded,
-            detail: `Retrying ${failed} failed ${failed === 1 ? "file" : "files"}...`,
-        });
-        passes.push(await runIndexPass(ctx, report, "Scanning retry candidates...", timings, progressBase));
-    }
-    const finalPass = passes[passes.length - 1];
-    throwIfIndexCancelled(ctx);
-    reportIndexFinalizing(ctx, report, finalPass, progressBase);
-    await timings.time("index_optimize", () => optimizeStorage(ctx));
-    const result = buildIndexResult(ctx, passes, Date.now() - start, timings);
-    if (result.filesFailed > 0) {
-        report({ phase: "done", detail: "Indexing completed with failed files" });
-        throw filesFailedError(ctx, result, finalPass, passes.length);
-    }
-    report({ phase: "done", detail: "Indexing complete" });
-    return result;
-}
-async function indexWorkspacePathsUnchecked(ctx, changedPaths) {
-    const start = Date.now();
-    const report = ctx.onProgress ?? (() => undefined);
-    const timings = new TimingCollector();
-    const normalizedPaths = [...new Set(changedPaths.map(normalizePath))];
-    throwIfIndexCancelled(ctx);
-    const firstPass = await runPathIndexPass(ctx, report, normalizedPaths, timings);
-    const passes = [firstPass];
-    let progressBase;
-    if (firstPass.stats.filesFailed > 0) {
-        progressBase = retryProgressBase(firstPass);
-        passes.push(await runPathIndexPass(ctx, report, normalizedPaths, timings, progressBase));
-    }
-    const finalPass = passes[passes.length - 1];
-    throwIfIndexCancelled(ctx);
-    reportIndexFinalizing(ctx, report, finalPass, progressBase);
-    await timings.time("index_optimize", () => optimizeStorage(ctx));
-    const result = buildIndexResult(ctx, passes, Date.now() - start, timings);
-    if (result.filesFailed > 0) {
-        throw filesFailedError(ctx, result, finalPass, passes.length);
-    }
-    report({ phase: "done", detail: "Indexing complete" });
-    return result;
-}
-function filesFailedError(ctx, result, finalPass, passCount) {
-    return new EngineError(`Indexing completed with ${result.filesFailed} failed ${result.filesFailed === 1 ? "file" : "files"}`, {
-        code: "ZVEC_GREP.ENGINE.INDEXING.FILES_FAILED",
-        context: errorDetails([
-            workspaceIndexDetail(ctx.workspaceIndex.name),
-            detail("filesFailed", result.filesFailed),
-            detail("filesScanned", result.filesScanned),
-            detail("failedFiles", summarizeFailedFiles(finalPass.stats.failedFiles)),
-            detail("failedReasons", summarizeFailedFiles(finalPass.stats.failedFileReasons)),
-            detail("hint", passCount > 1
-                ? "Retried failed files once automatically; if failures persist, fix the failed files or embedding configuration."
-                : "If the failure was transient, rerun the same indexing command; if it persists, fix the failed files or embedding configuration."),
-        ]),
-    });
-}
-async function runPathIndexPass(ctx, report, changedPaths, timings, progressBase) {
-    reportScanning(report, "Scanning changed paths...", progressBase);
-    const scanned = await timings.time("index_scan_paths", async () => {
-        const files = [];
-        const diagnostics = emptyScanDiagnostics();
-        for (const path of changedPaths) {
-            throwIfIndexCancelled(ctx);
-            const info = await lstat(path).catch(() => null);
-            const scan = info?.isDirectory()
-                ? await scanDirectoryPath(ctx.workspaceIndex.id, ctx.workspaceIndex.rootPaths, path, { signal: ctx.signal })
-                : await scanFilePath(ctx.workspaceIndex.id, ctx.workspaceIndex.rootPaths, path, {
-                    signal: ctx.signal,
-                });
-            files.push(...scan.files);
-            mergeScanDiagnostics(diagnostics, scan.diagnostics);
-        }
-        return {
-            files: [...new Map(files.map((file) => [file.id, file])).values()],
-            diagnostics,
-        };
-    });
-    throwIfIndexCancelled(ctx);
-    const exactExisting = [];
-    const prefixPaths = [];
-    for (const path of changedPaths) {
-        const exact = ctx.storage.getFileByPath(path);
-        if (exact) {
-            exactExisting.push(exact);
-        }
-        else {
-            prefixPaths.push(path);
-        }
-    }
-    const existing = [
-        ...new Map([
-            ...exactExisting,
-            ...ctx.storage.listFilesByPathPrefixes(prefixPaths),
-        ].map((file) => [file.id, file])).values(),
-    ];
-    return runDiffPass(ctx, report, scanned.files, existing, timings, progressBase, scanned.diagnostics);
-}
-async function runIndexPass(ctx, report, scanningDetail, timings, progressBase) {
-    reportScanning(report, scanningDetail, progressBase);
-    const existing = ctx.storage.listFiles();
-    const scan = await timings.time("index_scan", () => scanRootPaths(ctx.workspaceIndex.id, ctx.workspaceIndex.rootPaths, {
-        signal: ctx.signal,
-        knownFiles: existing,
-    }));
-    throwIfIndexCancelled(ctx);
-    return runDiffPass(ctx, report, scan.files, existing, timings, progressBase, scan.diagnostics);
-}
-async function runDiffPass(ctx, report, scannedFiles, existingFiles, timings, progressBase, scanDiagnostics = emptyScanDiagnostics()) {
-    const diff = await timings.time("index_diff", () => computeDiffFromFiles(scannedFiles, existingFiles));
-    throwIfIndexCancelled(ctx);
-    const pending = [...diff.added, ...diff.modified, ...diff.pending];
-    report({
-        phase: "scanning",
-        filesTotal: progressBase?.filesTotal ?? scannedFiles.length,
-        filesIndexed: progressBase?.filesSucceeded ?? 0,
-        detail: `${diff.added.length} added, ${diff.modified.length} modified, ${diff.pending.length} pending, ${diff.deleted.length} deleted, ${diff.unchanged.length} unchanged`,
-    });
-    timings.timeSync("index_delete_stale", () => {
-        for (const file of diff.deleted) {
-            throwIfIndexCancelled(ctx);
-            try {
-                ctx.storage.deleteFile(file.id);
-            }
-            catch (error) {
-                throw toEngineError(error, "Indexing failed to delete stale file records", {
-                    code: "ZVEC_GREP.ENGINE.INDEXING.DELETE_FILE_FAILED",
-                    context: fileContext(file),
-                });
-            }
-        }
-    });
-    const reportIndexing = (currentStats, detail, embedding) => {
-        report({
-            phase: "indexing",
-            filesTotal: progressBase?.filesTotal ?? pending.length,
-            filesIndexed: (progressBase?.filesSucceeded ?? 0) +
-                currentStats.filesIndexed +
-                currentStats.filesFailed,
-            filesFailed: currentStats.filesFailed,
-            detail,
-            embedding,
-        });
-    };
-    reportIndexing({
-        filesIndexed: 0,
-        filesFailed: 0,
-        failedFiles: [],
-        failedFileReasons: [],
-        entitiesCreated: 0,
-    });
-    const stats = await indexFiles(pending, ctx, reportIndexing, timings);
-    throwIfIndexCancelled(ctx);
-    return {
-        filesScanned: scannedFiles.length,
-        scanDiagnostics,
-        diff,
-        stats,
-    };
-}
-function reportIndexFinalizing(ctx, report, pass, progressBase) {
-    report({
-        phase: "indexing",
-        filesTotal: progressBase?.filesTotal ??
-            pass.diff.added.length +
-                pass.diff.modified.length +
-                pass.diff.pending.length,
-        filesIndexed: (progressBase?.filesSucceeded ?? 0) +
-            pass.stats.filesIndexed +
-            pass.stats.filesFailed,
-        filesFailed: pass.stats.filesFailed,
-        detail: "finalizing index",
-    });
-}
-function retryProgressBase(pass) {
-    return {
-        filesSucceeded: pass.stats.filesIndexed,
-        filesTotal: pass.diff.added.length +
-            pass.diff.modified.length +
-            pass.diff.pending.length,
-    };
-}
-function reportScanning(report, detail, progressBase) {
-    report({
-        phase: "scanning",
-        ...(progressBase
-            ? {
-                filesTotal: progressBase.filesTotal,
-                filesIndexed: progressBase.filesSucceeded,
-            }
-            : {}),
-        detail,
-    });
-}
-function buildIndexResult(ctx, passes, durationMs, timings) {
-    const firstPass = passes[0];
-    const retryPasses = passes.slice(1);
-    const finalPass = passes[passes.length - 1];
-    return {
-        filesScanned: finalPass.filesScanned,
-        filesAdded: firstPass.diff.added.length +
-            retryPasses.reduce((count, pass) => count + pass.diff.added.length, 0),
-        filesModified: firstPass.diff.modified.length +
-            retryPasses.reduce((count, pass) => count + pass.diff.modified.length, 0),
-        filesPending: firstPass.diff.pending.length +
-            retryPasses.reduce((count, pass) => count + pass.diff.pending.length, 0),
-        filesDeleted: firstPass.diff.deleted.length +
-            retryPasses.reduce((count, pass) => count + pass.diff.deleted.length, 0),
-        filesUnchanged: firstPass.diff.unchanged.length,
-        filesFailed: finalPass.stats.filesFailed,
-        entitiesCreated: passes.reduce((count, pass) => count + pass.stats.entitiesCreated, 0),
-        durationMs,
-        timings: timings.entries(),
-        ...(finalPass.scanDiagnostics.skippedFiles > 0
-            ? { scanDiagnostics: finalPass.scanDiagnostics }
-            : {}),
-    };
-}
-const MAX_SKIPPED_FILE_SAMPLES = 20;
-function emptyScanDiagnostics() {
-    return {
-        skippedFiles: 0,
-        skippedByReason: {
-            empty: 0,
-            too_large: 0,
-            unsupported: 0,
-            binary: 0,
-        },
-        skippedSamples: [],
-    };
-}
-function mergeScanDiagnostics(target, source) {
-    target.skippedFiles += source.skippedFiles;
-    for (const reason of [
-        "empty",
-        "too_large",
-        "unsupported",
-        "binary",
-    ]) {
-        target.skippedByReason[reason] += source.skippedByReason[reason];
-    }
-    const remaining = MAX_SKIPPED_FILE_SAMPLES - target.skippedSamples.length;
-    if (remaining > 0) {
-        target.skippedSamples.push(...source.skippedSamples.slice(0, remaining));
-    }
-}
-async function optimizeStorage(ctx) {
-    try {
-        await ctx.storage.finalizeWrites();
-    }
-    catch (error) {
-        throw toEngineError(error, "Indexing failed to finalize storage", {
-            code: "ZVEC_GREP.ENGINE.INDEXING.OPTIMIZE_FAILED",
-            context: workspaceIndexContext(ctx.workspaceIndex),
-        });
-    }
-}
-async function computeDiffFromFiles(scannedFiles, existingFiles) {
-    const existingById = new Map(existingFiles.map((file) => [file.id, file]));
-    const seen = new Set();
-    const added = [];
-    const modified = [];
-    const pending = [];
-    const unchanged = [];
-    for (const file of scannedFiles) {
-        seen.add(file.id);
-        const existing = existingById.get(file.id);
-        if (!existing) {
-            added.push(await withContentHash(file));
-            continue;
-        }
-        if (existing.indexStatus?.indexedTime === null) {
-            pending.push(await withContentHash(file));
-            continue;
-        }
-        if (existing.sizeBytes === file.sizeBytes &&
-            existing.lastModifiedTime === file.lastModifiedTime &&
-            existing.contentHash) {
-            unchanged.push(existing);
-            continue;
-        }
-        const hashed = await withContentHash(file);
-        if (existing.sizeBytes === hashed.sizeBytes &&
-            existing.contentHash === hashed.contentHash) {
-            unchanged.push(existing);
-            continue;
-        }
-        modified.push(hashed);
-    }
-    const deleted = [...existingById.values()].filter((file) => !seen.has(file.id));
-    return { added, modified, pending, deleted, unchanged };
-}
-async function indexFiles(files, ctx, onProgress, timings) {
-    const stats = {
-        filesIndexed: 0,
-        filesFailed: 0,
-        failedFiles: [],
-        failedFileReasons: [],
-        entitiesCreated: 0,
-    };
-    const batchFiles = [];
-    let batchFragmentCount = 0;
-    const embeddingScheduler = createEmbeddingScheduler(ctx.embeddingConcurrency, ctx.embeddingModel);
-    const embeddingTiming = new ConcurrentTiming(timings, "index_embedding");
-    const runningEmbeddings = new Set();
-    let firstEmbeddingError;
-    const prepareModel = ctx.embeddingModel.info.provider === "local"
-        ? ctx.embeddingModel.prepare?.bind(ctx.embeddingModel)
-        : undefined;
-    let modelPrepared = false;
-    const reportEmbeddingProgress = (currentStats, detail) => {
-        onProgress(currentStats, detail, embeddingScheduler.snapshot());
-    };
-    const throwFirstEmbeddingError = () => {
-        const failure = firstEmbeddingError;
-        if (failure) {
-            throw failure.error;
-        }
-    };
-    const flushBatch = async () => {
-        throwFirstEmbeddingError();
-        throwIfIndexCancelled(ctx);
-        if (batchFiles.length === 0) {
-            return;
-        }
-        const filesToEmbed = batchFiles.splice(0);
-        batchFragmentCount = 0;
-        await scheduleEmbeddingTask(() => embedAndCommitBatch(filesToEmbed, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming));
-    };
-    const scheduleEmbeddingTask = async (task) => {
-        throwFirstEmbeddingError();
-        throwIfIndexCancelled(ctx);
-        if (!modelPrepared && prepareModel) {
-            // Finish shared initialization before any file or fragment batch is queued.
-            try {
-                await embeddingTiming.time(() => prepareModel({
-                    signal: ctx.signal,
-                    onProgress: (progress) => reportModelDownloadProgress(stats, progress, onProgress, embeddingScheduler),
-                }));
-            }
-            catch (error) {
-                throwIfIndexCancelled(ctx);
-                throw error;
-            }
-            throwIfIndexCancelled(ctx);
-        }
-        modelPrepared = true;
-        const cleanup = () => {
-            runningEmbeddings.delete(promise);
-        };
-        const promise = Promise.resolve()
-            .then(task)
-            .then(cleanup, (error) => {
-            if (!firstEmbeddingError) {
-                firstEmbeddingError = { error };
-            }
-            cleanup();
-        });
-        runningEmbeddings.add(promise);
-        if (runningEmbeddings.size >= embeddingScheduler.taskConcurrency) {
-            await Promise.race(runningEmbeddings);
-        }
-        throwFirstEmbeddingError();
-        throwIfIndexCancelled(ctx);
-    };
-    try {
-        for (const file of files) {
-            throwFirstEmbeddingError();
-            throwIfIndexCancelled(ctx);
-            onProgress(stats, `reading ${file.relativePath}`);
-            const prepared = await timings.time("index_prepare", () => prepareFile(file, ctx));
-            throwFirstEmbeddingError();
-            throwIfIndexCancelled(ctx);
-            if ("failedReason" in prepared) {
-                recordFileFailed(stats, file, prepared.failedReason);
-                onProgress(stats, `failed ${file.relativePath}`);
-                continue;
-            }
-            if (prepared.fragments.length === 0) {
-                const committed = commitFile(prepared, [], ctx, stats);
-                onProgress(stats, finishedFileDetail(committed, file.relativePath));
-                continue;
-            }
-            if (prepared.fragments.length > ctx.embeddingModel.info.limits.maxBatchSize) {
-                await flushBatch();
-                await scheduleEmbeddingTask(async () => {
-                    throwIfIndexCancelled(ctx);
-                    reportEmbeddingProgress(stats, `embedding ${file.relativePath}`);
-                    await embedAndCommitFile(prepared, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming);
-                });
-                continue;
-            }
-            if (batchFragmentCount > 0 &&
-                batchFragmentCount + prepared.fragments.length >
-                    ctx.embeddingModel.info.limits.maxBatchSize) {
-                await flushBatch();
-            }
-            batchFiles.push(prepared);
-            batchFragmentCount += prepared.fragments.length;
-            if (batchFragmentCount === ctx.embeddingModel.info.limits.maxBatchSize) {
-                await flushBatch();
-            }
-        }
-        await flushBatch();
-        await Promise.all(runningEmbeddings);
-        throwFirstEmbeddingError();
-        throwIfIndexCancelled(ctx);
-    }
-    catch (error) {
-        await Promise.allSettled(runningEmbeddings);
-        throw error;
-    }
-    throwIfIndexCancelled(ctx);
-    return stats;
-}
-async function prepareFile(file, ctx) {
-    try {
-        throwIfIndexCancelled(ctx);
-        const source = await readSource(file);
-        const chunkOptions = indexChunkOptions(ctx.embeddingModel.info.limits.maxInputTokens, source.kind === "text" ? source.text : undefined);
-        const extracted = await extractForIndexing(source, chunkOptions);
-        throwIfIndexCancelled(ctx);
-        const fragments = extracted
-            .filter(({ fragment }) => ctx.embeddingModel.info.inputKinds.includes(fragment.content.kind))
-            .map(({ fragment, embeddingSource }) => ({
-            fragment,
-            embeddingContent: vectorContentForFragment(fragment, embeddingSource, chunkOptions.maxChunkChars),
-        }));
-        return { file, fragments };
-    }
-    catch (error) {
-        if (indexIsCancelled(ctx)) {
-            throw indexCancellationError(ctx);
-        }
-        return {
-            file,
-            failedReason: markFileFailed(ctx, file, error, "prepare"),
-        };
-    }
-}
-async function embedAndCommitBatch(files, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming) {
-    try {
-        throwIfIndexCancelled(ctx);
-        const contents = files.flatMap((file) => file.fragments.map((fragment) => fragment.embeddingContent));
-        onProgress(stats, `embedding ${describePreparedFiles(files)}`, embeddingScheduler.snapshot());
-        const embedding = await embeddingTiming.time(() => embedContentsWithRetry(contents, ctx.embeddingModel, embeddingScheduler, ctx.signal, (progress) => reportModelDownloadProgress(stats, progress, onProgress, embeddingScheduler)));
-        const truncatedInputIndexes = new Set(embedding.truncated);
-        throwIfIndexCancelled(ctx);
-        let offset = 0;
-        for (const file of files) {
-            throwIfIndexCancelled(ctx);
-            const end = offset + file.fragments.length;
-            const fileVectors = embedding.vectors.slice(offset, end);
-            let truncatedFragmentCount = 0;
-            for (let index = offset; index < end; index++) {
-                if (truncatedInputIndexes.has(index)) {
-                    truncatedFragmentCount++;
-                }
-            }
-            offset += file.fragments.length;
-            const committed = timings.timeSync("index_commit", () => commitFile(file, fileVectors, ctx, stats, truncatedFragmentCount));
-            onProgress(stats, finishedFileDetail(committed, file.file.relativePath));
-        }
-    }
-    catch (error) {
-        if (indexIsCancelled(ctx)) {
-            throw indexCancellationError(ctx);
-        }
-        if (shouldFailFastEmbeddingError(error, ctx.embeddingModel)) {
-            throw error;
-        }
-        for (const file of files) {
-            onProgress(stats, `embedding ${file.file.relativePath}`, embeddingScheduler.snapshot());
-            await embedAndCommitFile(file, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming);
-        }
-    }
-}
-async function embedAndCommitFile(file, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming) {
-    try {
-        throwIfIndexCancelled(ctx);
-        const embedding = await embeddingTiming.time(() => embedFragments(file.fragments, ctx.embeddingModel, embeddingScheduler, ctx.signal, (progress) => reportModelDownloadProgress(stats, progress, onProgress, embeddingScheduler)));
-        throwIfIndexCancelled(ctx);
-        const committed = timings.timeSync("index_commit", () => commitFile(file, embedding.vectors, ctx, stats, embedding.truncated.length));
-        onProgress(stats, finishedFileDetail(committed, file.file.relativePath));
-    }
-    catch (error) {
-        if (indexIsCancelled(ctx)) {
-            throw indexCancellationError(ctx);
-        }
-        if (shouldFailFastEmbeddingError(error, ctx.embeddingModel)) {
-            throw error;
-        }
-        const reason = markFileFailed(ctx, file.file, error, "embed");
-        recordFileFailed(stats, file.file, reason);
-        onProgress(stats, finishedFileDetail(false, file.file.relativePath));
-    }
-}
-function commitFile(file, vectors, ctx, stats, truncatedFragmentCount = 0) {
-    try {
-        throwIfIndexCancelled(ctx);
-        if (file.fragments.length !== vectors.length) {
-            throw new EngineError("Embedding returned mismatched entity/vector counts", {
-                code: "ZVEC_GREP.ENGINE.STORAGE.ENTITY_VECTOR_COUNT_MISMATCH",
-                context: `fileId=${file.file.id} fragmentCount=${file.fragments.length} vectorCount=${vectors.length}`,
-            });
-        }
-        ctx.storage.replaceFile(file.file, file.fragments.map(({ fragment }, index) => ({
-            fragment,
-            vector: vectors[index],
-        })), {
-            truncatedFragmentCount,
-        });
-        stats.filesIndexed++;
-        stats.entitiesCreated += countPublicEntities(file.fragments.map(({ fragment }) => fragment));
-        return true;
-    }
-    catch (error) {
-        if (indexIsCancelled(ctx)) {
-            throw indexCancellationError(ctx);
-        }
-        const reason = markFileFailed(ctx, file.file, error, "commit");
-        recordFileFailed(stats, file.file, reason);
-        return false;
-    }
-}
-function throwIfIndexCancelled(ctx) {
-    if (!indexIsCancelled(ctx)) {
-        return;
-    }
-    throw indexCancellationError(ctx);
-}
-function indexIsCancelled(ctx) {
-    return ctx.signal?.aborted === true;
-}
-function indexCancellationError(ctx) {
-    return ctx.signal?.reason instanceof Error
-        ? ctx.signal.reason
-        : new EngineError("Indexing was cancelled.", {
-            code: "ZVEC_GREP.ENGINE.INDEXING.CANCELLED",
-            context: workspaceIndexContext(ctx.workspaceIndex),
-        });
-}
-function recordFileFailed(stats, file, reason) {
-    stats.filesFailed++;
-    stats.failedFiles.push(file.relativePath);
-    if (reason) {
-        stats.failedFileReasons.push(`${file.relativePath}: ${reason}`);
-    }
-}
-function summarizeFailedFiles(files) {
-    const shown = files.slice(0, 5);
-    const remaining = files.length - shown.length;
-    return remaining > 0
-        ? `${shown.join(", ")} and ${remaining} more`
-        : shown.join(", ");
-}
-function describePreparedFiles(files) {
-    if (files.length === 0) {
-        return "0 files";
-    }
-    if (files.length === 1) {
-        return files[0].file.relativePath;
-    }
-    return `${files.length} files, starting with ${files[0].file.relativePath}`;
-}
-function finishedFileDetail(succeeded, relativePath) {
-    return succeeded ? `indexed ${relativePath}` : `failed ${relativePath}`;
-}
-async function readSource(file) {
-    let bytes;
-    try {
-        bytes = await readFile(file.absolutePath);
-    }
-    catch (error) {
-        throw toEngineError(error, "Indexing failed to read source file", {
-            code: "ZVEC_GREP.ENGINE.INDEXING.READ_SOURCE_FAILED",
-            context: fileContext(file),
-        });
-    }
-    if (file.kind === "image") {
-        return {
-            kind: "image",
-            file,
-            data: bytes,
-            format: file.format,
-        };
-    }
-    return {
-        kind: "text",
-        file,
-        text: bytes.toString("utf8"),
-    };
-}
-async function embedFragments(fragments, model, embeddingScheduler, signal, onModelProgress) {
-    const batches = [];
-    for (let start = 0; start < fragments.length; start += model.info.limits.maxBatchSize) {
-        const batch = fragments.slice(start, start + model.info.limits.maxBatchSize);
-        batches.push({ start, fragments: batch });
-    }
-    const vectors = new Array(fragments.length);
-    const truncatedInputIndexes = [];
-    const batchAbortController = new AbortController();
-    const batchSignal = signal
-        ? AbortSignal.any([signal, batchAbortController.signal])
-        : batchAbortController.signal;
-    const results = await Promise.allSettled(batches.map((batch) => embedFragmentBatch(batch.fragments, model, batch.start, embeddingScheduler, batchSignal, onModelProgress, (error) => batchAbortController.abort(error))));
-    let firstError;
-    for (const [index, result] of results.entries()) {
-        if (result.status === "rejected") {
-            firstError ??= result.reason;
-            continue;
-        }
-        const start = batches[index].start;
-        for (const [offset, vector] of result.value.vectors.entries()) {
-            vectors[start + offset] = vector;
-        }
-        truncatedInputIndexes.push(...result.value.truncated.map((inputIndex) => start + inputIndex));
-    }
-    if (firstError) {
-        throw firstError;
-    }
-    return {
-        vectors,
-        truncated: truncatedInputIndexes,
-    };
-}
-async function embedFragmentBatch(fragments, model, startIndex, embeddingScheduler, signal, onModelProgress, onTerminalFailure) {
-    const contents = fragments.map((fragment) => fragment.embeddingContent);
-    try {
-        return await embedContentsWithRetry(contents, model, embeddingScheduler, signal, onModelProgress, onTerminalFailure);
-    }
-    catch (error) {
-        if (fragments.length === 1 || shouldFailFastEmbeddingError(error, model)) {
-            throw error;
-        }
-        return await embedFragmentBatchOneByOne(fragments, model, startIndex, embeddingScheduler, signal, onModelProgress, onTerminalFailure);
-    }
-}
-async function embedFragmentBatchOneByOne(fragments, model, startIndex, embeddingScheduler, signal, onModelProgress, onTerminalFailure) {
-    const vectors = [];
-    const truncatedInputIndexes = [];
-    for (const [index, fragment] of fragments.entries()) {
-        try {
-            const result = await embedContentsWithRetry([fragment.embeddingContent], model, embeddingScheduler, signal, onModelProgress, onTerminalFailure);
-            vectors.push(result.vectors[0]);
-            if (result.truncated.length > 0) {
-                truncatedInputIndexes.push(index);
-            }
-        }
-        catch (error) {
-            if (shouldFailFastEmbeddingError(error, model)) {
-                throw error;
-            }
-            throw new EngineError("Embedding entity fragment failed after one-by-one fallback", {
-                code: "ZVEC_GREP.ENGINE.INDEXING.EMBEDDING_FRAGMENT_FAILED",
-                context: errorDetails([
-                    detail("model", model.info.reference),
-                    detail("fragmentId", fragment.fragment.id),
-                    detail("fragmentIndex", startIndex + index),
-                    isEngineError(error) ? detail("causeCode", error.code) : null,
-                    error instanceof Error ? detail("cause", error.message) : null,
-                    isEngineError(error) ? error.context : null,
-                ]),
-                cause: error,
-            });
-        }
-    }
-    return {
-        vectors,
-        truncated: truncatedInputIndexes,
-    };
-}
-async function embedContentsWithRetry(contents, model, embeddingScheduler, signal, onModelProgress, onTerminalFailure) {
-    let attempt = 0;
-    while (true) {
-        let retry = nonRetryableEmbeddingError();
-        let delayMs = 0;
-        try {
-            throwIfAborted(signal);
-            const result = await embeddingScheduler.run(() => model.embed(contents, {
-                purpose: "document",
-                signal,
-                onProgress: onModelProgress,
-            }), signal, (error) => {
-                retry = classifyEmbeddingRetry(error, model);
-                delayMs = retryDelayMs(attempt, retry);
-                if (retry.retryable) {
-                    embeddingScheduler.recordRetryableFailure({
-                        rateLimited: retry.rateLimited,
-                        delayMs,
-                    });
-                }
-                if (retry.failFast &&
-                    (!retry.retryable || attempt >= maxRetryAttempts(retry))) {
-                    onTerminalFailure?.(error);
-                }
-            });
-            embeddingScheduler.recordSuccess();
-            return result;
-        }
-        catch (error) {
-            if (signal?.aborted) {
-                throw signal.reason instanceof Error
-                    ? signal.reason
-                    : new Error("Embedding was cancelled.");
-            }
-            if (!retry.retryable) {
-                retry = classifyEmbeddingRetry(error, model);
-                delayMs = retryDelayMs(attempt, retry);
-            }
-            if (attempt >= maxRetryAttempts(retry) || !retry.retryable) {
-                throw error;
-            }
-            await abortableDelay(delayMs, signal);
-            attempt++;
-        }
-    }
-}
-function reportModelDownloadProgress(stats, progress, onProgress, embeddingScheduler) {
-    onProgress(stats, `downloading ${progress.model}`, {
-        ...embeddingScheduler.snapshot(),
-        ...progress,
-    });
-}
-function createEmbeddingScheduler(requestedConcurrency, model) {
-    return new AdaptiveEmbeddingScheduler(resolveEmbeddingConcurrencyPolicy(requestedConcurrency, model));
-}
-function resolveEmbeddingConcurrencyPolicy(requestedConcurrency, model) {
-    if (requestedConcurrency !== undefined &&
-        Number.isInteger(requestedConcurrency) &&
-        requestedConcurrency > 0) {
-        return {
-            initial: requestedConcurrency,
-            min: 1,
-            max: requestedConcurrency,
-            adaptive: requestedConcurrency > 1,
-        };
-    }
-    const remote = model.info.provider === "qwen";
-    const multimodal = model.info.inputKinds.includes("image");
-    const configuredLocalDefault = model.info.defaultConcurrency;
-    const localDefault = configuredLocalDefault !== undefined &&
-        Number.isInteger(configuredLocalDefault) &&
-        configuredLocalDefault > 0
-        ? configuredLocalDefault
-        : 1;
-    const initial = remote ? (multimodal ? 4 : 8) : localDefault;
-    const max = remote ? (multimodal ? 8 : 12) : localDefault;
-    const min = Math.min(initial, 4);
-    return {
-        initial,
-        min,
-        max: Math.max(initial, max),
-        adaptive: max > 1,
-    };
-}
-class AdaptiveEmbeddingScheduler {
-    policy;
-    active = 0;
-    cooldownUntil = 0;
-    currentConcurrency;
-    retryableFailures = 0;
-    successStreak = 0;
-    queue = [];
-    constructor(policy) {
-        this.policy = policy;
-        this.currentConcurrency = policy.initial;
-    }
-    get taskConcurrency() {
-        return this.policy.max;
-    }
-    async run(task, signal, onError) {
-        throwIfAborted(signal);
-        await this.waitForCooldown(signal);
-        await this.acquire(signal);
-        try {
-            await this.waitForCooldown(signal);
-            throwIfAborted(signal);
-            return await task();
-        }
-        catch (error) {
-            onError?.(error);
-            throw error;
-        }
-        finally {
-            this.release();
-        }
-    }
-    recordSuccess() {
-        if (!this.policy.adaptive || this.currentConcurrency >= this.policy.max) {
-            return;
-        }
-        this.successStreak++;
-        if (this.successStreak <
-            Math.max(EMBEDDING_SUCCESS_STREAK_MIN, this.currentConcurrency * 2)) {
-            return;
-        }
-        this.currentConcurrency++;
-        this.successStreak = 0;
-        this.drainQueue();
-    }
-    recordRetryableFailure(retry) {
-        this.retryableFailures++;
-        if (retry.rateLimited && retry.delayMs > 0) {
-            this.cooldownUntil = Math.max(this.cooldownUntil, Date.now() + retry.delayMs);
-        }
-        if (!this.policy.adaptive) {
-            return;
-        }
-        this.currentConcurrency = Math.max(this.policy.min, Math.floor(this.currentConcurrency / 2));
-        this.successStreak = 0;
-    }
-    snapshot() {
-        return {
-            concurrency: this.currentConcurrency,
-            maxConcurrency: this.policy.max,
-            retryableFailures: this.retryableFailures,
-        };
-    }
-    async waitForCooldown(signal) {
-        const remaining = this.cooldownUntil - Date.now();
-        if (remaining > 0) {
-            await abortableDelay(remaining, signal);
-        }
-    }
-    async acquire(signal) {
-        throwIfAborted(signal);
-        if (this.active < this.currentConcurrency) {
-            this.active++;
-            return;
-        }
-        await new Promise((resolve, reject) => {
-            const queued = () => {
-                signal?.removeEventListener("abort", abort);
-                resolve();
-            };
-            const abort = () => {
-                const index = this.queue.indexOf(queued);
-                if (index >= 0) {
-                    this.queue.splice(index, 1);
-                }
-                reject(abortError(signal));
-            };
-            if (signal?.aborted) {
-                reject(abortError(signal));
-                return;
-            }
-            signal?.addEventListener("abort", abort, { once: true });
-            this.queue.push(queued);
-        });
-    }
-    release() {
-        this.active--;
-        this.drainQueue();
-    }
-    drainQueue() {
-        while (this.active < this.currentConcurrency) {
-            const resolve = this.queue.shift();
-            if (!resolve) {
-                return;
-            }
-            this.active++;
-            resolve();
-        }
-    }
-}
-function oneLine(value) {
-    return value.replace(/\s+/g, " ").trim();
-}
-function shouldFailFastEmbeddingError(error, model) {
-    return classifyEmbeddingRetry(error, model).failFast;
-}
-function classifyEmbeddingRetry(error, model) {
-    const codes = errorCodes(error);
-    const text = errorChainText(error);
-    const status = httpStatusFromText(text);
-    const remoteEmbedding = model.info.provider !== "local";
-    const rateLimited = remoteEmbedding &&
-        (status === 429 ||
-            /rate limit|quota exceeded|too many requests|request rate increased too quickly/i.test(text));
-    const serverError = remoteEmbedding &&
-        typeof status === "number" &&
-        status >= 500 &&
-        status <= 599;
-    const requestTimeout = remoteEmbedding && status === 408;
-    const requestFailure = codes.some((code) => code.endsWith("_REQUEST_FAILED"));
-    const transientNetworkFailure = remoteEmbedding && requestFailure && isTransientNetworkFailure(text);
-    const sharedLocalModelFailure = codes.some((code) => code === "ZVEC_GREP.ENGINE.MODELS.MODEL2VEC_DOWNLOAD_FAILED" ||
-        code === "ZVEC_GREP.ENGINE.MODELS.MODEL2VEC_LOAD_FAILED" ||
-        code === "ZVEC_GREP.ENGINE.MODELS.MODEL2VEC_DISPOSED");
-    const retryable = !sharedLocalModelFailure &&
-        (rateLimited || serverError || requestTimeout || transientNetworkFailure);
-    const remoteConfigurationFailure = remoteEmbedding &&
-        (codes.some((code) => code.endsWith("_MISSING_API_KEY") || code.endsWith("_MISSING_ENDPOINT")) ||
-            status === 401 ||
-            status === 403 ||
-            status === 404 ||
-            /\b(?:invalid|missing|unauthorized|forbidden)[ _-]?(?:api[ _-]?)?key\b/i.test(text));
-    const permanentRemoteModelFailure = remoteEmbedding &&
-        (codes.includes("ZVEC_GREP.ENGINE.MODELS.EMBEDDING_DIMENSION_MISMATCH") ||
-            (status === 400 && isPermanentRemoteModelBadRequest(text)));
-    return {
-        retryable,
-        rateLimited,
-        failFast: retryable ||
-            sharedLocalModelFailure ||
-            remoteConfigurationFailure ||
-            permanentRemoteModelFailure ||
-            (remoteEmbedding && requestFailure),
-        retryAfterMs: retryAfterMsFromText(text),
-    };
-}
-function isPermanentRemoteModelBadRequest(text) {
-    const providerCode = /\bproviderCode=([^\s]+)/i.exec(text)?.[1];
-    const normalizedCode = providerCode
-        ?.replace(/[^a-z0-9]+/gi, "_")
-        .replace(/^_+|_+$/g, "")
-        .toLowerCase();
-    if (normalizedCode &&
-        PERMANENT_REMOTE_MODEL_PROVIDER_CODES.has(normalizedCode)) {
-        return true;
-    }
-    const providerMessage = /\bproviderMessage=(.*?)(?=\s+ZVEC_GREP\.|$)/i.exec(text)?.[1];
-    return (providerMessage !== undefined &&
-        /(?:\b(?:invalid|unsupported|unknown)\b.{0,48}\bmodel\b|\bmodel\b.{0,48}\b(?:invalid|unsupported|unknown|not found|does not exist)\b|\b(?:invalid|unsupported|out of range)\b.{0,48}\bdimensions?\b|\bdimensions?\b.{0,48}\b(?:invalid|unsupported|not supported|out of range|must|should|expected|between|only supports?)\b)/i.test(providerMessage));
-}
-function nonRetryableEmbeddingError() {
-    return {
-        retryable: false,
-        rateLimited: false,
-        failFast: false,
-    };
-}
-function errorCodes(error) {
-    const codes = [];
-    for (const current of errorChain(error)) {
-        if (typeof current === "object" &&
-            current !== null &&
-            "code" in current &&
-            typeof current.code === "string") {
-            codes.push(current.code);
-        }
-    }
-    return codes;
-}
-function errorChainText(error) {
-    const parts = [];
-    for (const current of errorChain(error)) {
-        if (current instanceof Error) {
-            parts.push(current.name, current.message);
-        }
-        else {
-            parts.push(String(current));
-        }
-        if (isEngineError(current) && current.context) {
-            parts.push(current.context);
-        }
-        if (typeof current === "object" &&
-            current !== null &&
-            "code" in current &&
-            typeof current.code === "string") {
-            parts.push(current.code);
-        }
-    }
-    return parts.join(" ");
-}
-function errorChain(error) {
-    const chain = [];
-    const seen = new Set();
-    let current = error;
-    while (current !== undefined && current !== null && !seen.has(current)) {
-        seen.add(current);
-        chain.push(current);
-        if (typeof current !== "object" ||
-            !("cause" in current) ||
-            current.cause === undefined) {
-            break;
-        }
-        current = current.cause;
-    }
-    return chain;
-}
-function isTransientNetworkFailure(text) {
-    return /\b(?:EAI_AGAIN|ECONNREFUSED|ECONNRESET|EHOSTUNREACH|ENETDOWN|ENETUNREACH|ENOTFOUND|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_SOCKET|TimeoutError)\b|connection (?:reset|timed out)|network connection (?:failed|was lost)|socket hang up|temporary failure/i.test(text);
-}
-function maxRetryAttempts(retry) {
-    return retry.rateLimited
-        ? EMBEDDING_RATE_LIMIT_MAX_RETRIES
-        : EMBEDDING_TRANSIENT_MAX_RETRIES;
-}
-function retryDelayMs(attempt, retry) {
-    if (typeof retry.retryAfterMs === "number") {
-        return retry.retryAfterMs;
-    }
-    const base = retry.rateLimited
-        ? EMBEDDING_RATE_LIMIT_RETRY_BASE_DELAY_MS
-        : EMBEDDING_TRANSIENT_RETRY_BASE_DELAY_MS;
-    const max = retry.rateLimited
-        ? EMBEDDING_RATE_LIMIT_RETRY_MAX_DELAY_MS
-        : EMBEDDING_TRANSIENT_RETRY_MAX_DELAY_MS;
-    const exponential = base * 2 ** attempt;
-    const jitter = Math.floor(Math.random() * EMBEDDING_RETRY_JITTER_MS);
-    return Math.min(exponential + jitter, max);
-}
-function httpStatusFromText(text) {
-    const match = /\bstatus=(\d{3})\b/i.exec(text);
-    if (!match) {
-        return undefined;
-    }
-    return Number(match[1]);
-}
-function retryAfterMsFromText(text) {
-    const milliseconds = /\bretryAfterMs=(\d+)\b/i.exec(text);
-    if (milliseconds) {
-        return Number(milliseconds[1]);
-    }
-    const seconds = /\bretryAfter=(\d+(?:\.\d+)?)\b/i.exec(text);
-    if (seconds) {
-        return Math.round(Number(seconds[1]) * 1000);
-    }
-    return undefined;
-}
-function abortableDelay(ms, signal) {
-    if (ms <= 0) {
-        return Promise.resolve();
-    }
-    throwIfAborted(signal);
-    return new Promise((resolve, reject) => {
-        const timer = setTimeout(() => {
-            signal?.removeEventListener("abort", abort);
-            resolve();
-        }, ms);
-        const abort = () => {
-            clearTimeout(timer);
-            reject(abortError(signal));
-        };
-        signal?.addEventListener("abort", abort, { once: true });
-    });
-}
-function throwIfAborted(signal) {
-    if (signal?.aborted) {
-        throw abortError(signal);
-    }
-}
-function abortError(signal) {
-    return signal?.reason instanceof Error
-        ? signal.reason
-        : new Error("Embedding was cancelled.");
-}
-function countPublicEntities(fragments) {
-    let count = 0;
-    for (const fragment of fragments) {
-        if (!fragment.group || fragment.group === fragment.id) {
-            count++;
-        }
-    }
-    return count;
-}
-async function withContentHash(file) {
-    let bytes;
-    try {
-        bytes = await readFile(file.absolutePath);
-    }
-    catch (error) {
-        throw toEngineError(error, "Indexing failed to compute file content hash", {
-            code: "ZVEC_GREP.ENGINE.INDEXING.CONTENT_HASH_FAILED",
-            context: fileContext(file),
-        });
-    }
-    return {
-        ...file,
-        contentHash: sha256Bytes(bytes),
-    };
-}
-function markFileFailed(ctx, file, error, stage) {
-    const reason = fileFailureReason(stage, error);
-    try {
-        ctx.storage.markFileFailed(file, reason);
-        return reason;
-    }
-    catch (markError) {
-        throw toEngineError(markError, "Indexing failed to record file failure", {
-            code: "ZVEC_GREP.ENGINE.INDEXING.MARK_FILE_FAILED",
-            context: `${fileContext(file)} stage=${stage} original=${reason}`,
-        });
-    }
-}
-function fileFailureReason(stage, error) {
-    return oneLine(`${stage}: ${errorToMessage(error)}`);
-}
-function toEngineError(error, message, options) {
-    if (isEngineError(error)) {
-        return error;
-    }
-    return new EngineError(message, {
-        ...options,
-        cause: error,
-    });
-}
-function workspaceIndexContext(workspaceIndex) {
-    return (errorDetails([
-        detail("workspace_index_id", workspaceIndex.id),
-        workspaceIndexDetail(workspaceIndex.name),
-    ]) ?? "");
-}
-function fileContext(file) {
-    return `fileId=${file.id} path=${file.relativePath}`;
-}
-function errorToMessage(error) {
-    if (isEngineError(error) && error.context) {
-        return `${error.code}: ${error.message} (${error.context})`;
-    }
-    if (isEngineError(error)) {
-        return `${error.code}: ${error.message}`;
-    }
-    return error instanceof Error ? error.message : String(error);
-}
-//# sourceMappingURL=index.js.map