@zvec/zvec-grep 0.1.5 → 0.1.7

package/dist/engine/pipeline/indexing/index.js

@@ -1,12 +1,11 @@
-import { readFile } from "node:fs/promises";
-import { collectionDetail, detail, EngineError, errorDetails, isEngineError, } from "../../errors/index.js";
+import { lstat, readFile } from "node:fs/promises";
+import { workspaceIndexDetail, detail, EngineError, errorDetails, isEngineError, } from "../../errors.js";
 import { sha256Bytes } from "../../utils/hash.js";
+import { normalizePath } from "../../utils/path.js";
 import { ConcurrentTiming, TimingCollector } from "../../utils/timing.js";
-import { ExtractorRegistry } from "../../extraction/index.js";
-import { scanRootPaths } from "./scanner/index.js";
-const DEFAULT_EMBEDDING_CONCURRENCY = 4;
-const DEFAULT_EMBEDDING_MIN_CONCURRENCY = 4;
-const DEFAULT_EMBEDDING_MAX_CONCURRENCY = 12;
+import { extract, vectorContentForFragment, } from "../../extraction/index.js";
+import { scanDirectoryPath, scanFilePath, scanRootPaths, } from "./scanner/index.js";
+import { indexChunkOptions } from "./input-budget.js";
 const EMBEDDING_TRANSIENT_MAX_RETRIES = 3;
 const EMBEDDING_RATE_LIMIT_MAX_RETRIES = 6;
 const EMBEDDING_TRANSIENT_RETRY_BASE_DELAY_MS = 500;
@@ -15,33 +14,44 @@ const EMBEDDING_TRANSIENT_RETRY_MAX_DELAY_MS = 8000;
 const EMBEDDING_RATE_LIMIT_RETRY_MAX_DELAY_MS = 30000;
 const EMBEDDING_RETRY_JITTER_MS = 500;
 const EMBEDDING_SUCCESS_STREAK_MIN = 4;
-export async function indexCollection(ctx) {
+export async function indexWorkspace(ctx) {
     try {
-        return await indexCollectionUnchecked(ctx);
+        return await indexWorkspaceUnchecked(ctx);
     }
     catch (error) {
-        throw toEngineError(error, "Indexing collection failed", {
-            code: "ZVEC_GREP.ENGINE.INDEXING.COLLECTION_FAILED",
-            context: collectionContext(ctx.collection),
+        throw toEngineError(error, "Indexing workspace failed", {
+            code: "ZVEC_GREP.ENGINE.INDEXING.WORKSPACE_FAILED",
+            context: workspaceIndexContext(ctx.workspaceIndex),
         });
     }
 }
-export async function getCollectionIndexStatus(collection, storedFiles) {
+export async function indexWorkspacePaths(ctx, changedPaths) {
     try {
-        const scan = await scanRootPaths(collection.id, collection.rootPaths);
+        return await indexWorkspacePathsUnchecked(ctx, changedPaths);
+    }
+    catch (error) {
+        throw toEngineError(error, "Indexing changed paths failed", {
+            code: "ZVEC_GREP.ENGINE.INDEXING.WORKSPACE_FAILED",
+            context: workspaceIndexContext(ctx.workspaceIndex),
+        });
+    }
+}
+export async function getWorkspaceIndexStatus(workspaceIndex, storedFiles) {
+    try {
+        const scan = await scanRootPaths(workspaceIndex.id, workspaceIndex.rootPaths);
         const diff = await computeDiffFromFiles(scan.files, storedFiles);
         const pendingFiles = storedFiles.filter((file) => file.indexStatus?.indexedTime === null);
         const failedFiles = pendingFiles.filter((file) => file.indexStatus?.error !== undefined);
         const indexedFiles = storedFiles.filter((file) => file.indexStatus?.indexedTime !== undefined &&
             file.indexStatus.indexedTime !== null);
         const entitiesIndexed = indexedFiles.reduce((count, file) => count + (file.indexStatus?.entityCount ?? 0), 0);
+        const fragmentsTruncated = indexedFiles.reduce((count, file) => count + (file.indexStatus?.truncatedFragmentCount ?? 0), 0);
         return {
-            collectionId: collection.id,
-            collectionName: collection.name,
             filesScanned: scan.files.length,
             filesStored: storedFiles.length,
             filesIndexed: indexedFiles.length,
             entitiesIndexed,
+            fragmentsTruncated,
             filesPending: pendingFiles.length,
             filesFailed: failedFiles.length,
             filesAdded: diff.added.length,
@@ -56,36 +66,42 @@ export async function getCollectionIndexStatus(collection, storedFiles) {
         };
     }
     catch (error) {
-        throw toEngineError(error, "Inspecting collection index status failed", {
+        throw toEngineError(error, "Inspecting workspace index status failed", {
             code: "ZVEC_GREP.ENGINE.INDEXING.STATUS_FAILED",
-            context: collectionContext(collection),
+            context: workspaceIndexContext(workspaceIndex),
         });
     }
 }
-async function indexCollectionUnchecked(ctx) {
+async function indexWorkspaceUnchecked(ctx) {
     const start = Date.now();
     const report = ctx.onProgress ?? (() => undefined);
     const timings = new TimingCollector();
+    throwIfIndexCancelled(ctx);
     const firstPass = await runIndexPass(ctx, report, "Scanning files...", timings);
     const passes = [firstPass];
+    let progressBase;
     if (firstPass.stats.filesFailed > 0) {
         const failed = firstPass.stats.filesFailed;
+        progressBase = retryProgressBase(firstPass);
         report({
             phase: "scanning",
+            filesTotal: progressBase.filesTotal,
+            filesIndexed: progressBase.filesSucceeded,
             detail: `Retrying ${failed} failed ${failed === 1 ? "file" : "files"}...`,
         });
-        passes.push(await runIndexPass(ctx, report, "Scanning retry candidates...", timings));
+        passes.push(await runIndexPass(ctx, report, "Scanning retry candidates...", timings, progressBase));
     }
     const finalPass = passes[passes.length - 1];
-    reportIndexFinalizing(ctx, report, finalPass);
-    timings.timeSync("index_optimize", () => optimizeStorage(ctx));
+    throwIfIndexCancelled(ctx);
+    reportIndexFinalizing(ctx, report, finalPass, progressBase);
+    await timings.time("index_optimize", () => optimizeStorage(ctx));
     const result = buildIndexResult(ctx, passes, Date.now() - start, timings);
     if (result.filesFailed > 0) {
         report({ phase: "done", detail: "Indexing completed with failed files" });
         throw new EngineError(`Indexing completed with ${result.filesFailed} failed ${result.filesFailed === 1 ? "file" : "files"}`, {
             code: "ZVEC_GREP.ENGINE.INDEXING.FILES_FAILED",
             context: errorDetails([
-                collectionDetail(ctx.collection.name),
+                workspaceIndexDetail(ctx.workspaceIndex.name),
                 detail("filesFailed", result.filesFailed),
                 detail("filesScanned", result.filesScanned),
                 detail("failedFiles", summarizeFailedFiles(finalPass.stats.failedFiles)),
@@ -99,18 +115,83 @@ async function indexCollectionUnchecked(ctx) {
     report({ phase: "done", detail: "Indexing complete" });
     return result;
 }
-async function runIndexPass(ctx, report, scanningDetail, timings) {
-    report({ phase: "scanning", detail: scanningDetail });
-    const scan = await timings.time("index_scan", () => scanRootPaths(ctx.collection.id, ctx.collection.rootPaths));
-    const diff = await timings.time("index_diff", () => computeDiff(scan.files, ctx.storage));
+async function indexWorkspacePathsUnchecked(ctx, changedPaths) {
+    const start = Date.now();
+    const report = ctx.onProgress ?? (() => undefined);
+    const timings = new TimingCollector();
+    const normalizedPaths = [...new Set(changedPaths.map(normalizePath))];
+    throwIfIndexCancelled(ctx);
+    const firstPass = await runPathIndexPass(ctx, report, normalizedPaths, timings);
+    const passes = [firstPass];
+    let progressBase;
+    if (firstPass.stats.filesFailed > 0) {
+        progressBase = retryProgressBase(firstPass);
+        passes.push(await runPathIndexPass(ctx, report, normalizedPaths, timings, progressBase));
+    }
+    const finalPass = passes[passes.length - 1];
+    throwIfIndexCancelled(ctx);
+    reportIndexFinalizing(ctx, report, finalPass, progressBase);
+    await timings.time("index_optimize", () => optimizeStorage(ctx));
+    const result = buildIndexResult(ctx, passes, Date.now() - start, timings);
+    if (result.filesFailed > 0) {
+        throw new EngineError(`Indexing completed with ${result.filesFailed} failed files`, {
+            code: "ZVEC_GREP.ENGINE.INDEXING.FILES_FAILED",
+            context: workspaceIndexContext(ctx.workspaceIndex),
+        });
+    }
+    report({ phase: "done", detail: "Indexing complete" });
+    return result;
+}
+async function runPathIndexPass(ctx, report, changedPaths, timings, progressBase) {
+    reportScanning(report, "Scanning changed paths...", progressBase);
+    const scanned = await timings.time("index_scan_paths", async () => {
+        const files = [];
+        const diagnostics = emptyScanDiagnostics();
+        for (const path of changedPaths) {
+            throwIfIndexCancelled(ctx);
+            const info = await lstat(path).catch(() => null);
+            const scan = info?.isDirectory()
+                ? await scanDirectoryPath(ctx.workspaceIndex.id, ctx.workspaceIndex.rootPaths, path, { signal: ctx.signal })
+                : await scanFilePath(ctx.workspaceIndex.id, ctx.workspaceIndex.rootPaths, path, {
+                    signal: ctx.signal,
+                });
+            files.push(...scan.files);
+            mergeScanDiagnostics(diagnostics, scan.diagnostics);
+        }
+        return {
+            files: [...new Map(files.map((file) => [file.id, file])).values()],
+            diagnostics,
+        };
+    });
+    throwIfIndexCancelled(ctx);
+    const existing = [
+        ...new Map(changedPaths
+            .flatMap((path) => ctx.storage.listFilesByPathPrefix(path))
+            .map((file) => [file.id, file])).values(),
+    ];
+    return runDiffPass(ctx, report, scanned.files, existing, timings, progressBase, scanned.diagnostics);
+}
+async function runIndexPass(ctx, report, scanningDetail, timings, progressBase) {
+    reportScanning(report, scanningDetail, progressBase);
+    const scan = await timings.time("index_scan", () => scanRootPaths(ctx.workspaceIndex.id, ctx.workspaceIndex.rootPaths, {
+        signal: ctx.signal,
+    }));
+    throwIfIndexCancelled(ctx);
+    return runDiffPass(ctx, report, scan.files, ctx.storage.listFiles(), timings, progressBase, scan.diagnostics);
+}
+async function runDiffPass(ctx, report, scannedFiles, existingFiles, timings, progressBase, scanDiagnostics = emptyScanDiagnostics()) {
+    const diff = await timings.time("index_diff", () => computeDiffFromFiles(scannedFiles, existingFiles));
+    throwIfIndexCancelled(ctx);
     const pending = [...diff.added, ...diff.modified, ...diff.pending];
     report({
         phase: "scanning",
-        filesTotal: scan.files.length,
+        filesTotal: progressBase?.filesTotal ?? scannedFiles.length,
+        filesIndexed: progressBase?.filesSucceeded ?? 0,
         detail: `${diff.added.length} added, ${diff.modified.length} modified, ${diff.pending.length} pending, ${diff.deleted.length} deleted, ${diff.unchanged.length} unchanged`,
     });
     timings.timeSync("index_delete_stale", () => {
         for (const file of diff.deleted) {
+            throwIfIndexCancelled(ctx);
             try {
                 ctx.storage.deleteFile(file.id);
             }
@@ -125,8 +206,10 @@ async function runIndexPass(ctx, report, scanningDetail, timings) {
     const reportIndexing = (currentStats, detail, embedding) => {
         report({
             phase: "indexing",
-            filesTotal: pending.length,
-            filesIndexed: currentStats.filesIndexed + currentStats.filesFailed,
+            filesTotal: progressBase?.filesTotal ?? pending.length,
+            filesIndexed: (progressBase?.filesSucceeded ?? 0) +
+                currentStats.filesIndexed +
+                currentStats.filesFailed,
             filesFailed: currentStats.filesFailed,
             detail,
             embedding,
@@ -140,21 +223,46 @@ async function runIndexPass(ctx, report, scanningDetail, timings) {
         entitiesCreated: 0,
     });
     const stats = await indexFiles(pending, ctx, reportIndexing, timings);
+    throwIfIndexCancelled(ctx);
     return {
-        filesScanned: scan.files.length,
+        filesScanned: scannedFiles.length,
+        scanDiagnostics,
         diff,
         stats,
     };
 }
-function reportIndexFinalizing(ctx, report, pass) {
+function reportIndexFinalizing(ctx, report, pass, progressBase) {
     report({
         phase: "indexing",
+        filesTotal: progressBase?.filesTotal ??
+            pass.diff.added.length +
+                pass.diff.modified.length +
+                pass.diff.pending.length,
+        filesIndexed: (progressBase?.filesSucceeded ?? 0) +
+            pass.stats.filesIndexed +
+            pass.stats.filesFailed,
+        filesFailed: pass.stats.filesFailed,
+        detail: "finalizing index",
+    });
+}
+function retryProgressBase(pass) {
+    return {
+        filesSucceeded: pass.stats.filesIndexed,
         filesTotal: pass.diff.added.length +
             pass.diff.modified.length +
             pass.diff.pending.length,
-        filesIndexed: pass.stats.filesIndexed + pass.stats.filesFailed,
-        filesFailed: pass.stats.filesFailed,
-        detail: "finalizing index",
+    };
+}
+function reportScanning(report, detail, progressBase) {
+    report({
+        phase: "scanning",
+        ...(progressBase
+            ? {
+                filesTotal: progressBase.filesTotal,
+                filesIndexed: progressBase.filesSucceeded,
+            }
+            : {}),
+        detail,
     });
 }
 function buildIndexResult(ctx, passes, durationMs, timings) {
@@ -162,8 +270,6 @@ function buildIndexResult(ctx, passes, durationMs, timings) {
     const retryPasses = passes.slice(1);
     const finalPass = passes[passes.length - 1];
     return {
-        collectionId: ctx.collection.id,
-        collectionName: ctx.collection.name,
         filesScanned: finalPass.filesScanned,
         filesAdded: firstPass.diff.added.length +
             retryPasses.reduce((count, pass) => count + pass.diff.added.length, 0),
@@ -178,22 +284,50 @@ function buildIndexResult(ctx, passes, durationMs, timings) {
         entitiesCreated: passes.reduce((count, pass) => count + pass.stats.entitiesCreated, 0),
         durationMs,
         timings: timings.entries(),
+        ...(finalPass.scanDiagnostics.skippedFiles > 0
+            ? { scanDiagnostics: finalPass.scanDiagnostics }
+            : {}),
+    };
+}
+const MAX_SKIPPED_FILE_SAMPLES = 20;
+function emptyScanDiagnostics() {
+    return {
+        skippedFiles: 0,
+        skippedByReason: {
+            empty: 0,
+            too_large: 0,
+            unsupported: 0,
+            binary: 0,
+        },
+        skippedSamples: [],
     };
 }
-function optimizeStorage(ctx) {
+function mergeScanDiagnostics(target, source) {
+    target.skippedFiles += source.skippedFiles;
+    for (const reason of [
+        "empty",
+        "too_large",
+        "unsupported",
+        "binary",
+    ]) {
+        target.skippedByReason[reason] += source.skippedByReason[reason];
+    }
+    const remaining = MAX_SKIPPED_FILE_SAMPLES - target.skippedSamples.length;
+    if (remaining > 0) {
+        target.skippedSamples.push(...source.skippedSamples.slice(0, remaining));
+    }
+}
+async function optimizeStorage(ctx) {
     try {
-        ctx.storage.optimize();
+        await ctx.storage.finalizeWrites();
     }
     catch (error) {
         throw toEngineError(error, "Indexing failed to finalize storage", {
             code: "ZVEC_GREP.ENGINE.INDEXING.OPTIMIZE_FAILED",
-            context: collectionContext(ctx.collection),
+            context: workspaceIndexContext(ctx.workspaceIndex),
         });
     }
 }
-async function computeDiff(scannedFiles, storage) {
-    return computeDiffFromFiles(scannedFiles, storage.listFiles());
-}
 async function computeDiffFromFiles(scannedFiles, existingFiles) {
     const existingById = new Map(existingFiles.map((file) => [file.id, file]));
     const seen = new Set();
@@ -246,6 +380,7 @@ async function indexFiles(files, ctx, onProgress, timings) {
         onProgress(currentStats, detail, embeddingScheduler.snapshot());
     };
     const flushBatch = async () => {
+        throwIfIndexCancelled(ctx);
         if (batchFiles.length === 0) {
             return;
         }
@@ -254,6 +389,7 @@ async function indexFiles(files, ctx, onProgress, timings) {
         await scheduleEmbeddingTask(() => embedAndCommitBatch(filesToEmbed, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming));
     };
     const scheduleEmbeddingTask = async (task) => {
+        throwIfIndexCancelled(ctx);
         const promise = task().finally(() => {
             runningEmbeddings.delete(promise);
         });
@@ -262,50 +398,66 @@ async function indexFiles(files, ctx, onProgress, timings) {
             await Promise.race(runningEmbeddings);
         }
     };
-    for (const file of files) {
-        onProgress(stats, `reading ${file.relativePath}`);
-        const prepared = await timings.time("index_prepare", () => prepareFile(file, ctx));
-        if ("failedReason" in prepared) {
-            recordFileFailed(stats, file, prepared.failedReason);
-            onProgress(stats, `failed ${file.relativePath}`);
-            continue;
-        }
-        if (prepared.fragments.length === 0) {
-            const committed = commitFile(prepared, [], ctx, stats);
-            onProgress(stats, finishedFileDetail(committed, file.relativePath));
-            continue;
-        }
-        if (prepared.fragments.length > ctx.embeddingModel.limits.maxBatchSize) {
-            await flushBatch();
-            await scheduleEmbeddingTask(async () => {
-                reportEmbeddingProgress(stats, `embedding ${file.relativePath}`);
-                await embedAndCommitFile(prepared, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming);
-            });
-            continue;
-        }
-        if (batchFragmentCount > 0 &&
-            batchFragmentCount + prepared.fragments.length >
-                ctx.embeddingModel.limits.maxBatchSize) {
-            await flushBatch();
-        }
-        batchFiles.push(prepared);
-        batchFragmentCount += prepared.fragments.length;
-        if (batchFragmentCount === ctx.embeddingModel.limits.maxBatchSize) {
-            await flushBatch();
+    try {
+        for (const file of files) {
+            throwIfIndexCancelled(ctx);
+            onProgress(stats, `reading ${file.relativePath}`);
+            const prepared = await timings.time("index_prepare", () => prepareFile(file, ctx));
+            throwIfIndexCancelled(ctx);
+            if ("failedReason" in prepared) {
+                recordFileFailed(stats, file, prepared.failedReason);
+                onProgress(stats, `failed ${file.relativePath}`);
+                continue;
+            }
+            if (prepared.fragments.length === 0) {
+                const committed = commitFile(prepared, [], ctx, stats);
+                onProgress(stats, finishedFileDetail(committed, file.relativePath));
+                continue;
+            }
+            if (prepared.fragments.length > ctx.embeddingModel.info.limits.maxBatchSize) {
+                await flushBatch();
+                await scheduleEmbeddingTask(async () => {
+                    throwIfIndexCancelled(ctx);
+                    reportEmbeddingProgress(stats, `embedding ${file.relativePath}`);
+                    await embedAndCommitFile(prepared, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming);
+                });
+                continue;
+            }
+            if (batchFragmentCount > 0 &&
+                batchFragmentCount + prepared.fragments.length >
+                    ctx.embeddingModel.info.limits.maxBatchSize) {
+                await flushBatch();
+            }
+            batchFiles.push(prepared);
+            batchFragmentCount += prepared.fragments.length;
+            if (batchFragmentCount === ctx.embeddingModel.info.limits.maxBatchSize) {
+                await flushBatch();
+            }
         }
+        await flushBatch();
+        await Promise.all(runningEmbeddings);
     }
-    await flushBatch();
-    await Promise.all(runningEmbeddings);
+    catch (error) {
+        await Promise.allSettled(runningEmbeddings);
+        throw error;
+    }
+    throwIfIndexCancelled(ctx);
     return stats;
 }
 async function prepareFile(file, ctx) {
     try {
+        throwIfIndexCancelled(ctx);
         const source = await readSource(file);
-        const extracted = await new ExtractorRegistry().extract(source);
-        const fragments = extracted.filter((fragment) => ctx.embeddingModel.supportedContentKinds.includes(fragment.content.kind));
-        return { file, fragments };
+        const chunkOptions = indexChunkOptions(ctx.embeddingModel.info.limits.maxInputTokens, source.kind === "text" ? source.text : undefined);
+        const extracted = await extract(source, chunkOptions);
+        throwIfIndexCancelled(ctx);
+        const fragments = extracted.filter((fragment) => ctx.embeddingModel.info.inputKinds.includes(fragment.content.kind));
+        return { file, fragments, maxChars: chunkOptions.maxChunkChars };
     }
     catch (error) {
+        if (indexIsCancelled(ctx)) {
+            throw indexCancellationError(ctx);
+        }
         return {
             file,
             failedReason: markFileFailed(ctx, file, error, "prepare"),
@@ -313,20 +465,33 @@ async function prepareFile(file, ctx) {
     }
 }
 async function embedAndCommitBatch(files, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming) {
-    const fragments = files.flatMap((file) => file.fragments);
     try {
-        const contents = fragments.map(vectorContentForFragment);
+        throwIfIndexCancelled(ctx);
+        const contents = files.flatMap((file) => file.fragments.map((fragment) => vectorContentForFragment(fragment, file.maxChars)));
         onProgress(stats, `embedding ${describePreparedFiles(files)}`, embeddingScheduler.snapshot());
-        const vectors = await embeddingTiming.time(() => embedContentsWithRetry(contents, ctx.embeddingModel, embeddingScheduler));
+        const embedding = await embeddingTiming.time(() => embedContentsWithRetry(contents, ctx.embeddingModel, embeddingScheduler, ctx.signal, (progress) => reportModelDownloadProgress(stats, progress, onProgress, embeddingScheduler)));
+        const truncatedInputIndexes = new Set(embedding.truncated);
+        throwIfIndexCancelled(ctx);
         let offset = 0;
         for (const file of files) {
-            const fileVectors = vectors.slice(offset, offset + file.fragments.length);
+            throwIfIndexCancelled(ctx);
+            const end = offset + file.fragments.length;
+            const fileVectors = embedding.vectors.slice(offset, end);
+            let truncatedFragmentCount = 0;
+            for (let index = offset; index < end; index++) {
+                if (truncatedInputIndexes.has(index)) {
+                    truncatedFragmentCount++;
+                }
+            }
             offset += file.fragments.length;
-            const committed = timings.timeSync("index_commit", () => commitFile(file, fileVectors, ctx, stats));
+            const committed = timings.timeSync("index_commit", () => commitFile(file, fileVectors, ctx, stats, truncatedFragmentCount));
             onProgress(stats, finishedFileDetail(committed, file.file.relativePath));
         }
     }
     catch (error) {
+        if (indexIsCancelled(ctx)) {
+            throw indexCancellationError(ctx);
+        }
         if (isRetryableEmbeddingError(error)) {
             for (const file of files) {
                 const reason = markFileFailed(ctx, file.file, error, "embed");
@@ -343,29 +508,66 @@ async function embedAndCommitBatch(files, ctx, stats, onProgress, embeddingSched
 }
 async function embedAndCommitFile(file, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming) {
     try {
-        const vectors = await embeddingTiming.time(() => embedFragments(file.fragments, ctx.embeddingModel, embeddingScheduler));
-        const committed = timings.timeSync("index_commit", () => commitFile(file, vectors, ctx, stats));
+        throwIfIndexCancelled(ctx);
+        const embedding = await embeddingTiming.time(() => embedFragments(file.fragments, ctx.embeddingModel, embeddingScheduler, file.maxChars, ctx.signal, (progress) => reportModelDownloadProgress(stats, progress, onProgress, embeddingScheduler)));
+        throwIfIndexCancelled(ctx);
+        const committed = timings.timeSync("index_commit", () => commitFile(file, embedding.vectors, ctx, stats, embedding.truncated.length));
         onProgress(stats, finishedFileDetail(committed, file.file.relativePath));
     }
     catch (error) {
+        if (indexIsCancelled(ctx)) {
+            throw indexCancellationError(ctx);
+        }
         const reason = markFileFailed(ctx, file.file, error, "embed");
         recordFileFailed(stats, file.file, reason);
         onProgress(stats, finishedFileDetail(false, file.file.relativePath));
     }
 }
-function commitFile(file, vectors, ctx, stats) {
+function commitFile(file, vectors, ctx, stats, truncatedFragmentCount = 0) {
     try {
-        ctx.storage.upsertFile(file.file, file.fragments, vectors);
+        throwIfIndexCancelled(ctx);
+        if (file.fragments.length !== vectors.length) {
+            throw new EngineError("Embedding returned mismatched entity/vector counts", {
+                code: "ZVEC_GREP.ENGINE.STORAGE.ENTITY_VECTOR_COUNT_MISMATCH",
+                context: `fileId=${file.file.id} fragmentCount=${file.fragments.length} vectorCount=${vectors.length}`,
+            });
+        }
+        ctx.storage.replaceFile(file.file, file.fragments.map((fragment, index) => ({
+            fragment,
+            vector: vectors[index],
+        })), {
+            truncatedFragmentCount,
+        });
         stats.filesIndexed++;
         stats.entitiesCreated += countPublicEntities(file.fragments);
         return true;
     }
     catch (error) {
+        if (indexIsCancelled(ctx)) {
+            throw indexCancellationError(ctx);
+        }
         const reason = markFileFailed(ctx, file.file, error, "commit");
         recordFileFailed(stats, file.file, reason);
         return false;
     }
 }
+function throwIfIndexCancelled(ctx) {
+    if (!indexIsCancelled(ctx)) {
+        return;
+    }
+    throw indexCancellationError(ctx);
+}
+function indexIsCancelled(ctx) {
+    return ctx.signal?.aborted === true;
+}
+function indexCancellationError(ctx) {
+    return ctx.signal?.reason instanceof Error
+        ? ctx.signal.reason
+        : new EngineError("Indexing was cancelled.", {
+            code: "ZVEC_GREP.ENGINE.INDEXING.CANCELLED",
+            context: workspaceIndexContext(ctx.workspaceIndex),
+        });
+}
 function recordFileFailed(stats, file, reason) {
     stats.filesFailed++;
     stats.failedFiles.push(file.relativePath);
@@ -417,14 +619,15 @@ async function readSource(file) {
         text: bytes.toString("utf8"),
     };
 }
-async function embedFragments(fragments, model, embeddingScheduler) {
+async function embedFragments(fragments, model, embeddingScheduler, maxChars, signal, onModelProgress) {
     const batches = [];
-    for (let start = 0; start < fragments.length; start += model.limits.maxBatchSize) {
-        const batch = fragments.slice(start, start + model.limits.maxBatchSize);
+    for (let start = 0; start < fragments.length; start += model.info.limits.maxBatchSize) {
+        const batch = fragments.slice(start, start + model.info.limits.maxBatchSize);
         batches.push({ start, fragments: batch });
     }
     const vectors = new Array(fragments.length);
-    const results = await Promise.allSettled(batches.map((batch) => embedFragmentBatch(batch.fragments, model, batch.start, embeddingScheduler)));
+    const truncatedInputIndexes = [];
+    const results = await Promise.allSettled(batches.map((batch) => embedFragmentBatch(batch.fragments, model, batch.start, embeddingScheduler, maxChars, signal, onModelProgress)));
     let firstError;
     for (const [index, result] of results.entries()) {
         if (result.status === "rejected") {
@@ -432,94 +635,74 @@ async function embedFragments(fragments, model, embeddingScheduler) {
             continue;
         }
         const start = batches[index].start;
-        for (const [offset, vector] of result.value.entries()) {
+        for (const [offset, vector] of result.value.vectors.entries()) {
             vectors[start + offset] = vector;
         }
+        truncatedInputIndexes.push(...result.value.truncated.map((inputIndex) => start + inputIndex));
     }
     if (firstError) {
         throw firstError;
     }
-    return vectors;
+    return {
+        vectors,
+        truncated: truncatedInputIndexes,
+    };
 }
-async function embedFragmentBatch(fragments, model, startIndex, embeddingScheduler) {
-    const contents = fragments.map(vectorContentForFragment);
+async function embedFragmentBatch(fragments, model, startIndex, embeddingScheduler, maxChars, signal, onModelProgress) {
+    const contents = fragments.map((fragment) => vectorContentForFragment(fragment, maxChars));
     try {
-        return await embedContentsWithRetry(contents, model, embeddingScheduler);
+        return await embedContentsWithRetry(contents, model, embeddingScheduler, signal, onModelProgress);
     }
     catch (error) {
         if (fragments.length === 1 || isRetryableEmbeddingError(error)) {
             throw error;
         }
-        return await embedFragmentBatchOneByOne(fragments, model, startIndex, embeddingScheduler);
+        return await embedFragmentBatchOneByOne(fragments, model, startIndex, embeddingScheduler, maxChars, signal, onModelProgress);
     }
 }
-async function embedFragmentBatchOneByOne(fragments, model, startIndex, embeddingScheduler) {
+async function embedFragmentBatchOneByOne(fragments, model, startIndex, embeddingScheduler, maxChars, signal, onModelProgress) {
     const vectors = [];
+    const truncatedInputIndexes = [];
     for (const [index, fragment] of fragments.entries()) {
         try {
-            const [vector] = await embedContentsWithRetry([vectorContentForFragment(fragment)], model, embeddingScheduler);
-            vectors.push(vector);
+            const result = await embedContentsWithRetry([vectorContentForFragment(fragment, maxChars)], model, embeddingScheduler, signal, onModelProgress);
+            vectors.push(result.vectors[0]);
+            if (result.truncated.length > 0) {
+                truncatedInputIndexes.push(index);
+            }
         }
         catch (error) {
             throw new EngineError("Embedding entity fragment failed after one-by-one fallback", {
                 code: "ZVEC_GREP.ENGINE.INDEXING.EMBEDDING_FRAGMENT_FAILED",
-                context: `model=${model.ref.model} fragmentId=${fragment.id} fragmentIndex=${startIndex + index}`,
+                context: errorDetails([
+                    detail("model", model.info.reference),
+                    detail("fragmentId", fragment.id),
+                    detail("fragmentIndex", startIndex + index),
+                    isEngineError(error) ? detail("causeCode", error.code) : null,
+                    error instanceof Error ? detail("cause", error.message) : null,
+                    isEngineError(error) ? error.context : null,
+                ]),
                 cause: error,
             });
         }
     }
-    return vectors;
-}
-function vectorContentForFragment(fragment) {
-    if (fragment.content.kind !== "text") {
-        return fragment.content;
-    }
-    const metadata = vectorMetadataText(fragment.metadata);
-    if (metadata.length === 0) {
-        return fragment.content;
-    }
     return {
-        kind: "text",
-        text: `${metadata}\n${fragment.content.text}`,
+        vectors,
+        truncated: truncatedInputIndexes,
     };
 }
-function vectorMetadataText(metadata) {
-    if (!metadata) {
-        return "";
-    }
-    if (metadata.kind === "code") {
-        return compactMetadataLines([
-            metadata.symbolName
-                ? `symbol: ${metadata.symbolType} ${metadata.symbolName}`
-                : `symbol: ${metadata.symbolType}`,
-            metadata.scope ? `scope: ${metadata.scope}` : null,
-            metadata.signature ? `signature: ${oneLine(metadata.signature)}` : null,
-            metadata.modifiers.length > 0
-                ? `modifiers: ${metadata.modifiers.join(" ")}`
-                : null,
-            metadata.doc ? `doc: ${oneLine(metadata.doc)}` : null,
-        ]);
-    }
-    return compactMetadataLines([
-        metadata.heading ? `heading: ${metadata.heading}` : null,
-        typeof metadata.level === "number"
-            ? `heading_level: ${metadata.level}`
-            : null,
-        metadata.scope ? `scope: ${metadata.scope}` : null,
-    ]);
-}
-function compactMetadataLines(lines) {
-    return lines
-        .filter((line) => line !== null && line.trim().length > 0)
-        .join("\n");
-}
-async function embedContentsWithRetry(contents, model, embeddingScheduler) {
+async function embedContentsWithRetry(contents, model, embeddingScheduler, signal, onModelProgress) {
     let attempt = 0;
     while (true) {
         let retry = nonRetryableEmbeddingError();
         let delayMs = 0;
         try {
-            const vectors = await embeddingScheduler.run(() => model.embed(contents, { purpose: "document" }), (error) => {
+            throwIfAborted(signal);
+            const result = await embeddingScheduler.run(() => model.embed(contents, {
+                purpose: "document",
+                signal,
+                onProgress: onModelProgress,
+            }), signal, (error) => {
                 retry = classifyEmbeddingRetry(error);
                 delayMs = retryDelayMs(attempt, retry);
                 if (retry.retryable) {
@@ -530,9 +713,14 @@ async function embedContentsWithRetry(contents, model, embeddingScheduler) {
                 }
             });
             embeddingScheduler.recordSuccess();
-            return vectors;
+            return result;
         }
         catch (error) {
+            if (signal?.aborted) {
+                throw signal.reason instanceof Error
+                    ? signal.reason
+                    : new Error("Embedding was cancelled.");
+            }
             if (!retry.retryable) {
                 retry = classifyEmbeddingRetry(error);
                 delayMs = retryDelayMs(attempt, retry);
@@ -540,11 +728,17 @@ async function embedContentsWithRetry(contents, model, embeddingScheduler) {
             if (attempt >= maxRetryAttempts(retry) || !retry.retryable) {
                 throw error;
             }
-            await delay(delayMs);
+            await abortableDelay(delayMs, signal);
             attempt++;
         }
     }
 }
+function reportModelDownloadProgress(stats, progress, onProgress, embeddingScheduler) {
+    onProgress(stats, `downloading ${progress.model}`, {
+        ...embeddingScheduler.snapshot(),
+        ...progress,
+    });
+}
 function createEmbeddingScheduler(requestedConcurrency, model) {
     return new AdaptiveEmbeddingScheduler(resolveEmbeddingConcurrencyPolicy(requestedConcurrency, model));
 }
@@ -559,10 +753,17 @@ function resolveEmbeddingConcurrencyPolicy(requestedConcurrency, model) {
             adaptive: requestedConcurrency > 1,
         };
     }
-    const initial = model.recommendedIndexConcurrency ?? DEFAULT_EMBEDDING_CONCURRENCY;
-    const max = model.maxIndexConcurrency ??
-        (initial <= 1 ? initial : DEFAULT_EMBEDDING_MAX_CONCURRENCY);
-    const min = Math.min(initial, DEFAULT_EMBEDDING_MIN_CONCURRENCY);
+    const remote = model.info.provider === "qwen";
+    const multimodal = model.info.inputKinds.includes("image");
+    const configuredLocalDefault = model.info.defaultConcurrency;
+    const localDefault = configuredLocalDefault !== undefined &&
+        Number.isInteger(configuredLocalDefault) &&
+        configuredLocalDefault > 0
+        ? configuredLocalDefault
+        : 1;
+    const initial = remote ? (multimodal ? 4 : 8) : localDefault;
+    const max = remote ? (multimodal ? 8 : 12) : localDefault;
+    const min = Math.min(initial, 4);
     return {
         initial,
         min,
@@ -585,11 +786,13 @@ class AdaptiveEmbeddingScheduler {
     get taskConcurrency() {
         return this.policy.max;
     }
-    async run(task, onError) {
-        await this.waitForCooldown();
-        await this.acquire();
+    async run(task, signal, onError) {
+        throwIfAborted(signal);
+        await this.waitForCooldown(signal);
+        await this.acquire(signal);
         try {
-            await this.waitForCooldown();
+            await this.waitForCooldown(signal);
+            throwIfAborted(signal);
             return await task();
         }
         catch (error) {
@@ -631,19 +834,36 @@ class AdaptiveEmbeddingScheduler {
             retryableFailures: this.retryableFailures,
         };
     }
-    async waitForCooldown() {
+    async waitForCooldown(signal) {
         const remaining = this.cooldownUntil - Date.now();
         if (remaining > 0) {
-            await delay(remaining);
+            await abortableDelay(remaining, signal);
         }
     }
-    async acquire() {
+    async acquire(signal) {
+        throwIfAborted(signal);
         if (this.active < this.currentConcurrency) {
             this.active++;
             return;
         }
-        await new Promise((resolve) => {
-            this.queue.push(resolve);
+        await new Promise((resolve, reject) => {
+            const queued = () => {
+                signal?.removeEventListener("abort", abort);
+                resolve();
+            };
+            const abort = () => {
+                const index = this.queue.indexOf(queued);
+                if (index >= 0) {
+                    this.queue.splice(index, 1);
+                }
+                reject(abortError(signal));
+            };
+            if (signal?.aborted) {
+                reject(abortError(signal));
+                return;
+            }
+            signal?.addEventListener("abort", abort, { once: true });
+            this.queue.push(queued);
         });
     }
     release() {
@@ -724,8 +944,32 @@ function retryAfterMsFromText(text) {
     }
     return undefined;
 }
-function delay(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
+function abortableDelay(ms, signal) {
+    if (ms <= 0) {
+        return Promise.resolve();
+    }
+    throwIfAborted(signal);
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            signal?.removeEventListener("abort", abort);
+            resolve();
+        }, ms);
+        const abort = () => {
+            clearTimeout(timer);
+            reject(abortError(signal));
+        };
+        signal?.addEventListener("abort", abort, { once: true });
+    });
+}
+function throwIfAborted(signal) {
+    if (signal?.aborted) {
+        throw abortError(signal);
+    }
+}
+function abortError(signal) {
+    return signal?.reason instanceof Error
+        ? signal.reason
+        : new Error("Embedding was cancelled.");
 }
 function countPublicEntities(fragments) {
     let count = 0;
@@ -777,10 +1021,10 @@ function toEngineError(error, message, options) {
         cause: error,
     });
 }
-function collectionContext(collection) {
+function workspaceIndexContext(workspaceIndex) {
     return (errorDetails([
-        detail("collection_id", collection.id),
-        collectionDetail(collection.name),
+        detail("workspace_index_id", workspaceIndex.id),
+        workspaceIndexDetail(workspaceIndex.name),
     ]) ?? "");
 }
 function fileContext(file) {