@zvec/zvec-grep 0.1.4 → 0.1.6

package/dist/engine/pipeline/indexing/index.js

@@ -1,12 +1,12 @@
-import { readFile } from "node:fs/promises";
-import { collectionDetail, detail, EngineError, errorDetails, isEngineError, } from "../../errors/index.js";
+import { lstat, readFile } from "node:fs/promises";
+import { workspaceIndexDetail, detail, EngineError, errorDetails, isEngineError, } from "../../errors.js";
 import { sha256Bytes } from "../../utils/hash.js";
+import { normalizePath } from "../../utils/path.js";
 import { ConcurrentTiming, TimingCollector } from "../../utils/timing.js";
-import { ExtractorRegistry } from "../../extraction/index.js";
-import { scanRootPaths } from "./scanner/index.js";
-const DEFAULT_EMBEDDING_CONCURRENCY = 4;
-const DEFAULT_EMBEDDING_MIN_CONCURRENCY = 4;
-const DEFAULT_EMBEDDING_MAX_CONCURRENCY = 12;
+import { extract, } from "../../extraction/index.js";
+import { scanDirectoryPath, scanFilePath, scanRootPaths, } from "./scanner/index.js";
+const APPROXIMATE_CHARS_PER_TOKEN = 2;
+const CHUNK_OVERLAP_RATIO = 0.15;
 const EMBEDDING_TRANSIENT_MAX_RETRIES = 3;
 const EMBEDDING_RATE_LIMIT_MAX_RETRIES = 6;
 const EMBEDDING_TRANSIENT_RETRY_BASE_DELAY_MS = 500;
@@ -15,33 +15,44 @@ const EMBEDDING_TRANSIENT_RETRY_MAX_DELAY_MS = 8000;
 const EMBEDDING_RATE_LIMIT_RETRY_MAX_DELAY_MS = 30000;
 const EMBEDDING_RETRY_JITTER_MS = 500;
 const EMBEDDING_SUCCESS_STREAK_MIN = 4;
-export async function indexCollection(ctx) {
+export async function indexWorkspace(ctx) {
     try {
-        return await indexCollectionUnchecked(ctx);
+        return await indexWorkspaceUnchecked(ctx);
     }
     catch (error) {
-        throw toEngineError(error, "Indexing collection failed", {
-            code: "ZVEC_GREP.ENGINE.INDEXING.COLLECTION_FAILED",
-            context: collectionContext(ctx.collection),
+        throw toEngineError(error, "Indexing workspace failed", {
+            code: "ZVEC_GREP.ENGINE.INDEXING.WORKSPACE_FAILED",
+            context: workspaceIndexContext(ctx.workspaceIndex),
         });
     }
 }
-export async function getCollectionIndexStatus(collection, storedFiles) {
+export async function indexWorkspacePaths(ctx, changedPaths) {
     try {
-        const scan = await scanRootPaths(collection.id, collection.rootPaths);
+        return await indexWorkspacePathsUnchecked(ctx, changedPaths);
+    }
+    catch (error) {
+        throw toEngineError(error, "Indexing changed paths failed", {
+            code: "ZVEC_GREP.ENGINE.INDEXING.WORKSPACE_FAILED",
+            context: workspaceIndexContext(ctx.workspaceIndex),
+        });
+    }
+}
+export async function getWorkspaceIndexStatus(workspaceIndex, storedFiles) {
+    try {
+        const scan = await scanRootPaths(workspaceIndex.id, workspaceIndex.rootPaths);
         const diff = await computeDiffFromFiles(scan.files, storedFiles);
         const pendingFiles = storedFiles.filter((file) => file.indexStatus?.indexedTime === null);
         const failedFiles = pendingFiles.filter((file) => file.indexStatus?.error !== undefined);
-        const indexedFiles = storedFiles.filter((file) => file.indexStatus?.indexedTime !== undefined
-            && file.indexStatus.indexedTime !== null);
+        const indexedFiles = storedFiles.filter((file) => file.indexStatus?.indexedTime !== undefined &&
+            file.indexStatus.indexedTime !== null);
         const entitiesIndexed = indexedFiles.reduce((count, file) => count + (file.indexStatus?.entityCount ?? 0), 0);
+        const fragmentsTruncated = indexedFiles.reduce((count, file) => count + (file.indexStatus?.truncatedFragmentCount ?? 0), 0);
         return {
-            collectionId: collection.id,
-            collectionName: collection.name,
             filesScanned: scan.files.length,
             filesStored: storedFiles.length,
             filesIndexed: indexedFiles.length,
             entitiesIndexed,
+            fragmentsTruncated,
             filesPending: pendingFiles.length,
             filesFailed: failedFiles.length,
             filesAdded: diff.added.length,
@@ -56,36 +67,42 @@ export async function getCollectionIndexStatus(collection, storedFiles) {
         };
     }
     catch (error) {
-        throw toEngineError(error, "Inspecting collection index status failed", {
+        throw toEngineError(error, "Inspecting workspace index status failed", {
             code: "ZVEC_GREP.ENGINE.INDEXING.STATUS_FAILED",
-            context: collectionContext(collection),
+            context: workspaceIndexContext(workspaceIndex),
         });
     }
 }
-async function indexCollectionUnchecked(ctx) {
+async function indexWorkspaceUnchecked(ctx) {
     const start = Date.now();
     const report = ctx.onProgress ?? (() => undefined);
     const timings = new TimingCollector();
+    throwIfIndexCancelled(ctx);
     const firstPass = await runIndexPass(ctx, report, "Scanning files...", timings);
     const passes = [firstPass];
+    let progressBase;
     if (firstPass.stats.filesFailed > 0) {
         const failed = firstPass.stats.filesFailed;
+        progressBase = retryProgressBase(firstPass);
         report({
             phase: "scanning",
+            filesTotal: progressBase.filesTotal,
+            filesIndexed: progressBase.filesSucceeded,
             detail: `Retrying ${failed} failed ${failed === 1 ? "file" : "files"}...`,
         });
-        passes.push(await runIndexPass(ctx, report, "Scanning retry candidates...", timings));
+        passes.push(await runIndexPass(ctx, report, "Scanning retry candidates...", timings, progressBase));
     }
     const finalPass = passes[passes.length - 1];
-    reportIndexFinalizing(ctx, report, finalPass);
-    timings.timeSync("index_optimize", () => optimizeStorage(ctx));
+    throwIfIndexCancelled(ctx);
+    reportIndexFinalizing(ctx, report, finalPass, progressBase);
+    await timings.time("index_optimize", () => optimizeStorage(ctx));
     const result = buildIndexResult(ctx, passes, Date.now() - start, timings);
     if (result.filesFailed > 0) {
         report({ phase: "done", detail: "Indexing completed with failed files" });
         throw new EngineError(`Indexing completed with ${result.filesFailed} failed ${result.filesFailed === 1 ? "file" : "files"}`, {
             code: "ZVEC_GREP.ENGINE.INDEXING.FILES_FAILED",
             context: errorDetails([
-                collectionDetail(ctx.collection.name),
+                workspaceIndexDetail(ctx.workspaceIndex.name),
                 detail("filesFailed", result.filesFailed),
                 detail("filesScanned", result.filesScanned),
                 detail("failedFiles", summarizeFailedFiles(finalPass.stats.failedFiles)),
@@ -99,18 +116,78 @@ async function indexCollectionUnchecked(ctx) {
     report({ phase: "done", detail: "Indexing complete" });
     return result;
 }
-async function runIndexPass(ctx, report, scanningDetail, timings) {
-    report({ phase: "scanning", detail: scanningDetail });
-    const scan = await timings.time("index_scan", () => scanRootPaths(ctx.collection.id, ctx.collection.rootPaths));
-    const diff = await timings.time("index_diff", () => computeDiff(scan.files, ctx.storage));
+async function indexWorkspacePathsUnchecked(ctx, changedPaths) {
+    const start = Date.now();
+    const report = ctx.onProgress ?? (() => undefined);
+    const timings = new TimingCollector();
+    const normalizedPaths = [...new Set(changedPaths.map(normalizePath))];
+    throwIfIndexCancelled(ctx);
+    const firstPass = await runPathIndexPass(ctx, report, normalizedPaths, timings);
+    const passes = [firstPass];
+    let progressBase;
+    if (firstPass.stats.filesFailed > 0) {
+        progressBase = retryProgressBase(firstPass);
+        passes.push(await runPathIndexPass(ctx, report, normalizedPaths, timings, progressBase));
+    }
+    const finalPass = passes[passes.length - 1];
+    throwIfIndexCancelled(ctx);
+    reportIndexFinalizing(ctx, report, finalPass, progressBase);
+    await timings.time("index_optimize", () => optimizeStorage(ctx));
+    const result = buildIndexResult(ctx, passes, Date.now() - start, timings);
+    if (result.filesFailed > 0) {
+        throw new EngineError(`Indexing completed with ${result.filesFailed} failed files`, {
+            code: "ZVEC_GREP.ENGINE.INDEXING.FILES_FAILED",
+            context: workspaceIndexContext(ctx.workspaceIndex),
+        });
+    }
+    report({ phase: "done", detail: "Indexing complete" });
+    return result;
+}
+async function runPathIndexPass(ctx, report, changedPaths, timings, progressBase) {
+    reportScanning(report, "Scanning changed paths...", progressBase);
+    const scanned = await timings.time("index_scan_paths", async () => {
+        const files = [];
+        for (const path of changedPaths) {
+            throwIfIndexCancelled(ctx);
+            const info = await lstat(path).catch(() => null);
+            const scan = info?.isDirectory()
+                ? await scanDirectoryPath(ctx.workspaceIndex.id, ctx.workspaceIndex.rootPaths, path, { signal: ctx.signal })
+                : await scanFilePath(ctx.workspaceIndex.id, ctx.workspaceIndex.rootPaths, path, {
+                    signal: ctx.signal,
+                });
+            files.push(...scan.files);
+        }
+        return [...new Map(files.map((file) => [file.id, file])).values()];
+    });
+    throwIfIndexCancelled(ctx);
+    const existing = [
+        ...new Map(changedPaths
+            .flatMap((path) => ctx.storage.listFilesByPathPrefix(path))
+            .map((file) => [file.id, file])).values(),
+    ];
+    return runDiffPass(ctx, report, scanned, existing, timings, progressBase);
+}
+async function runIndexPass(ctx, report, scanningDetail, timings, progressBase) {
+    reportScanning(report, scanningDetail, progressBase);
+    const scan = await timings.time("index_scan", () => scanRootPaths(ctx.workspaceIndex.id, ctx.workspaceIndex.rootPaths, {
+        signal: ctx.signal,
+    }));
+    throwIfIndexCancelled(ctx);
+    return runDiffPass(ctx, report, scan.files, ctx.storage.listFiles(), timings, progressBase);
+}
+async function runDiffPass(ctx, report, scannedFiles, existingFiles, timings, progressBase) {
+    const diff = await timings.time("index_diff", () => computeDiffFromFiles(scannedFiles, existingFiles));
+    throwIfIndexCancelled(ctx);
     const pending = [...diff.added, ...diff.modified, ...diff.pending];
     report({
         phase: "scanning",
-        filesTotal: scan.files.length,
+        filesTotal: progressBase?.filesTotal ?? scannedFiles.length,
+        filesIndexed: progressBase?.filesSucceeded ?? 0,
         detail: `${diff.added.length} added, ${diff.modified.length} modified, ${diff.pending.length} pending, ${diff.deleted.length} deleted, ${diff.unchanged.length} unchanged`,
     });
     timings.timeSync("index_delete_stale", () => {
         for (const file of diff.deleted) {
+            throwIfIndexCancelled(ctx);
             try {
                 ctx.storage.deleteFile(file.id);
             }
@@ -125,46 +202,78 @@ async function runIndexPass(ctx, report, scanningDetail, timings) {
     const reportIndexing = (currentStats, detail, embedding) => {
         report({
             phase: "indexing",
-            filesTotal: pending.length,
-            filesIndexed: currentStats.filesIndexed + currentStats.filesFailed,
+            filesTotal: progressBase?.filesTotal ?? pending.length,
+            filesIndexed: (progressBase?.filesSucceeded ?? 0) +
+                currentStats.filesIndexed +
+                currentStats.filesFailed,
             filesFailed: currentStats.filesFailed,
             detail,
             embedding,
         });
     };
-    reportIndexing({ filesIndexed: 0, filesFailed: 0, failedFiles: [], failedFileReasons: [], entitiesCreated: 0 });
+    reportIndexing({
+        filesIndexed: 0,
+        filesFailed: 0,
+        failedFiles: [],
+        failedFileReasons: [],
+        entitiesCreated: 0,
+    });
     const stats = await indexFiles(pending, ctx, reportIndexing, timings);
+    throwIfIndexCancelled(ctx);
     return {
-        filesScanned: scan.files.length,
+        filesScanned: scannedFiles.length,
         diff,
         stats,
     };
 }
-function reportIndexFinalizing(ctx, report, pass) {
+function reportIndexFinalizing(ctx, report, pass, progressBase) {
     report({
         phase: "indexing",
-        filesTotal: pass.diff.added.length + pass.diff.modified.length + pass.diff.pending.length,
-        filesIndexed: pass.stats.filesIndexed + pass.stats.filesFailed,
+        filesTotal: progressBase?.filesTotal ??
+            pass.diff.added.length +
+                pass.diff.modified.length +
+                pass.diff.pending.length,
+        filesIndexed: (progressBase?.filesSucceeded ?? 0) +
+            pass.stats.filesIndexed +
+            pass.stats.filesFailed,
         filesFailed: pass.stats.filesFailed,
         detail: "finalizing index",
     });
 }
+function retryProgressBase(pass) {
+    return {
+        filesSucceeded: pass.stats.filesIndexed,
+        filesTotal: pass.diff.added.length +
+            pass.diff.modified.length +
+            pass.diff.pending.length,
+    };
+}
+function reportScanning(report, detail, progressBase) {
+    report({
+        phase: "scanning",
+        ...(progressBase
+            ? {
+                filesTotal: progressBase.filesTotal,
+                filesIndexed: progressBase.filesSucceeded,
+            }
+            : {}),
+        detail,
+    });
+}
 function buildIndexResult(ctx, passes, durationMs, timings) {
     const firstPass = passes[0];
     const retryPasses = passes.slice(1);
     const finalPass = passes[passes.length - 1];
     return {
-        collectionId: ctx.collection.id,
-        collectionName: ctx.collection.name,
         filesScanned: finalPass.filesScanned,
-        filesAdded: firstPass.diff.added.length
-            + retryPasses.reduce((count, pass) => count + pass.diff.added.length, 0),
-        filesModified: firstPass.diff.modified.length
-            + retryPasses.reduce((count, pass) => count + pass.diff.modified.length, 0),
-        filesPending: firstPass.diff.pending.length
-            + retryPasses.reduce((count, pass) => count + pass.diff.pending.length, 0),
-        filesDeleted: firstPass.diff.deleted.length
-            + retryPasses.reduce((count, pass) => count + pass.diff.deleted.length, 0),
+        filesAdded: firstPass.diff.added.length +
+            retryPasses.reduce((count, pass) => count + pass.diff.added.length, 0),
+        filesModified: firstPass.diff.modified.length +
+            retryPasses.reduce((count, pass) => count + pass.diff.modified.length, 0),
+        filesPending: firstPass.diff.pending.length +
+            retryPasses.reduce((count, pass) => count + pass.diff.pending.length, 0),
+        filesDeleted: firstPass.diff.deleted.length +
+            retryPasses.reduce((count, pass) => count + pass.diff.deleted.length, 0),
         filesUnchanged: firstPass.diff.unchanged.length,
         filesFailed: finalPass.stats.filesFailed,
         entitiesCreated: passes.reduce((count, pass) => count + pass.stats.entitiesCreated, 0),
@@ -172,20 +281,17 @@ function buildIndexResult(ctx, passes, durationMs, timings) {
         timings: timings.entries(),
     };
 }
-function optimizeStorage(ctx) {
+async function optimizeStorage(ctx) {
     try {
-        ctx.storage.optimize();
+        await ctx.storage.optimize();
     }
     catch (error) {
         throw toEngineError(error, "Indexing failed to finalize storage", {
             code: "ZVEC_GREP.ENGINE.INDEXING.OPTIMIZE_FAILED",
-            context: collectionContext(ctx.collection),
+            context: workspaceIndexContext(ctx.workspaceIndex),
         });
     }
 }
-async function computeDiff(scannedFiles, storage) {
-    return computeDiffFromFiles(scannedFiles, storage.listFiles());
-}
 async function computeDiffFromFiles(scannedFiles, existingFiles) {
     const existingById = new Map(existingFiles.map((file) => [file.id, file]));
     const seen = new Set();
@@ -204,15 +310,15 @@ async function computeDiffFromFiles(scannedFiles, existingFiles) {
             pending.push(await withContentHash(file));
             continue;
         }
-        if (existing.sizeBytes === file.sizeBytes
-            && existing.lastModifiedTime === file.lastModifiedTime
-            && existing.contentHash) {
+        if (existing.sizeBytes === file.sizeBytes &&
+            existing.lastModifiedTime === file.lastModifiedTime &&
+            existing.contentHash) {
             unchanged.push(existing);
             continue;
         }
         const hashed = await withContentHash(file);
-        if (existing.sizeBytes === hashed.sizeBytes
-            && existing.contentHash === hashed.contentHash) {
+        if (existing.sizeBytes === hashed.sizeBytes &&
+            existing.contentHash === hashed.contentHash) {
             unchanged.push(existing);
             continue;
         }
@@ -234,10 +340,12 @@ async function indexFiles(files, ctx, onProgress, timings) {
     const embeddingScheduler = createEmbeddingScheduler(ctx.embeddingConcurrency, ctx.embeddingModel);
     const embeddingTiming = new ConcurrentTiming(timings, "index_embedding");
     const runningEmbeddings = new Set();
+    const chunkOptions = indexChunkOptions(ctx);
     const reportEmbeddingProgress = (currentStats, detail) => {
         onProgress(currentStats, detail, embeddingScheduler.snapshot());
     };
     const flushBatch = async () => {
+        throwIfIndexCancelled(ctx);
         if (batchFiles.length === 0) {
             return;
         }
@@ -246,8 +354,8 @@ async function indexFiles(files, ctx, onProgress, timings) {
         await scheduleEmbeddingTask(() => embedAndCommitBatch(filesToEmbed, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming));
     };
     const scheduleEmbeddingTask = async (task) => {
-        const promise = task()
-            .finally(() => {
+        throwIfIndexCancelled(ctx);
+        const promise = task().finally(() => {
             runningEmbeddings.delete(promise);
         });
         runningEmbeddings.add(promise);
@@ -255,70 +363,109 @@ async function indexFiles(files, ctx, onProgress, timings) {
             await Promise.race(runningEmbeddings);
         }
     };
-    for (const file of files) {
-        onProgress(stats, `reading ${file.relativePath}`);
-        const prepared = await timings.time("index_prepare", () => prepareFile(file, ctx));
-        if ("failedReason" in prepared) {
-            recordFileFailed(stats, file, prepared.failedReason);
-            onProgress(stats, `failed ${file.relativePath}`);
-            continue;
-        }
-        if (prepared.fragments.length === 0) {
-            const committed = commitFile(prepared, [], ctx, stats);
-            onProgress(stats, finishedFileDetail(committed, file.relativePath));
-            continue;
-        }
-        if (prepared.fragments.length > ctx.embeddingModel.limits.maxBatchSize) {
-            await flushBatch();
-            await scheduleEmbeddingTask(async () => {
-                reportEmbeddingProgress(stats, `embedding ${file.relativePath}`);
-                await embedAndCommitFile(prepared, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming);
-            });
-            continue;
-        }
-        if (batchFragmentCount > 0
-            && batchFragmentCount + prepared.fragments.length > ctx.embeddingModel.limits.maxBatchSize) {
-            await flushBatch();
-        }
-        batchFiles.push(prepared);
-        batchFragmentCount += prepared.fragments.length;
-        if (batchFragmentCount === ctx.embeddingModel.limits.maxBatchSize) {
-            await flushBatch();
+    try {
+        for (const file of files) {
+            throwIfIndexCancelled(ctx);
+            onProgress(stats, `reading ${file.relativePath}`);
+            const prepared = await timings.time("index_prepare", () => prepareFile(file, ctx, chunkOptions));
+            throwIfIndexCancelled(ctx);
+            if ("failedReason" in prepared) {
+                recordFileFailed(stats, file, prepared.failedReason);
+                onProgress(stats, `failed ${file.relativePath}`);
+                continue;
+            }
+            if (prepared.fragments.length === 0) {
+                const committed = commitFile(prepared, [], ctx, stats);
+                onProgress(stats, finishedFileDetail(committed, file.relativePath));
+                continue;
+            }
+            if (prepared.fragments.length > ctx.embeddingModel.info.limits.maxBatchSize) {
+                await flushBatch();
+                await scheduleEmbeddingTask(async () => {
+                    throwIfIndexCancelled(ctx);
+                    reportEmbeddingProgress(stats, `embedding ${file.relativePath}`);
+                    await embedAndCommitFile(prepared, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming);
+                });
+                continue;
+            }
+            if (batchFragmentCount > 0 &&
+                batchFragmentCount + prepared.fragments.length >
+                    ctx.embeddingModel.info.limits.maxBatchSize) {
+                await flushBatch();
+            }
+            batchFiles.push(prepared);
+            batchFragmentCount += prepared.fragments.length;
+            if (batchFragmentCount === ctx.embeddingModel.info.limits.maxBatchSize) {
+                await flushBatch();
+            }
         }
+        await flushBatch();
+        await Promise.all(runningEmbeddings);
     }
-    await flushBatch();
-    await Promise.all(runningEmbeddings);
+    catch (error) {
+        await Promise.allSettled(runningEmbeddings);
+        throw error;
+    }
+    throwIfIndexCancelled(ctx);
     return stats;
 }
-async function prepareFile(file, ctx) {
+async function prepareFile(file, ctx, chunkOptions) {
     try {
+        throwIfIndexCancelled(ctx);
         const source = await readSource(file);
-        const extracted = await new ExtractorRegistry().extract(source);
-        const fragments = extracted.filter((fragment) => ctx.embeddingModel.supportedContentKinds.includes(fragment.content.kind));
+        const extracted = await extract(source, chunkOptions);
+        throwIfIndexCancelled(ctx);
+        const fragments = extracted.filter((fragment) => ctx.embeddingModel.info.inputKinds.includes(fragment.content.kind));
         return { file, fragments };
     }
     catch (error) {
+        if (indexIsCancelled(ctx)) {
+            throw indexCancellationError(ctx);
+        }
         return {
             file,
             failedReason: markFileFailed(ctx, file, error, "prepare"),
         };
     }
 }
+function indexChunkOptions(ctx) {
+    const maxInputTokens = ctx.embeddingModel.info.limits.maxInputTokens;
+    if (maxInputTokens === undefined) {
+        return {};
+    }
+    const maxChunkChars = Math.floor(maxInputTokens * APPROXIMATE_CHARS_PER_TOKEN);
+    const chunkOverlapChars = Math.floor(maxChunkChars * CHUNK_OVERLAP_RATIO);
+    return { maxChunkChars, chunkOverlapChars };
+}
 async function embedAndCommitBatch(files, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming) {
     const fragments = files.flatMap((file) => file.fragments);
     try {
+        throwIfIndexCancelled(ctx);
         const contents = fragments.map(vectorContentForFragment);
         onProgress(stats, `embedding ${describePreparedFiles(files)}`, embeddingScheduler.snapshot());
-        const vectors = await embeddingTiming.time(() => embedContentsWithRetry(contents, ctx.embeddingModel, embeddingScheduler));
+        const embedding = await embeddingTiming.time(() => embedContentsWithRetry(contents, ctx.embeddingModel, embeddingScheduler, ctx.signal, (progress) => reportModelDownloadProgress(stats, progress, onProgress, embeddingScheduler)));
+        const truncatedInputIndexes = new Set(embedding.truncated);
+        throwIfIndexCancelled(ctx);
         let offset = 0;
         for (const file of files) {
-            const fileVectors = vectors.slice(offset, offset + file.fragments.length);
+            throwIfIndexCancelled(ctx);
+            const end = offset + file.fragments.length;
+            const fileVectors = embedding.vectors.slice(offset, end);
+            let truncatedFragmentCount = 0;
+            for (let index = offset; index < end; index++) {
+                if (truncatedInputIndexes.has(index)) {
+                    truncatedFragmentCount++;
+                }
+            }
             offset += file.fragments.length;
-            const committed = timings.timeSync("index_commit", () => commitFile(file, fileVectors, ctx, stats));
+            const committed = timings.timeSync("index_commit", () => commitFile(file, fileVectors, ctx, stats, truncatedFragmentCount));
             onProgress(stats, finishedFileDetail(committed, file.file.relativePath));
         }
     }
     catch (error) {
+        if (indexIsCancelled(ctx)) {
+            throw indexCancellationError(ctx);
+        }
         if (isRetryableEmbeddingError(error)) {
             for (const file of files) {
                 const reason = markFileFailed(ctx, file.file, error, "embed");
@@ -335,29 +482,57 @@ async function embedAndCommitBatch(files, ctx, stats, onProgress, embeddingSched
 }
 async function embedAndCommitFile(file, ctx, stats, onProgress, embeddingScheduler, timings, embeddingTiming) {
     try {
-        const vectors = await embeddingTiming.time(() => embedFragments(file.fragments, ctx.embeddingModel, embeddingScheduler));
-        const committed = timings.timeSync("index_commit", () => commitFile(file, vectors, ctx, stats));
+        throwIfIndexCancelled(ctx);
+        const embedding = await embeddingTiming.time(() => embedFragments(file.fragments, ctx.embeddingModel, embeddingScheduler, ctx.signal, (progress) => reportModelDownloadProgress(stats, progress, onProgress, embeddingScheduler)));
+        throwIfIndexCancelled(ctx);
+        const committed = timings.timeSync("index_commit", () => commitFile(file, embedding.vectors, ctx, stats, embedding.truncated.length));
         onProgress(stats, finishedFileDetail(committed, file.file.relativePath));
     }
     catch (error) {
+        if (indexIsCancelled(ctx)) {
+            throw indexCancellationError(ctx);
+        }
         const reason = markFileFailed(ctx, file.file, error, "embed");
         recordFileFailed(stats, file.file, reason);
         onProgress(stats, finishedFileDetail(false, file.file.relativePath));
     }
 }
-function commitFile(file, vectors, ctx, stats) {
+function commitFile(file, vectors, ctx, stats, truncatedFragmentCount = 0) {
     try {
-        ctx.storage.upsertFile(file.file, file.fragments, vectors);
+        throwIfIndexCancelled(ctx);
+        ctx.storage.upsertFile(file.file, file.fragments, vectors, {
+            truncatedFragmentCount,
+        });
         stats.filesIndexed++;
         stats.entitiesCreated += countPublicEntities(file.fragments);
         return true;
     }
     catch (error) {
+        if (indexIsCancelled(ctx)) {
+            throw indexCancellationError(ctx);
+        }
         const reason = markFileFailed(ctx, file.file, error, "commit");
         recordFileFailed(stats, file.file, reason);
         return false;
     }
 }
+function throwIfIndexCancelled(ctx) {
+    if (!indexIsCancelled(ctx)) {
+        return;
+    }
+    throw indexCancellationError(ctx);
+}
+function indexIsCancelled(ctx) {
+    return ctx.signal?.aborted === true;
+}
+function indexCancellationError(ctx) {
+    return ctx.signal?.reason instanceof Error
+        ? ctx.signal.reason
+        : new EngineError("Indexing was cancelled.", {
+            code: "ZVEC_GREP.ENGINE.INDEXING.CANCELLED",
+            context: workspaceIndexContext(ctx.workspaceIndex),
+        });
+}
 function recordFileFailed(stats, file, reason) {
     stats.filesFailed++;
     stats.failedFiles.push(file.relativePath);
@@ -409,14 +584,15 @@ async function readSource(file) {
         text: bytes.toString("utf8"),
     };
 }
-async function embedFragments(fragments, model, embeddingScheduler) {
+async function embedFragments(fragments, model, embeddingScheduler, signal, onModelProgress) {
     const batches = [];
-    for (let start = 0; start < fragments.length; start += model.limits.maxBatchSize) {
-        const batch = fragments.slice(start, start + model.limits.maxBatchSize);
+    for (let start = 0; start < fragments.length; start += model.info.limits.maxBatchSize) {
+        const batch = fragments.slice(start, start + model.info.limits.maxBatchSize);
         batches.push({ start, fragments: batch });
     }
     const vectors = new Array(fragments.length);
-    const results = await Promise.allSettled(batches.map((batch) => embedFragmentBatch(batch.fragments, model, batch.start, embeddingScheduler)));
+    const truncatedInputIndexes = [];
+    const results = await Promise.allSettled(batches.map((batch) => embedFragmentBatch(batch.fragments, model, batch.start, embeddingScheduler, signal, onModelProgress)));
     let firstError;
     for (const [index, result] of results.entries()) {
         if (result.status === "rejected") {
@@ -424,43 +600,54 @@ async function embedFragments(fragments, model, embeddingScheduler) {
             continue;
         }
         const start = batches[index].start;
-        for (const [offset, vector] of result.value.entries()) {
+        for (const [offset, vector] of result.value.vectors.entries()) {
             vectors[start + offset] = vector;
         }
+        truncatedInputIndexes.push(...result.value.truncated.map((inputIndex) => start + inputIndex));
     }
     if (firstError) {
         throw firstError;
     }
-    return vectors;
+    return {
+        vectors,
+        truncated: truncatedInputIndexes,
+    };
 }
-async function embedFragmentBatch(fragments, model, startIndex, embeddingScheduler) {
+async function embedFragmentBatch(fragments, model, startIndex, embeddingScheduler, signal, onModelProgress) {
     const contents = fragments.map(vectorContentForFragment);
     try {
-        return await embedContentsWithRetry(contents, model, embeddingScheduler);
+        return await embedContentsWithRetry(contents, model, embeddingScheduler, signal, onModelProgress);
     }
     catch (error) {
         if (fragments.length === 1 || isRetryableEmbeddingError(error)) {
             throw error;
         }
-        return await embedFragmentBatchOneByOne(fragments, model, startIndex, embeddingScheduler);
+        return await embedFragmentBatchOneByOne(fragments, model, startIndex, embeddingScheduler, signal, onModelProgress);
     }
 }
-async function embedFragmentBatchOneByOne(fragments, model, startIndex, embeddingScheduler) {
+async function embedFragmentBatchOneByOne(fragments, model, startIndex, embeddingScheduler, signal, onModelProgress) {
     const vectors = [];
+    const truncatedInputIndexes = [];
     for (const [index, fragment] of fragments.entries()) {
         try {
-            const [vector] = await embedContentsWithRetry([vectorContentForFragment(fragment)], model, embeddingScheduler);
-            vectors.push(vector);
+            const result = await embedContentsWithRetry([vectorContentForFragment(fragment)], model, embeddingScheduler, signal, onModelProgress);
+            vectors.push(result.vectors[0]);
+            if (result.truncated.length > 0) {
+                truncatedInputIndexes.push(index);
+            }
         }
         catch (error) {
             throw new EngineError("Embedding entity fragment failed after one-by-one fallback", {
                 code: "ZVEC_GREP.ENGINE.INDEXING.EMBEDDING_FRAGMENT_FAILED",
-                context: `model=${model.ref.model} fragmentId=${fragment.id} fragmentIndex=${startIndex + index}`,
+                context: `model=${model.info.reference} fragmentId=${fragment.id} fragmentIndex=${startIndex + index}`,
                 cause: error,
             });
         }
     }
-    return vectors;
+    return {
+        vectors,
+        truncated: truncatedInputIndexes,
+    };
 }
 function vectorContentForFragment(fragment) {
     if (fragment.content.kind !== "text") {
@@ -486,13 +673,17 @@ function vectorMetadataText(metadata) {
                 : `symbol: ${metadata.symbolType}`,
             metadata.scope ? `scope: ${metadata.scope}` : null,
             metadata.signature ? `signature: ${oneLine(metadata.signature)}` : null,
-            metadata.modifiers.length > 0 ? `modifiers: ${metadata.modifiers.join(" ")}` : null,
+            metadata.modifiers.length > 0
+                ? `modifiers: ${metadata.modifiers.join(" ")}`
+                : null,
             metadata.doc ? `doc: ${oneLine(metadata.doc)}` : null,
         ]);
     }
     return compactMetadataLines([
         metadata.heading ? `heading: ${metadata.heading}` : null,
-        typeof metadata.level === "number" ? `heading_level: ${metadata.level}` : null,
+        typeof metadata.level === "number"
+            ? `heading_level: ${metadata.level}`
+            : null,
         metadata.scope ? `scope: ${metadata.scope}` : null,
     ]);
 }
@@ -501,13 +692,18 @@ function compactMetadataLines(lines) {
         .filter((line) => line !== null && line.trim().length > 0)
         .join("\n");
 }
-async function embedContentsWithRetry(contents, model, embeddingScheduler) {
+async function embedContentsWithRetry(contents, model, embeddingScheduler, signal, onModelProgress) {
     let attempt = 0;
     while (true) {
         let retry = nonRetryableEmbeddingError();
         let delayMs = 0;
         try {
-            const vectors = await embeddingScheduler.run(() => model.embed(contents, { purpose: "document" }), (error) => {
+            throwIfAborted(signal);
+            const result = await embeddingScheduler.run(() => model.embed(contents, {
+                purpose: "document",
+                signal,
+                onProgress: onModelProgress,
+            }), signal, (error) => {
                 retry = classifyEmbeddingRetry(error);
                 delayMs = retryDelayMs(attempt, retry);
                 if (retry.retryable) {
@@ -518,9 +714,14 @@ async function embedContentsWithRetry(contents, model, embeddingScheduler) {
                 }
             });
             embeddingScheduler.recordSuccess();
-            return vectors;
+            return result;
         }
         catch (error) {
+            if (signal?.aborted) {
+                throw signal.reason instanceof Error
+                    ? signal.reason
+                    : new Error("Embedding was cancelled.");
+            }
             if (!retry.retryable) {
                 retry = classifyEmbeddingRetry(error);
                 delayMs = retryDelayMs(attempt, retry);
@@ -528,18 +729,24 @@ async function embedContentsWithRetry(contents, model, embeddingScheduler) {
             if (attempt >= maxRetryAttempts(retry) || !retry.retryable) {
                 throw error;
             }
-            await delay(delayMs);
+            await abortableDelay(delayMs, signal);
             attempt++;
         }
     }
 }
+function reportModelDownloadProgress(stats, progress, onProgress, embeddingScheduler) {
+    onProgress(stats, `downloading ${progress.model}`, {
+        ...embeddingScheduler.snapshot(),
+        ...progress,
+    });
+}
 function createEmbeddingScheduler(requestedConcurrency, model) {
     return new AdaptiveEmbeddingScheduler(resolveEmbeddingConcurrencyPolicy(requestedConcurrency, model));
 }
 function resolveEmbeddingConcurrencyPolicy(requestedConcurrency, model) {
-    if (requestedConcurrency !== undefined
-        && Number.isInteger(requestedConcurrency)
-        && requestedConcurrency > 0) {
+    if (requestedConcurrency !== undefined &&
+        Number.isInteger(requestedConcurrency) &&
+        requestedConcurrency > 0) {
         return {
             initial: requestedConcurrency,
             min: 1,
@@ -547,10 +754,11 @@ function resolveEmbeddingConcurrencyPolicy(requestedConcurrency, model) {
             adaptive: requestedConcurrency > 1,
         };
     }
-    const initial = model.recommendedIndexConcurrency ?? DEFAULT_EMBEDDING_CONCURRENCY;
-    const max = model.maxIndexConcurrency
-        ?? (initial <= 1 ? initial : DEFAULT_EMBEDDING_MAX_CONCURRENCY);
-    const min = Math.min(initial, DEFAULT_EMBEDDING_MIN_CONCURRENCY);
+    const remote = model.info.provider === "qwen";
+    const multimodal = model.info.inputKinds.includes("image");
+    const initial = remote ? (multimodal ? 4 : 8) : 1;
+    const max = remote ? (multimodal ? 8 : 12) : 1;
+    const min = Math.min(initial, 4);
     return {
         initial,
         min,
@@ -573,11 +781,13 @@ class AdaptiveEmbeddingScheduler {
     get taskConcurrency() {
         return this.policy.max;
     }
-    async run(task, onError) {
-        await this.waitForCooldown();
-        await this.acquire();
+    async run(task, signal, onError) {
+        throwIfAborted(signal);
+        await this.waitForCooldown(signal);
+        await this.acquire(signal);
         try {
-            await this.waitForCooldown();
+            await this.waitForCooldown(signal);
+            throwIfAborted(signal);
             return await task();
         }
         catch (error) {
@@ -593,7 +803,8 @@ class AdaptiveEmbeddingScheduler {
             return;
         }
         this.successStreak++;
-        if (this.successStreak < Math.max(EMBEDDING_SUCCESS_STREAK_MIN, this.currentConcurrency * 2)) {
+        if (this.successStreak <
+            Math.max(EMBEDDING_SUCCESS_STREAK_MIN, this.currentConcurrency * 2)) {
             return;
         }
         this.currentConcurrency++;
@@ -618,19 +829,36 @@ class AdaptiveEmbeddingScheduler {
             retryableFailures: this.retryableFailures,
         };
     }
-    async waitForCooldown() {
+    async waitForCooldown(signal) {
         const remaining = this.cooldownUntil - Date.now();
         if (remaining > 0) {
-            await delay(remaining);
+            await abortableDelay(remaining, signal);
         }
     }
-    async acquire() {
+    async acquire(signal) {
+        throwIfAborted(signal);
         if (this.active < this.currentConcurrency) {
             this.active++;
             return;
         }
-        await new Promise((resolve) => {
-            this.queue.push(resolve);
+        await new Promise((resolve, reject) => {
+            const queued = () => {
+                signal?.removeEventListener("abort", abort);
+                resolve();
+            };
+            const abort = () => {
+                const index = this.queue.indexOf(queued);
+                if (index >= 0) {
+                    this.queue.splice(index, 1);
+                }
+                reject(abortError(signal));
+            };
+            if (signal?.aborted) {
+                reject(abortError(signal));
+                return;
+            }
+            signal?.addEventListener("abort", abort, { once: true });
+            this.queue.push(queued);
         });
     }
     release() {
@@ -659,8 +887,8 @@ function classifyEmbeddingRetry(error) {
     const context = isEngineError(error) && error.context ? ` ${error.context}` : "";
     const text = `${message}${context}`;
     const status = httpStatusFromText(text);
-    const rateLimited = status === 429
-        || /rate limit|quota exceeded|too many requests|request rate increased too quickly/i.test(text);
+    const rateLimited = status === 429 ||
+        /rate limit|quota exceeded|too many requests|request rate increased too quickly/i.test(text);
     const serverError = typeof status === "number" && status >= 500 && status <= 599;
     return {
         retryable: rateLimited || serverError,
@@ -711,8 +939,32 @@ function retryAfterMsFromText(text) {
     }
     return undefined;
 }
-function delay(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
+function abortableDelay(ms, signal) {
+    if (ms <= 0) {
+        return Promise.resolve();
+    }
+    throwIfAborted(signal);
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            signal?.removeEventListener("abort", abort);
+            resolve();
+        }, ms);
+        const abort = () => {
+            clearTimeout(timer);
+            reject(abortError(signal));
+        };
+        signal?.addEventListener("abort", abort, { once: true });
+    });
+}
+function throwIfAborted(signal) {
+    if (signal?.aborted) {
+        throw abortError(signal);
+    }
+}
+function abortError(signal) {
+    return signal?.reason instanceof Error
+        ? signal.reason
+        : new Error("Embedding was cancelled.");
 }
 function countPublicEntities(fragments) {
     let count = 0;
@@ -764,11 +1016,11 @@ function toEngineError(error, message, options) {
         cause: error,
     });
 }
-function collectionContext(collection) {
-    return errorDetails([
-        detail("collection_id", collection.id),
-        collectionDetail(collection.name),
-    ]) ?? "";
+function workspaceIndexContext(workspaceIndex) {
+    return (errorDetails([
+        detail("workspace_index_id", workspaceIndex.id),
+        workspaceIndexDetail(workspaceIndex.name),
+    ]) ?? "");
 }
 function fileContext(file) {
     return `fileId=${file.id} path=${file.relativePath}`;