@jcode.labs/ragmir 2.16.2 → 2.17.1

package/dist/ingest.js

@@ -8,12 +8,14 @@ import { countSkippedByReason, inventorySourceFiles, summarizeUnsupportedExtensi
 import { INDEX_SCHEMA_VERSION } from "./index-diagnostics.js";
 import { indexPolicyFingerprint } from "./index-policy.js";
 import { withIndexWriteLock } from "./index-write-lock.js";
+import { canResumeIngestion, createIngestionRunState, finishIngestionState, generationTableName, ingestionProgress, readIngestionState, removeStagedIndexManifest, resumeIngestionState, updateIngestionFile, writeIngestionState, writeStagedIndexManifest, } from "./ingestion-state.js";
 import { operationSignal, throwIfAborted } from "./operation.js";
 import { parseFile } from "./parsing.js";
 import { redactText, totalRedactions } from "./redaction.js";
-import { openRowsTable, readEmptyTextFiles, readIndexManifest, updateRows, writeEmptyTextFiles, writeIndexManifest, writeRows, } from "./store.js";
+import { activeIndexTableName, dropRowsTable, openRowsTable, openRowsTableByName, readEmptyTextFiles, readIndexManifest, updateRowsInTable, writeEmptyTextFiles, writeIndexManifest, } from "./store.js";
 import { VERSION } from "./version.js";
 const MAX_SOURCE_DIAGNOSTIC_ITEMS = 20;
+const DEFAULT_INGEST_FILE_BATCH_SIZE = 25;
 const ARCHIVE_PATH_PATTERNS = [
     /(^|[/_-])archive(s)?([/_-]|$)/iu,
     /(^|[/_-])backup(s)?([/_-]|$)/iu,
@@ -37,77 +39,250 @@ export async function ingestWithConfig(config, options = {}, connection) {
     return withIndexWriteLock(config.storageDir, signal, () => ingestUnlocked(config, options, connection, signal));
 }
 async function ingestUnlocked(config, options, connection, signal) {
-    throwIfAborted(signal);
-    const policyFingerprint = indexPolicyFingerprint(config);
-    const existingManifest = await readIndexManifest(config);
-    const manifestCompatible = !options.rebuild &&
-        existingManifest?.schemaVersion === INDEX_SCHEMA_VERSION &&
-        existingManifest.indexPolicyFingerprint === policyFingerprint &&
-        existingManifest.indexedFiles !== undefined;
-    const existingTable = manifestCompatible ? await openRowsTable(config, connection) : null;
-    const storedEmptyFiles = manifestCompatible ? await readEmptyTextFiles(config) : [];
-    const knownFiles = new Map([...(existingManifest?.indexedFiles ?? []), ...storedEmptyFiles].map((file) => [
-        file.relativePath,
-        file,
-    ]));
-    const inventory = await inventorySourceFiles(config, { knownFiles });
-    const files = inventory.supportedFiles;
-    const inventoryMetrics = sourceInventoryMetrics(files);
-    const currentFiles = new Map(files.map((file) => [file.relativePath, file]));
-    const policyRebuild = !options.rebuild &&
-        existingManifest !== null &&
-        (existingManifest.schemaVersion !== INDEX_SCHEMA_VERSION ||
-            existingManifest.indexPolicyFingerprint !== policyFingerprint);
-    const canReuse = manifestCompatible && existingTable !== null;
-    const previousIndexedFiles = canReuse ? (existingManifest.indexedFiles ?? []) : [];
-    const previousEmptyFiles = canReuse ? storedEmptyFiles : [];
-    const reusableIndexedFiles = previousIndexedFiles.filter((file) => currentFiles.get(file.relativePath)?.checksum === file.checksum);
-    const reusableEmptyFiles = previousEmptyFiles.filter((file) => currentFiles.get(file.relativePath)?.checksum === file.checksum);
-    const reusableFiles = new Set([
-        ...reusableIndexedFiles.map((file) => file.relativePath),
-        ...reusableEmptyFiles.map((file) => file.relativePath),
-    ]);
-    const filesToIndex = files.filter((file) => !reusableFiles.has(file.relativePath));
-    const allChunks = [];
-    const errors = [];
-    const redactionCounts = [];
-    const emptyTextFiles = [];
-    const results = await mapLimit(filesToIndex, config.ingestConcurrency, signal, async (file) => {
-        try {
-            const parsed = await parseFile(file, config);
-            throwIfAborted(signal);
-            const redacted = redactText(parsed.text, config);
-            const chunks = chunkDocument({ ...parsed, text: redacted.text }, config.chunkSize, config.chunkOverlap);
-            return { path: file.relativePath, chunks, redactions: redacted.counts, error: null };
+    let state = null;
+    try {
+        throwIfAborted(signal);
+        const requestedBatchSize = ingestFileBatchSize(options.batchSize);
+        const policyFingerprint = indexPolicyFingerprint(config);
+        const existingManifest = await readIndexManifest(config);
+        const storedState = await readIngestionState(config);
+        const storedEmptyFiles = await readEmptyTextFiles(config);
+        const inventory = await inventorySourceFiles(config, signal ? { signal } : {});
+        const files = inventory.supportedFiles;
+        const inventoryMetrics = sourceInventoryMetrics(files);
+        const currentFiles = new Map(files.map((file) => [file.relativePath, file]));
+        const existingTable = await openRowsTable(config, connection);
+        const manifestCompatible = !options.rebuild &&
+            existingManifest?.schemaVersion === INDEX_SCHEMA_VERSION &&
+            existingManifest.indexPolicyFingerprint === policyFingerprint &&
+            existingManifest.indexedFiles !== undefined;
+        const policyRebuild = !options.rebuild &&
+            existingManifest !== null &&
+            (existingManifest.schemaVersion !== INDEX_SCHEMA_VERSION ||
+                existingManifest.indexPolicyFingerprint !== policyFingerprint);
+        const canReuse = manifestCompatible && existingTable !== null;
+        const previousIndexedFiles = canReuse ? (existingManifest.indexedFiles ?? []) : [];
+        const previousEmptyFiles = canReuse ? storedEmptyFiles : [];
+        const reusableIndexedFiles = previousIndexedFiles.filter((file) => currentFiles.get(file.relativePath)?.checksum === file.checksum);
+        const reusableEmptyFiles = previousEmptyFiles.filter((file) => currentFiles.get(file.relativePath)?.checksum === file.checksum);
+        const reusablePaths = new Set([
+            ...reusableIndexedFiles.map((file) => file.relativePath),
+            ...reusableEmptyFiles.map((file) => file.relativePath),
+        ]);
+        const activeTableName = await activeIndexTableName(config);
+        const resumableTable = storedState
+            ? await openRowsTableByName(storedState.tableName, config, connection)
+            : null;
+        const resumableTableAvailable = !storedState?.files.some((file) => file.state === "indexed" && file.chunkCount > 0) ||
+            resumableTable !== null;
+        if (storedState &&
+            resumableTableAvailable &&
+            canResumeIngestion(storedState, files, policyFingerprint, options.rebuild === true)) {
+            state = resumeIngestionState(await reconcileCommittedFiles(storedState, config, connection));
+        }
+        else {
+            if (storedState?.mode === "rebuild" &&
+                storedState.tableName !== activeTableName &&
+                storedState.status !== "completed" &&
+                storedState.status !== "completed_with_errors") {
+                await dropRowsTable(storedState.tableName, config, connection);
+                await removeStagedIndexManifest(storedState.runId, config);
+            }
+            const mode = options.rebuild || (existingTable !== null && !manifestCompatible)
+                ? "rebuild"
+                : "incremental";
+            const reusableChunkCounts = new Map([
+                ...reusableIndexedFiles.map((file) => [file.relativePath, file.chunkCount]),
+                ...reusableEmptyFiles.map((file) => [file.relativePath, 0]),
+            ]);
+            state = createIngestionRunState({
+                mode,
+                tableName: activeTableName,
+                previousTableName: mode === "rebuild" ? activeTableName : null,
+                policyFingerprint,
+                batchSize: requestedBatchSize,
+                files,
+                reusablePaths: mode === "incremental" ? reusablePaths : new Set(),
+                reusableChunkCounts,
+            });
+            if (mode === "rebuild") {
+                state = {
+                    ...state,
+                    tableName: generationTableName(config.tableName, state.runId),
+                    files: state.files.map((file) => ({
+                        ...file,
+                        state: "pending",
+                        chunkCount: 0,
+                        reused: false,
+                    })),
+                };
+            }
         }
-        catch (error) {
+        await persistIngestionProgress(state, config, options);
+        throwIfAborted(signal);
+        const previousPaths = new Set(previousIndexedFiles.map((file) => file.relativePath));
+        const currentPaths = new Set(files.map((file) => file.relativePath));
+        const removedPaths = state.mode === "incremental"
+            ? [...previousPaths].filter((relativePath) => !currentPaths.has(relativePath))
+            : [];
+        const pendingFiles = state.files
+            .filter((file) => file.state !== "indexed")
+            .flatMap((file) => {
+            const source = currentFiles.get(file.relativePath);
+            return source ? [source] : [];
+        });
+        let removalApplied = false;
+        let lexicalIndexWarning = null;
+        for (const fileBatch of valueBatches(pendingFiles, state.batchSize)) {
+            throwIfAborted(signal);
+            const parsedBatch = await mapLimit(fileBatch, config.ingestConcurrency, signal, async (file) => parseSourceFile(file, config, signal));
+            for (const parsed of parsedBatch) {
+                state = updateIngestionFile(state, parsed.file.relativePath, parsed.error
+                    ? { state: "error", chunkCount: 0, redactions: 0, error: parsed.error }
+                    : {
+                        state: "parsed",
+                        chunkCount: parsed.chunks.length,
+                        redactions: parsed.redactions,
+                        error: null,
+                    });
+            }
+            await persistIngestionProgress(state, config, options);
+            throwIfAborted(signal);
+            const successfulFiles = parsedBatch.filter((parsed) => parsed.error === null);
+            const allChunks = successfulFiles.flatMap((parsed) => parsed.chunks);
+            const rows = await vectorRowsForChunks(allChunks, config, signal);
+            for (const parsed of successfulFiles) {
+                state = updateIngestionFile(state, parsed.file.relativePath, { state: "embedded" });
+            }
+            await persistIngestionProgress(state, config, options);
+            throwIfAborted(signal);
+            const replacePaths = [
+                ...fileBatch.map((file) => file.relativePath),
+                ...(!removalApplied ? removedPaths : []),
+            ];
+            const writeResult = await updateRowsInTable(rows, replacePaths, state.tableName, config, connection);
+            removalApplied = true;
+            lexicalIndexWarning ??= writeResult.lexicalIndexWarning;
+            for (const parsed of successfulFiles) {
+                state = updateIngestionFile(state, parsed.file.relativePath, { state: "indexed" });
+            }
+            await writeProgressManifest(state, config, connection);
+            await persistIngestionProgress(state, config, options);
             throwIfAborted(signal);
-            return {
-                path: file.relativePath,
-                chunks: [],
-                redactions: [],
-                error: {
-                    path: file.relativePath,
-                    message: error instanceof Error ? error.message : String(error),
-                },
-            };
         }
-    });
-    for (const result of results) {
-        if (result.error) {
-            errors.push(result.error);
-            continue;
+        if (!removalApplied && removedPaths.length > 0) {
+            const writeResult = await updateRowsInTable([], removedPaths, state.tableName, config, connection);
+            lexicalIndexWarning ??= writeResult.lexicalIndexWarning;
         }
-        redactionCounts.push(...result.redactions);
-        if (result.chunks.length === 0) {
-            emptyTextFiles.push(result.path);
+        const manifest = await manifestForState(state, config, connection);
+        await validateIngestionTable(state, manifest, config, connection);
+        await writeEmptyTextFiles(emptyTextRecords(state), config);
+        await writeIndexManifest(manifest, config);
+        await removeStagedIndexManifest(state.runId, config);
+        const errors = ingestionErrors(state);
+        state = finishIngestionState(state, errors.length > 0 ? "completed_with_errors" : "completed");
+        await persistIngestionProgress(state, config, options);
+        const indexedFiles = manifest.indexedFiles ?? [];
+        const emptyTextFiles = emptyTextRecords(state).map((file) => file.relativePath);
+        const redactions = state.files.reduce((sum, file) => sum + file.redactions, 0);
+        await recordAccess(config, {
+            action: "ingest",
+            resultCount: manifest.chunkCount,
+            redactions,
+        });
+        return {
+            runId: state.runId,
+            resumed: state.resumed,
+            batchSize: state.batchSize,
+            indexedFiles: indexedFiles.length,
+            rebuiltFiles: state.files.filter((file) => file.state === "indexed" && file.chunkCount > 0 && !file.reused).length,
+            reusedFiles: state.files.filter((file) => file.reused).length,
+            policyRebuild,
+            chunks: manifest.chunkCount,
+            discoveredFiles: inventory.discoveredFiles,
+            supportedFiles: files.length,
+            supportedBytes: inventoryMetrics.supportedBytes,
+            largestFileBytes: inventoryMetrics.largestFileBytes,
+            skippedFiles: inventory.skippedFiles.length + emptyTextFiles.length,
+            unsupportedFiles: countSkippedByReason(inventory.skippedFiles, "unsupported-extension"),
+            oversizedFiles: countSkippedByReason(inventory.skippedFiles, "oversized"),
+            sensitiveFiles: countSkippedByReason(inventory.skippedFiles, "sensitive-name"),
+            emptyTextFiles,
+            unsupportedExtensions: summarizeUnsupportedExtensions(inventory.skippedFiles),
+            redactions,
+            vectorIndexWarning: null,
+            lexicalIndexWarning,
+            errors,
+        };
+    }
+    catch (error) {
+        if (state?.status === "running") {
+            state = finishIngestionState(state, signal?.aborted ? "interrupted" : "failed");
+            try {
+                await writeIngestionState(state, config);
+            }
+            catch {
+                // Preserve the original ingestion failure.
+            }
         }
-        allChunks.push(...result.chunks);
+        throw error;
+    }
+}
+async function reconcileCommittedFiles(state, config, connection) {
+    const filesToReconcile = state.files.filter((file) => file.state !== "indexed" && file.state !== "error");
+    if (filesToReconcile.length === 0) {
+        return state;
+    }
+    const table = await openRowsTableByName(state.tableName, config, connection);
+    const rows = table
+        ? (await table.query().select(["relativePath", "checksum"]).toArray())
+        : [];
+    const rowCounts = new Map();
+    for (const row of rows) {
+        const key = `${row.relativePath}\0${row.checksum}`;
+        rowCounts.set(key, (rowCounts.get(key) ?? 0) + 1);
+    }
+    const committedEmptyFiles = state.mode === "incremental"
+        ? new Set((await readEmptyTextFiles(config)).map((file) => `${file.relativePath}\0${file.checksum}`))
+        : new Set();
+    return {
+        ...state,
+        files: state.files.map((file) => {
+            if (file.state === "indexed" || file.state === "error") {
+                return file;
+            }
+            const key = `${file.relativePath}\0${file.checksum}`;
+            const committed = file.chunkCount > 0 ? rowCounts.get(key) === file.chunkCount : committedEmptyFiles.has(key);
+            return committed ? { ...file, state: "indexed", error: null } : file;
+        }),
+    };
+}
+async function parseSourceFile(file, config, signal) {
+    try {
+        const parsed = await parseFile(file, { ...config, ...(signal ? { signal } : {}) });
+        throwIfAborted(signal);
+        const redacted = redactText(parsed.text, config);
+        return {
+            file,
+            chunks: chunkDocument({ ...parsed, text: redacted.text }, config.chunkSize, config.chunkOverlap),
+            redactions: totalRedactions(redacted.counts),
+            error: null,
+        };
+    }
+    catch (error) {
+        throwIfAborted(signal);
+        return {
+            file,
+            chunks: [],
+            redactions: 0,
+            error: error instanceof Error ? error.message : String(error),
+        };
     }
+}
+async function vectorRowsForChunks(chunks, config, signal) {
     const rows = [];
-    for (let i = 0; i < allChunks.length; i += config.embeddingBatchSize) {
+    for (const batch of valueBatches(chunks, config.embeddingBatchSize)) {
         throwIfAborted(signal);
-        const batch = allChunks.slice(i, i + config.embeddingBatchSize);
         const embeddings = await embedTexts(batch.map(chunkSearchText), config);
         throwIfAborted(signal);
         for (const [index, chunk] of batch.entries()) {
@@ -124,118 +299,129 @@ async function ingestUnlocked(config, options, connection, signal) {
             });
         }
     }
-    const rebuiltIndexedFiles = indexedFileRecords(rows);
-    const indexedFiles = [...reusableIndexedFiles, ...rebuiltIndexedFiles].sort((a, b) => a.relativePath.localeCompare(b.relativePath));
+    return rows;
+}
+async function persistIngestionProgress(state, config, options) {
+    await writeIngestionState(state, config);
+    await options.onProgress?.(ingestionProgress(state));
+}
+async function writeProgressManifest(state, config, connection) {
+    const manifest = await manifestForState(state, config, connection);
+    if (state.mode === "rebuild") {
+        await writeStagedIndexManifest(manifest, state.runId, config);
+        return;
+    }
+    await writeEmptyTextFiles(emptyTextRecords(state), config);
+    await writeIndexManifest(manifest, config);
+}
+async function manifestForState(state, config, connection) {
+    const indexedFiles = indexedFilesFromState(state);
     const chunkCount = indexedFiles.reduce((sum, file) => sum + file.chunkCount, 0);
-    const previousPaths = new Set(previousIndexedFiles.map((file) => file.relativePath));
-    const currentPaths = new Set(files.map((file) => file.relativePath));
-    const replacePaths = [
-        ...filesToIndex.map((file) => file.relativePath),
-        ...[...previousPaths].filter((relativePath) => !currentPaths.has(relativePath)),
-    ];
-    throwIfAborted(signal);
-    const writeResult = !canReuse || chunkCount === 0
-        ? await writeRows(rows, config, connection)
-        : replacePaths.length > 0
-            ? await updateRows(rows, replacePaths, config, connection)
-            : { vectorIndexWarning: null, lexicalIndexWarning: null };
-    if (chunkCount > 0) {
-        const firstRow = rows[0] ?? (await firstStoredRow(config, connection));
-        if (!firstRow) {
-            throw new Error("Cannot write an index manifest without indexed rows.");
-        }
-        await writeIndexManifest({
-            schemaVersion: INDEX_SCHEMA_VERSION,
-            createdAt: new Date().toISOString(),
-            ragmirVersion: VERSION,
-            embeddingProvider: config.embeddingProvider,
-            embeddingModel: config.embeddingModel,
-            indexPolicyFingerprint: policyFingerprint,
-            vectorDimension: firstRow.vector.length,
-            vectorDistanceMetric: VECTOR_DISTANCE_METRIC,
-            chunkSize: config.chunkSize,
-            chunkOverlap: config.chunkOverlap,
-            fileCount: indexedFiles.length,
-            chunkCount,
-            indexedFiles,
-        }, config);
+    const table = await openRowsTableByName(state.tableName, config, connection);
+    const [firstRow] = table ? (await table.query().limit(1).toArray()) : [];
+    if (chunkCount > 0 && !firstRow) {
+        throw new Error("Cannot write an index manifest without indexed rows.");
     }
-    await writeEmptyTextFiles([
-        ...reusableEmptyFiles,
-        ...emptyTextFiles.flatMap((relativePath) => {
-            const file = currentFiles.get(relativePath);
-            return file
-                ? [
-                    {
-                        relativePath,
-                        checksum: file.checksum,
-                        bytes: file.bytes,
-                        mtimeMs: file.mtimeMs,
-                    },
-                ]
-                : [];
-        }),
-    ], config);
-    await recordAccess(config, {
-        action: "ingest",
-        resultCount: chunkCount,
-        redactions: totalRedactions(redactionCounts),
-    });
     return {
-        indexedFiles: indexedFiles.length,
-        rebuiltFiles: new Set(rows.map((row) => row.relativePath)).size,
-        reusedFiles: reusableFiles.size,
-        policyRebuild,
-        chunks: chunkCount,
-        discoveredFiles: inventory.discoveredFiles,
-        supportedFiles: files.length,
-        supportedBytes: inventoryMetrics.supportedBytes,
-        largestFileBytes: inventoryMetrics.largestFileBytes,
-        skippedFiles: inventory.skippedFiles.length + emptyTextFiles.length,
-        unsupportedFiles: countSkippedByReason(inventory.skippedFiles, "unsupported-extension"),
-        oversizedFiles: countSkippedByReason(inventory.skippedFiles, "oversized"),
-        sensitiveFiles: countSkippedByReason(inventory.skippedFiles, "sensitive-name"),
-        emptyTextFiles: [
-            ...reusableEmptyFiles.map((file) => file.relativePath),
-            ...emptyTextFiles,
-        ].sort(),
-        unsupportedExtensions: summarizeUnsupportedExtensions(inventory.skippedFiles),
-        redactions: totalRedactions(redactionCounts),
-        vectorIndexWarning: writeResult.vectorIndexWarning,
-        lexicalIndexWarning: writeResult.lexicalIndexWarning,
-        errors,
+        schemaVersion: INDEX_SCHEMA_VERSION,
+        createdAt: new Date().toISOString(),
+        ragmirVersion: VERSION,
+        embeddingProvider: config.embeddingProvider,
+        embeddingModel: config.embeddingModel,
+        indexPolicyFingerprint: state.policyFingerprint,
+        ...(firstRow ? { vectorDimension: firstRow.vector.length } : {}),
+        vectorDistanceMetric: VECTOR_DISTANCE_METRIC,
+        chunkSize: config.chunkSize,
+        chunkOverlap: config.chunkOverlap,
+        fileCount: indexedFiles.length,
+        chunkCount,
+        tableName: state.tableName,
+        indexedFiles,
     };
 }
-function indexedFileRecords(rows) {
-    const records = new Map();
+function indexedFilesFromState(state) {
+    return state.files
+        .filter((file) => file.state === "indexed" && file.chunkCount > 0)
+        .map((file) => ({
+        relativePath: file.relativePath,
+        checksum: file.checksum,
+        chunkCount: file.chunkCount,
+        bytes: file.bytes,
+        mtimeMs: file.mtimeMs,
+    }))
+        .sort((left, right) => left.relativePath.localeCompare(right.relativePath));
+}
+function emptyTextRecords(state) {
+    return state.files
+        .filter((file) => file.state === "indexed" && file.chunkCount === 0)
+        .map((file) => ({
+        relativePath: file.relativePath,
+        checksum: file.checksum,
+        bytes: file.bytes,
+        mtimeMs: file.mtimeMs,
+    }))
+        .sort((left, right) => left.relativePath.localeCompare(right.relativePath));
+}
+function ingestionErrors(state) {
+    return state.files.flatMap((file) => file.state === "error" && file.error ? [{ path: file.relativePath, message: file.error }] : []);
+}
+async function validateIngestionTable(state, manifest, config, connection) {
+    const table = await openRowsTableByName(state.tableName, config, connection);
+    if (!table) {
+        if (manifest.chunkCount === 0) {
+            return;
+        }
+        throw new Error("Ingestion validation failed because the generated table is missing.");
+    }
+    const rows = (await table.query().select(["id", "relativePath", "checksum"]).toArray());
+    if (rows.length !== manifest.chunkCount) {
+        throw new Error(`Ingestion validation expected ${manifest.chunkCount} rows but found ${rows.length}.`);
+    }
+    if (new Set(rows.map((row) => row.id)).size !== rows.length) {
+        throw new Error("Ingestion validation found duplicate chunk identifiers.");
+    }
+    const expectedFiles = new Map((manifest.indexedFiles ?? []).map((file) => [
+        `${file.relativePath}\0${file.checksum}`,
+        file.chunkCount,
+    ]));
+    const actualFiles = new Map();
     for (const row of rows) {
-        const current = records.get(row.relativePath);
-        records.set(row.relativePath, {
-            relativePath: row.relativePath,
-            checksum: row.checksum,
-            chunkCount: (current?.chunkCount ?? 0) + 1,
-            bytes: row.bytes,
-            mtimeMs: row.mtimeMs,
-        });
+        const key = `${row.relativePath}\0${row.checksum}`;
+        actualFiles.set(key, (actualFiles.get(key) ?? 0) + 1);
+    }
+    if (expectedFiles.size !== actualFiles.size ||
+        [...expectedFiles].some(([key, count]) => actualFiles.get(key) !== count)) {
+        throw new Error("Ingestion validation found rows that do not match the generated manifest.");
     }
-    return [...records.values()];
 }
-async function firstStoredRow(config, connection) {
-    const table = await openRowsTable(config, connection);
-    if (!table) {
-        return null;
+function ingestFileBatchSize(value) {
+    const batchSize = value ?? DEFAULT_INGEST_FILE_BATCH_SIZE;
+    if (!Number.isInteger(batchSize) || batchSize <= 0) {
+        throw new Error("batchSize must be a positive integer.");
+    }
+    return batchSize;
+}
+function valueBatches(values, batchSize) {
+    const batches = [];
+    for (let index = 0; index < values.length; index += batchSize) {
+        batches.push(values.slice(index, index + batchSize));
     }
-    const [row] = (await table.query().limit(1).toArray());
-    return row ?? null;
+    return batches;
 }
-export async function audit(cwd = process.cwd()) {
+export async function audit(cwd = process.cwd(), options = {}) {
+    const signal = operationSignal(options);
+    throwIfAborted(signal);
     const config = await loadConfig(cwd);
-    const inventory = await inventorySourceFiles(config);
+    throwIfAborted(signal);
+    const inventory = await inventorySourceFiles(config, signal ? { signal } : {});
+    throwIfAborted(signal);
     const files = inventory.supportedFiles;
     const inventoryMetrics = sourceInventoryMetrics(files);
     const supportedFiles = files.map((file) => file.relativePath);
     const table = await openRowsTable(config);
-    const emptyTextFiles = await currentEmptyTextFiles(config, files);
+    throwIfAborted(signal);
+    const emptyTextFiles = await currentEmptyTextFiles(config, files, signal);
+    throwIfAborted(signal);
     if (!table) {
         return {
             discoveredFiles: inventory.discoveredFiles,
@@ -257,9 +443,11 @@ export async function audit(cwd = process.cwd()) {
         .query()
         .select(["relativePath", "checksum", "contextPath", "text"])
         .toArray());
+    throwIfAborted(signal);
     const counts = new Map();
     const checksums = new Map();
     for (const row of rows) {
+        throwIfAborted(signal);
         counts.set(row.relativePath, (counts.get(row.relativePath) ?? 0) + 1);
         if (row.checksum) {
             const fileChecksums = checksums.get(row.relativePath) ?? new Set();
@@ -270,6 +458,7 @@ export async function audit(cwd = process.cwd()) {
     const supportedSet = new Set(supportedFiles);
     const indexedSet = new Set(counts.keys());
     const currentChecksums = new Map(files.map((file) => [file.relativePath, file.checksum]));
+    throwIfAborted(signal);
     return {
         discoveredFiles: inventory.discoveredFiles,
         supportedBytes: inventoryMetrics.supportedBytes,
@@ -343,10 +532,14 @@ function groupedDuplicates(groups) {
         .filter(([, files]) => files.length > 1)
         .map(([key, files]) => ({ key, files: [...new Set(files)].sort() }));
 }
-async function currentEmptyTextFiles(config, files) {
+async function currentEmptyTextFiles(config, files, signal) {
+    throwIfAborted(signal);
     const currentChecksums = new Map(files.map((file) => [file.relativePath, file.checksum]));
     const emptyTextFiles = new Set();
-    for (const record of await readEmptyTextFiles(config)) {
+    const records = await readEmptyTextFiles(config);
+    throwIfAborted(signal);
+    for (const record of records) {
+        throwIfAborted(signal);
         if (currentChecksums.get(record.relativePath) === record.checksum) {
             emptyTextFiles.add(record.relativePath);
         }