@autorag/librarian 2.13.5 → 2.13.6

package/dist/mcp/index.js

@@ -10374,195 +10374,225 @@ async function syncWithRegistry(options, registry, isolated) {
     });
     const pending = new Set(plan.decisions.filter((decision) => decision.action === "process" || decision.action === "retry").map((decision) => decision.key));
     const deferred = new Map(plan.decisions.filter((decision) => decision.deferred === true).map((decision) => [decision.key, decision]));
-    for (const decision of plan.decisions) {
-      if (decision.action !== "delete")
-        continue;
-      const path = pathsByKey.get(decision.key);
-      await session.delete(decision.key);
-      if (path === undefined)
-        continue;
-      if (previous.entries[path])
-        deleted += 1;
-      if (!currentPaths.has(path))
-        diagnostics.push({
-          code: "deleted-mirror",
-          severity: "info",
-          message: "A previously indexed document is gone; its parsed mirror was removed.",
-          source: path
-        });
-    }
-    await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
-    for (const [position, entry] of current.entries()) {
-      options.onProgress?.({
-        processed: position,
-        total: current.length,
-        written,
-        deleted,
-        skipped,
-        currentVirtualPath: entry.virtualPath
-      });
-      if (options.signal?.aborted)
-        throw abortReason(options.signal);
-      const omission = omitted.get(entry.virtualPath);
-      if (omission !== undefined) {
-        if (pending.has(mirrorStageKey(entry.virtualPath))) {
-          if (previous.entries[entry.virtualPath])
-            deleted += 1;
-          await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
-        }
-        skipped += 1;
-        if (omission.diagnostic)
-          diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
-        continue;
-      }
-      const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
-      if (deferredFailure !== undefined) {
-        skipped += 1;
-        const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
-        diagnostics.push({
-          code,
-          severity: "warning",
-          source: entry.virtualPath,
-          message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
-        });
-        continue;
-      }
-      const parser = registry.getForVirtualPath(entry.virtualPath);
-      const outputPath = parsedOutputPath(options.root, entry.virtualPath);
-      const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname12(entry.sourcePath)) : { kind: "local" };
-      if (cloudDecision.kind === "skip") {
-        skipped += 1;
-        diagnostics.push({
-          code: "cloud-placeholder-skipped",
-          severity: "info",
-          message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
-          source: entry.virtualPath,
-          ...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
-        });
-        continue;
-      }
-      if (cloudDecision.kind === "defer") {
-        skipped += 1;
-        diagnostics.push({
-          code: "cloud-download-deferred",
-          severity: "info",
-          message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
-          source: entry.virtualPath
-        });
-        continue;
-      }
-      if (pending.has(mirrorStageKey(entry.virtualPath))) {
-        let bytes;
-        try {
-          bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
-        } catch (error) {
-          const failure = sourceReadFailure(error);
-          await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
+    const applyDeleteDecisions = async () => {
+      for (const decision of plan.decisions) {
+        if (decision.action !== "delete")
           continue;
-        }
-        const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
-        if (tooSmallImage !== undefined) {
-          await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
+        const path = pathsByKey.get(decision.key);
+        await session.delete(decision.key);
+        if (path === undefined)
           continue;
-        }
-        let parsed;
-        try {
-          const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
-          parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
-        } catch (error) {
-          if (options.signal?.aborted)
-            throw abortReason(options.signal);
-          if (error instanceof ParseError)
-            await skipFile(entry, {
-              code: "parser-failed",
-              severity: "warning",
-              message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
-            }, "parser-failed");
-          else
-            await skipFile(entry, {
-              code: "parser-crashed",
-              severity: "warning",
-              message: PARSER_CRASH_MESSAGE,
-              ...error instanceof Error ? { reason: error.name } : {}
-            });
+        if (previous.entries[path])
+          deleted += 1;
+        if (!currentPaths.has(path))
+          diagnostics.push({
+            code: "deleted-mirror",
+            severity: "info",
+            message: "A previously indexed document is gone; its parsed mirror was removed.",
+            source: path
+          });
+      }
+    };
+    const deferDeletions = options.onBatch !== undefined;
+    const batchSize = Math.max(1, Math.floor(options.batchSize ?? DEFAULT_MIRROR_BATCH_SIZE));
+    const batchCount = Math.max(1, Math.ceil(current.length / batchSize));
+    let batchIndex = 0;
+    let batchStopped = false;
+    const runBatch = async (virtualPaths, final) => {
+      if (options.onBatch === undefined)
+        return;
+      batchIndex += 1;
+      const outcome = await options.onBatch({ index: batchIndex, total: batchCount, virtualPaths, final });
+      if (outcome?.stop === true)
+        batchStopped = true;
+    };
+    if (!deferDeletions)
+      await applyDeleteDecisions();
+    await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
+    for (let batchStart = 0;batchStart < Math.max(current.length, 1); batchStart += batchSize) {
+      const batchEnd = Math.min(batchStart + batchSize, current.length);
+      const finalBatch = batchEnd >= current.length;
+      const changedBefore = written + deleted;
+      for (let position = batchStart;position < batchEnd; position += 1) {
+        const entry = current[position];
+        options.onProgress?.({
+          processed: position,
+          total: current.length,
+          written,
+          deleted,
+          skipped,
+          currentVirtualPath: entry.virtualPath
+        });
+        if (options.signal?.aborted)
+          throw abortReason(options.signal);
+        const omission = omitted.get(entry.virtualPath);
+        if (omission !== undefined) {
+          if (pending.has(mirrorStageKey(entry.virtualPath))) {
+            if (previous.entries[entry.virtualPath])
+              deleted += 1;
+            await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
+          }
+          skipped += 1;
+          if (omission.diagnostic)
+            diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
           continue;
         }
-        let info;
-        try {
-          info = await stat3(entry.sourcePath, { bigint: true });
-        } catch (error) {
-          await skipFile(entry, sourceReadFailure(error));
+        const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
+        if (deferredFailure !== undefined) {
+          skipped += 1;
+          const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
+          diagnostics.push({
+            code,
+            severity: "warning",
+            source: entry.virtualPath,
+            message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
+          });
           continue;
         }
-        const unchanged = cloudDecision.kind === "download" ? contentIdentity(info) === entry.contentIdentity : statIdentity(info) === entry.statIdentity;
-        if (!unchanged) {
+        const parser = registry.getForVirtualPath(entry.virtualPath);
+        const outputPath = parsedOutputPath(options.root, entry.virtualPath);
+        const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname12(entry.sourcePath)) : { kind: "local" };
+        if (cloudDecision.kind === "skip") {
           skipped += 1;
           diagnostics.push({
-            code: "source-changed",
-            severity: "warning",
-            message: "The source changed while parsing; its last good mirror was kept and it will be retried on the next refresh.",
-            source: entry.virtualPath
+            code: "cloud-placeholder-skipped",
+            severity: "info",
+            message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
+            source: entry.virtualPath,
+            ...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
           });
-          await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
           continue;
         }
-        const markdown = normalizeMarkdown(parsed.markdown);
-        try {
-          await withTransientRetry(() => writeFileAtomic(outputPath, markdown), transientRetries, transientBackoffMs);
-        } catch (error) {
-          const code = errorCode(error);
-          if (code !== undefined && Object.hasOwn(WORKSPACE_ERROR_CODES, code))
-            throw new MirrorWorkspaceError(options.root, error);
-          await skipFile(entry, {
-            code: "mirror-write-failed",
-            severity: "warning",
-            message: error instanceof Error ? error.message : String(error),
-            ...code !== undefined ? { reason: code } : {}
+        if (cloudDecision.kind === "defer") {
+          skipped += 1;
+          diagnostics.push({
+            code: "cloud-download-deferred",
+            severity: "info",
+            message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
+            source: entry.virtualPath
           });
           continue;
         }
-        await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
-          path: basename10(outputPath),
-          size: Buffer.byteLength(markdown)
-        });
-        for (const diagnostic of parsed.diagnostics ?? [])
-          diagnostics.push({ ...diagnostic, source: entry.virtualPath });
-        written += 1;
-        if (fingerprintCloud && cloudFilePaths.has(entry.sourcePath)) {
-          const existing = cloudFingerprintEntries[entry.sourcePath];
-          if (existing === undefined || existing.sourceVersion !== entry.sourceVersion) {
-            try {
-              const fingerprintResult = await fingerprintWithDupey(entry.sourcePath, options.cloudFingerprints?.dupey ?? {});
-              cloudFingerprintEntries[entry.sourcePath] = {
-                sourceVersion: entry.sourceVersion,
-                exact: fingerprintResult.exact,
-                chars: fingerprintResult.chars,
-                modified: fingerprintResult.modified,
-                revision: fingerprintResult.revision
-              };
-              cloudFingerprintsDirty = true;
-            } catch (error) {
-              diagnostics.push({
-                code: "cloud-fingerprint-failed",
+        if (pending.has(mirrorStageKey(entry.virtualPath))) {
+          let bytes;
+          try {
+            bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
+          } catch (error) {
+            const failure = sourceReadFailure(error);
+            await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
+            continue;
+          }
+          const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
+          if (tooSmallImage !== undefined) {
+            await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
+            continue;
+          }
+          let parsed;
+          try {
+            const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
+            parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
+          } catch (error) {
+            if (options.signal?.aborted)
+              throw abortReason(options.signal);
+            if (error instanceof ParseError)
+              await skipFile(entry, {
+                code: "parser-failed",
                 severity: "warning",
-                message: `The cloud or network file was indexed but its duplicate fingerprint could not be read, so it will not take part in exact-duplicate exclusion: ${error instanceof Error ? error.message : String(error)}`,
-                source: entry.virtualPath
+                message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
+              }, "parser-failed");
+            else
+              await skipFile(entry, {
+                code: "parser-crashed",
+                severity: "warning",
+                message: PARSER_CRASH_MESSAGE,
+                ...error instanceof Error ? { reason: error.name } : {}
               });
-            }
+            continue;
           }
-        }
-        if (cloudDecision.kind === "download" && gate.evictAfterParse) {
-          const released = await gate.release(entry.sourcePath);
-          if (!released.ok) {
+          let info;
+          try {
+            info = await stat3(entry.sourcePath, { bigint: true });
+          } catch (error) {
+            await skipFile(entry, sourceReadFailure(error));
+            continue;
+          }
+          const unchanged = cloudDecision.kind === "download" ? contentIdentity(info) === entry.contentIdentity : statIdentity(info) === entry.statIdentity;
+          if (!unchanged) {
+            skipped += 1;
             diagnostics.push({
-              code: "cloud-evict-failed",
+              code: "source-changed",
               severity: "warning",
-              message: `The file was indexed but its local copy could not be released: ${released.message}`,
+              message: "The source changed while parsing; its last good mirror was kept and it will be retried on the next refresh.",
               source: entry.virtualPath
             });
+            await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
+            continue;
+          }
+          const markdown = normalizeMarkdown(parsed.markdown);
+          try {
+            await withTransientRetry(() => writeFileAtomic(outputPath, markdown), transientRetries, transientBackoffMs);
+          } catch (error) {
+            const code = errorCode(error);
+            if (code !== undefined && Object.hasOwn(WORKSPACE_ERROR_CODES, code))
+              throw new MirrorWorkspaceError(options.root, error);
+            await skipFile(entry, {
+              code: "mirror-write-failed",
+              severity: "warning",
+              message: error instanceof Error ? error.message : String(error),
+              ...code !== undefined ? { reason: code } : {}
+            });
+            continue;
+          }
+          await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
+            path: basename10(outputPath),
+            size: Buffer.byteLength(markdown)
+          });
+          for (const diagnostic of parsed.diagnostics ?? [])
+            diagnostics.push({ ...diagnostic, source: entry.virtualPath });
+          written += 1;
+          if (fingerprintCloud && cloudFilePaths.has(entry.sourcePath)) {
+            const existing = cloudFingerprintEntries[entry.sourcePath];
+            if (existing === undefined || existing.sourceVersion !== entry.sourceVersion) {
+              try {
+                const fingerprintResult = await fingerprintWithDupey(entry.sourcePath, options.cloudFingerprints?.dupey ?? {});
+                cloudFingerprintEntries[entry.sourcePath] = {
+                  sourceVersion: entry.sourceVersion,
+                  exact: fingerprintResult.exact,
+                  chars: fingerprintResult.chars,
+                  modified: fingerprintResult.modified,
+                  revision: fingerprintResult.revision
+                };
+                cloudFingerprintsDirty = true;
+              } catch (error) {
+                diagnostics.push({
+                  code: "cloud-fingerprint-failed",
+                  severity: "warning",
+                  message: `The cloud or network file was indexed but its duplicate fingerprint could not be read, so it will not take part in exact-duplicate exclusion: ${error instanceof Error ? error.message : String(error)}`,
+                  source: entry.virtualPath
+                });
+              }
+            }
+          }
+          if (cloudDecision.kind === "download" && gate.evictAfterParse) {
+            const released = await gate.release(entry.sourcePath);
+            if (!released.ok) {
+              diagnostics.push({
+                code: "cloud-evict-failed",
+                severity: "warning",
+                message: `The file was indexed but its local copy could not be released: ${released.message}`,
+                source: entry.virtualPath
+              });
+            }
           }
         }
       }
+      if (finalBatch && deferDeletions)
+        await applyDeleteDecisions();
+      if (finalBatch || written + deleted !== changedBefore) {
+        await runBatch(current.slice(batchStart, batchEnd).map((item) => item.virtualPath), finalBatch);
+      }
+      if (batchStopped)
+        break;
     }
     await session.finish();
     if (fingerprintCloud) {
@@ -10854,7 +10884,7 @@ async function collectFiles(state, directory, depth) {
     warnDirectoryUnreadable(state, directory, code);
   }
 }
-var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
+var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
 var init_sync = __esm(() => {
   init_cli2();
   init_cloud_placeholder();
@@ -38693,11 +38723,52 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
     };
     try {
       writeRefreshProgress(this.workspaceProjectRoot, progress);
+      const batchedMinSync = needsParsed && wants("minsync");
+      const minSyncBatches = [];
+      const minSyncBatchFailures = [];
+      let minSyncSyncedBase = 0;
+      let minSyncTotalBase = 0;
+      const minSyncBatch = async (batch) => {
+        signal?.throwIfAborted();
+        progress = updateRefreshProgress(progress, { phase: "minsync" });
+        writeRefreshProgress(this.workspaceProjectRoot, progress);
+        let batchTotal = minSyncTotalBase;
+        const result = await this.syncMinSync(force && batch.index === 1, signal, (minsyncProgress) => {
+          batchTotal = minSyncTotalBase + minsyncProgress.total;
+          progress = updateRefreshProgress(progress, {
+            minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: batchTotal }
+          });
+          writeRefreshProgress(this.workspaceProjectRoot, progress);
+          emitProgress({
+            phase: "minsync",
+            processed: minSyncSyncedBase + minsyncProgress.processed,
+            total: batchTotal
+          });
+        });
+        minSyncBatches.push(result);
+        if (result.ok)
+          minSyncBatchFailures.length = 0;
+        else if (result.reason !== "lock-held") {
+          for (const virtualPath of batch.virtualPaths) {
+            minSyncBatchFailures.push({
+              code: "minsync-sync-failed",
+              severity: "warning",
+              message: `This file stayed at its previous index entry: its batch could not be indexed and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
+              source: virtualPath
+            });
+          }
+        }
+        minSyncSyncedBase = batchTotal;
+        minSyncTotalBase = batchTotal;
+        return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
+      };
       const summary = needsParsed ? await this.syncParsedMirrors(force, {
         ...opts?.retrySkipped === true ? { retrySkipped: true } : {},
         ...signal === undefined ? {} : { signal },
+        ...batchedMinSync ? { onBatch: minSyncBatch } : {},
         onProgress: (parsedProgress) => {
           progress = updateRefreshProgress(progress, {
+            phase: "parsed",
             sourceFiles: { total: parsedProgress.total },
             parsedCounts: {
               scanned: parsedProgress.processed,
@@ -38719,7 +38790,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
       }) : await this.scanMirrorStaleness();
       signal?.throwIfAborted();
       progress = updateRefreshProgress(progress, {
-        phase: wants("minsync") ? "minsync" : wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
+        phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
         sourceFiles: { total: summary.scanned },
         parsedCounts: {
           scanned: summary.scanned,
@@ -38729,24 +38800,12 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
         }
       });
       writeRefreshProgress(this.workspaceProjectRoot, progress);
-      const minsync = wants("minsync") ? await this.syncMinSync(force, signal, (minsyncProgress) => {
-        progress = updateRefreshProgress(progress, {
-          minsync: { synced: minsyncProgress.processed, total: minsyncProgress.total }
-        });
-        writeRefreshProgress(this.workspaceProjectRoot, progress);
-        emitProgress({
-          phase: "minsync",
-          processed: minsyncProgress.processed,
-          total: minsyncProgress.total
-        });
-      }) : undefined;
-      signal?.throwIfAborted();
+      const minsync = batchedMinSync ? aggregateMinSyncBatches(minSyncBatches) : undefined;
       if (minsync !== undefined) {
         progress = updateRefreshProgress(progress, {
-          phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
           minsync: {
-            ...progress.minsync,
             synced: minsync.synced,
+            total: Math.max(minSyncTotalBase, minsync.synced),
             ...minsync.filesFailed !== undefined && minsync.filesFailed.length > 0 ? { filesSkipped: minsync.filesFailed.length } : {}
           }
         });
@@ -38808,7 +38867,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
         finishedAt: new Date().toISOString()
       });
       writeRefreshProgress(this.workspaceProjectRoot, progress);
-      const minsyncDiagnostics = minSyncRefreshDiagnostics(minsync);
+      const minsyncDiagnostics = [...minSyncRefreshDiagnostics(minsync), ...minSyncBatchFailures];
       const publicMinsync = minsync ? {
         ok: minsync.ok,
         synced: minsync.synced,
@@ -39613,6 +39672,12 @@ function fsearchRefreshDiagnostics(fsearch) {
     }
   ];
 }
+function aggregateMinSyncBatches(batches) {
+  const last = batches.at(-1);
+  if (last === undefined)
+    return;
+  return { ...last, synced: batches.reduce((total, batch) => total + batch.synced, 0) };
+}
 function minSyncRefreshDiagnostics(minsync) {
   if (!minsync)
     return [];

package/dist/mirror/sync.d.ts

@@ -10,6 +10,12 @@ export declare const DEFAULT_MAX_DELETE_RATIO = 0.3;
 export declare const DEFAULT_MAX_SCANNED_ENTRIES = 2000000;
 /** Deepest directory level (search root = 0) the walk descends into. */
 export declare const DEFAULT_MAX_WALK_DEPTH = 64;
+/**
+ * Sources parsed and committed per batch before {@link ParsedMirrorSyncOptions.onBatch}
+ * runs. Small enough that a corpus becomes searchable after its first slice,
+ * large enough to amortize one index sync's startup.
+ */
+export declare const DEFAULT_MIRROR_BATCH_SIZE = 25;
 export interface ParsedMirrorSyncOptions {
     readonly root: string;
     readonly searchPaths: readonly string[];
@@ -36,6 +42,20 @@ export interface ParsedMirrorSyncOptions {
     readonly signal?: AbortSignal;
     /** Called before each file and once more with `processed === total` when the parse phase ends. */
     readonly onProgress?: (progress: ParsedMirrorProgress) => void;
+    /**
+     * Sources parsed and committed per batch before {@link ParsedMirrorSyncOptions.onBatch}
+     * runs. Default {@link DEFAULT_MIRROR_BATCH_SIZE}.
+     */
+    readonly batchSize?: number;
+    /**
+     * Runs after each batch of sources is parsed and committed to the mirror, and
+     * before the next batch is read, so a caller can make the batch's results
+     * searchable (stage them and sync the index). A thrown error aborts the
+     * refresh; returning `stop` ends the loop after this batch. Deletions of
+     * genuinely-removed sources wait for the final batch, so a mid-refresh call
+     * never drops a not-yet-reached document from the index.
+     */
+    readonly onBatch?: (batch: ParsedMirrorBatch) => Promise<ParsedMirrorBatchOutcome | undefined>;
     /**
      * Largest share of previously indexed documents one refresh may delete because their source
      * disappeared. A larger deletion (above 100 documents) is suppressed with `mass-delete-suppressed`
@@ -115,6 +135,32 @@ export interface ParsedMirrorSyncResult {
     readonly indexPath: string;
     readonly diagnostics: readonly ParsedMirrorDiagnostic[];
 }
+/**
+ * One batch of the source listing, parsed and committed to the mirror before
+ * {@link ParsedMirrorSyncOptions.onBatch} runs. The last batch is the only one
+ * whose run reflects deletions of genuinely-removed sources.
+ */
+export interface ParsedMirrorBatch {
+    /** 1-based index of this batch among the batches handed to `onBatch`. */
+    readonly index: number;
+    /**
+     * Batches the listing was divided into. A batch that wrote and deleted no
+     * mirror output is not handed to `onBatch`; the final batch always is.
+     */
+    readonly total: number;
+    /** Virtual paths the refresh attempted in this batch. */
+    readonly virtualPaths: readonly string[];
+    /** The last batch: genuine deletions have by now been applied to the mirror. */
+    readonly final: boolean;
+}
+/**
+ * What {@link ParsedMirrorSyncOptions.onBatch} asks the refresh to do next.
+ * A thrown error aborts the refresh; `stop` ends the loop after this batch, so a
+ * systemic index failure never reads sources it knows it cannot index.
+ */
+export interface ParsedMirrorBatchOutcome {
+    readonly stop?: boolean;
+}
 /** Default retries for a transient per-file I/O error. Mirrors the `refresh.transientRetries` config default. */
 export declare const DEFAULT_TRANSIENT_RETRIES = 3;
 /**