@autorag/librarian 2.13.5 → 2.13.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -10373,195 +10373,225 @@ async function syncWithRegistry(options, registry, isolated) {
10373
10373
  });
10374
10374
  const pending = new Set(plan.decisions.filter((decision) => decision.action === "process" || decision.action === "retry").map((decision) => decision.key));
10375
10375
  const deferred = new Map(plan.decisions.filter((decision) => decision.deferred === true).map((decision) => [decision.key, decision]));
10376
- for (const decision of plan.decisions) {
10377
- if (decision.action !== "delete")
10378
- continue;
10379
- const path = pathsByKey.get(decision.key);
10380
- await session.delete(decision.key);
10381
- if (path === undefined)
10382
- continue;
10383
- if (previous.entries[path])
10384
- deleted += 1;
10385
- if (!currentPaths.has(path))
10386
- diagnostics.push({
10387
- code: "deleted-mirror",
10388
- severity: "info",
10389
- message: "A previously indexed document is gone; its parsed mirror was removed.",
10390
- source: path
10391
- });
10392
- }
10393
- await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
10394
- for (const [position, entry] of current.entries()) {
10395
- options.onProgress?.({
10396
- processed: position,
10397
- total: current.length,
10398
- written,
10399
- deleted,
10400
- skipped,
10401
- currentVirtualPath: entry.virtualPath
10402
- });
10403
- if (options.signal?.aborted)
10404
- throw abortReason(options.signal);
10405
- const omission = omitted.get(entry.virtualPath);
10406
- if (omission !== undefined) {
10407
- if (pending.has(mirrorStageKey(entry.virtualPath))) {
10408
- if (previous.entries[entry.virtualPath])
10409
- deleted += 1;
10410
- await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
10411
- }
10412
- skipped += 1;
10413
- if (omission.diagnostic)
10414
- diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
10415
- continue;
10416
- }
10417
- const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
10418
- if (deferredFailure !== undefined) {
10419
- skipped += 1;
10420
- const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
10421
- diagnostics.push({
10422
- code,
10423
- severity: "warning",
10424
- source: entry.virtualPath,
10425
- message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
10426
- });
10427
- continue;
10428
- }
10429
- const parser = registry.getForVirtualPath(entry.virtualPath);
10430
- const outputPath = parsedOutputPath(options.root, entry.virtualPath);
10431
- const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname12(entry.sourcePath)) : { kind: "local" };
10432
- if (cloudDecision.kind === "skip") {
10433
- skipped += 1;
10434
- diagnostics.push({
10435
- code: "cloud-placeholder-skipped",
10436
- severity: "info",
10437
- message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
10438
- source: entry.virtualPath,
10439
- ...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
10440
- });
10441
- continue;
10442
- }
10443
- if (cloudDecision.kind === "defer") {
10444
- skipped += 1;
10445
- diagnostics.push({
10446
- code: "cloud-download-deferred",
10447
- severity: "info",
10448
- message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
10449
- source: entry.virtualPath
10450
- });
10451
- continue;
10452
- }
10453
- if (pending.has(mirrorStageKey(entry.virtualPath))) {
10454
- let bytes;
10455
- try {
10456
- bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
10457
- } catch (error) {
10458
- const failure = sourceReadFailure(error);
10459
- await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
10376
+ const applyDeleteDecisions = async () => {
10377
+ for (const decision of plan.decisions) {
10378
+ if (decision.action !== "delete")
10460
10379
  continue;
10461
- }
10462
- const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
10463
- if (tooSmallImage !== undefined) {
10464
- await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
10380
+ const path = pathsByKey.get(decision.key);
10381
+ await session.delete(decision.key);
10382
+ if (path === undefined)
10465
10383
  continue;
10466
- }
10467
- let parsed;
10468
- try {
10469
- const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
10470
- parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
10471
- } catch (error) {
10472
- if (options.signal?.aborted)
10473
- throw abortReason(options.signal);
10474
- if (error instanceof ParseError)
10475
- await skipFile(entry, {
10476
- code: "parser-failed",
10477
- severity: "warning",
10478
- message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
10479
- }, "parser-failed");
10480
- else
10481
- await skipFile(entry, {
10482
- code: "parser-crashed",
10483
- severity: "warning",
10484
- message: PARSER_CRASH_MESSAGE,
10485
- ...error instanceof Error ? { reason: error.name } : {}
10486
- });
10384
+ if (previous.entries[path])
10385
+ deleted += 1;
10386
+ if (!currentPaths.has(path))
10387
+ diagnostics.push({
10388
+ code: "deleted-mirror",
10389
+ severity: "info",
10390
+ message: "A previously indexed document is gone; its parsed mirror was removed.",
10391
+ source: path
10392
+ });
10393
+ }
10394
+ };
10395
+ const deferDeletions = options.onBatch !== undefined;
10396
+ const batchSize = Math.max(1, Math.floor(options.batchSize ?? DEFAULT_MIRROR_BATCH_SIZE));
10397
+ const batchCount = Math.max(1, Math.ceil(current.length / batchSize));
10398
+ let batchIndex = 0;
10399
+ let batchStopped = false;
10400
+ const runBatch = async (virtualPaths, final) => {
10401
+ if (options.onBatch === undefined)
10402
+ return;
10403
+ batchIndex += 1;
10404
+ const outcome = await options.onBatch({ index: batchIndex, total: batchCount, virtualPaths, final });
10405
+ if (outcome?.stop === true)
10406
+ batchStopped = true;
10407
+ };
10408
+ if (!deferDeletions)
10409
+ await applyDeleteDecisions();
10410
+ await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
10411
+ for (let batchStart = 0;batchStart < Math.max(current.length, 1); batchStart += batchSize) {
10412
+ const batchEnd = Math.min(batchStart + batchSize, current.length);
10413
+ const finalBatch = batchEnd >= current.length;
10414
+ const changedBefore = written + deleted;
10415
+ for (let position = batchStart;position < batchEnd; position += 1) {
10416
+ const entry = current[position];
10417
+ options.onProgress?.({
10418
+ processed: position,
10419
+ total: current.length,
10420
+ written,
10421
+ deleted,
10422
+ skipped,
10423
+ currentVirtualPath: entry.virtualPath
10424
+ });
10425
+ if (options.signal?.aborted)
10426
+ throw abortReason(options.signal);
10427
+ const omission = omitted.get(entry.virtualPath);
10428
+ if (omission !== undefined) {
10429
+ if (pending.has(mirrorStageKey(entry.virtualPath))) {
10430
+ if (previous.entries[entry.virtualPath])
10431
+ deleted += 1;
10432
+ await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
10433
+ }
10434
+ skipped += 1;
10435
+ if (omission.diagnostic)
10436
+ diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
10487
10437
  continue;
10488
10438
  }
10489
- let info;
10490
- try {
10491
- info = await stat3(entry.sourcePath, { bigint: true });
10492
- } catch (error) {
10493
- await skipFile(entry, sourceReadFailure(error));
10439
+ const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
10440
+ if (deferredFailure !== undefined) {
10441
+ skipped += 1;
10442
+ const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
10443
+ diagnostics.push({
10444
+ code,
10445
+ severity: "warning",
10446
+ source: entry.virtualPath,
10447
+ message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
10448
+ });
10494
10449
  continue;
10495
10450
  }
10496
- const unchanged = cloudDecision.kind === "download" ? contentIdentity(info) === entry.contentIdentity : statIdentity(info) === entry.statIdentity;
10497
- if (!unchanged) {
10451
+ const parser = registry.getForVirtualPath(entry.virtualPath);
10452
+ const outputPath = parsedOutputPath(options.root, entry.virtualPath);
10453
+ const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname12(entry.sourcePath)) : { kind: "local" };
10454
+ if (cloudDecision.kind === "skip") {
10498
10455
  skipped += 1;
10499
10456
  diagnostics.push({
10500
- code: "source-changed",
10501
- severity: "warning",
10502
- message: "The source changed while parsing; its last good mirror was kept and it will be retried on the next refresh.",
10503
- source: entry.virtualPath
10457
+ code: "cloud-placeholder-skipped",
10458
+ severity: "info",
10459
+ message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
10460
+ source: entry.virtualPath,
10461
+ ...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
10504
10462
  });
10505
- await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
10506
10463
  continue;
10507
10464
  }
10508
- const markdown = normalizeMarkdown(parsed.markdown);
10509
- try {
10510
- await withTransientRetry(() => writeFileAtomic(outputPath, markdown), transientRetries, transientBackoffMs);
10511
- } catch (error) {
10512
- const code = errorCode(error);
10513
- if (code !== undefined && Object.hasOwn(WORKSPACE_ERROR_CODES, code))
10514
- throw new MirrorWorkspaceError(options.root, error);
10515
- await skipFile(entry, {
10516
- code: "mirror-write-failed",
10517
- severity: "warning",
10518
- message: error instanceof Error ? error.message : String(error),
10519
- ...code !== undefined ? { reason: code } : {}
10465
+ if (cloudDecision.kind === "defer") {
10466
+ skipped += 1;
10467
+ diagnostics.push({
10468
+ code: "cloud-download-deferred",
10469
+ severity: "info",
10470
+ message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
10471
+ source: entry.virtualPath
10520
10472
  });
10521
10473
  continue;
10522
10474
  }
10523
- await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
10524
- path: basename10(outputPath),
10525
- size: Buffer.byteLength(markdown)
10526
- });
10527
- for (const diagnostic of parsed.diagnostics ?? [])
10528
- diagnostics.push({ ...diagnostic, source: entry.virtualPath });
10529
- written += 1;
10530
- if (fingerprintCloud && cloudFilePaths.has(entry.sourcePath)) {
10531
- const existing = cloudFingerprintEntries[entry.sourcePath];
10532
- if (existing === undefined || existing.sourceVersion !== entry.sourceVersion) {
10533
- try {
10534
- const fingerprintResult = await fingerprintWithDupey(entry.sourcePath, options.cloudFingerprints?.dupey ?? {});
10535
- cloudFingerprintEntries[entry.sourcePath] = {
10536
- sourceVersion: entry.sourceVersion,
10537
- exact: fingerprintResult.exact,
10538
- chars: fingerprintResult.chars,
10539
- modified: fingerprintResult.modified,
10540
- revision: fingerprintResult.revision
10541
- };
10542
- cloudFingerprintsDirty = true;
10543
- } catch (error) {
10544
- diagnostics.push({
10545
- code: "cloud-fingerprint-failed",
10475
+ if (pending.has(mirrorStageKey(entry.virtualPath))) {
10476
+ let bytes;
10477
+ try {
10478
+ bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
10479
+ } catch (error) {
10480
+ const failure = sourceReadFailure(error);
10481
+ await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
10482
+ continue;
10483
+ }
10484
+ const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
10485
+ if (tooSmallImage !== undefined) {
10486
+ await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
10487
+ continue;
10488
+ }
10489
+ let parsed;
10490
+ try {
10491
+ const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
10492
+ parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
10493
+ } catch (error) {
10494
+ if (options.signal?.aborted)
10495
+ throw abortReason(options.signal);
10496
+ if (error instanceof ParseError)
10497
+ await skipFile(entry, {
10498
+ code: "parser-failed",
10546
10499
  severity: "warning",
10547
- message: `The cloud or network file was indexed but its duplicate fingerprint could not be read, so it will not take part in exact-duplicate exclusion: ${error instanceof Error ? error.message : String(error)}`,
10548
- source: entry.virtualPath
10500
+ message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
10501
+ }, "parser-failed");
10502
+ else
10503
+ await skipFile(entry, {
10504
+ code: "parser-crashed",
10505
+ severity: "warning",
10506
+ message: PARSER_CRASH_MESSAGE,
10507
+ ...error instanceof Error ? { reason: error.name } : {}
10549
10508
  });
10550
- }
10509
+ continue;
10551
10510
  }
10552
- }
10553
- if (cloudDecision.kind === "download" && gate.evictAfterParse) {
10554
- const released = await gate.release(entry.sourcePath);
10555
- if (!released.ok) {
10511
+ let info;
10512
+ try {
10513
+ info = await stat3(entry.sourcePath, { bigint: true });
10514
+ } catch (error) {
10515
+ await skipFile(entry, sourceReadFailure(error));
10516
+ continue;
10517
+ }
10518
+ const unchanged = cloudDecision.kind === "download" ? contentIdentity(info) === entry.contentIdentity : statIdentity(info) === entry.statIdentity;
10519
+ if (!unchanged) {
10520
+ skipped += 1;
10556
10521
  diagnostics.push({
10557
- code: "cloud-evict-failed",
10522
+ code: "source-changed",
10558
10523
  severity: "warning",
10559
- message: `The file was indexed but its local copy could not be released: ${released.message}`,
10524
+ message: "The source changed while parsing; its last good mirror was kept and it will be retried on the next refresh.",
10560
10525
  source: entry.virtualPath
10561
10526
  });
10527
+ await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
10528
+ continue;
10529
+ }
10530
+ const markdown = normalizeMarkdown(parsed.markdown);
10531
+ try {
10532
+ await withTransientRetry(() => writeFileAtomic(outputPath, markdown), transientRetries, transientBackoffMs);
10533
+ } catch (error) {
10534
+ const code = errorCode(error);
10535
+ if (code !== undefined && Object.hasOwn(WORKSPACE_ERROR_CODES, code))
10536
+ throw new MirrorWorkspaceError(options.root, error);
10537
+ await skipFile(entry, {
10538
+ code: "mirror-write-failed",
10539
+ severity: "warning",
10540
+ message: error instanceof Error ? error.message : String(error),
10541
+ ...code !== undefined ? { reason: code } : {}
10542
+ });
10543
+ continue;
10544
+ }
10545
+ await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
10546
+ path: basename10(outputPath),
10547
+ size: Buffer.byteLength(markdown)
10548
+ });
10549
+ for (const diagnostic of parsed.diagnostics ?? [])
10550
+ diagnostics.push({ ...diagnostic, source: entry.virtualPath });
10551
+ written += 1;
10552
+ if (fingerprintCloud && cloudFilePaths.has(entry.sourcePath)) {
10553
+ const existing = cloudFingerprintEntries[entry.sourcePath];
10554
+ if (existing === undefined || existing.sourceVersion !== entry.sourceVersion) {
10555
+ try {
10556
+ const fingerprintResult = await fingerprintWithDupey(entry.sourcePath, options.cloudFingerprints?.dupey ?? {});
10557
+ cloudFingerprintEntries[entry.sourcePath] = {
10558
+ sourceVersion: entry.sourceVersion,
10559
+ exact: fingerprintResult.exact,
10560
+ chars: fingerprintResult.chars,
10561
+ modified: fingerprintResult.modified,
10562
+ revision: fingerprintResult.revision
10563
+ };
10564
+ cloudFingerprintsDirty = true;
10565
+ } catch (error) {
10566
+ diagnostics.push({
10567
+ code: "cloud-fingerprint-failed",
10568
+ severity: "warning",
10569
+ message: `The cloud or network file was indexed but its duplicate fingerprint could not be read, so it will not take part in exact-duplicate exclusion: ${error instanceof Error ? error.message : String(error)}`,
10570
+ source: entry.virtualPath
10571
+ });
10572
+ }
10573
+ }
10574
+ }
10575
+ if (cloudDecision.kind === "download" && gate.evictAfterParse) {
10576
+ const released = await gate.release(entry.sourcePath);
10577
+ if (!released.ok) {
10578
+ diagnostics.push({
10579
+ code: "cloud-evict-failed",
10580
+ severity: "warning",
10581
+ message: `The file was indexed but its local copy could not be released: ${released.message}`,
10582
+ source: entry.virtualPath
10583
+ });
10584
+ }
10562
10585
  }
10563
10586
  }
10564
10587
  }
10588
+ if (finalBatch && deferDeletions)
10589
+ await applyDeleteDecisions();
10590
+ if (finalBatch || written + deleted !== changedBefore) {
10591
+ await runBatch(current.slice(batchStart, batchEnd).map((item) => item.virtualPath), finalBatch);
10592
+ }
10593
+ if (batchStopped)
10594
+ break;
10565
10595
  }
10566
10596
  await session.finish();
10567
10597
  if (fingerprintCloud) {
@@ -10853,7 +10883,7 @@ async function collectFiles(state, directory, depth) {
10853
10883
  warnDirectoryUnreadable(state, directory, code);
10854
10884
  }
10855
10885
  }
10856
- var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
10886
+ var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
10857
10887
  var init_sync = __esm(() => {
10858
10888
  init_cli2();
10859
10889
  init_cloud_placeholder();
@@ -38692,11 +38722,52 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38692
38722
  };
38693
38723
  try {
38694
38724
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38725
+ const batchedMinSync = needsParsed && wants("minsync");
38726
+ const minSyncBatches = [];
38727
+ const minSyncBatchFailures = [];
38728
+ let minSyncSyncedBase = 0;
38729
+ let minSyncTotalBase = 0;
38730
+ const minSyncBatch = async (batch) => {
38731
+ signal?.throwIfAborted();
38732
+ progress = updateRefreshProgress(progress, { phase: "minsync" });
38733
+ writeRefreshProgress(this.workspaceProjectRoot, progress);
38734
+ let batchTotal = minSyncTotalBase;
38735
+ const result = await this.syncMinSync(force && batch.index === 1, signal, (minsyncProgress) => {
38736
+ batchTotal = minSyncTotalBase + minsyncProgress.total;
38737
+ progress = updateRefreshProgress(progress, {
38738
+ minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: batchTotal }
38739
+ });
38740
+ writeRefreshProgress(this.workspaceProjectRoot, progress);
38741
+ emitProgress({
38742
+ phase: "minsync",
38743
+ processed: minSyncSyncedBase + minsyncProgress.processed,
38744
+ total: batchTotal
38745
+ });
38746
+ });
38747
+ minSyncBatches.push(result);
38748
+ if (result.ok)
38749
+ minSyncBatchFailures.length = 0;
38750
+ else if (result.reason !== "lock-held") {
38751
+ for (const virtualPath of batch.virtualPaths) {
38752
+ minSyncBatchFailures.push({
38753
+ code: "minsync-sync-failed",
38754
+ severity: "warning",
38755
+ message: `This file stayed at its previous index entry: its batch could not be indexed and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
38756
+ source: virtualPath
38757
+ });
38758
+ }
38759
+ }
38760
+ minSyncSyncedBase = batchTotal;
38761
+ minSyncTotalBase = batchTotal;
38762
+ return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
38763
+ };
38695
38764
  const summary = needsParsed ? await this.syncParsedMirrors(force, {
38696
38765
  ...opts?.retrySkipped === true ? { retrySkipped: true } : {},
38697
38766
  ...signal === undefined ? {} : { signal },
38767
+ ...batchedMinSync ? { onBatch: minSyncBatch } : {},
38698
38768
  onProgress: (parsedProgress) => {
38699
38769
  progress = updateRefreshProgress(progress, {
38770
+ phase: "parsed",
38700
38771
  sourceFiles: { total: parsedProgress.total },
38701
38772
  parsedCounts: {
38702
38773
  scanned: parsedProgress.processed,
@@ -38718,7 +38789,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38718
38789
  }) : await this.scanMirrorStaleness();
38719
38790
  signal?.throwIfAborted();
38720
38791
  progress = updateRefreshProgress(progress, {
38721
- phase: wants("minsync") ? "minsync" : wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
38792
+ phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
38722
38793
  sourceFiles: { total: summary.scanned },
38723
38794
  parsedCounts: {
38724
38795
  scanned: summary.scanned,
@@ -38728,24 +38799,12 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38728
38799
  }
38729
38800
  });
38730
38801
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38731
- const minsync = wants("minsync") ? await this.syncMinSync(force, signal, (minsyncProgress) => {
38732
- progress = updateRefreshProgress(progress, {
38733
- minsync: { synced: minsyncProgress.processed, total: minsyncProgress.total }
38734
- });
38735
- writeRefreshProgress(this.workspaceProjectRoot, progress);
38736
- emitProgress({
38737
- phase: "minsync",
38738
- processed: minsyncProgress.processed,
38739
- total: minsyncProgress.total
38740
- });
38741
- }) : undefined;
38742
- signal?.throwIfAborted();
38802
+ const minsync = batchedMinSync ? aggregateMinSyncBatches(minSyncBatches) : undefined;
38743
38803
  if (minsync !== undefined) {
38744
38804
  progress = updateRefreshProgress(progress, {
38745
- phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
38746
38805
  minsync: {
38747
- ...progress.minsync,
38748
38806
  synced: minsync.synced,
38807
+ total: Math.max(minSyncTotalBase, minsync.synced),
38749
38808
  ...minsync.filesFailed !== undefined && minsync.filesFailed.length > 0 ? { filesSkipped: minsync.filesFailed.length } : {}
38750
38809
  }
38751
38810
  });
@@ -38807,7 +38866,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38807
38866
  finishedAt: new Date().toISOString()
38808
38867
  });
38809
38868
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38810
- const minsyncDiagnostics = minSyncRefreshDiagnostics(minsync);
38869
+ const minsyncDiagnostics = [...minSyncRefreshDiagnostics(minsync), ...minSyncBatchFailures];
38811
38870
  const publicMinsync = minsync ? {
38812
38871
  ok: minsync.ok,
38813
38872
  synced: minsync.synced,
@@ -39612,6 +39671,12 @@ function fsearchRefreshDiagnostics(fsearch) {
39612
39671
  }
39613
39672
  ];
39614
39673
  }
39674
+ function aggregateMinSyncBatches(batches) {
39675
+ const last = batches.at(-1);
39676
+ if (last === undefined)
39677
+ return;
39678
+ return { ...last, synced: batches.reduce((total, batch) => total + batch.synced, 0) };
39679
+ }
39615
39680
  function minSyncRefreshDiagnostics(minsync) {
39616
39681
  if (!minsync)
39617
39682
  return [];