@autorag/librarian 2.13.5 → 2.13.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/profiles.js CHANGED
@@ -23989,195 +23989,225 @@ async function syncWithRegistry(options, registry, isolated) {
23989
23989
  });
23990
23990
  const pending = new Set(plan.decisions.filter((decision) => decision.action === "process" || decision.action === "retry").map((decision) => decision.key));
23991
23991
  const deferred = new Map(plan.decisions.filter((decision) => decision.deferred === true).map((decision) => [decision.key, decision]));
23992
- for (const decision of plan.decisions) {
23993
- if (decision.action !== "delete")
23994
- continue;
23995
- const path = pathsByKey.get(decision.key);
23996
- await session.delete(decision.key);
23997
- if (path === undefined)
23998
- continue;
23999
- if (previous.entries[path])
24000
- deleted += 1;
24001
- if (!currentPaths.has(path))
24002
- diagnostics.push({
24003
- code: "deleted-mirror",
24004
- severity: "info",
24005
- message: "A previously indexed document is gone; its parsed mirror was removed.",
24006
- source: path
24007
- });
24008
- }
24009
- await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
24010
- for (const [position, entry] of current.entries()) {
24011
- options.onProgress?.({
24012
- processed: position,
24013
- total: current.length,
24014
- written,
24015
- deleted,
24016
- skipped,
24017
- currentVirtualPath: entry.virtualPath
24018
- });
24019
- if (options.signal?.aborted)
24020
- throw abortReason(options.signal);
24021
- const omission = omitted.get(entry.virtualPath);
24022
- if (omission !== undefined) {
24023
- if (pending.has(mirrorStageKey(entry.virtualPath))) {
24024
- if (previous.entries[entry.virtualPath])
24025
- deleted += 1;
24026
- await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
24027
- }
24028
- skipped += 1;
24029
- if (omission.diagnostic)
24030
- diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
24031
- continue;
24032
- }
24033
- const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
24034
- if (deferredFailure !== undefined) {
24035
- skipped += 1;
24036
- const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
24037
- diagnostics.push({
24038
- code,
24039
- severity: "warning",
24040
- source: entry.virtualPath,
24041
- message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
24042
- });
24043
- continue;
24044
- }
24045
- const parser = registry.getForVirtualPath(entry.virtualPath);
24046
- const outputPath = parsedOutputPath(options.root, entry.virtualPath);
24047
- const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname21(entry.sourcePath)) : { kind: "local" };
24048
- if (cloudDecision.kind === "skip") {
24049
- skipped += 1;
24050
- diagnostics.push({
24051
- code: "cloud-placeholder-skipped",
24052
- severity: "info",
24053
- message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
24054
- source: entry.virtualPath,
24055
- ...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
24056
- });
24057
- continue;
24058
- }
24059
- if (cloudDecision.kind === "defer") {
24060
- skipped += 1;
24061
- diagnostics.push({
24062
- code: "cloud-download-deferred",
24063
- severity: "info",
24064
- message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
24065
- source: entry.virtualPath
24066
- });
24067
- continue;
24068
- }
24069
- if (pending.has(mirrorStageKey(entry.virtualPath))) {
24070
- let bytes;
24071
- try {
24072
- bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
24073
- } catch (error) {
24074
- const failure = sourceReadFailure(error);
24075
- await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
23992
+ const applyDeleteDecisions = async () => {
23993
+ for (const decision of plan.decisions) {
23994
+ if (decision.action !== "delete")
24076
23995
  continue;
24077
- }
24078
- const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
24079
- if (tooSmallImage !== undefined) {
24080
- await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
23996
+ const path = pathsByKey.get(decision.key);
23997
+ await session.delete(decision.key);
23998
+ if (path === undefined)
24081
23999
  continue;
24082
- }
24083
- let parsed;
24084
- try {
24085
- const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
24086
- parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
24087
- } catch (error) {
24088
- if (options.signal?.aborted)
24089
- throw abortReason(options.signal);
24090
- if (error instanceof ParseError)
24091
- await skipFile(entry, {
24092
- code: "parser-failed",
24093
- severity: "warning",
24094
- message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
24095
- }, "parser-failed");
24096
- else
24097
- await skipFile(entry, {
24098
- code: "parser-crashed",
24099
- severity: "warning",
24100
- message: PARSER_CRASH_MESSAGE,
24101
- ...error instanceof Error ? { reason: error.name } : {}
24102
- });
24000
+ if (previous.entries[path])
24001
+ deleted += 1;
24002
+ if (!currentPaths.has(path))
24003
+ diagnostics.push({
24004
+ code: "deleted-mirror",
24005
+ severity: "info",
24006
+ message: "A previously indexed document is gone; its parsed mirror was removed.",
24007
+ source: path
24008
+ });
24009
+ }
24010
+ };
24011
+ const deferDeletions = options.onBatch !== undefined;
24012
+ const batchSize = Math.max(1, Math.floor(options.batchSize ?? DEFAULT_MIRROR_BATCH_SIZE));
24013
+ const batchCount = Math.max(1, Math.ceil(current.length / batchSize));
24014
+ let batchIndex = 0;
24015
+ let batchStopped = false;
24016
+ const runBatch = async (virtualPaths, final) => {
24017
+ if (options.onBatch === undefined)
24018
+ return;
24019
+ batchIndex += 1;
24020
+ const outcome = await options.onBatch({ index: batchIndex, total: batchCount, virtualPaths, final });
24021
+ if (outcome?.stop === true)
24022
+ batchStopped = true;
24023
+ };
24024
+ if (!deferDeletions)
24025
+ await applyDeleteDecisions();
24026
+ await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
24027
+ for (let batchStart = 0;batchStart < Math.max(current.length, 1); batchStart += batchSize) {
24028
+ const batchEnd = Math.min(batchStart + batchSize, current.length);
24029
+ const finalBatch = batchEnd >= current.length;
24030
+ const changedBefore = written + deleted;
24031
+ for (let position = batchStart;position < batchEnd; position += 1) {
24032
+ const entry = current[position];
24033
+ options.onProgress?.({
24034
+ processed: position,
24035
+ total: current.length,
24036
+ written,
24037
+ deleted,
24038
+ skipped,
24039
+ currentVirtualPath: entry.virtualPath
24040
+ });
24041
+ if (options.signal?.aborted)
24042
+ throw abortReason(options.signal);
24043
+ const omission = omitted.get(entry.virtualPath);
24044
+ if (omission !== undefined) {
24045
+ if (pending.has(mirrorStageKey(entry.virtualPath))) {
24046
+ if (previous.entries[entry.virtualPath])
24047
+ deleted += 1;
24048
+ await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
24049
+ }
24050
+ skipped += 1;
24051
+ if (omission.diagnostic)
24052
+ diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
24103
24053
  continue;
24104
24054
  }
24105
- let info;
24106
- try {
24107
- info = await stat4(entry.sourcePath, { bigint: true });
24108
- } catch (error) {
24109
- await skipFile(entry, sourceReadFailure(error));
24055
+ const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
24056
+ if (deferredFailure !== undefined) {
24057
+ skipped += 1;
24058
+ const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
24059
+ diagnostics.push({
24060
+ code,
24061
+ severity: "warning",
24062
+ source: entry.virtualPath,
24063
+ message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
24064
+ });
24110
24065
  continue;
24111
24066
  }
24112
- const unchanged = cloudDecision.kind === "download" ? contentIdentity(info) === entry.contentIdentity : statIdentity(info) === entry.statIdentity;
24113
- if (!unchanged) {
24067
+ const parser = registry.getForVirtualPath(entry.virtualPath);
24068
+ const outputPath = parsedOutputPath(options.root, entry.virtualPath);
24069
+ const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname21(entry.sourcePath)) : { kind: "local" };
24070
+ if (cloudDecision.kind === "skip") {
24114
24071
  skipped += 1;
24115
24072
  diagnostics.push({
24116
- code: "source-changed",
24117
- severity: "warning",
24118
- message: "The source changed while parsing; its last good mirror was kept and it will be retried on the next refresh.",
24119
- source: entry.virtualPath
24073
+ code: "cloud-placeholder-skipped",
24074
+ severity: "info",
24075
+ message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
24076
+ source: entry.virtualPath,
24077
+ ...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
24120
24078
  });
24121
- await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
24122
24079
  continue;
24123
24080
  }
24124
- const markdown = normalizeMarkdown(parsed.markdown);
24125
- try {
24126
- await withTransientRetry(() => writeFileAtomic(outputPath, markdown), transientRetries, transientBackoffMs);
24127
- } catch (error) {
24128
- const code = errorCode(error);
24129
- if (code !== undefined && Object.hasOwn(WORKSPACE_ERROR_CODES, code))
24130
- throw new MirrorWorkspaceError(options.root, error);
24131
- await skipFile(entry, {
24132
- code: "mirror-write-failed",
24133
- severity: "warning",
24134
- message: error instanceof Error ? error.message : String(error),
24135
- ...code !== undefined ? { reason: code } : {}
24081
+ if (cloudDecision.kind === "defer") {
24082
+ skipped += 1;
24083
+ diagnostics.push({
24084
+ code: "cloud-download-deferred",
24085
+ severity: "info",
24086
+ message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
24087
+ source: entry.virtualPath
24136
24088
  });
24137
24089
  continue;
24138
24090
  }
24139
- await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
24140
- path: basename14(outputPath),
24141
- size: Buffer.byteLength(markdown)
24142
- });
24143
- for (const diagnostic of parsed.diagnostics ?? [])
24144
- diagnostics.push({ ...diagnostic, source: entry.virtualPath });
24145
- written += 1;
24146
- if (fingerprintCloud && cloudFilePaths.has(entry.sourcePath)) {
24147
- const existing = cloudFingerprintEntries[entry.sourcePath];
24148
- if (existing === undefined || existing.sourceVersion !== entry.sourceVersion) {
24149
- try {
24150
- const fingerprintResult = await fingerprintWithDupey(entry.sourcePath, options.cloudFingerprints?.dupey ?? {});
24151
- cloudFingerprintEntries[entry.sourcePath] = {
24152
- sourceVersion: entry.sourceVersion,
24153
- exact: fingerprintResult.exact,
24154
- chars: fingerprintResult.chars,
24155
- modified: fingerprintResult.modified,
24156
- revision: fingerprintResult.revision
24157
- };
24158
- cloudFingerprintsDirty = true;
24159
- } catch (error) {
24160
- diagnostics.push({
24161
- code: "cloud-fingerprint-failed",
24091
+ if (pending.has(mirrorStageKey(entry.virtualPath))) {
24092
+ let bytes;
24093
+ try {
24094
+ bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
24095
+ } catch (error) {
24096
+ const failure = sourceReadFailure(error);
24097
+ await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
24098
+ continue;
24099
+ }
24100
+ const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
24101
+ if (tooSmallImage !== undefined) {
24102
+ await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
24103
+ continue;
24104
+ }
24105
+ let parsed;
24106
+ try {
24107
+ const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
24108
+ parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
24109
+ } catch (error) {
24110
+ if (options.signal?.aborted)
24111
+ throw abortReason(options.signal);
24112
+ if (error instanceof ParseError)
24113
+ await skipFile(entry, {
24114
+ code: "parser-failed",
24162
24115
  severity: "warning",
24163
- message: `The cloud or network file was indexed but its duplicate fingerprint could not be read, so it will not take part in exact-duplicate exclusion: ${error instanceof Error ? error.message : String(error)}`,
24164
- source: entry.virtualPath
24116
+ message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
24117
+ }, "parser-failed");
24118
+ else
24119
+ await skipFile(entry, {
24120
+ code: "parser-crashed",
24121
+ severity: "warning",
24122
+ message: PARSER_CRASH_MESSAGE,
24123
+ ...error instanceof Error ? { reason: error.name } : {}
24165
24124
  });
24166
- }
24125
+ continue;
24167
24126
  }
24168
- }
24169
- if (cloudDecision.kind === "download" && gate.evictAfterParse) {
24170
- const released = await gate.release(entry.sourcePath);
24171
- if (!released.ok) {
24127
+ let info;
24128
+ try {
24129
+ info = await stat4(entry.sourcePath, { bigint: true });
24130
+ } catch (error) {
24131
+ await skipFile(entry, sourceReadFailure(error));
24132
+ continue;
24133
+ }
24134
+ const unchanged = cloudDecision.kind === "download" ? contentIdentity(info) === entry.contentIdentity : statIdentity(info) === entry.statIdentity;
24135
+ if (!unchanged) {
24136
+ skipped += 1;
24172
24137
  diagnostics.push({
24173
- code: "cloud-evict-failed",
24138
+ code: "source-changed",
24174
24139
  severity: "warning",
24175
- message: `The file was indexed but its local copy could not be released: ${released.message}`,
24140
+ message: "The source changed while parsing; its last good mirror was kept and it will be retried on the next refresh.",
24176
24141
  source: entry.virtualPath
24177
24142
  });
24143
+ await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
24144
+ continue;
24145
+ }
24146
+ const markdown = normalizeMarkdown(parsed.markdown);
24147
+ try {
24148
+ await withTransientRetry(() => writeFileAtomic(outputPath, markdown), transientRetries, transientBackoffMs);
24149
+ } catch (error) {
24150
+ const code = errorCode(error);
24151
+ if (code !== undefined && Object.hasOwn(WORKSPACE_ERROR_CODES, code))
24152
+ throw new MirrorWorkspaceError(options.root, error);
24153
+ await skipFile(entry, {
24154
+ code: "mirror-write-failed",
24155
+ severity: "warning",
24156
+ message: error instanceof Error ? error.message : String(error),
24157
+ ...code !== undefined ? { reason: code } : {}
24158
+ });
24159
+ continue;
24160
+ }
24161
+ await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
24162
+ path: basename14(outputPath),
24163
+ size: Buffer.byteLength(markdown)
24164
+ });
24165
+ for (const diagnostic of parsed.diagnostics ?? [])
24166
+ diagnostics.push({ ...diagnostic, source: entry.virtualPath });
24167
+ written += 1;
24168
+ if (fingerprintCloud && cloudFilePaths.has(entry.sourcePath)) {
24169
+ const existing = cloudFingerprintEntries[entry.sourcePath];
24170
+ if (existing === undefined || existing.sourceVersion !== entry.sourceVersion) {
24171
+ try {
24172
+ const fingerprintResult = await fingerprintWithDupey(entry.sourcePath, options.cloudFingerprints?.dupey ?? {});
24173
+ cloudFingerprintEntries[entry.sourcePath] = {
24174
+ sourceVersion: entry.sourceVersion,
24175
+ exact: fingerprintResult.exact,
24176
+ chars: fingerprintResult.chars,
24177
+ modified: fingerprintResult.modified,
24178
+ revision: fingerprintResult.revision
24179
+ };
24180
+ cloudFingerprintsDirty = true;
24181
+ } catch (error) {
24182
+ diagnostics.push({
24183
+ code: "cloud-fingerprint-failed",
24184
+ severity: "warning",
24185
+ message: `The cloud or network file was indexed but its duplicate fingerprint could not be read, so it will not take part in exact-duplicate exclusion: ${error instanceof Error ? error.message : String(error)}`,
24186
+ source: entry.virtualPath
24187
+ });
24188
+ }
24189
+ }
24190
+ }
24191
+ if (cloudDecision.kind === "download" && gate.evictAfterParse) {
24192
+ const released = await gate.release(entry.sourcePath);
24193
+ if (!released.ok) {
24194
+ diagnostics.push({
24195
+ code: "cloud-evict-failed",
24196
+ severity: "warning",
24197
+ message: `The file was indexed but its local copy could not be released: ${released.message}`,
24198
+ source: entry.virtualPath
24199
+ });
24200
+ }
24178
24201
  }
24179
24202
  }
24180
24203
  }
24204
+ if (finalBatch && deferDeletions)
24205
+ await applyDeleteDecisions();
24206
+ if (finalBatch || written + deleted !== changedBefore) {
24207
+ await runBatch(current.slice(batchStart, batchEnd).map((item) => item.virtualPath), finalBatch);
24208
+ }
24209
+ if (batchStopped)
24210
+ break;
24181
24211
  }
24182
24212
  await session.finish();
24183
24213
  if (fingerprintCloud) {
@@ -24469,7 +24499,7 @@ async function collectFiles(state, directory, depth) {
24469
24499
  warnDirectoryUnreadable(state, directory, code);
24470
24500
  }
24471
24501
  }
24472
- var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
24502
+ var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
24473
24503
  var init_sync = __esm(() => {
24474
24504
  init_cli2();
24475
24505
  init_cloud_placeholder();
@@ -38634,11 +38664,52 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38634
38664
  };
38635
38665
  try {
38636
38666
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38667
+ const batchedMinSync = needsParsed && wants("minsync");
38668
+ const minSyncBatches = [];
38669
+ const minSyncBatchFailures = [];
38670
+ let minSyncSyncedBase = 0;
38671
+ let minSyncTotalBase = 0;
38672
+ const minSyncBatch = async (batch) => {
38673
+ signal?.throwIfAborted();
38674
+ progress = updateRefreshProgress(progress, { phase: "minsync" });
38675
+ writeRefreshProgress(this.workspaceProjectRoot, progress);
38676
+ let batchTotal = minSyncTotalBase;
38677
+ const result = await this.syncMinSync(force && batch.index === 1, signal, (minsyncProgress) => {
38678
+ batchTotal = minSyncTotalBase + minsyncProgress.total;
38679
+ progress = updateRefreshProgress(progress, {
38680
+ minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: batchTotal }
38681
+ });
38682
+ writeRefreshProgress(this.workspaceProjectRoot, progress);
38683
+ emitProgress({
38684
+ phase: "minsync",
38685
+ processed: minSyncSyncedBase + minsyncProgress.processed,
38686
+ total: batchTotal
38687
+ });
38688
+ });
38689
+ minSyncBatches.push(result);
38690
+ if (result.ok)
38691
+ minSyncBatchFailures.length = 0;
38692
+ else if (result.reason !== "lock-held") {
38693
+ for (const virtualPath of batch.virtualPaths) {
38694
+ minSyncBatchFailures.push({
38695
+ code: "minsync-sync-failed",
38696
+ severity: "warning",
38697
+ message: `This file stayed at its previous index entry: its batch could not be indexed and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
38698
+ source: virtualPath
38699
+ });
38700
+ }
38701
+ }
38702
+ minSyncSyncedBase = batchTotal;
38703
+ minSyncTotalBase = batchTotal;
38704
+ return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
38705
+ };
38637
38706
  const summary = needsParsed ? await this.syncParsedMirrors(force, {
38638
38707
  ...opts?.retrySkipped === true ? { retrySkipped: true } : {},
38639
38708
  ...signal === undefined ? {} : { signal },
38709
+ ...batchedMinSync ? { onBatch: minSyncBatch } : {},
38640
38710
  onProgress: (parsedProgress) => {
38641
38711
  progress = updateRefreshProgress(progress, {
38712
+ phase: "parsed",
38642
38713
  sourceFiles: { total: parsedProgress.total },
38643
38714
  parsedCounts: {
38644
38715
  scanned: parsedProgress.processed,
@@ -38660,7 +38731,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38660
38731
  }) : await this.scanMirrorStaleness();
38661
38732
  signal?.throwIfAborted();
38662
38733
  progress = updateRefreshProgress(progress, {
38663
- phase: wants("minsync") ? "minsync" : wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
38734
+ phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
38664
38735
  sourceFiles: { total: summary.scanned },
38665
38736
  parsedCounts: {
38666
38737
  scanned: summary.scanned,
@@ -38670,24 +38741,12 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38670
38741
  }
38671
38742
  });
38672
38743
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38673
- const minsync = wants("minsync") ? await this.syncMinSync(force, signal, (minsyncProgress) => {
38674
- progress = updateRefreshProgress(progress, {
38675
- minsync: { synced: minsyncProgress.processed, total: minsyncProgress.total }
38676
- });
38677
- writeRefreshProgress(this.workspaceProjectRoot, progress);
38678
- emitProgress({
38679
- phase: "minsync",
38680
- processed: minsyncProgress.processed,
38681
- total: minsyncProgress.total
38682
- });
38683
- }) : undefined;
38684
- signal?.throwIfAborted();
38744
+ const minsync = batchedMinSync ? aggregateMinSyncBatches(minSyncBatches) : undefined;
38685
38745
  if (minsync !== undefined) {
38686
38746
  progress = updateRefreshProgress(progress, {
38687
- phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
38688
38747
  minsync: {
38689
- ...progress.minsync,
38690
38748
  synced: minsync.synced,
38749
+ total: Math.max(minSyncTotalBase, minsync.synced),
38691
38750
  ...minsync.filesFailed !== undefined && minsync.filesFailed.length > 0 ? { filesSkipped: minsync.filesFailed.length } : {}
38692
38751
  }
38693
38752
  });
@@ -38749,7 +38808,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38749
38808
  finishedAt: new Date().toISOString()
38750
38809
  });
38751
38810
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38752
- const minsyncDiagnostics = minSyncRefreshDiagnostics(minsync);
38811
+ const minsyncDiagnostics = [...minSyncRefreshDiagnostics(minsync), ...minSyncBatchFailures];
38753
38812
  const publicMinsync = minsync ? {
38754
38813
  ok: minsync.ok,
38755
38814
  synced: minsync.synced,
@@ -39554,6 +39613,12 @@ function fsearchRefreshDiagnostics(fsearch) {
39554
39613
  }
39555
39614
  ];
39556
39615
  }
39616
+ function aggregateMinSyncBatches(batches) {
39617
+ const last = batches.at(-1);
39618
+ if (last === undefined)
39619
+ return;
39620
+ return { ...last, synced: batches.reduce((total, batch) => total + batch.synced, 0) };
39621
+ }
39557
39622
  function minSyncRefreshDiagnostics(minsync) {
39558
39623
  if (!minsync)
39559
39624
  return [];
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@autorag/librarian",
3
- "version": "2.13.5",
3
+ "version": "2.13.6",
4
4
  "description": "Personal librarian agent for office work on the Pi framework, with pluggable retrieval methods",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {