@autorag/librarian 2.13.5 → 2.13.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/agent.d.ts +1 -1
- package/dist/cli/index.js +247 -182
- package/dist/core.js +247 -182
- package/dist/index.js +247 -182
- package/dist/mcp/index.js +247 -182
- package/dist/mirror/sync.d.ts +46 -0
- package/dist/profiles.js +247 -182
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -10373,195 +10373,225 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
10373
10373
|
});
|
|
10374
10374
|
const pending = new Set(plan.decisions.filter((decision) => decision.action === "process" || decision.action === "retry").map((decision) => decision.key));
|
|
10375
10375
|
const deferred = new Map(plan.decisions.filter((decision) => decision.deferred === true).map((decision) => [decision.key, decision]));
|
|
10376
|
-
|
|
10377
|
-
|
|
10378
|
-
|
|
10379
|
-
const path = pathsByKey.get(decision.key);
|
|
10380
|
-
await session.delete(decision.key);
|
|
10381
|
-
if (path === undefined)
|
|
10382
|
-
continue;
|
|
10383
|
-
if (previous.entries[path])
|
|
10384
|
-
deleted += 1;
|
|
10385
|
-
if (!currentPaths.has(path))
|
|
10386
|
-
diagnostics.push({
|
|
10387
|
-
code: "deleted-mirror",
|
|
10388
|
-
severity: "info",
|
|
10389
|
-
message: "A previously indexed document is gone; its parsed mirror was removed.",
|
|
10390
|
-
source: path
|
|
10391
|
-
});
|
|
10392
|
-
}
|
|
10393
|
-
await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
|
|
10394
|
-
for (const [position, entry] of current.entries()) {
|
|
10395
|
-
options.onProgress?.({
|
|
10396
|
-
processed: position,
|
|
10397
|
-
total: current.length,
|
|
10398
|
-
written,
|
|
10399
|
-
deleted,
|
|
10400
|
-
skipped,
|
|
10401
|
-
currentVirtualPath: entry.virtualPath
|
|
10402
|
-
});
|
|
10403
|
-
if (options.signal?.aborted)
|
|
10404
|
-
throw abortReason(options.signal);
|
|
10405
|
-
const omission = omitted.get(entry.virtualPath);
|
|
10406
|
-
if (omission !== undefined) {
|
|
10407
|
-
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
10408
|
-
if (previous.entries[entry.virtualPath])
|
|
10409
|
-
deleted += 1;
|
|
10410
|
-
await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
|
|
10411
|
-
}
|
|
10412
|
-
skipped += 1;
|
|
10413
|
-
if (omission.diagnostic)
|
|
10414
|
-
diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
|
|
10415
|
-
continue;
|
|
10416
|
-
}
|
|
10417
|
-
const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
|
|
10418
|
-
if (deferredFailure !== undefined) {
|
|
10419
|
-
skipped += 1;
|
|
10420
|
-
const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
|
|
10421
|
-
diagnostics.push({
|
|
10422
|
-
code,
|
|
10423
|
-
severity: "warning",
|
|
10424
|
-
source: entry.virtualPath,
|
|
10425
|
-
message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
|
|
10426
|
-
});
|
|
10427
|
-
continue;
|
|
10428
|
-
}
|
|
10429
|
-
const parser = registry.getForVirtualPath(entry.virtualPath);
|
|
10430
|
-
const outputPath = parsedOutputPath(options.root, entry.virtualPath);
|
|
10431
|
-
const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname12(entry.sourcePath)) : { kind: "local" };
|
|
10432
|
-
if (cloudDecision.kind === "skip") {
|
|
10433
|
-
skipped += 1;
|
|
10434
|
-
diagnostics.push({
|
|
10435
|
-
code: "cloud-placeholder-skipped",
|
|
10436
|
-
severity: "info",
|
|
10437
|
-
message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
|
|
10438
|
-
source: entry.virtualPath,
|
|
10439
|
-
...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
|
|
10440
|
-
});
|
|
10441
|
-
continue;
|
|
10442
|
-
}
|
|
10443
|
-
if (cloudDecision.kind === "defer") {
|
|
10444
|
-
skipped += 1;
|
|
10445
|
-
diagnostics.push({
|
|
10446
|
-
code: "cloud-download-deferred",
|
|
10447
|
-
severity: "info",
|
|
10448
|
-
message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
|
|
10449
|
-
source: entry.virtualPath
|
|
10450
|
-
});
|
|
10451
|
-
continue;
|
|
10452
|
-
}
|
|
10453
|
-
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
10454
|
-
let bytes;
|
|
10455
|
-
try {
|
|
10456
|
-
bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
|
|
10457
|
-
} catch (error) {
|
|
10458
|
-
const failure = sourceReadFailure(error);
|
|
10459
|
-
await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
|
|
10376
|
+
const applyDeleteDecisions = async () => {
|
|
10377
|
+
for (const decision of plan.decisions) {
|
|
10378
|
+
if (decision.action !== "delete")
|
|
10460
10379
|
continue;
|
|
10461
|
-
|
|
10462
|
-
|
|
10463
|
-
if (
|
|
10464
|
-
await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
|
|
10380
|
+
const path = pathsByKey.get(decision.key);
|
|
10381
|
+
await session.delete(decision.key);
|
|
10382
|
+
if (path === undefined)
|
|
10465
10383
|
continue;
|
|
10466
|
-
|
|
10467
|
-
|
|
10468
|
-
|
|
10469
|
-
|
|
10470
|
-
|
|
10471
|
-
|
|
10472
|
-
|
|
10473
|
-
|
|
10474
|
-
|
|
10475
|
-
|
|
10476
|
-
|
|
10477
|
-
|
|
10478
|
-
|
|
10479
|
-
|
|
10480
|
-
|
|
10481
|
-
|
|
10482
|
-
|
|
10483
|
-
|
|
10484
|
-
|
|
10485
|
-
|
|
10486
|
-
|
|
10384
|
+
if (previous.entries[path])
|
|
10385
|
+
deleted += 1;
|
|
10386
|
+
if (!currentPaths.has(path))
|
|
10387
|
+
diagnostics.push({
|
|
10388
|
+
code: "deleted-mirror",
|
|
10389
|
+
severity: "info",
|
|
10390
|
+
message: "A previously indexed document is gone; its parsed mirror was removed.",
|
|
10391
|
+
source: path
|
|
10392
|
+
});
|
|
10393
|
+
}
|
|
10394
|
+
};
|
|
10395
|
+
const deferDeletions = options.onBatch !== undefined;
|
|
10396
|
+
const batchSize = Math.max(1, Math.floor(options.batchSize ?? DEFAULT_MIRROR_BATCH_SIZE));
|
|
10397
|
+
const batchCount = Math.max(1, Math.ceil(current.length / batchSize));
|
|
10398
|
+
let batchIndex = 0;
|
|
10399
|
+
let batchStopped = false;
|
|
10400
|
+
const runBatch = async (virtualPaths, final) => {
|
|
10401
|
+
if (options.onBatch === undefined)
|
|
10402
|
+
return;
|
|
10403
|
+
batchIndex += 1;
|
|
10404
|
+
const outcome = await options.onBatch({ index: batchIndex, total: batchCount, virtualPaths, final });
|
|
10405
|
+
if (outcome?.stop === true)
|
|
10406
|
+
batchStopped = true;
|
|
10407
|
+
};
|
|
10408
|
+
if (!deferDeletions)
|
|
10409
|
+
await applyDeleteDecisions();
|
|
10410
|
+
await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
|
|
10411
|
+
for (let batchStart = 0;batchStart < Math.max(current.length, 1); batchStart += batchSize) {
|
|
10412
|
+
const batchEnd = Math.min(batchStart + batchSize, current.length);
|
|
10413
|
+
const finalBatch = batchEnd >= current.length;
|
|
10414
|
+
const changedBefore = written + deleted;
|
|
10415
|
+
for (let position = batchStart;position < batchEnd; position += 1) {
|
|
10416
|
+
const entry = current[position];
|
|
10417
|
+
options.onProgress?.({
|
|
10418
|
+
processed: position,
|
|
10419
|
+
total: current.length,
|
|
10420
|
+
written,
|
|
10421
|
+
deleted,
|
|
10422
|
+
skipped,
|
|
10423
|
+
currentVirtualPath: entry.virtualPath
|
|
10424
|
+
});
|
|
10425
|
+
if (options.signal?.aborted)
|
|
10426
|
+
throw abortReason(options.signal);
|
|
10427
|
+
const omission = omitted.get(entry.virtualPath);
|
|
10428
|
+
if (omission !== undefined) {
|
|
10429
|
+
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
10430
|
+
if (previous.entries[entry.virtualPath])
|
|
10431
|
+
deleted += 1;
|
|
10432
|
+
await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
|
|
10433
|
+
}
|
|
10434
|
+
skipped += 1;
|
|
10435
|
+
if (omission.diagnostic)
|
|
10436
|
+
diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
|
|
10487
10437
|
continue;
|
|
10488
10438
|
}
|
|
10489
|
-
|
|
10490
|
-
|
|
10491
|
-
|
|
10492
|
-
|
|
10493
|
-
|
|
10439
|
+
const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
|
|
10440
|
+
if (deferredFailure !== undefined) {
|
|
10441
|
+
skipped += 1;
|
|
10442
|
+
const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
|
|
10443
|
+
diagnostics.push({
|
|
10444
|
+
code,
|
|
10445
|
+
severity: "warning",
|
|
10446
|
+
source: entry.virtualPath,
|
|
10447
|
+
message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
|
|
10448
|
+
});
|
|
10494
10449
|
continue;
|
|
10495
10450
|
}
|
|
10496
|
-
const
|
|
10497
|
-
|
|
10451
|
+
const parser = registry.getForVirtualPath(entry.virtualPath);
|
|
10452
|
+
const outputPath = parsedOutputPath(options.root, entry.virtualPath);
|
|
10453
|
+
const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname12(entry.sourcePath)) : { kind: "local" };
|
|
10454
|
+
if (cloudDecision.kind === "skip") {
|
|
10498
10455
|
skipped += 1;
|
|
10499
10456
|
diagnostics.push({
|
|
10500
|
-
code: "
|
|
10501
|
-
severity: "
|
|
10502
|
-
message:
|
|
10503
|
-
source: entry.virtualPath
|
|
10457
|
+
code: "cloud-placeholder-skipped",
|
|
10458
|
+
severity: "info",
|
|
10459
|
+
message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
|
|
10460
|
+
source: entry.virtualPath,
|
|
10461
|
+
...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
|
|
10504
10462
|
});
|
|
10505
|
-
await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
|
|
10506
10463
|
continue;
|
|
10507
10464
|
}
|
|
10508
|
-
|
|
10509
|
-
|
|
10510
|
-
|
|
10511
|
-
|
|
10512
|
-
|
|
10513
|
-
|
|
10514
|
-
|
|
10515
|
-
await skipFile(entry, {
|
|
10516
|
-
code: "mirror-write-failed",
|
|
10517
|
-
severity: "warning",
|
|
10518
|
-
message: error instanceof Error ? error.message : String(error),
|
|
10519
|
-
...code !== undefined ? { reason: code } : {}
|
|
10465
|
+
if (cloudDecision.kind === "defer") {
|
|
10466
|
+
skipped += 1;
|
|
10467
|
+
diagnostics.push({
|
|
10468
|
+
code: "cloud-download-deferred",
|
|
10469
|
+
severity: "info",
|
|
10470
|
+
message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
|
|
10471
|
+
source: entry.virtualPath
|
|
10520
10472
|
});
|
|
10521
10473
|
continue;
|
|
10522
10474
|
}
|
|
10523
|
-
|
|
10524
|
-
|
|
10525
|
-
|
|
10526
|
-
|
|
10527
|
-
|
|
10528
|
-
|
|
10529
|
-
|
|
10530
|
-
|
|
10531
|
-
|
|
10532
|
-
|
|
10533
|
-
|
|
10534
|
-
|
|
10535
|
-
|
|
10536
|
-
|
|
10537
|
-
|
|
10538
|
-
|
|
10539
|
-
|
|
10540
|
-
|
|
10541
|
-
|
|
10542
|
-
|
|
10543
|
-
|
|
10544
|
-
|
|
10545
|
-
|
|
10475
|
+
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
10476
|
+
let bytes;
|
|
10477
|
+
try {
|
|
10478
|
+
bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
|
|
10479
|
+
} catch (error) {
|
|
10480
|
+
const failure = sourceReadFailure(error);
|
|
10481
|
+
await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
|
|
10482
|
+
continue;
|
|
10483
|
+
}
|
|
10484
|
+
const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
|
|
10485
|
+
if (tooSmallImage !== undefined) {
|
|
10486
|
+
await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
|
|
10487
|
+
continue;
|
|
10488
|
+
}
|
|
10489
|
+
let parsed;
|
|
10490
|
+
try {
|
|
10491
|
+
const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
|
|
10492
|
+
parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
|
|
10493
|
+
} catch (error) {
|
|
10494
|
+
if (options.signal?.aborted)
|
|
10495
|
+
throw abortReason(options.signal);
|
|
10496
|
+
if (error instanceof ParseError)
|
|
10497
|
+
await skipFile(entry, {
|
|
10498
|
+
code: "parser-failed",
|
|
10546
10499
|
severity: "warning",
|
|
10547
|
-
message:
|
|
10548
|
-
|
|
10500
|
+
message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
|
|
10501
|
+
}, "parser-failed");
|
|
10502
|
+
else
|
|
10503
|
+
await skipFile(entry, {
|
|
10504
|
+
code: "parser-crashed",
|
|
10505
|
+
severity: "warning",
|
|
10506
|
+
message: PARSER_CRASH_MESSAGE,
|
|
10507
|
+
...error instanceof Error ? { reason: error.name } : {}
|
|
10549
10508
|
});
|
|
10550
|
-
|
|
10509
|
+
continue;
|
|
10551
10510
|
}
|
|
10552
|
-
|
|
10553
|
-
|
|
10554
|
-
|
|
10555
|
-
|
|
10511
|
+
let info;
|
|
10512
|
+
try {
|
|
10513
|
+
info = await stat3(entry.sourcePath, { bigint: true });
|
|
10514
|
+
} catch (error) {
|
|
10515
|
+
await skipFile(entry, sourceReadFailure(error));
|
|
10516
|
+
continue;
|
|
10517
|
+
}
|
|
10518
|
+
const unchanged = cloudDecision.kind === "download" ? contentIdentity(info) === entry.contentIdentity : statIdentity(info) === entry.statIdentity;
|
|
10519
|
+
if (!unchanged) {
|
|
10520
|
+
skipped += 1;
|
|
10556
10521
|
diagnostics.push({
|
|
10557
|
-
code: "
|
|
10522
|
+
code: "source-changed",
|
|
10558
10523
|
severity: "warning",
|
|
10559
|
-
message:
|
|
10524
|
+
message: "The source changed while parsing; its last good mirror was kept and it will be retried on the next refresh.",
|
|
10560
10525
|
source: entry.virtualPath
|
|
10561
10526
|
});
|
|
10527
|
+
await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
|
|
10528
|
+
continue;
|
|
10529
|
+
}
|
|
10530
|
+
const markdown = normalizeMarkdown(parsed.markdown);
|
|
10531
|
+
try {
|
|
10532
|
+
await withTransientRetry(() => writeFileAtomic(outputPath, markdown), transientRetries, transientBackoffMs);
|
|
10533
|
+
} catch (error) {
|
|
10534
|
+
const code = errorCode(error);
|
|
10535
|
+
if (code !== undefined && Object.hasOwn(WORKSPACE_ERROR_CODES, code))
|
|
10536
|
+
throw new MirrorWorkspaceError(options.root, error);
|
|
10537
|
+
await skipFile(entry, {
|
|
10538
|
+
code: "mirror-write-failed",
|
|
10539
|
+
severity: "warning",
|
|
10540
|
+
message: error instanceof Error ? error.message : String(error),
|
|
10541
|
+
...code !== undefined ? { reason: code } : {}
|
|
10542
|
+
});
|
|
10543
|
+
continue;
|
|
10544
|
+
}
|
|
10545
|
+
await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
|
|
10546
|
+
path: basename10(outputPath),
|
|
10547
|
+
size: Buffer.byteLength(markdown)
|
|
10548
|
+
});
|
|
10549
|
+
for (const diagnostic of parsed.diagnostics ?? [])
|
|
10550
|
+
diagnostics.push({ ...diagnostic, source: entry.virtualPath });
|
|
10551
|
+
written += 1;
|
|
10552
|
+
if (fingerprintCloud && cloudFilePaths.has(entry.sourcePath)) {
|
|
10553
|
+
const existing = cloudFingerprintEntries[entry.sourcePath];
|
|
10554
|
+
if (existing === undefined || existing.sourceVersion !== entry.sourceVersion) {
|
|
10555
|
+
try {
|
|
10556
|
+
const fingerprintResult = await fingerprintWithDupey(entry.sourcePath, options.cloudFingerprints?.dupey ?? {});
|
|
10557
|
+
cloudFingerprintEntries[entry.sourcePath] = {
|
|
10558
|
+
sourceVersion: entry.sourceVersion,
|
|
10559
|
+
exact: fingerprintResult.exact,
|
|
10560
|
+
chars: fingerprintResult.chars,
|
|
10561
|
+
modified: fingerprintResult.modified,
|
|
10562
|
+
revision: fingerprintResult.revision
|
|
10563
|
+
};
|
|
10564
|
+
cloudFingerprintsDirty = true;
|
|
10565
|
+
} catch (error) {
|
|
10566
|
+
diagnostics.push({
|
|
10567
|
+
code: "cloud-fingerprint-failed",
|
|
10568
|
+
severity: "warning",
|
|
10569
|
+
message: `The cloud or network file was indexed but its duplicate fingerprint could not be read, so it will not take part in exact-duplicate exclusion: ${error instanceof Error ? error.message : String(error)}`,
|
|
10570
|
+
source: entry.virtualPath
|
|
10571
|
+
});
|
|
10572
|
+
}
|
|
10573
|
+
}
|
|
10574
|
+
}
|
|
10575
|
+
if (cloudDecision.kind === "download" && gate.evictAfterParse) {
|
|
10576
|
+
const released = await gate.release(entry.sourcePath);
|
|
10577
|
+
if (!released.ok) {
|
|
10578
|
+
diagnostics.push({
|
|
10579
|
+
code: "cloud-evict-failed",
|
|
10580
|
+
severity: "warning",
|
|
10581
|
+
message: `The file was indexed but its local copy could not be released: ${released.message}`,
|
|
10582
|
+
source: entry.virtualPath
|
|
10583
|
+
});
|
|
10584
|
+
}
|
|
10562
10585
|
}
|
|
10563
10586
|
}
|
|
10564
10587
|
}
|
|
10588
|
+
if (finalBatch && deferDeletions)
|
|
10589
|
+
await applyDeleteDecisions();
|
|
10590
|
+
if (finalBatch || written + deleted !== changedBefore) {
|
|
10591
|
+
await runBatch(current.slice(batchStart, batchEnd).map((item) => item.virtualPath), finalBatch);
|
|
10592
|
+
}
|
|
10593
|
+
if (batchStopped)
|
|
10594
|
+
break;
|
|
10565
10595
|
}
|
|
10566
10596
|
await session.finish();
|
|
10567
10597
|
if (fingerprintCloud) {
|
|
@@ -10853,7 +10883,7 @@ async function collectFiles(state, directory, depth) {
|
|
|
10853
10883
|
warnDirectoryUnreadable(state, directory, code);
|
|
10854
10884
|
}
|
|
10855
10885
|
}
|
|
10856
|
-
var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
|
|
10886
|
+
var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
|
|
10857
10887
|
var init_sync = __esm(() => {
|
|
10858
10888
|
init_cli2();
|
|
10859
10889
|
init_cloud_placeholder();
|
|
@@ -38692,11 +38722,52 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38692
38722
|
};
|
|
38693
38723
|
try {
|
|
38694
38724
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38725
|
+
const batchedMinSync = needsParsed && wants("minsync");
|
|
38726
|
+
const minSyncBatches = [];
|
|
38727
|
+
const minSyncBatchFailures = [];
|
|
38728
|
+
let minSyncSyncedBase = 0;
|
|
38729
|
+
let minSyncTotalBase = 0;
|
|
38730
|
+
const minSyncBatch = async (batch) => {
|
|
38731
|
+
signal?.throwIfAborted();
|
|
38732
|
+
progress = updateRefreshProgress(progress, { phase: "minsync" });
|
|
38733
|
+
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38734
|
+
let batchTotal = minSyncTotalBase;
|
|
38735
|
+
const result = await this.syncMinSync(force && batch.index === 1, signal, (minsyncProgress) => {
|
|
38736
|
+
batchTotal = minSyncTotalBase + minsyncProgress.total;
|
|
38737
|
+
progress = updateRefreshProgress(progress, {
|
|
38738
|
+
minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: batchTotal }
|
|
38739
|
+
});
|
|
38740
|
+
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38741
|
+
emitProgress({
|
|
38742
|
+
phase: "minsync",
|
|
38743
|
+
processed: minSyncSyncedBase + minsyncProgress.processed,
|
|
38744
|
+
total: batchTotal
|
|
38745
|
+
});
|
|
38746
|
+
});
|
|
38747
|
+
minSyncBatches.push(result);
|
|
38748
|
+
if (result.ok)
|
|
38749
|
+
minSyncBatchFailures.length = 0;
|
|
38750
|
+
else if (result.reason !== "lock-held") {
|
|
38751
|
+
for (const virtualPath of batch.virtualPaths) {
|
|
38752
|
+
minSyncBatchFailures.push({
|
|
38753
|
+
code: "minsync-sync-failed",
|
|
38754
|
+
severity: "warning",
|
|
38755
|
+
message: `This file stayed at its previous index entry: its batch could not be indexed and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
|
|
38756
|
+
source: virtualPath
|
|
38757
|
+
});
|
|
38758
|
+
}
|
|
38759
|
+
}
|
|
38760
|
+
minSyncSyncedBase = batchTotal;
|
|
38761
|
+
minSyncTotalBase = batchTotal;
|
|
38762
|
+
return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
|
|
38763
|
+
};
|
|
38695
38764
|
const summary = needsParsed ? await this.syncParsedMirrors(force, {
|
|
38696
38765
|
...opts?.retrySkipped === true ? { retrySkipped: true } : {},
|
|
38697
38766
|
...signal === undefined ? {} : { signal },
|
|
38767
|
+
...batchedMinSync ? { onBatch: minSyncBatch } : {},
|
|
38698
38768
|
onProgress: (parsedProgress) => {
|
|
38699
38769
|
progress = updateRefreshProgress(progress, {
|
|
38770
|
+
phase: "parsed",
|
|
38700
38771
|
sourceFiles: { total: parsedProgress.total },
|
|
38701
38772
|
parsedCounts: {
|
|
38702
38773
|
scanned: parsedProgress.processed,
|
|
@@ -38718,7 +38789,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38718
38789
|
}) : await this.scanMirrorStaleness();
|
|
38719
38790
|
signal?.throwIfAborted();
|
|
38720
38791
|
progress = updateRefreshProgress(progress, {
|
|
38721
|
-
phase: wants("
|
|
38792
|
+
phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
|
|
38722
38793
|
sourceFiles: { total: summary.scanned },
|
|
38723
38794
|
parsedCounts: {
|
|
38724
38795
|
scanned: summary.scanned,
|
|
@@ -38728,24 +38799,12 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38728
38799
|
}
|
|
38729
38800
|
});
|
|
38730
38801
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38731
|
-
const minsync =
|
|
38732
|
-
progress = updateRefreshProgress(progress, {
|
|
38733
|
-
minsync: { synced: minsyncProgress.processed, total: minsyncProgress.total }
|
|
38734
|
-
});
|
|
38735
|
-
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38736
|
-
emitProgress({
|
|
38737
|
-
phase: "minsync",
|
|
38738
|
-
processed: minsyncProgress.processed,
|
|
38739
|
-
total: minsyncProgress.total
|
|
38740
|
-
});
|
|
38741
|
-
}) : undefined;
|
|
38742
|
-
signal?.throwIfAborted();
|
|
38802
|
+
const minsync = batchedMinSync ? aggregateMinSyncBatches(minSyncBatches) : undefined;
|
|
38743
38803
|
if (minsync !== undefined) {
|
|
38744
38804
|
progress = updateRefreshProgress(progress, {
|
|
38745
|
-
phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
|
|
38746
38805
|
minsync: {
|
|
38747
|
-
...progress.minsync,
|
|
38748
38806
|
synced: minsync.synced,
|
|
38807
|
+
total: Math.max(minSyncTotalBase, minsync.synced),
|
|
38749
38808
|
...minsync.filesFailed !== undefined && minsync.filesFailed.length > 0 ? { filesSkipped: minsync.filesFailed.length } : {}
|
|
38750
38809
|
}
|
|
38751
38810
|
});
|
|
@@ -38807,7 +38866,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38807
38866
|
finishedAt: new Date().toISOString()
|
|
38808
38867
|
});
|
|
38809
38868
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38810
|
-
const minsyncDiagnostics = minSyncRefreshDiagnostics(minsync);
|
|
38869
|
+
const minsyncDiagnostics = [...minSyncRefreshDiagnostics(minsync), ...minSyncBatchFailures];
|
|
38811
38870
|
const publicMinsync = minsync ? {
|
|
38812
38871
|
ok: minsync.ok,
|
|
38813
38872
|
synced: minsync.synced,
|
|
@@ -39612,6 +39671,12 @@ function fsearchRefreshDiagnostics(fsearch) {
|
|
|
39612
39671
|
}
|
|
39613
39672
|
];
|
|
39614
39673
|
}
|
|
39674
|
+
function aggregateMinSyncBatches(batches) {
|
|
39675
|
+
const last = batches.at(-1);
|
|
39676
|
+
if (last === undefined)
|
|
39677
|
+
return;
|
|
39678
|
+
return { ...last, synced: batches.reduce((total, batch) => total + batch.synced, 0) };
|
|
39679
|
+
}
|
|
39615
39680
|
function minSyncRefreshDiagnostics(minsync) {
|
|
39616
39681
|
if (!minsync)
|
|
39617
39682
|
return [];
|