@autorag/librarian 2.13.5 → 2.13.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/agent.d.ts +1 -1
- package/dist/cli/index.js +247 -182
- package/dist/core.js +247 -182
- package/dist/index.js +247 -182
- package/dist/mcp/index.js +247 -182
- package/dist/mirror/sync.d.ts +46 -0
- package/dist/profiles.js +247 -182
- package/package.json +1 -1
package/dist/profiles.js
CHANGED
|
@@ -23989,195 +23989,225 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
23989
23989
|
});
|
|
23990
23990
|
const pending = new Set(plan.decisions.filter((decision) => decision.action === "process" || decision.action === "retry").map((decision) => decision.key));
|
|
23991
23991
|
const deferred = new Map(plan.decisions.filter((decision) => decision.deferred === true).map((decision) => [decision.key, decision]));
|
|
23992
|
-
|
|
23993
|
-
|
|
23994
|
-
|
|
23995
|
-
const path = pathsByKey.get(decision.key);
|
|
23996
|
-
await session.delete(decision.key);
|
|
23997
|
-
if (path === undefined)
|
|
23998
|
-
continue;
|
|
23999
|
-
if (previous.entries[path])
|
|
24000
|
-
deleted += 1;
|
|
24001
|
-
if (!currentPaths.has(path))
|
|
24002
|
-
diagnostics.push({
|
|
24003
|
-
code: "deleted-mirror",
|
|
24004
|
-
severity: "info",
|
|
24005
|
-
message: "A previously indexed document is gone; its parsed mirror was removed.",
|
|
24006
|
-
source: path
|
|
24007
|
-
});
|
|
24008
|
-
}
|
|
24009
|
-
await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
|
|
24010
|
-
for (const [position, entry] of current.entries()) {
|
|
24011
|
-
options.onProgress?.({
|
|
24012
|
-
processed: position,
|
|
24013
|
-
total: current.length,
|
|
24014
|
-
written,
|
|
24015
|
-
deleted,
|
|
24016
|
-
skipped,
|
|
24017
|
-
currentVirtualPath: entry.virtualPath
|
|
24018
|
-
});
|
|
24019
|
-
if (options.signal?.aborted)
|
|
24020
|
-
throw abortReason(options.signal);
|
|
24021
|
-
const omission = omitted.get(entry.virtualPath);
|
|
24022
|
-
if (omission !== undefined) {
|
|
24023
|
-
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
24024
|
-
if (previous.entries[entry.virtualPath])
|
|
24025
|
-
deleted += 1;
|
|
24026
|
-
await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
|
|
24027
|
-
}
|
|
24028
|
-
skipped += 1;
|
|
24029
|
-
if (omission.diagnostic)
|
|
24030
|
-
diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
|
|
24031
|
-
continue;
|
|
24032
|
-
}
|
|
24033
|
-
const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
|
|
24034
|
-
if (deferredFailure !== undefined) {
|
|
24035
|
-
skipped += 1;
|
|
24036
|
-
const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
|
|
24037
|
-
diagnostics.push({
|
|
24038
|
-
code,
|
|
24039
|
-
severity: "warning",
|
|
24040
|
-
source: entry.virtualPath,
|
|
24041
|
-
message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
|
|
24042
|
-
});
|
|
24043
|
-
continue;
|
|
24044
|
-
}
|
|
24045
|
-
const parser = registry.getForVirtualPath(entry.virtualPath);
|
|
24046
|
-
const outputPath = parsedOutputPath(options.root, entry.virtualPath);
|
|
24047
|
-
const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname21(entry.sourcePath)) : { kind: "local" };
|
|
24048
|
-
if (cloudDecision.kind === "skip") {
|
|
24049
|
-
skipped += 1;
|
|
24050
|
-
diagnostics.push({
|
|
24051
|
-
code: "cloud-placeholder-skipped",
|
|
24052
|
-
severity: "info",
|
|
24053
|
-
message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
|
|
24054
|
-
source: entry.virtualPath,
|
|
24055
|
-
...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
|
|
24056
|
-
});
|
|
24057
|
-
continue;
|
|
24058
|
-
}
|
|
24059
|
-
if (cloudDecision.kind === "defer") {
|
|
24060
|
-
skipped += 1;
|
|
24061
|
-
diagnostics.push({
|
|
24062
|
-
code: "cloud-download-deferred",
|
|
24063
|
-
severity: "info",
|
|
24064
|
-
message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
|
|
24065
|
-
source: entry.virtualPath
|
|
24066
|
-
});
|
|
24067
|
-
continue;
|
|
24068
|
-
}
|
|
24069
|
-
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
24070
|
-
let bytes;
|
|
24071
|
-
try {
|
|
24072
|
-
bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
|
|
24073
|
-
} catch (error) {
|
|
24074
|
-
const failure = sourceReadFailure(error);
|
|
24075
|
-
await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
|
|
23992
|
+
const applyDeleteDecisions = async () => {
|
|
23993
|
+
for (const decision of plan.decisions) {
|
|
23994
|
+
if (decision.action !== "delete")
|
|
24076
23995
|
continue;
|
|
24077
|
-
|
|
24078
|
-
|
|
24079
|
-
if (
|
|
24080
|
-
await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
|
|
23996
|
+
const path = pathsByKey.get(decision.key);
|
|
23997
|
+
await session.delete(decision.key);
|
|
23998
|
+
if (path === undefined)
|
|
24081
23999
|
continue;
|
|
24082
|
-
|
|
24083
|
-
|
|
24084
|
-
|
|
24085
|
-
|
|
24086
|
-
|
|
24087
|
-
|
|
24088
|
-
|
|
24089
|
-
|
|
24090
|
-
|
|
24091
|
-
|
|
24092
|
-
|
|
24093
|
-
|
|
24094
|
-
|
|
24095
|
-
|
|
24096
|
-
|
|
24097
|
-
|
|
24098
|
-
|
|
24099
|
-
|
|
24100
|
-
|
|
24101
|
-
|
|
24102
|
-
|
|
24000
|
+
if (previous.entries[path])
|
|
24001
|
+
deleted += 1;
|
|
24002
|
+
if (!currentPaths.has(path))
|
|
24003
|
+
diagnostics.push({
|
|
24004
|
+
code: "deleted-mirror",
|
|
24005
|
+
severity: "info",
|
|
24006
|
+
message: "A previously indexed document is gone; its parsed mirror was removed.",
|
|
24007
|
+
source: path
|
|
24008
|
+
});
|
|
24009
|
+
}
|
|
24010
|
+
};
|
|
24011
|
+
const deferDeletions = options.onBatch !== undefined;
|
|
24012
|
+
const batchSize = Math.max(1, Math.floor(options.batchSize ?? DEFAULT_MIRROR_BATCH_SIZE));
|
|
24013
|
+
const batchCount = Math.max(1, Math.ceil(current.length / batchSize));
|
|
24014
|
+
let batchIndex = 0;
|
|
24015
|
+
let batchStopped = false;
|
|
24016
|
+
const runBatch = async (virtualPaths, final) => {
|
|
24017
|
+
if (options.onBatch === undefined)
|
|
24018
|
+
return;
|
|
24019
|
+
batchIndex += 1;
|
|
24020
|
+
const outcome = await options.onBatch({ index: batchIndex, total: batchCount, virtualPaths, final });
|
|
24021
|
+
if (outcome?.stop === true)
|
|
24022
|
+
batchStopped = true;
|
|
24023
|
+
};
|
|
24024
|
+
if (!deferDeletions)
|
|
24025
|
+
await applyDeleteDecisions();
|
|
24026
|
+
await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
|
|
24027
|
+
for (let batchStart = 0;batchStart < Math.max(current.length, 1); batchStart += batchSize) {
|
|
24028
|
+
const batchEnd = Math.min(batchStart + batchSize, current.length);
|
|
24029
|
+
const finalBatch = batchEnd >= current.length;
|
|
24030
|
+
const changedBefore = written + deleted;
|
|
24031
|
+
for (let position = batchStart;position < batchEnd; position += 1) {
|
|
24032
|
+
const entry = current[position];
|
|
24033
|
+
options.onProgress?.({
|
|
24034
|
+
processed: position,
|
|
24035
|
+
total: current.length,
|
|
24036
|
+
written,
|
|
24037
|
+
deleted,
|
|
24038
|
+
skipped,
|
|
24039
|
+
currentVirtualPath: entry.virtualPath
|
|
24040
|
+
});
|
|
24041
|
+
if (options.signal?.aborted)
|
|
24042
|
+
throw abortReason(options.signal);
|
|
24043
|
+
const omission = omitted.get(entry.virtualPath);
|
|
24044
|
+
if (omission !== undefined) {
|
|
24045
|
+
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
24046
|
+
if (previous.entries[entry.virtualPath])
|
|
24047
|
+
deleted += 1;
|
|
24048
|
+
await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
|
|
24049
|
+
}
|
|
24050
|
+
skipped += 1;
|
|
24051
|
+
if (omission.diagnostic)
|
|
24052
|
+
diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
|
|
24103
24053
|
continue;
|
|
24104
24054
|
}
|
|
24105
|
-
|
|
24106
|
-
|
|
24107
|
-
|
|
24108
|
-
|
|
24109
|
-
|
|
24055
|
+
const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
|
|
24056
|
+
if (deferredFailure !== undefined) {
|
|
24057
|
+
skipped += 1;
|
|
24058
|
+
const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
|
|
24059
|
+
diagnostics.push({
|
|
24060
|
+
code,
|
|
24061
|
+
severity: "warning",
|
|
24062
|
+
source: entry.virtualPath,
|
|
24063
|
+
message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
|
|
24064
|
+
});
|
|
24110
24065
|
continue;
|
|
24111
24066
|
}
|
|
24112
|
-
const
|
|
24113
|
-
|
|
24067
|
+
const parser = registry.getForVirtualPath(entry.virtualPath);
|
|
24068
|
+
const outputPath = parsedOutputPath(options.root, entry.virtualPath);
|
|
24069
|
+
const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname21(entry.sourcePath)) : { kind: "local" };
|
|
24070
|
+
if (cloudDecision.kind === "skip") {
|
|
24114
24071
|
skipped += 1;
|
|
24115
24072
|
diagnostics.push({
|
|
24116
|
-
code: "
|
|
24117
|
-
severity: "
|
|
24118
|
-
message:
|
|
24119
|
-
source: entry.virtualPath
|
|
24073
|
+
code: "cloud-placeholder-skipped",
|
|
24074
|
+
severity: "info",
|
|
24075
|
+
message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
|
|
24076
|
+
source: entry.virtualPath,
|
|
24077
|
+
...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
|
|
24120
24078
|
});
|
|
24121
|
-
await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
|
|
24122
24079
|
continue;
|
|
24123
24080
|
}
|
|
24124
|
-
|
|
24125
|
-
|
|
24126
|
-
|
|
24127
|
-
|
|
24128
|
-
|
|
24129
|
-
|
|
24130
|
-
|
|
24131
|
-
await skipFile(entry, {
|
|
24132
|
-
code: "mirror-write-failed",
|
|
24133
|
-
severity: "warning",
|
|
24134
|
-
message: error instanceof Error ? error.message : String(error),
|
|
24135
|
-
...code !== undefined ? { reason: code } : {}
|
|
24081
|
+
if (cloudDecision.kind === "defer") {
|
|
24082
|
+
skipped += 1;
|
|
24083
|
+
diagnostics.push({
|
|
24084
|
+
code: "cloud-download-deferred",
|
|
24085
|
+
severity: "info",
|
|
24086
|
+
message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
|
|
24087
|
+
source: entry.virtualPath
|
|
24136
24088
|
});
|
|
24137
24089
|
continue;
|
|
24138
24090
|
}
|
|
24139
|
-
|
|
24140
|
-
|
|
24141
|
-
|
|
24142
|
-
|
|
24143
|
-
|
|
24144
|
-
|
|
24145
|
-
|
|
24146
|
-
|
|
24147
|
-
|
|
24148
|
-
|
|
24149
|
-
|
|
24150
|
-
|
|
24151
|
-
|
|
24152
|
-
|
|
24153
|
-
|
|
24154
|
-
|
|
24155
|
-
|
|
24156
|
-
|
|
24157
|
-
|
|
24158
|
-
|
|
24159
|
-
|
|
24160
|
-
|
|
24161
|
-
|
|
24091
|
+
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
24092
|
+
let bytes;
|
|
24093
|
+
try {
|
|
24094
|
+
bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
|
|
24095
|
+
} catch (error) {
|
|
24096
|
+
const failure = sourceReadFailure(error);
|
|
24097
|
+
await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
|
|
24098
|
+
continue;
|
|
24099
|
+
}
|
|
24100
|
+
const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
|
|
24101
|
+
if (tooSmallImage !== undefined) {
|
|
24102
|
+
await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
|
|
24103
|
+
continue;
|
|
24104
|
+
}
|
|
24105
|
+
let parsed;
|
|
24106
|
+
try {
|
|
24107
|
+
const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
|
|
24108
|
+
parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
|
|
24109
|
+
} catch (error) {
|
|
24110
|
+
if (options.signal?.aborted)
|
|
24111
|
+
throw abortReason(options.signal);
|
|
24112
|
+
if (error instanceof ParseError)
|
|
24113
|
+
await skipFile(entry, {
|
|
24114
|
+
code: "parser-failed",
|
|
24162
24115
|
severity: "warning",
|
|
24163
|
-
message:
|
|
24164
|
-
|
|
24116
|
+
message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
|
|
24117
|
+
}, "parser-failed");
|
|
24118
|
+
else
|
|
24119
|
+
await skipFile(entry, {
|
|
24120
|
+
code: "parser-crashed",
|
|
24121
|
+
severity: "warning",
|
|
24122
|
+
message: PARSER_CRASH_MESSAGE,
|
|
24123
|
+
...error instanceof Error ? { reason: error.name } : {}
|
|
24165
24124
|
});
|
|
24166
|
-
|
|
24125
|
+
continue;
|
|
24167
24126
|
}
|
|
24168
|
-
|
|
24169
|
-
|
|
24170
|
-
|
|
24171
|
-
|
|
24127
|
+
let info;
|
|
24128
|
+
try {
|
|
24129
|
+
info = await stat4(entry.sourcePath, { bigint: true });
|
|
24130
|
+
} catch (error) {
|
|
24131
|
+
await skipFile(entry, sourceReadFailure(error));
|
|
24132
|
+
continue;
|
|
24133
|
+
}
|
|
24134
|
+
const unchanged = cloudDecision.kind === "download" ? contentIdentity(info) === entry.contentIdentity : statIdentity(info) === entry.statIdentity;
|
|
24135
|
+
if (!unchanged) {
|
|
24136
|
+
skipped += 1;
|
|
24172
24137
|
diagnostics.push({
|
|
24173
|
-
code: "
|
|
24138
|
+
code: "source-changed",
|
|
24174
24139
|
severity: "warning",
|
|
24175
|
-
message:
|
|
24140
|
+
message: "The source changed while parsing; its last good mirror was kept and it will be retried on the next refresh.",
|
|
24176
24141
|
source: entry.virtualPath
|
|
24177
24142
|
});
|
|
24143
|
+
await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
|
|
24144
|
+
continue;
|
|
24145
|
+
}
|
|
24146
|
+
const markdown = normalizeMarkdown(parsed.markdown);
|
|
24147
|
+
try {
|
|
24148
|
+
await withTransientRetry(() => writeFileAtomic(outputPath, markdown), transientRetries, transientBackoffMs);
|
|
24149
|
+
} catch (error) {
|
|
24150
|
+
const code = errorCode(error);
|
|
24151
|
+
if (code !== undefined && Object.hasOwn(WORKSPACE_ERROR_CODES, code))
|
|
24152
|
+
throw new MirrorWorkspaceError(options.root, error);
|
|
24153
|
+
await skipFile(entry, {
|
|
24154
|
+
code: "mirror-write-failed",
|
|
24155
|
+
severity: "warning",
|
|
24156
|
+
message: error instanceof Error ? error.message : String(error),
|
|
24157
|
+
...code !== undefined ? { reason: code } : {}
|
|
24158
|
+
});
|
|
24159
|
+
continue;
|
|
24160
|
+
}
|
|
24161
|
+
await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
|
|
24162
|
+
path: basename14(outputPath),
|
|
24163
|
+
size: Buffer.byteLength(markdown)
|
|
24164
|
+
});
|
|
24165
|
+
for (const diagnostic of parsed.diagnostics ?? [])
|
|
24166
|
+
diagnostics.push({ ...diagnostic, source: entry.virtualPath });
|
|
24167
|
+
written += 1;
|
|
24168
|
+
if (fingerprintCloud && cloudFilePaths.has(entry.sourcePath)) {
|
|
24169
|
+
const existing = cloudFingerprintEntries[entry.sourcePath];
|
|
24170
|
+
if (existing === undefined || existing.sourceVersion !== entry.sourceVersion) {
|
|
24171
|
+
try {
|
|
24172
|
+
const fingerprintResult = await fingerprintWithDupey(entry.sourcePath, options.cloudFingerprints?.dupey ?? {});
|
|
24173
|
+
cloudFingerprintEntries[entry.sourcePath] = {
|
|
24174
|
+
sourceVersion: entry.sourceVersion,
|
|
24175
|
+
exact: fingerprintResult.exact,
|
|
24176
|
+
chars: fingerprintResult.chars,
|
|
24177
|
+
modified: fingerprintResult.modified,
|
|
24178
|
+
revision: fingerprintResult.revision
|
|
24179
|
+
};
|
|
24180
|
+
cloudFingerprintsDirty = true;
|
|
24181
|
+
} catch (error) {
|
|
24182
|
+
diagnostics.push({
|
|
24183
|
+
code: "cloud-fingerprint-failed",
|
|
24184
|
+
severity: "warning",
|
|
24185
|
+
message: `The cloud or network file was indexed but its duplicate fingerprint could not be read, so it will not take part in exact-duplicate exclusion: ${error instanceof Error ? error.message : String(error)}`,
|
|
24186
|
+
source: entry.virtualPath
|
|
24187
|
+
});
|
|
24188
|
+
}
|
|
24189
|
+
}
|
|
24190
|
+
}
|
|
24191
|
+
if (cloudDecision.kind === "download" && gate.evictAfterParse) {
|
|
24192
|
+
const released = await gate.release(entry.sourcePath);
|
|
24193
|
+
if (!released.ok) {
|
|
24194
|
+
diagnostics.push({
|
|
24195
|
+
code: "cloud-evict-failed",
|
|
24196
|
+
severity: "warning",
|
|
24197
|
+
message: `The file was indexed but its local copy could not be released: ${released.message}`,
|
|
24198
|
+
source: entry.virtualPath
|
|
24199
|
+
});
|
|
24200
|
+
}
|
|
24178
24201
|
}
|
|
24179
24202
|
}
|
|
24180
24203
|
}
|
|
24204
|
+
if (finalBatch && deferDeletions)
|
|
24205
|
+
await applyDeleteDecisions();
|
|
24206
|
+
if (finalBatch || written + deleted !== changedBefore) {
|
|
24207
|
+
await runBatch(current.slice(batchStart, batchEnd).map((item) => item.virtualPath), finalBatch);
|
|
24208
|
+
}
|
|
24209
|
+
if (batchStopped)
|
|
24210
|
+
break;
|
|
24181
24211
|
}
|
|
24182
24212
|
await session.finish();
|
|
24183
24213
|
if (fingerprintCloud) {
|
|
@@ -24469,7 +24499,7 @@ async function collectFiles(state, directory, depth) {
|
|
|
24469
24499
|
warnDirectoryUnreadable(state, directory, code);
|
|
24470
24500
|
}
|
|
24471
24501
|
}
|
|
24472
|
-
var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
|
|
24502
|
+
var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
|
|
24473
24503
|
var init_sync = __esm(() => {
|
|
24474
24504
|
init_cli2();
|
|
24475
24505
|
init_cloud_placeholder();
|
|
@@ -38634,11 +38664,52 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38634
38664
|
};
|
|
38635
38665
|
try {
|
|
38636
38666
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38667
|
+
const batchedMinSync = needsParsed && wants("minsync");
|
|
38668
|
+
const minSyncBatches = [];
|
|
38669
|
+
const minSyncBatchFailures = [];
|
|
38670
|
+
let minSyncSyncedBase = 0;
|
|
38671
|
+
let minSyncTotalBase = 0;
|
|
38672
|
+
const minSyncBatch = async (batch) => {
|
|
38673
|
+
signal?.throwIfAborted();
|
|
38674
|
+
progress = updateRefreshProgress(progress, { phase: "minsync" });
|
|
38675
|
+
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38676
|
+
let batchTotal = minSyncTotalBase;
|
|
38677
|
+
const result = await this.syncMinSync(force && batch.index === 1, signal, (minsyncProgress) => {
|
|
38678
|
+
batchTotal = minSyncTotalBase + minsyncProgress.total;
|
|
38679
|
+
progress = updateRefreshProgress(progress, {
|
|
38680
|
+
minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: batchTotal }
|
|
38681
|
+
});
|
|
38682
|
+
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38683
|
+
emitProgress({
|
|
38684
|
+
phase: "minsync",
|
|
38685
|
+
processed: minSyncSyncedBase + minsyncProgress.processed,
|
|
38686
|
+
total: batchTotal
|
|
38687
|
+
});
|
|
38688
|
+
});
|
|
38689
|
+
minSyncBatches.push(result);
|
|
38690
|
+
if (result.ok)
|
|
38691
|
+
minSyncBatchFailures.length = 0;
|
|
38692
|
+
else if (result.reason !== "lock-held") {
|
|
38693
|
+
for (const virtualPath of batch.virtualPaths) {
|
|
38694
|
+
minSyncBatchFailures.push({
|
|
38695
|
+
code: "minsync-sync-failed",
|
|
38696
|
+
severity: "warning",
|
|
38697
|
+
message: `This file stayed at its previous index entry: its batch could not be indexed and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
|
|
38698
|
+
source: virtualPath
|
|
38699
|
+
});
|
|
38700
|
+
}
|
|
38701
|
+
}
|
|
38702
|
+
minSyncSyncedBase = batchTotal;
|
|
38703
|
+
minSyncTotalBase = batchTotal;
|
|
38704
|
+
return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
|
|
38705
|
+
};
|
|
38637
38706
|
const summary = needsParsed ? await this.syncParsedMirrors(force, {
|
|
38638
38707
|
...opts?.retrySkipped === true ? { retrySkipped: true } : {},
|
|
38639
38708
|
...signal === undefined ? {} : { signal },
|
|
38709
|
+
...batchedMinSync ? { onBatch: minSyncBatch } : {},
|
|
38640
38710
|
onProgress: (parsedProgress) => {
|
|
38641
38711
|
progress = updateRefreshProgress(progress, {
|
|
38712
|
+
phase: "parsed",
|
|
38642
38713
|
sourceFiles: { total: parsedProgress.total },
|
|
38643
38714
|
parsedCounts: {
|
|
38644
38715
|
scanned: parsedProgress.processed,
|
|
@@ -38660,7 +38731,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38660
38731
|
}) : await this.scanMirrorStaleness();
|
|
38661
38732
|
signal?.throwIfAborted();
|
|
38662
38733
|
progress = updateRefreshProgress(progress, {
|
|
38663
|
-
phase: wants("
|
|
38734
|
+
phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
|
|
38664
38735
|
sourceFiles: { total: summary.scanned },
|
|
38665
38736
|
parsedCounts: {
|
|
38666
38737
|
scanned: summary.scanned,
|
|
@@ -38670,24 +38741,12 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38670
38741
|
}
|
|
38671
38742
|
});
|
|
38672
38743
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38673
|
-
const minsync =
|
|
38674
|
-
progress = updateRefreshProgress(progress, {
|
|
38675
|
-
minsync: { synced: minsyncProgress.processed, total: minsyncProgress.total }
|
|
38676
|
-
});
|
|
38677
|
-
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38678
|
-
emitProgress({
|
|
38679
|
-
phase: "minsync",
|
|
38680
|
-
processed: minsyncProgress.processed,
|
|
38681
|
-
total: minsyncProgress.total
|
|
38682
|
-
});
|
|
38683
|
-
}) : undefined;
|
|
38684
|
-
signal?.throwIfAborted();
|
|
38744
|
+
const minsync = batchedMinSync ? aggregateMinSyncBatches(minSyncBatches) : undefined;
|
|
38685
38745
|
if (minsync !== undefined) {
|
|
38686
38746
|
progress = updateRefreshProgress(progress, {
|
|
38687
|
-
phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
|
|
38688
38747
|
minsync: {
|
|
38689
|
-
...progress.minsync,
|
|
38690
38748
|
synced: minsync.synced,
|
|
38749
|
+
total: Math.max(minSyncTotalBase, minsync.synced),
|
|
38691
38750
|
...minsync.filesFailed !== undefined && minsync.filesFailed.length > 0 ? { filesSkipped: minsync.filesFailed.length } : {}
|
|
38692
38751
|
}
|
|
38693
38752
|
});
|
|
@@ -38749,7 +38808,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38749
38808
|
finishedAt: new Date().toISOString()
|
|
38750
38809
|
});
|
|
38751
38810
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38752
|
-
const minsyncDiagnostics = minSyncRefreshDiagnostics(minsync);
|
|
38811
|
+
const minsyncDiagnostics = [...minSyncRefreshDiagnostics(minsync), ...minSyncBatchFailures];
|
|
38753
38812
|
const publicMinsync = minsync ? {
|
|
38754
38813
|
ok: minsync.ok,
|
|
38755
38814
|
synced: minsync.synced,
|
|
@@ -39554,6 +39613,12 @@ function fsearchRefreshDiagnostics(fsearch) {
|
|
|
39554
39613
|
}
|
|
39555
39614
|
];
|
|
39556
39615
|
}
|
|
39616
|
+
function aggregateMinSyncBatches(batches) {
|
|
39617
|
+
const last = batches.at(-1);
|
|
39618
|
+
if (last === undefined)
|
|
39619
|
+
return;
|
|
39620
|
+
return { ...last, synced: batches.reduce((total, batch) => total + batch.synced, 0) };
|
|
39621
|
+
}
|
|
39557
39622
|
function minSyncRefreshDiagnostics(minsync) {
|
|
39558
39623
|
if (!minsync)
|
|
39559
39624
|
return [];
|
package/package.json
CHANGED