@autorag/librarian 2.13.5 → 2.13.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/agent.d.ts +5 -5
- package/dist/agent/dupey-tool.d.ts +0 -2
- package/dist/agent/pipelined-sync.d.ts +34 -0
- package/dist/agent/search-documents.d.ts +1 -1
- package/dist/cli/commands/duplicates.d.ts +2 -3
- package/dist/cli/index.js +1721 -1542
- package/dist/core.js +2505 -2320
- package/dist/datasource/skills/factory.d.ts +4 -1
- package/dist/dupey/filter.d.ts +0 -2
- package/dist/dupey/index.d.ts +0 -1
- package/dist/dupey/version.d.ts +17 -0
- package/dist/index.js +2302 -2122
- package/dist/mcp/index.js +2302 -2120
- package/dist/mcp/server.d.ts +0 -3
- package/dist/minsync/installer.d.ts +1 -1
- package/dist/minsync/version.d.ts +3 -2
- package/dist/mirror/cloud-fingerprints.d.ts +6 -5
- package/dist/mirror/cloud.d.ts +9 -6
- package/dist/mirror/sync.d.ts +53 -3
- package/dist/profiles.js +1714 -1529
- package/package.json +1 -1
- package/dist/dupey/scan-plan.d.ts +0 -23
package/dist/cli/index.js
CHANGED
|
@@ -5032,7 +5032,7 @@ function outdatedMinSyncVersion(binaryPath) {
|
|
|
5032
5032
|
}
|
|
5033
5033
|
return version !== undefined && !isSupportedMinSyncVersion(version) ? version : undefined;
|
|
5034
5034
|
}
|
|
5035
|
-
var MINSYNC_MIN_VERSION = "0.
|
|
5035
|
+
var MINSYNC_MIN_VERSION = "0.7.1", VERSION_PROBE_TIMEOUT_MS = 5000, VERSION_LINE, probeCache;
|
|
5036
5036
|
var init_version = __esm(() => {
|
|
5037
5037
|
VERSION_LINE = /^minsync\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
|
|
5038
5038
|
probeCache = new Map;
|
|
@@ -5276,7 +5276,7 @@ function isGitHubAsset(value) {
|
|
|
5276
5276
|
function isRecord2(value) {
|
|
5277
5277
|
return typeof value === "object" && value !== null;
|
|
5278
5278
|
}
|
|
5279
|
-
var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.
|
|
5279
|
+
var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.7.3", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
|
|
5280
5280
|
var init_installer = __esm(() => {
|
|
5281
5281
|
init_atomic_write();
|
|
5282
5282
|
init_process();
|
|
@@ -14567,7 +14567,8 @@ class WombatMethod {
|
|
|
14567
14567
|
return matchesDatasourceScope(datasourceSourcePath("wombat", this.instanceId, hit.chunkId), options.scope);
|
|
14568
14568
|
}).map((hit) => ({
|
|
14569
14569
|
id: `wombat:${this.instanceId}:${hit.chunkId}`,
|
|
14570
|
-
content: hit
|
|
14570
|
+
content: `${sessionLabel(hit)}
|
|
14571
|
+
${hit.snippet}`,
|
|
14571
14572
|
source: datasourceSourcePath("wombat", this.instanceId, hit.chunkId),
|
|
14572
14573
|
score: hit.score,
|
|
14573
14574
|
metadata: {
|
|
@@ -14590,6 +14591,20 @@ class WombatMethod {
|
|
|
14590
14591
|
})).slice(0, topK);
|
|
14591
14592
|
}
|
|
14592
14593
|
}
|
|
14594
|
+
function oneLine(value) {
|
|
14595
|
+
return value.replace(/\s+/gu, " ").trim();
|
|
14596
|
+
}
|
|
14597
|
+
function sessionLabel(hit) {
|
|
14598
|
+
const turns = hit.startTurn === undefined ? undefined : hit.endTurn === undefined || hit.endTurn === hit.startTurn ? `turn ${hit.startTurn}` : `turns ${hit.startTurn}-${hit.endTurn}`;
|
|
14599
|
+
const parts = [
|
|
14600
|
+
oneLine(hit.catalogKey),
|
|
14601
|
+
turns,
|
|
14602
|
+
hit.startedAt === undefined ? undefined : hit.startedAt.slice(0, 10),
|
|
14603
|
+
hit.title.length === 0 ? undefined : oneLine(hit.title),
|
|
14604
|
+
hit.cwd === undefined ? undefined : oneLine(hit.cwd)
|
|
14605
|
+
].filter((part) => part !== undefined && part.length > 0);
|
|
14606
|
+
return `[${parts.join(" | ")}]`;
|
|
14607
|
+
}
|
|
14593
14608
|
var WOMBAT_MODES, METHOD_TYPE;
|
|
14594
14609
|
var init_methods7 = __esm(() => {
|
|
14595
14610
|
init_errors2();
|
|
@@ -14702,7 +14717,7 @@ class WombatSkill {
|
|
|
14702
14717
|
`);
|
|
14703
14718
|
return {
|
|
14704
14719
|
name: "datasource-wombat",
|
|
14705
|
-
description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built.",
|
|
14720
|
+
description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built. Each hit names its session. Load this skill to open that session's full conversation or its original log with the tool calls that the search does not index.",
|
|
14706
14721
|
content: [
|
|
14707
14722
|
"# wombat coding-agent session datasource",
|
|
14708
14723
|
"",
|
|
@@ -14716,10 +14731,10 @@ class WombatSkill {
|
|
|
14716
14731
|
scopes,
|
|
14717
14732
|
"",
|
|
14718
14733
|
"The selectable retrieval modes are keyword (SPLADE), semantic, and hybrid. Scope arguments can only narrow trusted access. Source identifiers remain opaque and hierarchical.",
|
|
14719
|
-
"
|
|
14734
|
+
"Every result starts with a label line, `[<sessionKey> | turns A-B | date | title | project]`, followed by the snippet. `sessionKey` is `<agent>:<session id>`.",
|
|
14720
14735
|
"",
|
|
14721
14736
|
"## Native CLI",
|
|
14722
|
-
"The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the result's
|
|
14737
|
+
"The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the `sessionKey` from the result's label:",
|
|
14723
14738
|
"- `wombat show <sessionKey> [--turn N]`: the full cleaned conversation, or one turn.",
|
|
14724
14739
|
"- `wombat trace <sessionKey> --turn N --context K`: the original session log around that turn, including the tool calls and outputs that the index leaves out.",
|
|
14725
14740
|
"- `wombat chunk context <chunk id> --json`: the turns before and after a result.",
|
|
@@ -14873,6 +14888,7 @@ function common(config, workspaceRoot) {
|
|
|
14873
14888
|
function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
|
|
14874
14889
|
const skills = [];
|
|
14875
14890
|
const unknown = [];
|
|
14891
|
+
const retired = [];
|
|
14876
14892
|
for (const [name, raw] of Object.entries(config ?? {})) {
|
|
14877
14893
|
if (raw === false)
|
|
14878
14894
|
continue;
|
|
@@ -14884,6 +14900,10 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
|
|
|
14884
14900
|
if (entry.enabled === false)
|
|
14885
14901
|
continue;
|
|
14886
14902
|
const templateName = entry.type ?? name;
|
|
14903
|
+
if (Object.hasOwn(RETIRED_TEMPLATES, templateName)) {
|
|
14904
|
+
retired.push(name);
|
|
14905
|
+
continue;
|
|
14906
|
+
}
|
|
14887
14907
|
const builder = Object.hasOwn(BUILDERS, templateName) && datasourceSkillSupported(templateName) ? BUILDERS[templateName] : undefined;
|
|
14888
14908
|
if (builder === undefined) {
|
|
14889
14909
|
unknown.push(name);
|
|
@@ -14907,9 +14927,9 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
|
|
|
14907
14927
|
}
|
|
14908
14928
|
skills.push(typeof entry.description === "string" && entry.description.trim().length > 0 ? new DescribedDatasourceSkill(aliased, entry.description.trim()) : aliased);
|
|
14909
14929
|
}
|
|
14910
|
-
return { skills, unknown };
|
|
14930
|
+
return { skills, unknown, retired };
|
|
14911
14931
|
}
|
|
14912
|
-
var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES;
|
|
14932
|
+
var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES, RETIRED_TEMPLATES;
|
|
14913
14933
|
var init_factory = __esm(() => {
|
|
14914
14934
|
init_aliased_skill();
|
|
14915
14935
|
init_config3();
|
|
@@ -15065,6 +15085,7 @@ var init_factory = __esm(() => {
|
|
|
15065
15085
|
};
|
|
15066
15086
|
PLATFORM_ONLY_SKILLS = { kakao: "darwin" };
|
|
15067
15087
|
BUILTIN_DATASOURCE_SKILL_NAMES = builtinDatasourceSkillNames();
|
|
15088
|
+
RETIRED_TEMPLATES = { spotlight: true };
|
|
15068
15089
|
});
|
|
15069
15090
|
|
|
15070
15091
|
// src/language.ts
|
|
@@ -16297,7 +16318,7 @@ function buildAgentOptions(config, agentDir) {
|
|
|
16297
16318
|
const embedder = createEmbedderFromConfig({ embedder: config.minSync?.embedder });
|
|
16298
16319
|
opts.embedder = embedder;
|
|
16299
16320
|
if (config.datasources !== undefined) {
|
|
16300
|
-
const { skills, unknown } = buildDatasourceSkills(config.datasources, config.workspacePath, {
|
|
16321
|
+
const { skills, unknown, retired } = buildDatasourceSkills(config.datasources, config.workspacePath, {
|
|
16301
16322
|
embedder,
|
|
16302
16323
|
discrawl: async () => discrawlEmbeddingRuntimeFor(await embedder.describeEndpoint()),
|
|
16303
16324
|
cliEndpoint: embeddingEndpointSource(embedder),
|
|
@@ -16310,16 +16331,27 @@ function buildAgentOptions(config, agentDir) {
|
|
|
16310
16331
|
});
|
|
16311
16332
|
if (skills.length > 0)
|
|
16312
16333
|
opts.datasourceSkills = skills;
|
|
16334
|
+
const diagnostics = [];
|
|
16335
|
+
if (retired.length > 0) {
|
|
16336
|
+
const names = retired.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
|
|
16337
|
+
diagnostics.push({
|
|
16338
|
+
code: "retired-datasource-skill",
|
|
16339
|
+
severity: "warning",
|
|
16340
|
+
message: `Datasource config ignored because the Spotlight datasource was removed: ${names.join(", ")}. Delete these entries; use fsearch/mirror search (searchPaths) instead.`,
|
|
16341
|
+
source: "datasources"
|
|
16342
|
+
});
|
|
16343
|
+
}
|
|
16313
16344
|
if (unknown.length > 0) {
|
|
16314
|
-
const
|
|
16315
|
-
|
|
16345
|
+
const names = unknown.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
|
|
16346
|
+
diagnostics.push({
|
|
16316
16347
|
code: "unknown-datasource-skill",
|
|
16317
16348
|
severity: "warning",
|
|
16318
|
-
message: `Unknown datasource skill(s) in config were skipped: ${
|
|
16349
|
+
message: `Unknown datasource skill(s) in config were skipped: ${names.join(", ")}`,
|
|
16319
16350
|
source: "datasources"
|
|
16320
|
-
};
|
|
16321
|
-
opts.startupDiagnostics = [diagnostic];
|
|
16351
|
+
});
|
|
16322
16352
|
}
|
|
16353
|
+
if (diagnostics.length > 0)
|
|
16354
|
+
opts.startupDiagnostics = diagnostics;
|
|
16323
16355
|
}
|
|
16324
16356
|
return opts;
|
|
16325
16357
|
}
|
|
@@ -17832,11 +17864,49 @@ var init_matrix = __esm(() => {
|
|
|
17832
17864
|
};
|
|
17833
17865
|
});
|
|
17834
17866
|
|
|
17867
|
+
// src/dupey/version.ts
|
|
17868
|
+
function parseDupeyVersion(output) {
|
|
17869
|
+
return VERSION_LINE2.exec(output)?.[1];
|
|
17870
|
+
}
|
|
17871
|
+
function isSupportedDupeyVersion(version) {
|
|
17872
|
+
const have = version.split(".").map((part) => Number.parseInt(part, 10));
|
|
17873
|
+
const need = DUPEY_MIN_VERSION.split(".").map((part) => Number.parseInt(part, 10));
|
|
17874
|
+
for (let index = 0;index < need.length; index += 1) {
|
|
17875
|
+
const left = have[index] ?? 0;
|
|
17876
|
+
const right = need[index] ?? 0;
|
|
17877
|
+
if (left !== right)
|
|
17878
|
+
return left > right;
|
|
17879
|
+
}
|
|
17880
|
+
return true;
|
|
17881
|
+
}
|
|
17882
|
+
var DUPEY_MIN_VERSION = "0.2.0", VERSION_LINE2;
|
|
17883
|
+
var init_version2 = __esm(() => {
|
|
17884
|
+
VERSION_LINE2 = /^dupey\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
|
|
17885
|
+
});
|
|
17886
|
+
|
|
17835
17887
|
// src/dupey/cli.ts
|
|
17836
17888
|
import { spawn as spawn14 } from "node:child_process";
|
|
17837
17889
|
import { resolve as resolve9 } from "node:path";
|
|
17838
|
-
function
|
|
17839
|
-
|
|
17890
|
+
function checkDupeyVersion(executable, options) {
|
|
17891
|
+
const cached = versionChecks.get(executable);
|
|
17892
|
+
if (cached !== undefined)
|
|
17893
|
+
return cached;
|
|
17894
|
+
const check = spawnDupey(executable, ["--version"], options).then((stdout) => {
|
|
17895
|
+
const version = parseDupeyVersion(stdout);
|
|
17896
|
+
if (version !== undefined && !isSupportedDupeyVersion(version)) {
|
|
17897
|
+
throw new DupeyCliError(`dupey ${version} at ${executable} is older than the ${DUPEY_MIN_VERSION} AutoRAG requires; run \`cargo install dupey\` to update it.`);
|
|
17898
|
+
}
|
|
17899
|
+
});
|
|
17900
|
+
check.catch(() => versionChecks.delete(executable));
|
|
17901
|
+
versionChecks.set(executable, check);
|
|
17902
|
+
return check;
|
|
17903
|
+
}
|
|
17904
|
+
async function runDupey(args, options) {
|
|
17905
|
+
if (options.run)
|
|
17906
|
+
return options.run(args);
|
|
17907
|
+
const executable = options.executable ?? "dupey";
|
|
17908
|
+
await checkDupeyVersion(executable, options);
|
|
17909
|
+
return spawnDupey(executable, args, options);
|
|
17840
17910
|
}
|
|
17841
17911
|
async function scanWithDupey(dir, options = {}) {
|
|
17842
17912
|
const absoluteDir = resolve9(dir);
|
|
@@ -17970,28 +18040,26 @@ function spawnDupey(executable, args, options) {
|
|
|
17970
18040
|
});
|
|
17971
18041
|
});
|
|
17972
18042
|
}
|
|
17973
|
-
var DupeyCliError, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
|
|
18043
|
+
var DupeyCliError, versionChecks, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
|
|
17974
18044
|
var init_cli2 = __esm(() => {
|
|
17975
18045
|
init_portable_spawn();
|
|
18046
|
+
init_version2();
|
|
17976
18047
|
DupeyCliError = class DupeyCliError extends Error {
|
|
17977
18048
|
constructor(message) {
|
|
17978
18049
|
super(message);
|
|
17979
18050
|
this.name = "DupeyCliError";
|
|
17980
18051
|
}
|
|
17981
18052
|
};
|
|
18053
|
+
versionChecks = new Map;
|
|
17982
18054
|
});
|
|
17983
18055
|
|
|
17984
18056
|
// src/dupey/filter.ts
|
|
17985
18057
|
import { stat as stat2 } from "node:fs/promises";
|
|
17986
18058
|
import { resolve as resolve10 } from "node:path";
|
|
17987
|
-
function extractedNoText(file) {
|
|
17988
|
-
const signals = file.signals;
|
|
17989
|
-
return typeof signals === "object" && signals !== null && "chars" in signals && signals.chars === 0;
|
|
17990
|
-
}
|
|
17991
18059
|
async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerprints) {
|
|
17992
18060
|
const byHash = new Map;
|
|
17993
18061
|
for (const file of scan.files) {
|
|
17994
|
-
if (typeof file.content_hash !== "string" || file.content_hash.length === 0
|
|
18062
|
+
if (typeof file.content_hash !== "string" || file.content_hash.length === 0)
|
|
17995
18063
|
continue;
|
|
17996
18064
|
const path = resolve10(root, file.path);
|
|
17997
18065
|
const group = byHash.get(file.content_hash) ?? [];
|
|
@@ -18000,7 +18068,7 @@ async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerp
|
|
|
18000
18068
|
}
|
|
18001
18069
|
const fingerprintModified = new Map;
|
|
18002
18070
|
for (const [path, fingerprint] of fingerprints ?? []) {
|
|
18003
|
-
if (fingerprint.exact.length === 0
|
|
18071
|
+
if (fingerprint.exact.length === 0)
|
|
18004
18072
|
continue;
|
|
18005
18073
|
fingerprintModified.set(path, fingerprint.modified);
|
|
18006
18074
|
const group = byHash.get(fingerprint.exact) ?? [];
|
|
@@ -18067,1397 +18135,1368 @@ async function rankExactCandidates(paths, dupeyRanking, fingerprintModified, err
|
|
|
18067
18135
|
}
|
|
18068
18136
|
var init_filter = () => {};
|
|
18069
18137
|
|
|
18070
|
-
// src/
|
|
18071
|
-
|
|
18072
|
-
|
|
18073
|
-
|
|
18074
|
-
|
|
18075
|
-
|
|
18076
|
-
|
|
18077
|
-
function
|
|
18078
|
-
const
|
|
18079
|
-
|
|
18080
|
-
}
|
|
18081
|
-
|
|
18082
|
-
|
|
18138
|
+
// src/dupey/index.ts
|
|
18139
|
+
var init_dupey = __esm(() => {
|
|
18140
|
+
init_cli2();
|
|
18141
|
+
init_filter();
|
|
18142
|
+
});
|
|
18143
|
+
|
|
18144
|
+
// src/convert/relate.ts
|
|
18145
|
+
async function convertAndRelate(request, options = {}) {
|
|
18146
|
+
const result = await convertWithFilee(request, options.filee);
|
|
18147
|
+
if (options.dupey === false)
|
|
18148
|
+
return { result, links: [] };
|
|
18149
|
+
const dupeyOptions = options.dupey ?? {};
|
|
18150
|
+
const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
|
|
18151
|
+
const links = await Promise.all(pairs.map(async ({ source, output }) => {
|
|
18152
|
+
try {
|
|
18153
|
+
return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
|
|
18154
|
+
} catch (error) {
|
|
18155
|
+
if (!(error instanceof DupeyCliError))
|
|
18156
|
+
throw error;
|
|
18157
|
+
return { source, output, dupey: { relation: "unavailable", reason: error.message } };
|
|
18158
|
+
}
|
|
18159
|
+
}));
|
|
18160
|
+
return { result, links };
|
|
18083
18161
|
}
|
|
18084
|
-
|
|
18085
|
-
|
|
18162
|
+
var init_relate = __esm(() => {
|
|
18163
|
+
init_dupey();
|
|
18164
|
+
init_cli();
|
|
18165
|
+
});
|
|
18166
|
+
|
|
18167
|
+
// src/convert/index.ts
|
|
18168
|
+
var init_convert = __esm(() => {
|
|
18169
|
+
init_cli();
|
|
18170
|
+
init_installer2();
|
|
18171
|
+
init_matrix();
|
|
18172
|
+
init_relate();
|
|
18173
|
+
});
|
|
18174
|
+
|
|
18175
|
+
// src/datasource/diagnostics.ts
|
|
18176
|
+
function mapDatasourceDiagnostic(ds) {
|
|
18177
|
+
return {
|
|
18178
|
+
code: datasourceCodeToRetrieval(ds.code),
|
|
18179
|
+
severity: ds.severity,
|
|
18180
|
+
message: `[datasource:${ds.code}] ${ds.message}`,
|
|
18181
|
+
source: ds.source ?? (ds.instanceId ? `datasource:${ds.instanceId}` : "datasource")
|
|
18182
|
+
};
|
|
18086
18183
|
}
|
|
18087
|
-
function
|
|
18088
|
-
|
|
18089
|
-
if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
|
|
18090
|
-
return "google-drive";
|
|
18091
|
-
if (lower.includes("onedrive"))
|
|
18092
|
-
return "onedrive";
|
|
18093
|
-
if (lower.includes("dropbox"))
|
|
18094
|
-
return "dropbox";
|
|
18095
|
-
if (lower.includes("mobile documents") || lower.includes("clouddocs"))
|
|
18096
|
-
return "icloud";
|
|
18097
|
-
return "file-provider";
|
|
18184
|
+
function mapDatasourceDiagnostics(diagnostics) {
|
|
18185
|
+
return diagnostics.map(mapDatasourceDiagnostic);
|
|
18098
18186
|
}
|
|
18099
|
-
function
|
|
18100
|
-
|
|
18101
|
-
|
|
18102
|
-
|
|
18103
|
-
|
|
18104
|
-
|
|
18105
|
-
|
|
18106
|
-
|
|
18107
|
-
|
|
18108
|
-
|
|
18109
|
-
|
|
18187
|
+
function datasourceCodeToRetrieval(code) {
|
|
18188
|
+
switch (code) {
|
|
18189
|
+
case "datasource-unavailable":
|
|
18190
|
+
case "datasource-cli-error":
|
|
18191
|
+
case "datasource-empty":
|
|
18192
|
+
case "datasource-rate-limited":
|
|
18193
|
+
case "datasource-auth-error":
|
|
18194
|
+
case "datasource-embedding-egress-rejected":
|
|
18195
|
+
case "datasource-index-failed":
|
|
18196
|
+
case "datasource-index-partial":
|
|
18197
|
+
case "datasource-permission-denied":
|
|
18198
|
+
case "datasource-remote-embedding-rejected":
|
|
18199
|
+
return "retrieval-method-failed";
|
|
18200
|
+
}
|
|
18110
18201
|
}
|
|
18111
|
-
|
|
18112
|
-
|
|
18113
|
-
|
|
18114
|
-
|
|
18115
|
-
|
|
18202
|
+
|
|
18203
|
+
// src/datasource/retry.ts
|
|
18204
|
+
import { setTimeout as delay2 } from "node:timers/promises";
|
|
18205
|
+
function datasourceRetryDelayMs(retry, policy) {
|
|
18206
|
+
const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
|
|
18207
|
+
return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
|
|
18116
18208
|
}
|
|
18117
|
-
function
|
|
18118
|
-
const
|
|
18119
|
-
|
|
18120
|
-
|
|
18121
|
-
|
|
18122
|
-
|
|
18123
|
-
|
|
18209
|
+
async function indexWithRetry(skill, options = {}, signal) {
|
|
18210
|
+
const policy = {
|
|
18211
|
+
maxRetries: options.maxRetries ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxRetries,
|
|
18212
|
+
baseDelayMs: options.baseDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.baseDelayMs,
|
|
18213
|
+
maxDelayMs: options.maxDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxDelayMs
|
|
18214
|
+
};
|
|
18215
|
+
const sleep = options.sleep ?? ((ms) => delay2(ms, undefined, signal === undefined ? undefined : { signal }));
|
|
18216
|
+
let lastError = "";
|
|
18217
|
+
for (let attempt = 1;; attempt += 1) {
|
|
18218
|
+
const result = await runIndex(skill, signal);
|
|
18219
|
+
signal?.throwIfAborted();
|
|
18220
|
+
if (result.ok) {
|
|
18221
|
+
if (attempt === 1)
|
|
18222
|
+
return result;
|
|
18223
|
+
const recovered = {
|
|
18224
|
+
code: "datasource-index-failed",
|
|
18225
|
+
severity: "info",
|
|
18226
|
+
message: `recovered after ${attempt} attempts; last error: ${lastError}`,
|
|
18227
|
+
instanceId: result.instanceId,
|
|
18228
|
+
source: result.skill
|
|
18229
|
+
};
|
|
18230
|
+
return { ...result, attempts: attempt, diagnostics: [...result.diagnostics, recovered] };
|
|
18231
|
+
}
|
|
18232
|
+
lastError = result.message ?? result.error ?? "datasource-index-failed";
|
|
18233
|
+
if (attempt > policy.maxRetries) {
|
|
18234
|
+
if (attempt === 1)
|
|
18235
|
+
return { ...result, attempts: attempt };
|
|
18236
|
+
const suffix = ` (gave up after ${attempt} attempts)`;
|
|
18237
|
+
const last = result.diagnostics.at(-1);
|
|
18238
|
+
return {
|
|
18239
|
+
...result,
|
|
18240
|
+
attempts: attempt,
|
|
18241
|
+
message: `${lastError}${suffix}`,
|
|
18242
|
+
diagnostics: last === undefined ? [
|
|
18243
|
+
{
|
|
18244
|
+
code: result.code ?? "datasource-index-failed",
|
|
18245
|
+
severity: "error",
|
|
18246
|
+
message: `${lastError}${suffix}`,
|
|
18247
|
+
instanceId: result.instanceId,
|
|
18248
|
+
source: result.skill
|
|
18249
|
+
}
|
|
18250
|
+
] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
|
|
18251
|
+
};
|
|
18252
|
+
}
|
|
18253
|
+
await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
|
|
18254
|
+
signal?.throwIfAborted();
|
|
18255
|
+
}
|
|
18124
18256
|
}
|
|
18125
|
-
function
|
|
18257
|
+
async function runIndex(skill, signal) {
|
|
18126
18258
|
try {
|
|
18127
|
-
return
|
|
18128
|
-
} catch {
|
|
18129
|
-
|
|
18259
|
+
return await skill.index(signal === undefined ? undefined : { signal });
|
|
18260
|
+
} catch (error) {
|
|
18261
|
+
signal?.throwIfAborted();
|
|
18262
|
+
const descriptor = skill.describe();
|
|
18263
|
+
const message = error instanceof Error ? error.message : "Datasource indexing failed.";
|
|
18264
|
+
const failed = {
|
|
18265
|
+
ok: false,
|
|
18266
|
+
instanceId: descriptor.instanceId ?? "default",
|
|
18267
|
+
skill: descriptor.name,
|
|
18268
|
+
indexedAt: Date.now(),
|
|
18269
|
+
diagnostics: [
|
|
18270
|
+
{
|
|
18271
|
+
code: "datasource-index-failed",
|
|
18272
|
+
severity: "error",
|
|
18273
|
+
message,
|
|
18274
|
+
source: descriptor.name,
|
|
18275
|
+
instanceId: descriptor.instanceId
|
|
18276
|
+
}
|
|
18277
|
+
],
|
|
18278
|
+
error: "datasource-index-failed",
|
|
18279
|
+
code: "datasource-index-failed",
|
|
18280
|
+
message
|
|
18281
|
+
};
|
|
18282
|
+
return failed;
|
|
18130
18283
|
}
|
|
18131
18284
|
}
|
|
18132
|
-
|
|
18133
|
-
|
|
18134
|
-
|
|
18135
|
-
|
|
18136
|
-
|
|
18137
|
-
|
|
18138
|
-
for (const { raw, canonical } of forms) {
|
|
18139
|
-
if (isPathInside(normalized, raw))
|
|
18140
|
-
return `${canonical}${normalized.slice(raw.length)}`;
|
|
18141
|
-
}
|
|
18142
|
-
return path;
|
|
18285
|
+
var DEFAULT_DATASOURCE_RETRY_POLICY;
|
|
18286
|
+
var init_retry = __esm(() => {
|
|
18287
|
+
DEFAULT_DATASOURCE_RETRY_POLICY = {
|
|
18288
|
+
maxRetries: 2,
|
|
18289
|
+
baseDelayMs: 1000,
|
|
18290
|
+
maxDelayMs: 30000
|
|
18143
18291
|
};
|
|
18292
|
+
});
|
|
18293
|
+
|
|
18294
|
+
// src/everything/bundle.ts
|
|
18295
|
+
import { createHash as createHash11 } from "node:crypto";
|
|
18296
|
+
import { existsSync as existsSync18, mkdirSync as mkdirSync15, readFileSync as readFileSync18, rmSync as rmSync7, writeFileSync as writeFileSync10 } from "node:fs";
|
|
18297
|
+
import { dirname as dirname15, join as join28 } from "node:path";
|
|
18298
|
+
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
18299
|
+
import JSZip from "jszip";
|
|
18300
|
+
function resolveEverythingBundleDir() {
|
|
18301
|
+
let dir = dirname15(fileURLToPath2(import.meta.url));
|
|
18302
|
+
for (let depth = 0;depth < 6; depth += 1) {
|
|
18303
|
+
const candidate = join28(dir, "vendor", "everything");
|
|
18304
|
+
if (existsSync18(join28(candidate, MANIFEST_FILENAME)))
|
|
18305
|
+
return candidate;
|
|
18306
|
+
const parent = dirname15(dir);
|
|
18307
|
+
if (parent === dir)
|
|
18308
|
+
break;
|
|
18309
|
+
dir = parent;
|
|
18310
|
+
}
|
|
18311
|
+
return;
|
|
18144
18312
|
}
|
|
18145
|
-
function
|
|
18146
|
-
|
|
18147
|
-
|
|
18148
|
-
return
|
|
18149
|
-
insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
|
|
18150
|
-
nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
|
|
18151
|
-
};
|
|
18313
|
+
function everythingArch(arch) {
|
|
18314
|
+
if (arch === "x64" || arch === "arm64")
|
|
18315
|
+
return arch;
|
|
18316
|
+
return;
|
|
18152
18317
|
}
|
|
18153
|
-
function
|
|
18154
|
-
|
|
18318
|
+
function everythingBinaryPaths(root, version, arch) {
|
|
18319
|
+
const dir = join28(root, ".autorag", "everything", version, arch);
|
|
18320
|
+
return { everythingPath: join28(dir, "everything.exe"), esPath: join28(dir, "es.exe") };
|
|
18321
|
+
}
|
|
18322
|
+
function loadEverythingBundleManifest(bundleDir) {
|
|
18323
|
+
return JSON.parse(readFileSync18(join28(bundleDir, MANIFEST_FILENAME), "utf8"));
|
|
18324
|
+
}
|
|
18325
|
+
async function ensureEverythingBinaries(options) {
|
|
18326
|
+
const platform = options.platform ?? process.platform;
|
|
18327
|
+
const rawArch = options.arch ?? process.arch;
|
|
18328
|
+
if (platform !== "win32") {
|
|
18329
|
+
return {
|
|
18330
|
+
ok: false,
|
|
18331
|
+
reason: "unsupported-platform",
|
|
18332
|
+
message: `Everything is Windows-only; this host is ${platform}.`
|
|
18333
|
+
};
|
|
18334
|
+
}
|
|
18335
|
+
const arch = everythingArch(rawArch);
|
|
18336
|
+
const bundleDir = options.bundleDir ?? resolveEverythingBundleDir();
|
|
18337
|
+
if (bundleDir === undefined) {
|
|
18338
|
+
return {
|
|
18339
|
+
ok: false,
|
|
18340
|
+
reason: "bundle-missing",
|
|
18341
|
+
message: `Bundled Everything assets (vendor/everything/${MANIFEST_FILENAME}) were not found above ${dirname15(fileURLToPath2(import.meta.url))}.`
|
|
18342
|
+
};
|
|
18343
|
+
}
|
|
18344
|
+
let manifest;
|
|
18155
18345
|
try {
|
|
18156
|
-
|
|
18157
|
-
} catch {
|
|
18158
|
-
return;
|
|
18346
|
+
manifest = options.manifest ?? loadEverythingBundleManifest(bundleDir);
|
|
18347
|
+
} catch (error) {
|
|
18348
|
+
return { ok: false, reason: "bundle-missing", message: errorMessage2(error) };
|
|
18159
18349
|
}
|
|
18160
|
-
|
|
18161
|
-
|
|
18350
|
+
const assets = arch === undefined ? undefined : manifest.architectures[arch];
|
|
18351
|
+
if (arch === undefined || assets === undefined) {
|
|
18352
|
+
return {
|
|
18353
|
+
ok: false,
|
|
18354
|
+
reason: "unsupported-platform",
|
|
18355
|
+
message: `No bundled Everything build for Windows architecture ${rawArch}; bundled: ${Object.keys(manifest.architectures).join(", ")}.`
|
|
18356
|
+
};
|
|
18162
18357
|
}
|
|
18163
|
-
|
|
18164
|
-
|
|
18165
|
-
|
|
18166
|
-
|
|
18167
|
-
|
|
18168
|
-
}
|
|
18169
|
-
|
|
18170
|
-
|
|
18171
|
-
|
|
18172
|
-
|
|
18173
|
-
|
|
18174
|
-
|
|
18175
|
-
|
|
18176
|
-
|
|
18358
|
+
const { everythingPath, esPath } = everythingBinaryPaths(options.root, manifest.everythingVersion, arch);
|
|
18359
|
+
const everythingCached = fileHasSha256(everythingPath, assets.everythingSha256);
|
|
18360
|
+
const esCached = fileHasSha256(esPath, assets.esSha256);
|
|
18361
|
+
if (everythingCached && esCached) {
|
|
18362
|
+
return { ok: true, everythingPath, esPath, source: "cached" };
|
|
18363
|
+
}
|
|
18364
|
+
if ((options.mode ?? "install") === "cached") {
|
|
18365
|
+
const state = everythingCached || esCached ? "are incomplete or do not match their pinned SHA-256 digests" : "are not installed";
|
|
18366
|
+
return {
|
|
18367
|
+
ok: false,
|
|
18368
|
+
reason: "not-installed",
|
|
18369
|
+
message: `Everything ${manifest.everythingVersion} (${arch}) binaries ${state} under ${dirname15(everythingPath)}; run \`autorag refresh\` to (re)install and index them.`
|
|
18370
|
+
};
|
|
18371
|
+
}
|
|
18372
|
+
try {
|
|
18373
|
+
mkdirSync15(dirname15(everythingPath), { recursive: true });
|
|
18374
|
+
const everythingBytes = await extractVerified(join28(bundleDir, assets.everythingArchive), assets.everythingArchiveSha256, assets.everythingMember, assets.everythingSha256);
|
|
18375
|
+
const esBytes = await extractVerified(join28(bundleDir, assets.esArchive), assets.esArchiveSha256, assets.esMember, assets.esSha256);
|
|
18376
|
+
writeAtomic(everythingPath, everythingBytes);
|
|
18377
|
+
writeAtomic(esPath, esBytes);
|
|
18378
|
+
} catch (error) {
|
|
18379
|
+
return { ok: false, reason: "install-failed", message: errorMessage2(error) };
|
|
18380
|
+
}
|
|
18381
|
+
return { ok: true, everythingPath, esPath, source: "installed" };
|
|
18177
18382
|
}
|
|
18178
|
-
function
|
|
18179
|
-
const
|
|
18180
|
-
const
|
|
18181
|
-
if (
|
|
18182
|
-
|
|
18183
|
-
} else if (typeof failure.code === "number") {
|
|
18184
|
-
parts.push(`the helper exited with code ${failure.code}`);
|
|
18383
|
+
async function extractVerified(archivePath, archiveSha256, member, memberSha256) {
|
|
18384
|
+
const archive = readFileSync18(archivePath);
|
|
18385
|
+
const actualArchive = sha256(archive);
|
|
18386
|
+
if (actualArchive !== archiveSha256) {
|
|
18387
|
+
throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
|
|
18185
18388
|
}
|
|
18186
|
-
const
|
|
18187
|
-
|
|
18188
|
-
|
|
18189
|
-
|
|
18190
|
-
|
|
18191
|
-
|
|
18389
|
+
const zip = await JSZip.loadAsync(archive);
|
|
18390
|
+
const entry = zip.file(member);
|
|
18391
|
+
if (entry === null)
|
|
18392
|
+
throw new Error(`${archivePath} does not contain ${member}`);
|
|
18393
|
+
const bytes = await entry.async("nodebuffer");
|
|
18394
|
+
const actualMember = sha256(bytes);
|
|
18395
|
+
if (actualMember !== memberSha256) {
|
|
18396
|
+
throw new Error(`${member} from ${archivePath} SHA-256 ${actualMember} does not match pinned ${memberSha256}`);
|
|
18397
|
+
}
|
|
18398
|
+
return bytes;
|
|
18192
18399
|
}
|
|
18193
|
-
|
|
18194
|
-
|
|
18195
|
-
return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
|
|
18400
|
+
function fileHasSha256(path, expected) {
|
|
18401
|
+
return existsSync18(path) && sha256(readFileSync18(path)) === expected;
|
|
18196
18402
|
}
|
|
18197
|
-
function
|
|
18198
|
-
const
|
|
18199
|
-
|
|
18200
|
-
|
|
18201
|
-
|
|
18202
|
-
|
|
18203
|
-
|
|
18204
|
-
if (open <= 0 || !line.endsWith(")"))
|
|
18205
|
-
continue;
|
|
18206
|
-
const head = line.slice(0, open);
|
|
18207
|
-
const on = head.indexOf(" on ");
|
|
18208
|
-
if (on <= 0)
|
|
18209
|
-
continue;
|
|
18210
|
-
const mountPoint = head.slice(on + " on ".length);
|
|
18211
|
-
if (mountPoint === "")
|
|
18212
|
-
continue;
|
|
18213
|
-
const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
|
|
18214
|
-
mounts.push({
|
|
18215
|
-
device: head.slice(0, on),
|
|
18216
|
-
mountPoint,
|
|
18217
|
-
fileSystem: fields[0] ?? "",
|
|
18218
|
-
options: fields.slice(1)
|
|
18219
|
-
});
|
|
18403
|
+
function writeAtomic(path, bytes) {
|
|
18404
|
+
const temp = `${path}.${process.pid}.tmp`;
|
|
18405
|
+
try {
|
|
18406
|
+
writeFileSync10(temp, bytes);
|
|
18407
|
+
renameWithRetry(temp, path);
|
|
18408
|
+
} finally {
|
|
18409
|
+
rmSync7(temp, { force: true });
|
|
18220
18410
|
}
|
|
18221
|
-
return mounts;
|
|
18222
18411
|
}
|
|
18223
|
-
function
|
|
18224
|
-
|
|
18225
|
-
const seen = new Set;
|
|
18226
|
-
for (const mount of mounts) {
|
|
18227
|
-
if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
|
|
18228
|
-
continue;
|
|
18229
|
-
if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
|
|
18230
|
-
continue;
|
|
18231
|
-
if (seen.has(mount.mountPoint))
|
|
18232
|
-
continue;
|
|
18233
|
-
seen.add(mount.mountPoint);
|
|
18234
|
-
points.push(mount.mountPoint);
|
|
18235
|
-
}
|
|
18236
|
-
return points;
|
|
18412
|
+
function sha256(bytes) {
|
|
18413
|
+
return createHash11("sha256").update(bytes).digest("hex");
|
|
18237
18414
|
}
|
|
18238
|
-
function
|
|
18239
|
-
|
|
18240
|
-
|
|
18241
|
-
|
|
18242
|
-
|
|
18243
|
-
|
|
18244
|
-
|
|
18245
|
-
|
|
18246
|
-
|
|
18247
|
-
|
|
18248
|
-
|
|
18249
|
-
|
|
18250
|
-
|
|
18251
|
-
|
|
18252
|
-
|
|
18253
|
-
}
|
|
18254
|
-
|
|
18415
|
+
function errorMessage2(error) {
|
|
18416
|
+
return error instanceof Error ? error.message : String(error);
|
|
18417
|
+
}
|
|
18418
|
+
var MANIFEST_FILENAME = "manifest.json";
|
|
18419
|
+
var init_bundle = __esm(() => {
|
|
18420
|
+
init_atomic_write();
|
|
18421
|
+
});
|
|
18422
|
+
|
|
18423
|
+
// src/filesystem/path-exclusions.ts
|
|
18424
|
+
import { posix as posix3, win32 as win322 } from "node:path";
|
|
18425
|
+
function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
|
|
18426
|
+
const paths = platform === "win32" ? win322 : posix3;
|
|
18427
|
+
const normalize = (path) => {
|
|
18428
|
+
const absolute = paths.resolve(path);
|
|
18429
|
+
return platform === "win32" ? absolute.toLowerCase() : absolute;
|
|
18430
|
+
};
|
|
18431
|
+
const excluded = excludedPaths.map((path) => {
|
|
18432
|
+
const absolute = normalize(path);
|
|
18433
|
+
return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
|
|
18434
|
+
});
|
|
18435
|
+
return (path) => {
|
|
18436
|
+
const absolute = normalize(path);
|
|
18437
|
+
return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
|
|
18438
|
+
};
|
|
18255
18439
|
}
|
|
18440
|
+
var init_path_exclusions = () => {};
|
|
18256
18441
|
|
|
18257
|
-
|
|
18258
|
-
|
|
18259
|
-
|
|
18260
|
-
|
|
18261
|
-
|
|
18262
|
-
|
|
18263
|
-
|
|
18264
|
-
|
|
18442
|
+
// src/filesystem/shared-exclusive-queue.ts
|
|
18443
|
+
class SharedExclusiveQueue {
|
|
18444
|
+
sharedSinceExclusive = [];
|
|
18445
|
+
lastExclusive = Promise.resolve();
|
|
18446
|
+
shared(task) {
|
|
18447
|
+
const run = this.lastExclusive.then(task);
|
|
18448
|
+
const settled = run.then(() => {
|
|
18449
|
+
return;
|
|
18450
|
+
}, () => {
|
|
18451
|
+
return;
|
|
18452
|
+
});
|
|
18453
|
+
this.sharedSinceExclusive.push(settled);
|
|
18454
|
+
return run;
|
|
18265
18455
|
}
|
|
18266
|
-
|
|
18267
|
-
const
|
|
18268
|
-
const
|
|
18269
|
-
const
|
|
18270
|
-
const canonical = canonicalPath(path);
|
|
18271
|
-
const key = normalizeCloudPath(canonical);
|
|
18272
|
-
if (key === "" || seen.has(key))
|
|
18273
|
-
return;
|
|
18274
|
-
seen.add(key);
|
|
18275
|
-
roots.push({ path: canonical, kind });
|
|
18276
|
-
};
|
|
18277
|
-
const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
|
|
18278
|
-
for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
|
|
18279
|
-
if (entry.isDirectory())
|
|
18280
|
-
add(join28(cloudStorage, entry.name), "file-provider");
|
|
18281
|
-
}
|
|
18282
|
-
if (await this.tryReadDir(mobileDocuments) !== undefined)
|
|
18283
|
-
add(mobileDocuments, "file-provider");
|
|
18284
|
-
for (const mountPoint of await this.networkMountPoints())
|
|
18285
|
-
add(mountPoint, "network");
|
|
18286
|
-
for (const directory of await this.fileProviderDirectories(searchRoots))
|
|
18287
|
-
add(directory, "file-provider");
|
|
18288
|
-
return roots;
|
|
18289
|
-
}
|
|
18290
|
-
async networkMountPoints() {
|
|
18291
|
-
try {
|
|
18292
|
-
const { stdout } = await this.exec("/sbin/mount", [], { timeoutMs: 1e4, maxBuffer: 4 * 1024 * 1024 });
|
|
18293
|
-
return networkMountPoints(parseDarwinMounts(stdout));
|
|
18294
|
-
} catch {
|
|
18295
|
-
return [];
|
|
18296
|
-
}
|
|
18297
|
-
}
|
|
18298
|
-
async fileProviderDirectories(searchRoots) {
|
|
18299
|
-
if (searchRoots.length === 0)
|
|
18300
|
-
return [];
|
|
18301
|
-
const candidates = new Set;
|
|
18302
|
-
for (const searchRoot of searchRoots) {
|
|
18303
|
-
const canonical = canonicalPath(searchRoot);
|
|
18304
|
-
for (let cursor = canonical;; cursor = dirname15(cursor)) {
|
|
18305
|
-
candidates.add(cursor);
|
|
18306
|
-
const parent = dirname15(cursor);
|
|
18307
|
-
if (parent === cursor)
|
|
18308
|
-
break;
|
|
18309
|
-
}
|
|
18310
|
-
for (const entry of await this.tryReadDir(canonical) ?? []) {
|
|
18311
|
-
if (entry.isDirectory())
|
|
18312
|
-
candidates.add(join28(canonical, entry.name));
|
|
18313
|
-
}
|
|
18314
|
-
}
|
|
18315
|
-
const paths = [...candidates];
|
|
18316
|
-
const managed = [];
|
|
18317
|
-
for (let start = 0;start < paths.length; start += XATTR_BATCH) {
|
|
18318
|
-
for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
|
|
18319
|
-
managed.push(path);
|
|
18320
|
-
}
|
|
18321
|
-
return managed;
|
|
18322
|
-
}
|
|
18323
|
-
async xattrValues(paths) {
|
|
18324
|
-
try {
|
|
18325
|
-
const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
|
|
18326
|
-
timeoutMs: 15000,
|
|
18327
|
-
maxBuffer: 8 * 1024 * 1024
|
|
18328
|
-
});
|
|
18329
|
-
return parseXattrValues(stdout, paths);
|
|
18330
|
-
} catch (error) {
|
|
18331
|
-
const failure = error;
|
|
18332
|
-
return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
|
|
18333
|
-
}
|
|
18334
|
-
}
|
|
18335
|
-
async tryReadDir(path) {
|
|
18336
|
-
try {
|
|
18337
|
-
return await this.readDir(path);
|
|
18338
|
-
} catch {
|
|
18456
|
+
exclusive(task) {
|
|
18457
|
+
const earlier = Promise.all([this.lastExclusive, ...this.sharedSinceExclusive]);
|
|
18458
|
+
const run = earlier.then(task);
|
|
18459
|
+
const settled = run.then(() => {
|
|
18339
18460
|
return;
|
|
18340
|
-
}
|
|
18341
|
-
|
|
18342
|
-
|
|
18343
|
-
|
|
18344
|
-
|
|
18345
|
-
|
|
18346
|
-
const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
|
|
18347
|
-
timeoutMs: 30000,
|
|
18348
|
-
maxBuffer: 4 * 1024 * 1024
|
|
18349
|
-
});
|
|
18350
|
-
const flags = stdout.split(`
|
|
18351
|
-
`).filter(Boolean);
|
|
18352
|
-
for (const [index, path] of batch.entries()) {
|
|
18353
|
-
if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
|
|
18354
|
-
dataless.add(path);
|
|
18355
|
-
}
|
|
18356
|
-
}
|
|
18357
|
-
return dataless;
|
|
18358
|
-
}
|
|
18359
|
-
async providerOf(path) {
|
|
18360
|
-
const domain = await readFileProviderDomainUpwards(path);
|
|
18361
|
-
return domain === undefined ? undefined : cloudProviderFromDomain(domain);
|
|
18362
|
-
}
|
|
18363
|
-
canRelease(provider) {
|
|
18364
|
-
return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
|
|
18365
|
-
}
|
|
18366
|
-
async evict(path) {
|
|
18367
|
-
let stdout;
|
|
18368
|
-
try {
|
|
18369
|
-
stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
|
|
18370
|
-
timeoutMs: 30000
|
|
18371
|
-
})).stdout;
|
|
18372
|
-
} catch (error) {
|
|
18373
|
-
return { ok: false, message: describeReleaseHelperFailure(error) };
|
|
18374
|
-
}
|
|
18375
|
-
const outcome = parseReleaseOutcome(stdout);
|
|
18376
|
-
if (outcome === undefined) {
|
|
18377
|
-
const detail = stdout.trim();
|
|
18378
|
-
return {
|
|
18379
|
-
ok: false,
|
|
18380
|
-
message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
|
|
18381
|
-
};
|
|
18382
|
-
}
|
|
18383
|
-
return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
|
|
18384
|
-
}
|
|
18385
|
-
}
|
|
18386
|
-
function buildWindowsAttributeScript(paths) {
|
|
18387
|
-
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
18388
|
-
` + `$ErrorActionPreference='SilentlyContinue'
|
|
18389
|
-
` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
|
|
18390
|
-
`;
|
|
18391
|
-
}
|
|
18392
|
-
function buildWindowsSyncRootsScript() {
|
|
18393
|
-
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
18394
|
-
` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
|
|
18395
|
-
` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
|
|
18396
|
-
`;
|
|
18397
|
-
}
|
|
18398
|
-
function buildWindowsDehydrateScript(path) {
|
|
18399
|
-
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
18400
|
-
` + `$ErrorActionPreference='Stop'
|
|
18401
|
-
` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
|
|
18402
|
-
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
|
|
18403
|
-
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
|
|
18404
|
-
` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
|
|
18405
|
-
` + `'@
|
|
18406
|
-
` + `$h=[IntPtr]::Zero
|
|
18407
|
-
` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
|
|
18408
|
-
` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
|
|
18409
|
-
` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
|
|
18410
|
-
` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
|
|
18411
|
-
` + `[Autorag.CfApi]::CfCloseHandle($h)
|
|
18412
|
-
` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
|
|
18413
|
-
` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
|
|
18414
|
-
`;
|
|
18415
|
-
}
|
|
18416
|
-
function parseWindowsAttributeOutput(stdout, paths) {
|
|
18417
|
-
const lines = stdout.split(/\r?\n/);
|
|
18418
|
-
const attributes = new Map;
|
|
18419
|
-
for (const [index, path] of paths.entries()) {
|
|
18420
|
-
const value = Number.parseInt((lines[index] ?? "").trim(), 10);
|
|
18421
|
-
if (Number.isFinite(value))
|
|
18422
|
-
attributes.set(path, value);
|
|
18423
|
-
}
|
|
18424
|
-
return attributes;
|
|
18425
|
-
}
|
|
18426
|
-
function parseWindowsSyncRoots(stdout) {
|
|
18427
|
-
const roots = [];
|
|
18428
|
-
for (const line of stdout.split(/\r?\n/)) {
|
|
18429
|
-
const tab = line.indexOf("\t");
|
|
18430
|
-
if (tab <= 0)
|
|
18431
|
-
continue;
|
|
18432
|
-
const path = line.slice(0, tab).trim();
|
|
18433
|
-
if (path !== "")
|
|
18434
|
-
roots.push({ path, label: line.slice(tab + 1).trim() });
|
|
18461
|
+
}, () => {
|
|
18462
|
+
return;
|
|
18463
|
+
});
|
|
18464
|
+
this.sharedSinceExclusive = [];
|
|
18465
|
+
this.lastExclusive = settled;
|
|
18466
|
+
return run;
|
|
18435
18467
|
}
|
|
18436
|
-
return roots;
|
|
18437
18468
|
}
|
|
18438
|
-
|
|
18439
|
-
|
|
18440
|
-
|
|
18441
|
-
|
|
18442
|
-
|
|
18443
|
-
|
|
18444
|
-
|
|
18469
|
+
|
|
18470
|
+
// src/everything/client.ts
|
|
18471
|
+
import { spawn as spawn15 } from "node:child_process";
|
|
18472
|
+
import { createHash as createHash12 } from "node:crypto";
|
|
18473
|
+
import { mkdirSync as mkdirSync16, writeFileSync as writeFileSync11 } from "node:fs";
|
|
18474
|
+
import { join as join29, resolve as resolve11 } from "node:path";
|
|
18475
|
+
import { setTimeout as sleep2 } from "node:timers/promises";
|
|
18476
|
+
function everythingInstanceName(root) {
|
|
18477
|
+
return `autorag-${createHash12("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
|
|
18445
18478
|
}
|
|
18446
|
-
function
|
|
18447
|
-
|
|
18448
|
-
|
|
18449
|
-
|
|
18479
|
+
function buildEverythingIni(input) {
|
|
18480
|
+
const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
|
|
18481
|
+
const perFolder = (value) => input.folders.map(() => value).join(",");
|
|
18482
|
+
return [
|
|
18483
|
+
"[Everything]",
|
|
18484
|
+
"app_data=0",
|
|
18485
|
+
"run_as_admin=0",
|
|
18486
|
+
"run_in_background=1",
|
|
18487
|
+
"show_tray_icon=0",
|
|
18488
|
+
"show_in_taskbar=0",
|
|
18489
|
+
"check_for_updates_on_startup=0",
|
|
18490
|
+
"beta_updates=0",
|
|
18491
|
+
"language=1033",
|
|
18492
|
+
"allow_http_server=0",
|
|
18493
|
+
"allow_etp_server=0",
|
|
18494
|
+
"search_history_enabled=0",
|
|
18495
|
+
"run_history_enabled=0",
|
|
18496
|
+
"auto_include_fixed_volumes=0",
|
|
18497
|
+
"auto_include_removable_volumes=0",
|
|
18498
|
+
"auto_include_fixed_refs_volumes=0",
|
|
18499
|
+
"auto_include_removable_refs_volumes=0",
|
|
18500
|
+
"ntfs_volume_paths=",
|
|
18501
|
+
"ntfs_volume_includes=",
|
|
18502
|
+
"refs_volume_paths=",
|
|
18503
|
+
"refs_volume_includes=",
|
|
18504
|
+
"index_size=1",
|
|
18505
|
+
"index_date_modified=1",
|
|
18506
|
+
`folders=${list(input.folders)}`,
|
|
18507
|
+
`folder_monitor_changes=${perFolder("1")}`,
|
|
18508
|
+
`folder_rescan_if_full_list=${perFolder("1")}`,
|
|
18509
|
+
`folder_update_types=${perFolder("0")}`,
|
|
18510
|
+
"exclude_list_enabled=1",
|
|
18511
|
+
`exclude_folders=${list(input.excludeFolders)}`,
|
|
18512
|
+
""
|
|
18513
|
+
].join(`\r
|
|
18514
|
+
`);
|
|
18450
18515
|
}
|
|
18451
|
-
function
|
|
18452
|
-
const
|
|
18453
|
-
|
|
18454
|
-
|
|
18455
|
-
|
|
18456
|
-
|
|
18457
|
-
|
|
18458
|
-
|
|
18459
|
-
|
|
18460
|
-
|
|
18461
|
-
|
|
18462
|
-
|
|
18463
|
-
|
|
18464
|
-
|
|
18465
|
-
|
|
18466
|
-
|
|
18467
|
-
|
|
18516
|
+
function buildEverythingSearchArgs(instance, request, timeoutMs) {
|
|
18517
|
+
const args = [
|
|
18518
|
+
"-argv",
|
|
18519
|
+
"-instance",
|
|
18520
|
+
instance,
|
|
18521
|
+
"-cp",
|
|
18522
|
+
"65001",
|
|
18523
|
+
"-json",
|
|
18524
|
+
"-full-path-and-name",
|
|
18525
|
+
"-size",
|
|
18526
|
+
"-date-modified",
|
|
18527
|
+
"-date-format",
|
|
18528
|
+
"1"
|
|
18529
|
+
];
|
|
18530
|
+
if (timeoutMs !== undefined)
|
|
18531
|
+
args.push("-timeout", String(timeoutMs));
|
|
18532
|
+
if (request.matchCase)
|
|
18533
|
+
args.push("-case");
|
|
18534
|
+
if (request.matchPath)
|
|
18535
|
+
args.push("-match-path");
|
|
18536
|
+
if (request.wholeWord)
|
|
18537
|
+
args.push("-whole-word");
|
|
18538
|
+
if (request.kind === "files")
|
|
18539
|
+
args.push("/a-d");
|
|
18540
|
+
if (request.kind === "folders")
|
|
18541
|
+
args.push("/ad");
|
|
18542
|
+
if (request.path !== undefined)
|
|
18543
|
+
args.push("-path", request.path);
|
|
18544
|
+
if (request.sort !== undefined)
|
|
18545
|
+
args.push("-sort", request.sort);
|
|
18546
|
+
if (request.offset !== undefined && request.offset > 0)
|
|
18547
|
+
args.push("-offset", String(request.offset));
|
|
18548
|
+
args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
|
|
18549
|
+
if (request.regex)
|
|
18550
|
+
args.push("-regex", "--", request.query);
|
|
18551
|
+
else
|
|
18552
|
+
args.push("--", request.query);
|
|
18553
|
+
return args;
|
|
18468
18554
|
}
|
|
18469
|
-
function
|
|
18470
|
-
const
|
|
18471
|
-
|
|
18472
|
-
|
|
18473
|
-
|
|
18474
|
-
|
|
18475
|
-
|
|
18476
|
-
|
|
18477
|
-
|
|
18478
|
-
continue;
|
|
18479
|
-
}
|
|
18480
|
-
if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
|
|
18481
|
-
roots.push({ path, kind: "network" });
|
|
18482
|
-
continue;
|
|
18483
|
-
}
|
|
18484
|
-
if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
|
|
18485
|
-
roots.push({ path, kind: "cloud-files" });
|
|
18486
|
-
}
|
|
18555
|
+
function parseEverythingJson(stdout) {
|
|
18556
|
+
const text = stdout.trim();
|
|
18557
|
+
if (text.length === 0)
|
|
18558
|
+
return [];
|
|
18559
|
+
let parsed;
|
|
18560
|
+
try {
|
|
18561
|
+
parsed = JSON.parse(text);
|
|
18562
|
+
} catch {
|
|
18563
|
+
throw new Error(`ES did not return JSON: ${text.slice(0, 2000)}`);
|
|
18487
18564
|
}
|
|
18488
|
-
|
|
18489
|
-
}
|
|
18490
|
-
|
|
18491
|
-
|
|
18492
|
-
|
|
18493
|
-
|
|
18494
|
-
const rest = normalized.slice(2);
|
|
18495
|
-
const separator = rest.indexOf("\\");
|
|
18496
|
-
if (separator <= 0)
|
|
18497
|
-
return;
|
|
18498
|
-
const server = rest.slice(0, separator);
|
|
18499
|
-
const share = rest.slice(separator + 1).split("\\")[0] ?? "";
|
|
18500
|
-
if (server === "" || share === "")
|
|
18501
|
-
return;
|
|
18502
|
-
return `\\\\${server}\\${share}`;
|
|
18503
|
-
}
|
|
18504
|
-
function parseWindowsDehydrateOutput(result) {
|
|
18505
|
-
const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
18506
|
-
if (open === null)
|
|
18507
|
-
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
18508
|
-
if (Number.parseInt(open[1], 16) !== 0) {
|
|
18565
|
+
if (!Array.isArray(parsed))
|
|
18566
|
+
throw new Error(`ES did not return a JSON array: ${text.slice(0, 2000)}`);
|
|
18567
|
+
return parsed.map((item) => {
|
|
18568
|
+
const record = item;
|
|
18569
|
+
const filename = String(record.filename ?? "");
|
|
18570
|
+
const isFolder = filename.endsWith("\\") && filename.length > 3;
|
|
18509
18571
|
return {
|
|
18510
|
-
|
|
18511
|
-
|
|
18572
|
+
path: isFolder ? filename.slice(0, -1) : filename,
|
|
18573
|
+
type: filename.endsWith("\\") ? "folder" : "file",
|
|
18574
|
+
size: typeof record.size === "number" ? record.size : undefined,
|
|
18575
|
+
dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
|
|
18512
18576
|
};
|
|
18513
|
-
}
|
|
18514
|
-
const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
18515
|
-
if (dehydrate === null) {
|
|
18516
|
-
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
18517
|
-
}
|
|
18518
|
-
const hresult = dehydrate[1];
|
|
18519
|
-
if (Number.parseInt(hresult, 16) === 0)
|
|
18520
|
-
return { ok: true };
|
|
18521
|
-
const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
|
|
18522
|
-
return {
|
|
18523
|
-
ok: false,
|
|
18524
|
-
message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
|
|
18525
|
-
};
|
|
18526
|
-
}
|
|
18527
|
-
function windowsFailureMessage(result, fallback) {
|
|
18528
|
-
const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
|
|
18529
|
-
if (printed !== undefined && printed !== "")
|
|
18530
|
-
return printed;
|
|
18531
|
-
const stderr = result.stderr.trim();
|
|
18532
|
-
return stderr !== "" ? stderr : fallback;
|
|
18577
|
+
});
|
|
18533
18578
|
}
|
|
18534
18579
|
|
|
18535
|
-
class
|
|
18580
|
+
class EverythingClient {
|
|
18581
|
+
instanceName;
|
|
18582
|
+
options;
|
|
18583
|
+
platform;
|
|
18536
18584
|
run;
|
|
18537
|
-
|
|
18538
|
-
|
|
18539
|
-
|
|
18540
|
-
|
|
18585
|
+
launch;
|
|
18586
|
+
stateDir;
|
|
18587
|
+
isExcludedPath;
|
|
18588
|
+
binaries;
|
|
18589
|
+
queue = new SharedExclusiveQueue;
|
|
18590
|
+
constructor(options) {
|
|
18591
|
+
this.options = options;
|
|
18592
|
+
this.platform = options.platform ?? process.platform;
|
|
18593
|
+
this.instanceName = everythingInstanceName(options.root);
|
|
18594
|
+
this.run = options.run ?? runProcess;
|
|
18595
|
+
this.launch = options.launch ?? launchDetached;
|
|
18596
|
+
this.stateDir = join29(options.root, ".autorag", "everything");
|
|
18597
|
+
this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
|
|
18541
18598
|
}
|
|
18542
|
-
|
|
18543
|
-
|
|
18544
|
-
const seen = new Set;
|
|
18545
|
-
const add = (path, kind) => {
|
|
18546
|
-
const canonical = canonicalPath(path);
|
|
18547
|
-
const key = normalizeCloudPath(canonical).toLowerCase();
|
|
18548
|
-
if (key === "" || seen.has(key))
|
|
18549
|
-
return;
|
|
18550
|
-
seen.add(key);
|
|
18551
|
-
roots.push({ path: canonical, kind });
|
|
18552
|
-
};
|
|
18553
|
-
for (const root of await this.loadSyncRoots())
|
|
18554
|
-
add(root.path, "cloud-files");
|
|
18555
|
-
for (const root of await this.loadDiskRoots())
|
|
18556
|
-
add(root.path, root.kind);
|
|
18557
|
-
for (const searchRoot of searchRoots) {
|
|
18558
|
-
const share = uncShareRoot(searchRoot);
|
|
18559
|
-
if (share !== undefined)
|
|
18560
|
-
add(share, "network");
|
|
18561
|
-
}
|
|
18562
|
-
return roots;
|
|
18599
|
+
isSupported() {
|
|
18600
|
+
return this.platform === "win32";
|
|
18563
18601
|
}
|
|
18564
|
-
|
|
18565
|
-
|
|
18566
|
-
|
|
18567
|
-
|
|
18568
|
-
|
|
18569
|
-
|
|
18570
|
-
|
|
18602
|
+
search(request) {
|
|
18603
|
+
return this.queue.shared(async () => {
|
|
18604
|
+
const binaries = await this.resolveBinaries("cached");
|
|
18605
|
+
if (!binaries.ok)
|
|
18606
|
+
return binaries;
|
|
18607
|
+
const ping = await this.ping(binaries.esPath);
|
|
18608
|
+
if (ping.code !== 0) {
|
|
18609
|
+
return {
|
|
18610
|
+
ok: false,
|
|
18611
|
+
reason: "not-running",
|
|
18612
|
+
message: `Everything instance ${this.instanceName} is not running; run \`autorag refresh\` to start and index it.`
|
|
18613
|
+
};
|
|
18571
18614
|
}
|
|
18572
|
-
|
|
18573
|
-
|
|
18574
|
-
|
|
18615
|
+
const es = { ok: true, esPath: binaries.esPath };
|
|
18616
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
18617
|
+
if ((this.options.excludeFolders?.length ?? 0) > 0) {
|
|
18618
|
+
return this.searchWithExclusions(es.esPath, request, timeoutMs);
|
|
18575
18619
|
}
|
|
18576
|
-
|
|
18577
|
-
|
|
18578
|
-
|
|
18579
|
-
|
|
18580
|
-
|
|
18581
|
-
|
|
18582
|
-
|
|
18583
|
-
|
|
18584
|
-
|
|
18620
|
+
const args = buildEverythingSearchArgs(this.instanceName, request, timeoutMs);
|
|
18621
|
+
const result = await this.run(es.esPath, args, timeoutMs + 5000);
|
|
18622
|
+
if (result.code !== 0)
|
|
18623
|
+
return this.failure("search-failed", es.esPath, args, result);
|
|
18624
|
+
try {
|
|
18625
|
+
return { ok: true, results: parseEverythingJson(result.stdout) };
|
|
18626
|
+
} catch (error) {
|
|
18627
|
+
return { ok: false, reason: "search-failed", message: error.message };
|
|
18628
|
+
}
|
|
18629
|
+
});
|
|
18585
18630
|
}
|
|
18586
|
-
async
|
|
18587
|
-
|
|
18631
|
+
async searchWithExclusions(esPath, request, timeoutMs) {
|
|
18632
|
+
const deadline = Date.now() + timeoutMs;
|
|
18633
|
+
const offset = request.offset ?? 0;
|
|
18634
|
+
const maxResults = request.maxResults ?? DEFAULT_MAX_RESULTS;
|
|
18635
|
+
const results = [];
|
|
18636
|
+
let skipped = 0;
|
|
18637
|
+
let nativeOffset = 0;
|
|
18638
|
+
const timedOut = () => ({
|
|
18639
|
+
ok: false,
|
|
18640
|
+
reason: "search-failed",
|
|
18641
|
+
message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
|
|
18642
|
+
});
|
|
18643
|
+
for (;; ) {
|
|
18644
|
+
const remainingMs = deadline - Date.now();
|
|
18645
|
+
if (remainingMs <= 0)
|
|
18646
|
+
return timedOut();
|
|
18647
|
+
const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
|
|
18648
|
+
const response = await this.run(esPath, args, remainingMs + 5000);
|
|
18649
|
+
if (response.code !== 0)
|
|
18650
|
+
return this.failure("search-failed", esPath, args, response);
|
|
18651
|
+
if (Date.now() >= deadline)
|
|
18652
|
+
return timedOut();
|
|
18653
|
+
let entries;
|
|
18654
|
+
try {
|
|
18655
|
+
entries = parseEverythingJson(response.stdout);
|
|
18656
|
+
} catch (error) {
|
|
18657
|
+
return { ok: false, reason: "search-failed", message: error.message };
|
|
18658
|
+
}
|
|
18659
|
+
for (const entry of entries) {
|
|
18660
|
+
if (this.isExcludedPath(entry.path))
|
|
18661
|
+
continue;
|
|
18662
|
+
if (skipped < offset) {
|
|
18663
|
+
skipped += 1;
|
|
18664
|
+
continue;
|
|
18665
|
+
}
|
|
18666
|
+
results.push(entry);
|
|
18667
|
+
if (results.length >= maxResults)
|
|
18668
|
+
return { ok: true, results };
|
|
18669
|
+
}
|
|
18670
|
+
if (entries.length < MAX_FILTER_CANDIDATES)
|
|
18671
|
+
return { ok: true, results };
|
|
18672
|
+
nativeOffset += entries.length;
|
|
18673
|
+
}
|
|
18588
18674
|
}
|
|
18589
|
-
|
|
18590
|
-
|
|
18591
|
-
|
|
18675
|
+
index() {
|
|
18676
|
+
return this.queue.exclusive(async () => {
|
|
18677
|
+
const es = await this.ensureRunning(true);
|
|
18678
|
+
if (!es.ok)
|
|
18679
|
+
return es;
|
|
18680
|
+
const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
|
|
18681
|
+
const saveArgs = ["-instance", this.instanceName, "-save-db"];
|
|
18682
|
+
const saved = await this.run(es.esPath, saveArgs, timeoutMs);
|
|
18683
|
+
if (saved.code !== 0)
|
|
18684
|
+
return this.failure("index-failed", es.esPath, saveArgs, saved);
|
|
18685
|
+
const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
|
|
18686
|
+
const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
|
|
18687
|
+
if (counted.code !== 0)
|
|
18688
|
+
return this.failure("index-failed", es.esPath, countArgs, counted);
|
|
18689
|
+
return {
|
|
18690
|
+
ok: true,
|
|
18691
|
+
indexedFolders: [...this.options.folders],
|
|
18692
|
+
indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
|
|
18693
|
+
};
|
|
18694
|
+
});
|
|
18592
18695
|
}
|
|
18593
|
-
|
|
18594
|
-
|
|
18595
|
-
|
|
18696
|
+
stop() {
|
|
18697
|
+
return this.queue.exclusive(async () => {
|
|
18698
|
+
const binaries = await this.resolveBinaries("cached");
|
|
18699
|
+
if (!binaries.ok)
|
|
18700
|
+
return;
|
|
18701
|
+
await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
18702
|
+
});
|
|
18596
18703
|
}
|
|
18597
|
-
|
|
18598
|
-
|
|
18599
|
-
|
|
18600
|
-
|
|
18601
|
-
|
|
18602
|
-
|
|
18603
|
-
|
|
18604
|
-
if (prefix === "" || prefix.length <= matchLength)
|
|
18605
|
-
continue;
|
|
18606
|
-
if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
|
|
18607
|
-
match = root;
|
|
18608
|
-
matchLength = prefix.length;
|
|
18704
|
+
async resolveBinaries(mode) {
|
|
18705
|
+
if (!this.isSupported()) {
|
|
18706
|
+
return {
|
|
18707
|
+
ok: false,
|
|
18708
|
+
reason: "unsupported-platform",
|
|
18709
|
+
message: `Everything is Windows-only; this host is ${this.platform}.`
|
|
18710
|
+
};
|
|
18609
18711
|
}
|
|
18712
|
+
if (this.binaries?.ok)
|
|
18713
|
+
return this.binaries;
|
|
18714
|
+
const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
|
|
18715
|
+
const result = await resolve(mode);
|
|
18716
|
+
if (result.ok)
|
|
18717
|
+
this.binaries = result;
|
|
18718
|
+
return result;
|
|
18610
18719
|
}
|
|
18611
|
-
|
|
18612
|
-
|
|
18613
|
-
|
|
18614
|
-
|
|
18615
|
-
|
|
18616
|
-
|
|
18617
|
-
|
|
18618
|
-
|
|
18619
|
-
|
|
18620
|
-
|
|
18621
|
-
|
|
18622
|
-
|
|
18623
|
-
|
|
18624
|
-
|
|
18625
|
-
|
|
18626
|
-
|
|
18627
|
-
|
|
18628
|
-
|
|
18629
|
-
|
|
18630
|
-
|
|
18631
|
-
|
|
18632
|
-
|
|
18633
|
-
|
|
18634
|
-
|
|
18635
|
-
|
|
18720
|
+
async ensureRunning(reconfigure) {
|
|
18721
|
+
const binaries = await this.resolveBinaries("install");
|
|
18722
|
+
if (!binaries.ok)
|
|
18723
|
+
return binaries;
|
|
18724
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
18725
|
+
let ping = await this.ping(binaries.esPath);
|
|
18726
|
+
if (ping.code === 0 && !reconfigure)
|
|
18727
|
+
return { ok: true, esPath: binaries.esPath };
|
|
18728
|
+
if (ping.code === 0) {
|
|
18729
|
+
const exitArgs = ["-instance", this.instanceName, "-exit"];
|
|
18730
|
+
const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
|
|
18731
|
+
if (exited.code !== 0)
|
|
18732
|
+
return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
|
|
18733
|
+
}
|
|
18734
|
+
mkdirSync16(this.stateDir, { recursive: true });
|
|
18735
|
+
const iniPath = join29(this.stateDir, "Everything.ini");
|
|
18736
|
+
writeFileSync11(iniPath, buildEverythingIni({
|
|
18737
|
+
folders: this.options.folders,
|
|
18738
|
+
excludeFolders: [
|
|
18739
|
+
...new Set([
|
|
18740
|
+
join29(this.options.root, ".autorag"),
|
|
18741
|
+
...this.options.folders.map((folder) => join29(folder, ".autorag")),
|
|
18742
|
+
...this.options.excludeFolders ?? []
|
|
18743
|
+
])
|
|
18744
|
+
]
|
|
18745
|
+
}));
|
|
18746
|
+
const launchArgs = [
|
|
18747
|
+
"-instance",
|
|
18748
|
+
this.instanceName,
|
|
18749
|
+
"-config",
|
|
18750
|
+
iniPath,
|
|
18751
|
+
"-db",
|
|
18752
|
+
join29(this.stateDir, "Everything.db"),
|
|
18753
|
+
"-startup"
|
|
18754
|
+
];
|
|
18755
|
+
try {
|
|
18756
|
+
this.launch(binaries.everythingPath, launchArgs);
|
|
18757
|
+
} catch (error) {
|
|
18758
|
+
return {
|
|
18759
|
+
ok: false,
|
|
18760
|
+
reason: "startup-failed",
|
|
18761
|
+
message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
|
|
18762
|
+
};
|
|
18763
|
+
}
|
|
18764
|
+
const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
|
|
18765
|
+
const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
18766
|
+
for (;; ) {
|
|
18767
|
+
ping = await this.ping(binaries.esPath);
|
|
18768
|
+
if (ping.code === 0)
|
|
18769
|
+
return { ok: true, esPath: binaries.esPath };
|
|
18770
|
+
if (Date.now() >= deadline)
|
|
18771
|
+
break;
|
|
18772
|
+
await sleep2(pollMs);
|
|
18773
|
+
}
|
|
18636
18774
|
return {
|
|
18637
18775
|
ok: false,
|
|
18638
|
-
|
|
18639
|
-
|
|
18776
|
+
reason: "startup-failed",
|
|
18777
|
+
message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
|
|
18640
18778
|
};
|
|
18641
18779
|
}
|
|
18642
|
-
|
|
18643
|
-
|
|
18644
|
-
|
|
18645
|
-
|
|
18646
|
-
|
|
18647
|
-
return
|
|
18648
|
-
|
|
18649
|
-
|
|
18650
|
-
|
|
18651
|
-
|
|
18652
|
-
await lstat(target);
|
|
18653
|
-
const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
|
|
18654
|
-
timeout: 2000
|
|
18655
|
-
});
|
|
18656
|
-
const value = stdout.trim();
|
|
18657
|
-
return value.length > 0 ? value : undefined;
|
|
18658
|
-
} catch {
|
|
18659
|
-
return;
|
|
18780
|
+
async ping(esPath) {
|
|
18781
|
+
const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
18782
|
+
return result.code === 0 && !VERSION_PATTERN.test(result.stdout.trim()) ? { ...result, code: -1 } : result;
|
|
18783
|
+
}
|
|
18784
|
+
failure(reason, esPath, args, result) {
|
|
18785
|
+
return {
|
|
18786
|
+
ok: false,
|
|
18787
|
+
reason,
|
|
18788
|
+
message: `${esPath} ${args.join(" ")} exit ${result.code}: ${(result.stderr || result.stdout).trim()}`
|
|
18789
|
+
};
|
|
18660
18790
|
}
|
|
18661
18791
|
}
|
|
18662
|
-
|
|
18663
|
-
let
|
|
18664
|
-
|
|
18665
|
-
|
|
18666
|
-
|
|
18667
|
-
|
|
18668
|
-
|
|
18669
|
-
|
|
18792
|
+
function runProcess(command, args, timeoutMs) {
|
|
18793
|
+
let resolvePromise;
|
|
18794
|
+
const promise = new Promise((resolve) => {
|
|
18795
|
+
resolvePromise = resolve;
|
|
18796
|
+
});
|
|
18797
|
+
const child = spawn15(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
|
|
18798
|
+
const stdout = [];
|
|
18799
|
+
const stderr = [];
|
|
18800
|
+
let bytes = 0;
|
|
18801
|
+
let overflow = false;
|
|
18802
|
+
const collect = (target) => (chunk) => {
|
|
18803
|
+
bytes += chunk.length;
|
|
18804
|
+
if (bytes > MAX_OUTPUT_BYTES) {
|
|
18805
|
+
overflow = true;
|
|
18806
|
+
child.kill();
|
|
18670
18807
|
return;
|
|
18671
|
-
|
|
18672
|
-
|
|
18673
|
-
|
|
18808
|
+
}
|
|
18809
|
+
target.push(chunk);
|
|
18810
|
+
};
|
|
18811
|
+
child.stdout.on("data", collect(stdout));
|
|
18812
|
+
child.stderr.on("data", collect(stderr));
|
|
18813
|
+
const timer = setTimeout(() => child.kill(), timeoutMs);
|
|
18814
|
+
child.on("error", (error) => {
|
|
18815
|
+
clearTimeout(timer);
|
|
18816
|
+
resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
|
|
18817
|
+
});
|
|
18818
|
+
child.on("close", (code, signal) => {
|
|
18819
|
+
clearTimeout(timer);
|
|
18820
|
+
const err = Buffer.concat(stderr).toString("utf8");
|
|
18821
|
+
resolvePromise({
|
|
18822
|
+
code,
|
|
18823
|
+
stdout: Buffer.concat(stdout).toString("utf8"),
|
|
18824
|
+
stderr: overflow ? `${err}
|
|
18825
|
+
output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
|
|
18826
|
+
killed by ${signal} after ${timeoutMs}ms` : err
|
|
18827
|
+
});
|
|
18828
|
+
});
|
|
18829
|
+
return promise;
|
|
18674
18830
|
}
|
|
18675
|
-
function
|
|
18676
|
-
|
|
18831
|
+
function launchDetached(command, args) {
|
|
18832
|
+
const child = spawn15(command, args, { detached: true, stdio: "ignore", windowsHide: true });
|
|
18833
|
+
child.on("error", () => {});
|
|
18834
|
+
child.unref();
|
|
18677
18835
|
}
|
|
18678
|
-
var
|
|
18679
|
-
|
|
18680
|
-
|
|
18681
|
-
|
|
18682
|
-
|
|
18683
|
-
|
|
18684
|
-
|
|
18685
|
-
var released = $.NSFileManager.defaultManager.evictUbiquitousItemAtURLError(url, errorRef);
|
|
18686
|
-
if (released) return JSON.stringify({ ok: true });
|
|
18687
|
-
function resourceValue(key) {
|
|
18688
|
-
var ref = Ref();
|
|
18689
|
-
if (!url.getResourceValueForKeyError(ref, key, null)) return null;
|
|
18690
|
-
var value = ref[0];
|
|
18691
|
-
return value === undefined || value === null ? null : ObjC.unwrap(value);
|
|
18692
|
-
}
|
|
18693
|
-
return JSON.stringify({
|
|
18694
|
-
ok: false,
|
|
18695
|
-
isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
|
|
18696
|
-
isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
|
|
18697
|
-
downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
|
|
18698
|
-
});
|
|
18699
|
-
}`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
|
|
18700
|
-
var init_cloud_placeholder = __esm(() => {
|
|
18701
|
-
execFileAsync = promisify(execFile2);
|
|
18702
|
-
RELEASABLE_DARWIN_PROVIDERS = {
|
|
18703
|
-
icloud: true,
|
|
18704
|
-
"google-drive": true,
|
|
18705
|
-
onedrive: true,
|
|
18706
|
-
dropbox: true
|
|
18707
|
-
};
|
|
18708
|
-
DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
|
|
18709
|
-
WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
|
|
18710
|
-
WINDOWS_DEHYDRATE_REFUSALS = {
|
|
18711
|
-
"80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
|
|
18712
|
-
"80070188": "The file is pinned to stay on this device, so it is not released."
|
|
18713
|
-
};
|
|
18836
|
+
var DEFAULT_TIMEOUT_MS8 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
|
|
18837
|
+
var init_client9 = __esm(() => {
|
|
18838
|
+
init_path_exclusions();
|
|
18839
|
+
init_bundle();
|
|
18840
|
+
DEFAULT_INDEX_TIMEOUT_MS3 = 10 * 60000;
|
|
18841
|
+
MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
|
|
18842
|
+
VERSION_PATTERN = /^\d+\.\d+\.\d+\.\d+$/;
|
|
18714
18843
|
});
|
|
18715
18844
|
|
|
18716
|
-
// src/
|
|
18717
|
-
|
|
18718
|
-
|
|
18719
|
-
|
|
18720
|
-
|
|
18845
|
+
// src/everything/index.ts
|
|
18846
|
+
var init_everything = __esm(() => {
|
|
18847
|
+
init_bundle();
|
|
18848
|
+
init_client9();
|
|
18849
|
+
});
|
|
18850
|
+
|
|
18851
|
+
// src/filesystem/cloud-placeholder.ts
|
|
18852
|
+
import { execFile as execFile2 } from "node:child_process";
|
|
18853
|
+
import { realpathSync as realpathSync3 } from "node:fs";
|
|
18854
|
+
import { lstat, readdir } from "node:fs/promises";
|
|
18855
|
+
import { homedir as homedir6 } from "node:os";
|
|
18856
|
+
import { dirname as dirname16, join as join30 } from "node:path";
|
|
18857
|
+
import { promisify } from "node:util";
|
|
18858
|
+
function homeCloudRoots(home = homedir6()) {
|
|
18859
|
+
const normalizedHome = home.replaceAll("\\", "/").replace(/\/+$/, "");
|
|
18860
|
+
return [`${normalizedHome}/Library/CloudStorage`, `${normalizedHome}/Library/Mobile Documents`];
|
|
18861
|
+
}
|
|
18862
|
+
function datalessFromDarwinFlags(flags) {
|
|
18863
|
+
return Number.isFinite(flags) && (flags >>> 0 & SF_DATALESS) !== 0;
|
|
18864
|
+
}
|
|
18865
|
+
function datalessFromWindowsAttributes(attributes) {
|
|
18866
|
+
return Number.isFinite(attributes) && (attributes & WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) !== 0;
|
|
18867
|
+
}
|
|
18868
|
+
function cloudProviderFromPath(target) {
|
|
18869
|
+
const lower = target.toLowerCase();
|
|
18870
|
+
if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
|
|
18871
|
+
return "google-drive";
|
|
18872
|
+
if (lower.includes("onedrive"))
|
|
18873
|
+
return "onedrive";
|
|
18874
|
+
if (lower.includes("dropbox"))
|
|
18875
|
+
return "dropbox";
|
|
18876
|
+
if (lower.includes("mobile documents") || lower.includes("clouddocs"))
|
|
18877
|
+
return "icloud";
|
|
18878
|
+
return "file-provider";
|
|
18879
|
+
}
|
|
18880
|
+
function cloudProviderFromDomain(domain) {
|
|
18881
|
+
const lower = domain.toLowerCase();
|
|
18882
|
+
if (lower.includes("clouddocs") || lower.includes("icloud"))
|
|
18883
|
+
return "icloud";
|
|
18884
|
+
if (lower.includes("google"))
|
|
18885
|
+
return "google-drive";
|
|
18886
|
+
if (lower.includes("onedrive"))
|
|
18887
|
+
return "onedrive";
|
|
18888
|
+
if (lower.includes("dropbox"))
|
|
18889
|
+
return "dropbox";
|
|
18890
|
+
return "file-provider";
|
|
18891
|
+
}
|
|
18892
|
+
function normalizeCloudPath(value) {
|
|
18893
|
+
let normalized = value.replaceAll("\\", "/");
|
|
18894
|
+
if (normalized.length > 1)
|
|
18895
|
+
normalized = normalized.replace(/\/+$/u, "");
|
|
18896
|
+
return normalized;
|
|
18897
|
+
}
|
|
18898
|
+
function isPathInside(target, parent) {
|
|
18899
|
+
const normalizedParent = normalizeCloudPath(parent);
|
|
18900
|
+
if (normalizedParent === "")
|
|
18901
|
+
return false;
|
|
18902
|
+
const left = normalizeCloudPath(target);
|
|
18903
|
+
const [a, b] = process.platform === "win32" ? [left.toLowerCase(), normalizedParent.toLowerCase()] : [left, normalizedParent];
|
|
18904
|
+
return a === b || a.startsWith(`${b}/`);
|
|
18905
|
+
}
|
|
18906
|
+
function canonicalPath(value) {
|
|
18721
18907
|
try {
|
|
18722
|
-
return
|
|
18908
|
+
return realpathSync3(value);
|
|
18723
18909
|
} catch {
|
|
18724
|
-
return
|
|
18910
|
+
return value;
|
|
18725
18911
|
}
|
|
18726
18912
|
}
|
|
18727
|
-
function
|
|
18728
|
-
const
|
|
18729
|
-
|
|
18730
|
-
|
|
18731
|
-
|
|
18732
|
-
|
|
18733
|
-
|
|
18734
|
-
|
|
18913
|
+
function canonicalPathMapper(searchRoots) {
|
|
18914
|
+
const forms = searchRoots.map((root) => ({ raw: normalizeCloudPath(root), canonical: normalizeCloudPath(canonicalPath(root)) })).filter(({ raw, canonical }) => raw !== "" && raw !== canonical).sort((left, right) => right.raw.length - left.raw.length);
|
|
18915
|
+
if (forms.length === 0)
|
|
18916
|
+
return (path) => path;
|
|
18917
|
+
return (path) => {
|
|
18918
|
+
const normalized = normalizeCloudPath(path);
|
|
18919
|
+
for (const { raw, canonical } of forms) {
|
|
18920
|
+
if (isPathInside(normalized, raw))
|
|
18921
|
+
return `${canonical}${normalized.slice(raw.length)}`;
|
|
18735
18922
|
}
|
|
18736
|
-
|
|
18737
|
-
}
|
|
18738
|
-
return { targets, skipped };
|
|
18923
|
+
return path;
|
|
18924
|
+
};
|
|
18739
18925
|
}
|
|
18740
|
-
|
|
18741
|
-
|
|
18742
|
-
});
|
|
18743
|
-
|
|
18744
|
-
|
|
18745
|
-
|
|
18746
|
-
|
|
18747
|
-
init_filter();
|
|
18748
|
-
init_scan_plan();
|
|
18749
|
-
});
|
|
18750
|
-
|
|
18751
|
-
// src/convert/relate.ts
|
|
18752
|
-
async function convertAndRelate(request, options = {}) {
|
|
18753
|
-
const result = await convertWithFilee(request, options.filee);
|
|
18754
|
-
if (options.dupey === false)
|
|
18755
|
-
return { result, links: [] };
|
|
18756
|
-
const dupeyOptions = options.dupey ?? {};
|
|
18757
|
-
const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
|
|
18758
|
-
const links = await Promise.all(pairs.map(async ({ source, output }) => {
|
|
18759
|
-
try {
|
|
18760
|
-
return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
|
|
18761
|
-
} catch (error) {
|
|
18762
|
-
if (!(error instanceof DupeyCliError))
|
|
18763
|
-
throw error;
|
|
18764
|
-
return { source, output, dupey: { relation: "unavailable", reason: error.message } };
|
|
18765
|
-
}
|
|
18766
|
-
}));
|
|
18767
|
-
return { result, links };
|
|
18926
|
+
function cloudScope(searchRoot, cloudRoots) {
|
|
18927
|
+
const canonicalSearchRoot = canonicalPath(searchRoot);
|
|
18928
|
+
const roots = cloudRoots.map((root) => ({ path: canonicalPath(root.path), kind: root.kind }));
|
|
18929
|
+
return {
|
|
18930
|
+
insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
|
|
18931
|
+
nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
|
|
18932
|
+
};
|
|
18768
18933
|
}
|
|
18769
|
-
|
|
18770
|
-
|
|
18771
|
-
|
|
18772
|
-
|
|
18773
|
-
|
|
18774
|
-
|
|
18775
|
-
|
|
18776
|
-
|
|
18777
|
-
|
|
18778
|
-
|
|
18779
|
-
init_relate();
|
|
18780
|
-
});
|
|
18781
|
-
|
|
18782
|
-
// src/datasource/diagnostics.ts
|
|
18783
|
-
function mapDatasourceDiagnostic(ds) {
|
|
18934
|
+
function parseReleaseOutcome(stdout) {
|
|
18935
|
+
let parsed;
|
|
18936
|
+
try {
|
|
18937
|
+
parsed = JSON.parse(stdout.trim());
|
|
18938
|
+
} catch {
|
|
18939
|
+
return;
|
|
18940
|
+
}
|
|
18941
|
+
if (typeof parsed !== "object" || parsed === null || !("ok" in parsed) || typeof parsed.ok !== "boolean") {
|
|
18942
|
+
return;
|
|
18943
|
+
}
|
|
18784
18944
|
return {
|
|
18785
|
-
|
|
18786
|
-
|
|
18787
|
-
|
|
18788
|
-
|
|
18945
|
+
ok: parsed.ok,
|
|
18946
|
+
..."isUploaded" in parsed ? { isUploaded: parsed.isUploaded } : {},
|
|
18947
|
+
..."isUploading" in parsed ? { isUploading: parsed.isUploading } : {},
|
|
18948
|
+
..."downloadingStatus" in parsed ? { downloadingStatus: parsed.downloadingStatus } : {}
|
|
18789
18949
|
};
|
|
18790
18950
|
}
|
|
18791
|
-
function
|
|
18792
|
-
|
|
18951
|
+
function describeReleaseRefusal(outcome) {
|
|
18952
|
+
if (outcome.isUploading === true)
|
|
18953
|
+
return "The file is still uploading to the cloud, so it cannot be released yet.";
|
|
18954
|
+
if (outcome.isUploaded === false)
|
|
18955
|
+
return "The file has not finished uploading to the cloud, so it cannot be released yet.";
|
|
18956
|
+
const status = typeof outcome.downloadingStatus === "string" ? outcome.downloadingStatus : "unknown";
|
|
18957
|
+
return `The cloud provider declined to release the local copy (download status ${status}).`;
|
|
18793
18958
|
}
|
|
18794
|
-
function
|
|
18795
|
-
|
|
18796
|
-
|
|
18797
|
-
|
|
18798
|
-
|
|
18799
|
-
|
|
18800
|
-
|
|
18801
|
-
case "datasource-embedding-egress-rejected":
|
|
18802
|
-
case "datasource-index-failed":
|
|
18803
|
-
case "datasource-index-partial":
|
|
18804
|
-
case "datasource-permission-denied":
|
|
18805
|
-
case "datasource-remote-embedding-rejected":
|
|
18806
|
-
return "retrieval-method-failed";
|
|
18959
|
+
function describeReleaseHelperFailure(error) {
|
|
18960
|
+
const failure = error;
|
|
18961
|
+
const parts = [];
|
|
18962
|
+
if (failure.killed === true || typeof failure.signal === "string") {
|
|
18963
|
+
parts.push(`the helper was killed${typeof failure.signal === "string" ? ` by ${failure.signal}` : ""}`);
|
|
18964
|
+
} else if (typeof failure.code === "number") {
|
|
18965
|
+
parts.push(`the helper exited with code ${failure.code}`);
|
|
18807
18966
|
}
|
|
18967
|
+
const stderr = typeof failure.stderr === "string" ? failure.stderr.trim() : "";
|
|
18968
|
+
if (stderr !== "")
|
|
18969
|
+
parts.push(stderr);
|
|
18970
|
+
if (parts.length === 0)
|
|
18971
|
+
parts.push(error instanceof Error ? error.message : String(error));
|
|
18972
|
+
return `osascript could not release the local copy: ${parts.join("; ")}`;
|
|
18808
18973
|
}
|
|
18809
|
-
|
|
18810
|
-
|
|
18811
|
-
|
|
18812
|
-
function datasourceRetryDelayMs(retry, policy) {
|
|
18813
|
-
const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
|
|
18814
|
-
return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
|
|
18974
|
+
async function defaultDarwinReadDir(path) {
|
|
18975
|
+
const entries = await readdir(path, { withFileTypes: true });
|
|
18976
|
+
return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
|
|
18815
18977
|
}
|
|
18816
|
-
|
|
18817
|
-
const
|
|
18818
|
-
|
|
18819
|
-
|
|
18820
|
-
|
|
18821
|
-
|
|
18822
|
-
|
|
18823
|
-
|
|
18824
|
-
|
|
18825
|
-
const
|
|
18826
|
-
|
|
18827
|
-
if (
|
|
18828
|
-
|
|
18829
|
-
|
|
18830
|
-
|
|
18831
|
-
|
|
18832
|
-
|
|
18833
|
-
|
|
18834
|
-
|
|
18835
|
-
|
|
18836
|
-
|
|
18837
|
-
|
|
18838
|
-
}
|
|
18839
|
-
lastError = result.message ?? result.error ?? "datasource-index-failed";
|
|
18840
|
-
if (attempt > policy.maxRetries) {
|
|
18841
|
-
if (attempt === 1)
|
|
18842
|
-
return { ...result, attempts: attempt };
|
|
18843
|
-
const suffix = ` (gave up after ${attempt} attempts)`;
|
|
18844
|
-
const last = result.diagnostics.at(-1);
|
|
18845
|
-
return {
|
|
18846
|
-
...result,
|
|
18847
|
-
attempts: attempt,
|
|
18848
|
-
message: `${lastError}${suffix}`,
|
|
18849
|
-
diagnostics: last === undefined ? [
|
|
18850
|
-
{
|
|
18851
|
-
code: result.code ?? "datasource-index-failed",
|
|
18852
|
-
severity: "error",
|
|
18853
|
-
message: `${lastError}${suffix}`,
|
|
18854
|
-
instanceId: result.instanceId,
|
|
18855
|
-
source: result.skill
|
|
18856
|
-
}
|
|
18857
|
-
] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
|
|
18858
|
-
};
|
|
18859
|
-
}
|
|
18860
|
-
await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
|
|
18861
|
-
signal?.throwIfAborted();
|
|
18978
|
+
function parseDarwinMounts(stdout) {
|
|
18979
|
+
const mounts = [];
|
|
18980
|
+
for (const line of stdout.split(`
|
|
18981
|
+
`)) {
|
|
18982
|
+
if (line.trim() === "")
|
|
18983
|
+
continue;
|
|
18984
|
+
const open = line.lastIndexOf(" (");
|
|
18985
|
+
if (open <= 0 || !line.endsWith(")"))
|
|
18986
|
+
continue;
|
|
18987
|
+
const head = line.slice(0, open);
|
|
18988
|
+
const on = head.indexOf(" on ");
|
|
18989
|
+
if (on <= 0)
|
|
18990
|
+
continue;
|
|
18991
|
+
const mountPoint = head.slice(on + " on ".length);
|
|
18992
|
+
if (mountPoint === "")
|
|
18993
|
+
continue;
|
|
18994
|
+
const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
|
|
18995
|
+
mounts.push({
|
|
18996
|
+
device: head.slice(0, on),
|
|
18997
|
+
mountPoint,
|
|
18998
|
+
fileSystem: fields[0] ?? "",
|
|
18999
|
+
options: fields.slice(1)
|
|
19000
|
+
});
|
|
18862
19001
|
}
|
|
19002
|
+
return mounts;
|
|
18863
19003
|
}
|
|
18864
|
-
|
|
18865
|
-
|
|
18866
|
-
|
|
18867
|
-
|
|
18868
|
-
|
|
18869
|
-
|
|
18870
|
-
|
|
18871
|
-
|
|
18872
|
-
|
|
18873
|
-
|
|
18874
|
-
|
|
18875
|
-
|
|
18876
|
-
diagnostics: [
|
|
18877
|
-
{
|
|
18878
|
-
code: "datasource-index-failed",
|
|
18879
|
-
severity: "error",
|
|
18880
|
-
message,
|
|
18881
|
-
source: descriptor.name,
|
|
18882
|
-
instanceId: descriptor.instanceId
|
|
18883
|
-
}
|
|
18884
|
-
],
|
|
18885
|
-
error: "datasource-index-failed",
|
|
18886
|
-
code: "datasource-index-failed",
|
|
18887
|
-
message
|
|
18888
|
-
};
|
|
18889
|
-
return failed;
|
|
19004
|
+
function networkMountPoints(mounts) {
|
|
19005
|
+
const points = [];
|
|
19006
|
+
const seen = new Set;
|
|
19007
|
+
for (const mount of mounts) {
|
|
19008
|
+
if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
|
|
19009
|
+
continue;
|
|
19010
|
+
if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
|
|
19011
|
+
continue;
|
|
19012
|
+
if (seen.has(mount.mountPoint))
|
|
19013
|
+
continue;
|
|
19014
|
+
seen.add(mount.mountPoint);
|
|
19015
|
+
points.push(mount.mountPoint);
|
|
18890
19016
|
}
|
|
19017
|
+
return points;
|
|
18891
19018
|
}
|
|
18892
|
-
|
|
18893
|
-
|
|
18894
|
-
|
|
18895
|
-
|
|
18896
|
-
|
|
18897
|
-
|
|
18898
|
-
|
|
18899
|
-
|
|
18900
|
-
|
|
18901
|
-
|
|
18902
|
-
|
|
18903
|
-
|
|
18904
|
-
|
|
18905
|
-
|
|
18906
|
-
|
|
18907
|
-
function resolveEverythingBundleDir() {
|
|
18908
|
-
let dir = dirname16(fileURLToPath2(import.meta.url));
|
|
18909
|
-
for (let depth = 0;depth < 6; depth += 1) {
|
|
18910
|
-
const candidate = join29(dir, "vendor", "everything");
|
|
18911
|
-
if (existsSync18(join29(candidate, MANIFEST_FILENAME)))
|
|
18912
|
-
return candidate;
|
|
18913
|
-
const parent = dirname16(dir);
|
|
18914
|
-
if (parent === dir)
|
|
18915
|
-
break;
|
|
18916
|
-
dir = parent;
|
|
19019
|
+
function parseXattrValues(stdout, candidates) {
|
|
19020
|
+
const values = new Map;
|
|
19021
|
+
if (candidates.length === 0)
|
|
19022
|
+
return values;
|
|
19023
|
+
const ordered = [...candidates].sort((left, right) => right.length - left.length);
|
|
19024
|
+
for (const line of stdout.split(`
|
|
19025
|
+
`)) {
|
|
19026
|
+
if (line === "")
|
|
19027
|
+
continue;
|
|
19028
|
+
for (const candidate of ordered) {
|
|
19029
|
+
if (line.startsWith(`${candidate}: `)) {
|
|
19030
|
+
values.set(candidate, line.slice(candidate.length + 2));
|
|
19031
|
+
break;
|
|
19032
|
+
}
|
|
19033
|
+
}
|
|
18917
19034
|
}
|
|
18918
|
-
return;
|
|
18919
|
-
}
|
|
18920
|
-
function everythingArch(arch) {
|
|
18921
|
-
if (arch === "x64" || arch === "arm64")
|
|
18922
|
-
return arch;
|
|
18923
|
-
return;
|
|
18924
|
-
}
|
|
18925
|
-
function everythingBinaryPaths(root, version, arch) {
|
|
18926
|
-
const dir = join29(root, ".autorag", "everything", version, arch);
|
|
18927
|
-
return { everythingPath: join29(dir, "everything.exe"), esPath: join29(dir, "es.exe") };
|
|
18928
|
-
}
|
|
18929
|
-
function loadEverythingBundleManifest(bundleDir) {
|
|
18930
|
-
return JSON.parse(readFileSync18(join29(bundleDir, MANIFEST_FILENAME), "utf8"));
|
|
19035
|
+
return values;
|
|
18931
19036
|
}
|
|
18932
|
-
|
|
18933
|
-
|
|
18934
|
-
|
|
18935
|
-
|
|
18936
|
-
|
|
18937
|
-
|
|
18938
|
-
|
|
18939
|
-
|
|
18940
|
-
|
|
19037
|
+
|
|
19038
|
+
class DarwinCloudProbe {
|
|
19039
|
+
exec;
|
|
19040
|
+
readDir;
|
|
19041
|
+
home;
|
|
19042
|
+
constructor(options = {}) {
|
|
19043
|
+
this.exec = options.exec ?? defaultCloudExec;
|
|
19044
|
+
this.readDir = options.readDir ?? defaultDarwinReadDir;
|
|
19045
|
+
this.home = options.home ?? homedir6();
|
|
18941
19046
|
}
|
|
18942
|
-
|
|
18943
|
-
|
|
18944
|
-
|
|
18945
|
-
|
|
18946
|
-
|
|
18947
|
-
|
|
18948
|
-
|
|
19047
|
+
async cloudRoots(searchRoots = []) {
|
|
19048
|
+
const roots = [];
|
|
19049
|
+
const seen = new Set;
|
|
19050
|
+
const add = (path, kind) => {
|
|
19051
|
+
const canonical = canonicalPath(path);
|
|
19052
|
+
const key = normalizeCloudPath(canonical);
|
|
19053
|
+
if (key === "" || seen.has(key))
|
|
19054
|
+
return;
|
|
19055
|
+
seen.add(key);
|
|
19056
|
+
roots.push({ path: canonical, kind });
|
|
18949
19057
|
};
|
|
19058
|
+
const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
|
|
19059
|
+
for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
|
|
19060
|
+
if (entry.isDirectory())
|
|
19061
|
+
add(join30(cloudStorage, entry.name), "file-provider");
|
|
19062
|
+
}
|
|
19063
|
+
if (await this.tryReadDir(mobileDocuments) !== undefined)
|
|
19064
|
+
add(mobileDocuments, "file-provider");
|
|
19065
|
+
for (const mountPoint of await this.networkMountPoints())
|
|
19066
|
+
add(mountPoint, "network");
|
|
19067
|
+
for (const directory of await this.fileProviderDirectories(searchRoots))
|
|
19068
|
+
add(directory, "file-provider");
|
|
19069
|
+
return roots;
|
|
18950
19070
|
}
|
|
18951
|
-
|
|
18952
|
-
|
|
18953
|
-
|
|
18954
|
-
|
|
18955
|
-
|
|
19071
|
+
async networkMountPoints() {
|
|
19072
|
+
try {
|
|
19073
|
+
const { stdout } = await this.exec("/sbin/mount", [], { timeoutMs: 1e4, maxBuffer: 4 * 1024 * 1024 });
|
|
19074
|
+
return networkMountPoints(parseDarwinMounts(stdout));
|
|
19075
|
+
} catch {
|
|
19076
|
+
return [];
|
|
19077
|
+
}
|
|
18956
19078
|
}
|
|
18957
|
-
|
|
18958
|
-
|
|
18959
|
-
|
|
18960
|
-
|
|
18961
|
-
|
|
18962
|
-
|
|
18963
|
-
|
|
19079
|
+
async fileProviderDirectories(searchRoots) {
|
|
19080
|
+
if (searchRoots.length === 0)
|
|
19081
|
+
return [];
|
|
19082
|
+
const candidates = new Set;
|
|
19083
|
+
for (const searchRoot of searchRoots) {
|
|
19084
|
+
const canonical = canonicalPath(searchRoot);
|
|
19085
|
+
for (let cursor = canonical;; cursor = dirname16(cursor)) {
|
|
19086
|
+
candidates.add(cursor);
|
|
19087
|
+
const parent = dirname16(cursor);
|
|
19088
|
+
if (parent === cursor)
|
|
19089
|
+
break;
|
|
19090
|
+
}
|
|
19091
|
+
for (const entry of await this.tryReadDir(canonical) ?? []) {
|
|
19092
|
+
if (entry.isDirectory())
|
|
19093
|
+
candidates.add(join30(canonical, entry.name));
|
|
19094
|
+
}
|
|
19095
|
+
}
|
|
19096
|
+
const paths = [...candidates];
|
|
19097
|
+
const managed = [];
|
|
19098
|
+
for (let start = 0;start < paths.length; start += XATTR_BATCH) {
|
|
19099
|
+
for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
|
|
19100
|
+
managed.push(path);
|
|
19101
|
+
}
|
|
19102
|
+
return managed;
|
|
18964
19103
|
}
|
|
18965
|
-
|
|
18966
|
-
|
|
18967
|
-
|
|
18968
|
-
|
|
18969
|
-
|
|
19104
|
+
async xattrValues(paths) {
|
|
19105
|
+
try {
|
|
19106
|
+
const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
|
|
19107
|
+
timeoutMs: 15000,
|
|
19108
|
+
maxBuffer: 8 * 1024 * 1024
|
|
19109
|
+
});
|
|
19110
|
+
return parseXattrValues(stdout, paths);
|
|
19111
|
+
} catch (error) {
|
|
19112
|
+
const failure = error;
|
|
19113
|
+
return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
|
|
19114
|
+
}
|
|
18970
19115
|
}
|
|
18971
|
-
|
|
18972
|
-
|
|
18973
|
-
|
|
18974
|
-
|
|
18975
|
-
|
|
18976
|
-
|
|
18977
|
-
};
|
|
19116
|
+
async tryReadDir(path) {
|
|
19117
|
+
try {
|
|
19118
|
+
return await this.readDir(path);
|
|
19119
|
+
} catch {
|
|
19120
|
+
return;
|
|
19121
|
+
}
|
|
18978
19122
|
}
|
|
18979
|
-
|
|
18980
|
-
|
|
18981
|
-
|
|
18982
|
-
|
|
18983
|
-
|
|
18984
|
-
|
|
18985
|
-
|
|
18986
|
-
|
|
19123
|
+
async datalessAmong(paths) {
|
|
19124
|
+
const dataless = new Set;
|
|
19125
|
+
for (let start = 0;start < paths.length; start += STAT_BATCH) {
|
|
19126
|
+
const batch = paths.slice(start, start + STAT_BATCH);
|
|
19127
|
+
const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
|
|
19128
|
+
timeoutMs: 30000,
|
|
19129
|
+
maxBuffer: 4 * 1024 * 1024
|
|
19130
|
+
});
|
|
19131
|
+
const flags = stdout.split(`
|
|
19132
|
+
`).filter(Boolean);
|
|
19133
|
+
for (const [index, path] of batch.entries()) {
|
|
19134
|
+
if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
|
|
19135
|
+
dataless.add(path);
|
|
19136
|
+
}
|
|
19137
|
+
}
|
|
19138
|
+
return dataless;
|
|
18987
19139
|
}
|
|
18988
|
-
|
|
18989
|
-
|
|
18990
|
-
|
|
18991
|
-
const archive = readFileSync18(archivePath);
|
|
18992
|
-
const actualArchive = sha256(archive);
|
|
18993
|
-
if (actualArchive !== archiveSha256) {
|
|
18994
|
-
throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
|
|
19140
|
+
async providerOf(path) {
|
|
19141
|
+
const domain = await readFileProviderDomainUpwards(path);
|
|
19142
|
+
return domain === undefined ? undefined : cloudProviderFromDomain(domain);
|
|
18995
19143
|
}
|
|
18996
|
-
|
|
18997
|
-
|
|
18998
|
-
if (entry === null)
|
|
18999
|
-
throw new Error(`${archivePath} does not contain ${member}`);
|
|
19000
|
-
const bytes = await entry.async("nodebuffer");
|
|
19001
|
-
const actualMember = sha256(bytes);
|
|
19002
|
-
if (actualMember !== memberSha256) {
|
|
19003
|
-
throw new Error(`${member} from ${archivePath} SHA-256 ${actualMember} does not match pinned ${memberSha256}`);
|
|
19144
|
+
canRelease(provider) {
|
|
19145
|
+
return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
|
|
19004
19146
|
}
|
|
19005
|
-
|
|
19006
|
-
|
|
19007
|
-
|
|
19008
|
-
|
|
19009
|
-
|
|
19010
|
-
|
|
19011
|
-
|
|
19012
|
-
|
|
19013
|
-
|
|
19014
|
-
|
|
19015
|
-
|
|
19016
|
-
|
|
19147
|
+
async evict(path) {
|
|
19148
|
+
let stdout;
|
|
19149
|
+
try {
|
|
19150
|
+
stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
|
|
19151
|
+
timeoutMs: 30000
|
|
19152
|
+
})).stdout;
|
|
19153
|
+
} catch (error) {
|
|
19154
|
+
return { ok: false, message: describeReleaseHelperFailure(error) };
|
|
19155
|
+
}
|
|
19156
|
+
const outcome = parseReleaseOutcome(stdout);
|
|
19157
|
+
if (outcome === undefined) {
|
|
19158
|
+
const detail = stdout.trim();
|
|
19159
|
+
return {
|
|
19160
|
+
ok: false,
|
|
19161
|
+
message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
|
|
19162
|
+
};
|
|
19163
|
+
}
|
|
19164
|
+
return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
|
|
19017
19165
|
}
|
|
19018
19166
|
}
|
|
19019
|
-
function
|
|
19020
|
-
return
|
|
19167
|
+
function buildWindowsAttributeScript(paths) {
|
|
19168
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19169
|
+
` + `$ErrorActionPreference='SilentlyContinue'
|
|
19170
|
+
` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
|
|
19171
|
+
`;
|
|
19021
19172
|
}
|
|
19022
|
-
function
|
|
19023
|
-
return
|
|
19173
|
+
function buildWindowsSyncRootsScript() {
|
|
19174
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19175
|
+
` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
|
|
19176
|
+
` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
|
|
19177
|
+
`;
|
|
19024
19178
|
}
|
|
19025
|
-
|
|
19026
|
-
|
|
19027
|
-
|
|
19028
|
-
|
|
19029
|
-
|
|
19030
|
-
|
|
19031
|
-
|
|
19032
|
-
|
|
19033
|
-
|
|
19034
|
-
|
|
19035
|
-
|
|
19036
|
-
|
|
19037
|
-
|
|
19038
|
-
|
|
19039
|
-
|
|
19040
|
-
|
|
19041
|
-
|
|
19042
|
-
return (path) => {
|
|
19043
|
-
const absolute = normalize(path);
|
|
19044
|
-
return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
|
|
19045
|
-
};
|
|
19179
|
+
function buildWindowsDehydrateScript(path) {
|
|
19180
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19181
|
+
` + `$ErrorActionPreference='Stop'
|
|
19182
|
+
` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
|
|
19183
|
+
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
|
|
19184
|
+
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
|
|
19185
|
+
` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
|
|
19186
|
+
` + `'@
|
|
19187
|
+
` + `$h=[IntPtr]::Zero
|
|
19188
|
+
` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
|
|
19189
|
+
` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
|
|
19190
|
+
` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
|
|
19191
|
+
` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
|
|
19192
|
+
` + `[Autorag.CfApi]::CfCloseHandle($h)
|
|
19193
|
+
` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
|
|
19194
|
+
` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
|
|
19195
|
+
`;
|
|
19046
19196
|
}
|
|
19047
|
-
|
|
19048
|
-
|
|
19049
|
-
|
|
19050
|
-
|
|
19051
|
-
|
|
19052
|
-
|
|
19053
|
-
|
|
19054
|
-
const run = this.lastExclusive.then(task);
|
|
19055
|
-
const settled = run.then(() => {
|
|
19056
|
-
return;
|
|
19057
|
-
}, () => {
|
|
19058
|
-
return;
|
|
19059
|
-
});
|
|
19060
|
-
this.sharedSinceExclusive.push(settled);
|
|
19061
|
-
return run;
|
|
19197
|
+
function parseWindowsAttributeOutput(stdout, paths) {
|
|
19198
|
+
const lines = stdout.split(/\r?\n/);
|
|
19199
|
+
const attributes = new Map;
|
|
19200
|
+
for (const [index, path] of paths.entries()) {
|
|
19201
|
+
const value = Number.parseInt((lines[index] ?? "").trim(), 10);
|
|
19202
|
+
if (Number.isFinite(value))
|
|
19203
|
+
attributes.set(path, value);
|
|
19062
19204
|
}
|
|
19063
|
-
|
|
19064
|
-
|
|
19065
|
-
|
|
19066
|
-
|
|
19067
|
-
|
|
19068
|
-
|
|
19069
|
-
|
|
19070
|
-
|
|
19071
|
-
|
|
19072
|
-
|
|
19073
|
-
|
|
19205
|
+
return attributes;
|
|
19206
|
+
}
|
|
19207
|
+
function parseWindowsSyncRoots(stdout) {
|
|
19208
|
+
const roots = [];
|
|
19209
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
19210
|
+
const tab = line.indexOf("\t");
|
|
19211
|
+
if (tab <= 0)
|
|
19212
|
+
continue;
|
|
19213
|
+
const path = line.slice(0, tab).trim();
|
|
19214
|
+
if (path !== "")
|
|
19215
|
+
roots.push({ path, label: line.slice(tab + 1).trim() });
|
|
19074
19216
|
}
|
|
19217
|
+
return roots;
|
|
19075
19218
|
}
|
|
19076
|
-
|
|
19077
|
-
|
|
19078
|
-
|
|
19079
|
-
|
|
19080
|
-
|
|
19081
|
-
|
|
19082
|
-
|
|
19083
|
-
function everythingInstanceName(root) {
|
|
19084
|
-
return `autorag-${createHash12("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
|
|
19219
|
+
function providerFromWindowsSyncRoot(label) {
|
|
19220
|
+
const lower = label.toLowerCase();
|
|
19221
|
+
if (lower.includes("onedrive") || lower.includes("one drive"))
|
|
19222
|
+
return "onedrive";
|
|
19223
|
+
if (lower.includes("google"))
|
|
19224
|
+
return "google-drive";
|
|
19225
|
+
return "file-provider";
|
|
19085
19226
|
}
|
|
19086
|
-
function
|
|
19087
|
-
|
|
19088
|
-
|
|
19089
|
-
|
|
19090
|
-
"[Everything]",
|
|
19091
|
-
"app_data=0",
|
|
19092
|
-
"run_as_admin=0",
|
|
19093
|
-
"run_in_background=1",
|
|
19094
|
-
"show_tray_icon=0",
|
|
19095
|
-
"show_in_taskbar=0",
|
|
19096
|
-
"check_for_updates_on_startup=0",
|
|
19097
|
-
"beta_updates=0",
|
|
19098
|
-
"language=1033",
|
|
19099
|
-
"allow_http_server=0",
|
|
19100
|
-
"allow_etp_server=0",
|
|
19101
|
-
"search_history_enabled=0",
|
|
19102
|
-
"run_history_enabled=0",
|
|
19103
|
-
"auto_include_fixed_volumes=0",
|
|
19104
|
-
"auto_include_removable_volumes=0",
|
|
19105
|
-
"auto_include_fixed_refs_volumes=0",
|
|
19106
|
-
"auto_include_removable_refs_volumes=0",
|
|
19107
|
-
"ntfs_volume_paths=",
|
|
19108
|
-
"ntfs_volume_includes=",
|
|
19109
|
-
"refs_volume_paths=",
|
|
19110
|
-
"refs_volume_includes=",
|
|
19111
|
-
"index_size=1",
|
|
19112
|
-
"index_date_modified=1",
|
|
19113
|
-
`folders=${list(input.folders)}`,
|
|
19114
|
-
`folder_monitor_changes=${perFolder("1")}`,
|
|
19115
|
-
`folder_rescan_if_full_list=${perFolder("1")}`,
|
|
19116
|
-
`folder_update_types=${perFolder("0")}`,
|
|
19117
|
-
"exclude_list_enabled=1",
|
|
19118
|
-
`exclude_folders=${list(input.excludeFolders)}`,
|
|
19119
|
-
""
|
|
19120
|
-
].join(`\r
|
|
19121
|
-
`);
|
|
19227
|
+
function buildWindowsDiskScript() {
|
|
19228
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19229
|
+
` + "Get-CimInstance -ClassName Win32_LogicalDisk | ForEach-Object {" + `$_.DeviceID + [char]9 + $_.DriveType + [char]9 + $_.FileSystem + [char]9 + $_.VolumeName}
|
|
19230
|
+
`;
|
|
19122
19231
|
}
|
|
19123
|
-
function
|
|
19124
|
-
const
|
|
19125
|
-
|
|
19126
|
-
"
|
|
19127
|
-
|
|
19128
|
-
"
|
|
19129
|
-
"
|
|
19130
|
-
|
|
19131
|
-
|
|
19132
|
-
|
|
19133
|
-
|
|
19134
|
-
|
|
19135
|
-
|
|
19136
|
-
|
|
19137
|
-
|
|
19138
|
-
|
|
19139
|
-
|
|
19140
|
-
args.push("-case");
|
|
19141
|
-
if (request.matchPath)
|
|
19142
|
-
args.push("-match-path");
|
|
19143
|
-
if (request.wholeWord)
|
|
19144
|
-
args.push("-whole-word");
|
|
19145
|
-
if (request.kind === "files")
|
|
19146
|
-
args.push("/a-d");
|
|
19147
|
-
if (request.kind === "folders")
|
|
19148
|
-
args.push("/ad");
|
|
19149
|
-
if (request.path !== undefined)
|
|
19150
|
-
args.push("-path", request.path);
|
|
19151
|
-
if (request.sort !== undefined)
|
|
19152
|
-
args.push("-sort", request.sort);
|
|
19153
|
-
if (request.offset !== undefined && request.offset > 0)
|
|
19154
|
-
args.push("-offset", String(request.offset));
|
|
19155
|
-
args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
|
|
19156
|
-
if (request.regex)
|
|
19157
|
-
args.push("-regex", "--", request.query);
|
|
19158
|
-
else
|
|
19159
|
-
args.push("--", request.query);
|
|
19160
|
-
return args;
|
|
19232
|
+
function parseWindowsLogicalDisks(stdout) {
|
|
19233
|
+
const disks = [];
|
|
19234
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
19235
|
+
if (line.trim() === "")
|
|
19236
|
+
continue;
|
|
19237
|
+
const [deviceId, driveType, fileSystem, ...rest] = line.split("\t");
|
|
19238
|
+
const type = Number.parseInt((driveType ?? "").trim(), 10);
|
|
19239
|
+
if (!Number.isFinite(type))
|
|
19240
|
+
continue;
|
|
19241
|
+
disks.push({
|
|
19242
|
+
deviceId: (deviceId ?? "").trim(),
|
|
19243
|
+
driveType: type,
|
|
19244
|
+
fileSystem: (fileSystem ?? "").trim(),
|
|
19245
|
+
volumeName: rest.join("\t").trim()
|
|
19246
|
+
});
|
|
19247
|
+
}
|
|
19248
|
+
return disks;
|
|
19161
19249
|
}
|
|
19162
|
-
function
|
|
19163
|
-
const
|
|
19164
|
-
|
|
19165
|
-
|
|
19166
|
-
|
|
19167
|
-
|
|
19168
|
-
|
|
19169
|
-
|
|
19170
|
-
|
|
19250
|
+
function windowsRootsFromDisks(disks) {
|
|
19251
|
+
const roots = [];
|
|
19252
|
+
for (const disk of disks) {
|
|
19253
|
+
if (disk.deviceId === "")
|
|
19254
|
+
continue;
|
|
19255
|
+
const path = `${disk.deviceId}\\`;
|
|
19256
|
+
const fileSystem = disk.fileSystem.toLowerCase();
|
|
19257
|
+
if (disk.driveType === WINDOWS_DRIVE_TYPE_NETWORK) {
|
|
19258
|
+
roots.push({ path, kind: "network" });
|
|
19259
|
+
continue;
|
|
19260
|
+
}
|
|
19261
|
+
if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
|
|
19262
|
+
roots.push({ path, kind: "network" });
|
|
19263
|
+
continue;
|
|
19264
|
+
}
|
|
19265
|
+
if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
|
|
19266
|
+
roots.push({ path, kind: "cloud-files" });
|
|
19267
|
+
}
|
|
19171
19268
|
}
|
|
19172
|
-
|
|
19173
|
-
|
|
19174
|
-
|
|
19175
|
-
|
|
19176
|
-
|
|
19177
|
-
|
|
19269
|
+
return roots;
|
|
19270
|
+
}
|
|
19271
|
+
function uncShareRoot(target) {
|
|
19272
|
+
const normalized = target.replaceAll("/", "\\");
|
|
19273
|
+
if (!normalized.startsWith("\\\\") || normalized.startsWith("\\\\?\\") || normalized.startsWith("\\\\.\\"))
|
|
19274
|
+
return;
|
|
19275
|
+
const rest = normalized.slice(2);
|
|
19276
|
+
const separator = rest.indexOf("\\");
|
|
19277
|
+
if (separator <= 0)
|
|
19278
|
+
return;
|
|
19279
|
+
const server = rest.slice(0, separator);
|
|
19280
|
+
const share = rest.slice(separator + 1).split("\\")[0] ?? "";
|
|
19281
|
+
if (server === "" || share === "")
|
|
19282
|
+
return;
|
|
19283
|
+
return `\\\\${server}\\${share}`;
|
|
19284
|
+
}
|
|
19285
|
+
function parseWindowsDehydrateOutput(result) {
|
|
19286
|
+
const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
19287
|
+
if (open === null)
|
|
19288
|
+
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
19289
|
+
if (Number.parseInt(open[1], 16) !== 0) {
|
|
19178
19290
|
return {
|
|
19179
|
-
|
|
19180
|
-
|
|
19181
|
-
size: typeof record.size === "number" ? record.size : undefined,
|
|
19182
|
-
dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
|
|
19291
|
+
ok: false,
|
|
19292
|
+
message: windowsFailureMessage(result, `could not open the file for release (0x${open[1]})`)
|
|
19183
19293
|
};
|
|
19184
|
-
}
|
|
19294
|
+
}
|
|
19295
|
+
const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
19296
|
+
if (dehydrate === null) {
|
|
19297
|
+
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
19298
|
+
}
|
|
19299
|
+
const hresult = dehydrate[1];
|
|
19300
|
+
if (Number.parseInt(hresult, 16) === 0)
|
|
19301
|
+
return { ok: true };
|
|
19302
|
+
const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
|
|
19303
|
+
return {
|
|
19304
|
+
ok: false,
|
|
19305
|
+
message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
|
|
19306
|
+
};
|
|
19307
|
+
}
|
|
19308
|
+
function windowsFailureMessage(result, fallback) {
|
|
19309
|
+
const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
|
|
19310
|
+
if (printed !== undefined && printed !== "")
|
|
19311
|
+
return printed;
|
|
19312
|
+
const stderr = result.stderr.trim();
|
|
19313
|
+
return stderr !== "" ? stderr : fallback;
|
|
19185
19314
|
}
|
|
19186
19315
|
|
|
19187
|
-
class
|
|
19188
|
-
instanceName;
|
|
19189
|
-
options;
|
|
19190
|
-
platform;
|
|
19316
|
+
class WindowsCloudProbe {
|
|
19191
19317
|
run;
|
|
19192
|
-
|
|
19193
|
-
|
|
19194
|
-
|
|
19195
|
-
|
|
19196
|
-
queue = new SharedExclusiveQueue;
|
|
19197
|
-
constructor(options) {
|
|
19198
|
-
this.options = options;
|
|
19199
|
-
this.platform = options.platform ?? process.platform;
|
|
19200
|
-
this.instanceName = everythingInstanceName(options.root);
|
|
19201
|
-
this.run = options.run ?? runProcess;
|
|
19202
|
-
this.launch = options.launch ?? launchDetached;
|
|
19203
|
-
this.stateDir = join30(options.root, ".autorag", "everything");
|
|
19204
|
-
this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
|
|
19205
|
-
}
|
|
19206
|
-
isSupported() {
|
|
19207
|
-
return this.platform === "win32";
|
|
19318
|
+
syncRoots;
|
|
19319
|
+
disks;
|
|
19320
|
+
constructor(options = {}) {
|
|
19321
|
+
this.run = options.run ?? runPowerShell;
|
|
19208
19322
|
}
|
|
19209
|
-
|
|
19210
|
-
|
|
19211
|
-
|
|
19212
|
-
|
|
19213
|
-
|
|
19214
|
-
const
|
|
19215
|
-
if (
|
|
19216
|
-
return
|
|
19217
|
-
|
|
19218
|
-
|
|
19219
|
-
|
|
19220
|
-
|
|
19221
|
-
|
|
19222
|
-
|
|
19223
|
-
|
|
19224
|
-
|
|
19225
|
-
|
|
19226
|
-
|
|
19227
|
-
|
|
19228
|
-
|
|
19229
|
-
|
|
19230
|
-
return this.failure("search-failed", es.esPath, args, result);
|
|
19231
|
-
try {
|
|
19232
|
-
return { ok: true, results: parseEverythingJson(result.stdout) };
|
|
19233
|
-
} catch (error) {
|
|
19234
|
-
return { ok: false, reason: "search-failed", message: error.message };
|
|
19235
|
-
}
|
|
19236
|
-
});
|
|
19323
|
+
async cloudRoots(searchRoots = []) {
|
|
19324
|
+
const roots = [];
|
|
19325
|
+
const seen = new Set;
|
|
19326
|
+
const add = (path, kind) => {
|
|
19327
|
+
const canonical = canonicalPath(path);
|
|
19328
|
+
const key = normalizeCloudPath(canonical).toLowerCase();
|
|
19329
|
+
if (key === "" || seen.has(key))
|
|
19330
|
+
return;
|
|
19331
|
+
seen.add(key);
|
|
19332
|
+
roots.push({ path: canonical, kind });
|
|
19333
|
+
};
|
|
19334
|
+
for (const root of await this.loadSyncRoots())
|
|
19335
|
+
add(root.path, "cloud-files");
|
|
19336
|
+
for (const root of await this.loadDiskRoots())
|
|
19337
|
+
add(root.path, root.kind);
|
|
19338
|
+
for (const searchRoot of searchRoots) {
|
|
19339
|
+
const share = uncShareRoot(searchRoot);
|
|
19340
|
+
if (share !== undefined)
|
|
19341
|
+
add(share, "network");
|
|
19342
|
+
}
|
|
19343
|
+
return roots;
|
|
19237
19344
|
}
|
|
19238
|
-
async
|
|
19239
|
-
const
|
|
19240
|
-
|
|
19241
|
-
|
|
19242
|
-
|
|
19243
|
-
|
|
19244
|
-
|
|
19245
|
-
const timedOut = () => ({
|
|
19246
|
-
ok: false,
|
|
19247
|
-
reason: "search-failed",
|
|
19248
|
-
message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
|
|
19249
|
-
});
|
|
19250
|
-
for (;; ) {
|
|
19251
|
-
const remainingMs = deadline - Date.now();
|
|
19252
|
-
if (remainingMs <= 0)
|
|
19253
|
-
return timedOut();
|
|
19254
|
-
const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
|
|
19255
|
-
const response = await this.run(esPath, args, remainingMs + 5000);
|
|
19256
|
-
if (response.code !== 0)
|
|
19257
|
-
return this.failure("search-failed", esPath, args, response);
|
|
19258
|
-
if (Date.now() >= deadline)
|
|
19259
|
-
return timedOut();
|
|
19260
|
-
let entries;
|
|
19261
|
-
try {
|
|
19262
|
-
entries = parseEverythingJson(response.stdout);
|
|
19263
|
-
} catch (error) {
|
|
19264
|
-
return { ok: false, reason: "search-failed", message: error.message };
|
|
19345
|
+
async datalessAmong(paths) {
|
|
19346
|
+
const dataless = new Set;
|
|
19347
|
+
for (let start = 0;start < paths.length; start += WINDOWS_ATTRIBUTE_BATCH) {
|
|
19348
|
+
const batch = paths.slice(start, start + WINDOWS_ATTRIBUTE_BATCH);
|
|
19349
|
+
const result = await this.run(buildWindowsAttributeScript(batch), 30000);
|
|
19350
|
+
if (!result.ok) {
|
|
19351
|
+
throw new Error(`Could not read Windows cloud file attributes: ${result.stderr.trim() || "powershell.exe failed"}`);
|
|
19265
19352
|
}
|
|
19266
|
-
for (const
|
|
19267
|
-
if (
|
|
19268
|
-
|
|
19269
|
-
if (skipped < offset) {
|
|
19270
|
-
skipped += 1;
|
|
19271
|
-
continue;
|
|
19272
|
-
}
|
|
19273
|
-
results.push(entry);
|
|
19274
|
-
if (results.length >= maxResults)
|
|
19275
|
-
return { ok: true, results };
|
|
19353
|
+
for (const [path, attributes] of parseWindowsAttributeOutput(result.stdout, batch)) {
|
|
19354
|
+
if (datalessFromWindowsAttributes(attributes))
|
|
19355
|
+
dataless.add(path);
|
|
19276
19356
|
}
|
|
19277
|
-
if (entries.length < MAX_FILTER_CANDIDATES)
|
|
19278
|
-
return { ok: true, results };
|
|
19279
|
-
nativeOffset += entries.length;
|
|
19280
19357
|
}
|
|
19358
|
+
return dataless;
|
|
19281
19359
|
}
|
|
19282
|
-
|
|
19283
|
-
|
|
19284
|
-
|
|
19285
|
-
if (!es.ok)
|
|
19286
|
-
return es;
|
|
19287
|
-
const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
|
|
19288
|
-
const saveArgs = ["-instance", this.instanceName, "-save-db"];
|
|
19289
|
-
const saved = await this.run(es.esPath, saveArgs, timeoutMs);
|
|
19290
|
-
if (saved.code !== 0)
|
|
19291
|
-
return this.failure("index-failed", es.esPath, saveArgs, saved);
|
|
19292
|
-
const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
|
|
19293
|
-
const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
|
|
19294
|
-
if (counted.code !== 0)
|
|
19295
|
-
return this.failure("index-failed", es.esPath, countArgs, counted);
|
|
19296
|
-
return {
|
|
19297
|
-
ok: true,
|
|
19298
|
-
indexedFolders: [...this.options.folders],
|
|
19299
|
-
indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
|
|
19300
|
-
};
|
|
19301
|
-
});
|
|
19360
|
+
async providerOf(path) {
|
|
19361
|
+
const root = longestSyncRoot(await this.loadSyncRoots(), path);
|
|
19362
|
+
return root === undefined ? cloudProviderFromPath(path) : providerFromWindowsSyncRoot(root.label);
|
|
19302
19363
|
}
|
|
19303
|
-
|
|
19304
|
-
return
|
|
19305
|
-
const binaries = await this.resolveBinaries("cached");
|
|
19306
|
-
if (!binaries.ok)
|
|
19307
|
-
return;
|
|
19308
|
-
await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
19309
|
-
});
|
|
19364
|
+
canRelease(provider) {
|
|
19365
|
+
return provider !== undefined;
|
|
19310
19366
|
}
|
|
19311
|
-
async
|
|
19312
|
-
|
|
19313
|
-
return {
|
|
19314
|
-
ok: false,
|
|
19315
|
-
reason: "unsupported-platform",
|
|
19316
|
-
message: `Everything is Windows-only; this host is ${this.platform}.`
|
|
19317
|
-
};
|
|
19318
|
-
}
|
|
19319
|
-
if (this.binaries?.ok)
|
|
19320
|
-
return this.binaries;
|
|
19321
|
-
const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
|
|
19322
|
-
const result = await resolve(mode);
|
|
19323
|
-
if (result.ok)
|
|
19324
|
-
this.binaries = result;
|
|
19325
|
-
return result;
|
|
19367
|
+
async evict(path) {
|
|
19368
|
+
return parseWindowsDehydrateOutput(await this.run(buildWindowsDehydrateScript(path), 60000));
|
|
19326
19369
|
}
|
|
19327
|
-
|
|
19328
|
-
|
|
19329
|
-
|
|
19330
|
-
return binaries;
|
|
19331
|
-
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
19332
|
-
let ping = await this.ping(binaries.esPath);
|
|
19333
|
-
if (ping.code === 0 && !reconfigure)
|
|
19334
|
-
return { ok: true, esPath: binaries.esPath };
|
|
19335
|
-
if (ping.code === 0) {
|
|
19336
|
-
const exitArgs = ["-instance", this.instanceName, "-exit"];
|
|
19337
|
-
const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
|
|
19338
|
-
if (exited.code !== 0)
|
|
19339
|
-
return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
|
|
19340
|
-
}
|
|
19341
|
-
mkdirSync16(this.stateDir, { recursive: true });
|
|
19342
|
-
const iniPath = join30(this.stateDir, "Everything.ini");
|
|
19343
|
-
writeFileSync11(iniPath, buildEverythingIni({
|
|
19344
|
-
folders: this.options.folders,
|
|
19345
|
-
excludeFolders: [
|
|
19346
|
-
...new Set([
|
|
19347
|
-
join30(this.options.root, ".autorag"),
|
|
19348
|
-
...this.options.folders.map((folder) => join30(folder, ".autorag")),
|
|
19349
|
-
...this.options.excludeFolders ?? []
|
|
19350
|
-
])
|
|
19351
|
-
]
|
|
19352
|
-
}));
|
|
19353
|
-
const launchArgs = [
|
|
19354
|
-
"-instance",
|
|
19355
|
-
this.instanceName,
|
|
19356
|
-
"-config",
|
|
19357
|
-
iniPath,
|
|
19358
|
-
"-db",
|
|
19359
|
-
join30(this.stateDir, "Everything.db"),
|
|
19360
|
-
"-startup"
|
|
19361
|
-
];
|
|
19362
|
-
try {
|
|
19363
|
-
this.launch(binaries.everythingPath, launchArgs);
|
|
19364
|
-
} catch (error) {
|
|
19365
|
-
return {
|
|
19366
|
-
ok: false,
|
|
19367
|
-
reason: "startup-failed",
|
|
19368
|
-
message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
|
|
19369
|
-
};
|
|
19370
|
-
}
|
|
19371
|
-
const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
|
|
19372
|
-
const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
19373
|
-
for (;; ) {
|
|
19374
|
-
ping = await this.ping(binaries.esPath);
|
|
19375
|
-
if (ping.code === 0)
|
|
19376
|
-
return { ok: true, esPath: binaries.esPath };
|
|
19377
|
-
if (Date.now() >= deadline)
|
|
19378
|
-
break;
|
|
19379
|
-
await sleep2(pollMs);
|
|
19380
|
-
}
|
|
19381
|
-
return {
|
|
19382
|
-
ok: false,
|
|
19383
|
-
reason: "startup-failed",
|
|
19384
|
-
message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
|
|
19385
|
-
};
|
|
19370
|
+
loadSyncRoots() {
|
|
19371
|
+
this.syncRoots ??= this.run(buildWindowsSyncRootsScript(), 15000).then((result) => result.ok ? parseWindowsSyncRoots(result.stdout) : [], () => []);
|
|
19372
|
+
return this.syncRoots;
|
|
19386
19373
|
}
|
|
19387
|
-
|
|
19388
|
-
|
|
19389
|
-
return
|
|
19374
|
+
loadDiskRoots() {
|
|
19375
|
+
this.disks ??= this.run(buildWindowsDiskScript(), 30000).then((result) => result.ok ? windowsRootsFromDisks(parseWindowsLogicalDisks(result.stdout)) : [], () => []);
|
|
19376
|
+
return this.disks;
|
|
19390
19377
|
}
|
|
19391
|
-
|
|
19378
|
+
}
|
|
19379
|
+
function longestSyncRoot(roots, target) {
|
|
19380
|
+
const normalized = target.replaceAll("/", "\\").toLowerCase();
|
|
19381
|
+
let match;
|
|
19382
|
+
let matchLength = -1;
|
|
19383
|
+
for (const root of roots) {
|
|
19384
|
+
const prefix = root.path.replaceAll("/", "\\").replace(/\\+$/u, "").toLowerCase();
|
|
19385
|
+
if (prefix === "" || prefix.length <= matchLength)
|
|
19386
|
+
continue;
|
|
19387
|
+
if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
|
|
19388
|
+
match = root;
|
|
19389
|
+
matchLength = prefix.length;
|
|
19390
|
+
}
|
|
19391
|
+
}
|
|
19392
|
+
return match;
|
|
19393
|
+
}
|
|
19394
|
+
function powerShellLaunch(script) {
|
|
19395
|
+
return {
|
|
19396
|
+
args: [
|
|
19397
|
+
"-NoProfile",
|
|
19398
|
+
"-NonInteractive",
|
|
19399
|
+
"-Command",
|
|
19400
|
+
`[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; Invoke-Expression $env:${POWERSHELL_SCRIPT_VARIABLE}`
|
|
19401
|
+
],
|
|
19402
|
+
env: { [POWERSHELL_SCRIPT_VARIABLE]: script }
|
|
19403
|
+
};
|
|
19404
|
+
}
|
|
19405
|
+
async function runPowerShell(script, timeoutMs) {
|
|
19406
|
+
const launch = powerShellLaunch(script);
|
|
19407
|
+
try {
|
|
19408
|
+
const { stdout, stderr } = await execFileAsync("powershell.exe", [...launch.args], {
|
|
19409
|
+
timeout: timeoutMs,
|
|
19410
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
19411
|
+
windowsHide: true,
|
|
19412
|
+
env: { ...process.env, ...launch.env }
|
|
19413
|
+
});
|
|
19414
|
+
return { ok: true, stdout, stderr };
|
|
19415
|
+
} catch (error) {
|
|
19416
|
+
const record = typeof error === "object" && error !== null ? error : {};
|
|
19392
19417
|
return {
|
|
19393
19418
|
ok: false,
|
|
19394
|
-
|
|
19395
|
-
|
|
19419
|
+
stdout: typeof record.stdout === "string" ? record.stdout : "",
|
|
19420
|
+
stderr: typeof record.stderr === "string" ? record.stderr : String(error)
|
|
19396
19421
|
};
|
|
19397
19422
|
}
|
|
19398
19423
|
}
|
|
19399
|
-
function
|
|
19400
|
-
|
|
19401
|
-
|
|
19402
|
-
|
|
19403
|
-
|
|
19404
|
-
|
|
19405
|
-
|
|
19406
|
-
|
|
19407
|
-
|
|
19408
|
-
|
|
19409
|
-
|
|
19410
|
-
|
|
19411
|
-
if (bytes > MAX_OUTPUT_BYTES) {
|
|
19412
|
-
overflow = true;
|
|
19413
|
-
child.kill();
|
|
19414
|
-
return;
|
|
19415
|
-
}
|
|
19416
|
-
target.push(chunk);
|
|
19417
|
-
};
|
|
19418
|
-
child.stdout.on("data", collect(stdout));
|
|
19419
|
-
child.stderr.on("data", collect(stderr));
|
|
19420
|
-
const timer = setTimeout(() => child.kill(), timeoutMs);
|
|
19421
|
-
child.on("error", (error) => {
|
|
19422
|
-
clearTimeout(timer);
|
|
19423
|
-
resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
|
|
19424
|
-
});
|
|
19425
|
-
child.on("close", (code, signal) => {
|
|
19426
|
-
clearTimeout(timer);
|
|
19427
|
-
const err = Buffer.concat(stderr).toString("utf8");
|
|
19428
|
-
resolvePromise({
|
|
19429
|
-
code,
|
|
19430
|
-
stdout: Buffer.concat(stdout).toString("utf8"),
|
|
19431
|
-
stderr: overflow ? `${err}
|
|
19432
|
-
output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
|
|
19433
|
-
killed by ${signal} after ${timeoutMs}ms` : err
|
|
19424
|
+
function defaultCloudProbe() {
|
|
19425
|
+
if (process.platform === "darwin")
|
|
19426
|
+
return new DarwinCloudProbe;
|
|
19427
|
+
if (process.platform === "win32")
|
|
19428
|
+
return new WindowsCloudProbe;
|
|
19429
|
+
return;
|
|
19430
|
+
}
|
|
19431
|
+
async function readFileProviderDomain(target) {
|
|
19432
|
+
try {
|
|
19433
|
+
await lstat(target);
|
|
19434
|
+
const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
|
|
19435
|
+
timeout: 2000
|
|
19434
19436
|
});
|
|
19435
|
-
|
|
19436
|
-
|
|
19437
|
+
const value = stdout.trim();
|
|
19438
|
+
return value.length > 0 ? value : undefined;
|
|
19439
|
+
} catch {
|
|
19440
|
+
return;
|
|
19441
|
+
}
|
|
19437
19442
|
}
|
|
19438
|
-
function
|
|
19439
|
-
|
|
19440
|
-
|
|
19441
|
-
|
|
19443
|
+
async function readFileProviderDomainUpwards(target) {
|
|
19444
|
+
let cursor = target;
|
|
19445
|
+
for (let depth = 0;depth < 12; depth += 1) {
|
|
19446
|
+
const domain = await readFileProviderDomain(cursor);
|
|
19447
|
+
if (domain !== undefined)
|
|
19448
|
+
return domain;
|
|
19449
|
+
const parent = dirname16(cursor);
|
|
19450
|
+
if (parent === cursor)
|
|
19451
|
+
return;
|
|
19452
|
+
cursor = parent;
|
|
19453
|
+
}
|
|
19454
|
+
return;
|
|
19442
19455
|
}
|
|
19443
|
-
|
|
19444
|
-
|
|
19445
|
-
|
|
19446
|
-
|
|
19447
|
-
|
|
19448
|
-
|
|
19449
|
-
|
|
19450
|
-
|
|
19451
|
-
|
|
19452
|
-
|
|
19453
|
-
var
|
|
19454
|
-
|
|
19455
|
-
|
|
19456
|
+
function quotePowerShell(value) {
|
|
19457
|
+
return `'${value.replace(/['\u2018\u2019\u201A\u201B]/gu, (quote) => quote + quote)}'`;
|
|
19458
|
+
}
|
|
19459
|
+
var execFileAsync, SF_DATALESS = 1073741824, FILE_PROVIDER_XATTR = "com.apple.file-provider-domain-id", WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 4194304, STAT_BATCH = 200, defaultCloudExec = (file, args, options) => execFileAsync(file, [...args], {
|
|
19460
|
+
timeout: options?.timeoutMs ?? 30000,
|
|
19461
|
+
...options?.maxBuffer === undefined ? {} : { maxBuffer: options.maxBuffer }
|
|
19462
|
+
}), JXA_RELEASE_PROGRAM = `ObjC.import("Foundation");
|
|
19463
|
+
function run(argv) {
|
|
19464
|
+
var url = $.NSURL.fileURLWithPath(argv[0]);
|
|
19465
|
+
var errorRef = Ref();
|
|
19466
|
+
var released = $.NSFileManager.defaultManager.evictUbiquitousItemAtURLError(url, errorRef);
|
|
19467
|
+
if (released) return JSON.stringify({ ok: true });
|
|
19468
|
+
function resourceValue(key) {
|
|
19469
|
+
var ref = Ref();
|
|
19470
|
+
if (!url.getResourceValueForKeyError(ref, key, null)) return null;
|
|
19471
|
+
var value = ref[0];
|
|
19472
|
+
return value === undefined || value === null ? null : ObjC.unwrap(value);
|
|
19473
|
+
}
|
|
19474
|
+
return JSON.stringify({
|
|
19475
|
+
ok: false,
|
|
19476
|
+
isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
|
|
19477
|
+
isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
|
|
19478
|
+
downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
|
|
19479
|
+
});
|
|
19480
|
+
}`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
|
|
19481
|
+
var init_cloud_placeholder = __esm(() => {
|
|
19482
|
+
execFileAsync = promisify(execFile2);
|
|
19483
|
+
RELEASABLE_DARWIN_PROVIDERS = {
|
|
19484
|
+
icloud: true,
|
|
19485
|
+
"google-drive": true,
|
|
19486
|
+
onedrive: true,
|
|
19487
|
+
dropbox: true
|
|
19488
|
+
};
|
|
19489
|
+
DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
|
|
19490
|
+
WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
|
|
19491
|
+
WINDOWS_DEHYDRATE_REFUSALS = {
|
|
19492
|
+
"80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
|
|
19493
|
+
"80070188": "The file is pinned to stay on this device, so it is not released."
|
|
19494
|
+
};
|
|
19456
19495
|
});
|
|
19457
19496
|
|
|
19458
19497
|
// src/fsearch/walk.ts
|
|
19459
19498
|
import { readdir as readdir2, stat as stat3 } from "node:fs/promises";
|
|
19460
|
-
import { basename as
|
|
19499
|
+
import { basename as basename7, join as join31, sep as sep5 } from "node:path";
|
|
19461
19500
|
function compare(a, b) {
|
|
19462
19501
|
return a < b ? -1 : a > b ? 1 : 0;
|
|
19463
19502
|
}
|
|
@@ -19468,11 +19507,11 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
19468
19507
|
const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_DEADLINE_MS);
|
|
19469
19508
|
const matches = request.regex ? (() => {
|
|
19470
19509
|
const pattern = new RegExp(request.query, request.matchCase ? "" : "i");
|
|
19471
|
-
return (path) => pattern.test(request.matchPath ? path :
|
|
19510
|
+
return (path) => pattern.test(request.matchPath ? path : basename7(path));
|
|
19472
19511
|
})() : (() => {
|
|
19473
19512
|
const needle = request.matchCase ? request.query : request.query.toLowerCase();
|
|
19474
19513
|
return (path) => {
|
|
19475
|
-
const haystack = request.matchPath ? path :
|
|
19514
|
+
const haystack = request.matchPath ? path : basename7(path);
|
|
19476
19515
|
return (request.matchCase ? haystack : haystack.toLowerCase()).includes(needle);
|
|
19477
19516
|
};
|
|
19478
19517
|
})();
|
|
@@ -19527,7 +19566,7 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
19527
19566
|
continue;
|
|
19528
19567
|
collected.push({
|
|
19529
19568
|
path,
|
|
19530
|
-
name:
|
|
19569
|
+
name: basename7(path),
|
|
19531
19570
|
type: info.isDirectory() ? "folder" : "file",
|
|
19532
19571
|
size: info.isDirectory() ? undefined : info.size,
|
|
19533
19572
|
dateModified: info.mtime.toISOString()
|
|
@@ -19552,7 +19591,7 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
19552
19591
|
continue;
|
|
19553
19592
|
collected.push({
|
|
19554
19593
|
path: folder,
|
|
19555
|
-
name:
|
|
19594
|
+
name: basename7(folder),
|
|
19556
19595
|
type: "folder",
|
|
19557
19596
|
size: undefined,
|
|
19558
19597
|
dateModified: info.mtime.toISOString()
|
|
@@ -19585,7 +19624,7 @@ var init_walk = __esm(() => {
|
|
|
19585
19624
|
import { spawn as spawn16 } from "node:child_process";
|
|
19586
19625
|
import { createHash as createHash13 } from "node:crypto";
|
|
19587
19626
|
import { existsSync as existsSync19, mkdirSync as mkdirSync17, readFileSync as readFileSync19, realpathSync as realpathSync4, rmSync as rmSync8, writeFileSync as writeFileSync12 } from "node:fs";
|
|
19588
|
-
import { basename as
|
|
19627
|
+
import { basename as basename8, join as join32, resolve as resolve12, sep as sep6 } from "node:path";
|
|
19589
19628
|
import { setTimeout as sleep3 } from "node:timers/promises";
|
|
19590
19629
|
function fsearchSocketPath(dbPath) {
|
|
19591
19630
|
let canonical;
|
|
@@ -19649,7 +19688,7 @@ function parseFSearchJsonLines(stdout) {
|
|
|
19649
19688
|
const path = String(record.path ?? "");
|
|
19650
19689
|
entries.push({
|
|
19651
19690
|
path,
|
|
19652
|
-
name:
|
|
19691
|
+
name: basename8(path),
|
|
19653
19692
|
type: record.type === "folder" ? "folder" : "file",
|
|
19654
19693
|
size: typeof record.size === "number" ? record.size : undefined,
|
|
19655
19694
|
dateModified: typeof record.mtime === "number" ? new Date(record.mtime * 1000).toISOString() : undefined
|
|
@@ -20067,7 +20106,7 @@ var init_diagnostics = __esm(() => {
|
|
|
20067
20106
|
});
|
|
20068
20107
|
|
|
20069
20108
|
// src/jikji/retry-command.ts
|
|
20070
|
-
import { basename as
|
|
20109
|
+
import { basename as basename9 } from "node:path";
|
|
20071
20110
|
function shellWords(command) {
|
|
20072
20111
|
const words = [];
|
|
20073
20112
|
let current = "";
|
|
@@ -20134,7 +20173,7 @@ function parseJikjiRetryCommand(command, envelopeProof) {
|
|
|
20134
20173
|
const words = shellWords(command);
|
|
20135
20174
|
if (words === undefined || words.length < 4)
|
|
20136
20175
|
return;
|
|
20137
|
-
const program =
|
|
20176
|
+
const program = basename9(words[0]).replace(/\.exe$/iu, "");
|
|
20138
20177
|
if (program !== "jikji" || words[1] !== "discover")
|
|
20139
20178
|
return;
|
|
20140
20179
|
const root = words[2];
|
|
@@ -20542,7 +20581,7 @@ class SerialQueue {
|
|
|
20542
20581
|
|
|
20543
20582
|
// src/jikji/version.ts
|
|
20544
20583
|
function parseJikjiVersion(output) {
|
|
20545
|
-
return
|
|
20584
|
+
return VERSION_LINE3.exec(output)?.[1];
|
|
20546
20585
|
}
|
|
20547
20586
|
function parts(version) {
|
|
20548
20587
|
return version.split(".").map((part) => Number.parseInt(part, 10));
|
|
@@ -20558,9 +20597,9 @@ function isSupportedJikjiVersion(version) {
|
|
|
20558
20597
|
}
|
|
20559
20598
|
return true;
|
|
20560
20599
|
}
|
|
20561
|
-
var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1",
|
|
20562
|
-
var
|
|
20563
|
-
|
|
20600
|
+
var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1", VERSION_LINE3;
|
|
20601
|
+
var init_version3 = __esm(() => {
|
|
20602
|
+
VERSION_LINE3 = /^jikji(?:-cli)?\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
|
|
20564
20603
|
});
|
|
20565
20604
|
|
|
20566
20605
|
// src/jikji/client.ts
|
|
@@ -20928,7 +20967,7 @@ var init_client11 = __esm(() => {
|
|
|
20928
20967
|
init_answer_pack();
|
|
20929
20968
|
init_installer3();
|
|
20930
20969
|
init_prepare_budget();
|
|
20931
|
-
|
|
20970
|
+
init_version3();
|
|
20932
20971
|
ABORTED_BEFORE_START = { ok: false, reason: "aborted", stdout: "", stderr: "", code: null };
|
|
20933
20972
|
});
|
|
20934
20973
|
|
|
@@ -22013,7 +22052,7 @@ import {
|
|
|
22013
22052
|
rmSync as rmSync10,
|
|
22014
22053
|
writeFileSync as writeFileSync14
|
|
22015
22054
|
} from "node:fs";
|
|
22016
|
-
import { basename as
|
|
22055
|
+
import { basename as basename10, dirname as dirname19, isAbsolute as isAbsolute9, join as join37, relative as relative7, sep as sep8 } from "node:path";
|
|
22017
22056
|
function syncMinSyncWorkspace(root, options = {}) {
|
|
22018
22057
|
const workspacePath = options.workspacePath ?? minSyncWorkspaceRoot(root);
|
|
22019
22058
|
const filesRoot = join37(workspacePath, "files");
|
|
@@ -22054,7 +22093,7 @@ function syncMinSyncWorkspace(root, options = {}) {
|
|
|
22054
22093
|
return { workspacePath, entries, excluded };
|
|
22055
22094
|
}
|
|
22056
22095
|
function stagingStatePath(workspacePath) {
|
|
22057
|
-
return join37(dirname19(workspacePath), `.${
|
|
22096
|
+
return join37(dirname19(workspacePath), `.${basename10(workspacePath)}-autorag-staging.json`);
|
|
22058
22097
|
}
|
|
22059
22098
|
function loadStagingState2(workspacePath) {
|
|
22060
22099
|
const path = stagingStatePath(workspacePath);
|
|
@@ -22105,7 +22144,7 @@ function ensureManagedDirectory(filesRoot, directory) {
|
|
|
22105
22144
|
}
|
|
22106
22145
|
}
|
|
22107
22146
|
function copyFileAtomically(source, destination) {
|
|
22108
|
-
const temporaryPath = join37(dirname19(destination), `.${
|
|
22147
|
+
const temporaryPath = join37(dirname19(destination), `.${basename10(destination)}.${randomUUID6()}.tmp`);
|
|
22109
22148
|
try {
|
|
22110
22149
|
copyFileSync4(source, temporaryPath);
|
|
22111
22150
|
renameWithRetry(temporaryPath, destination);
|
|
@@ -22159,7 +22198,7 @@ var init_workspace = __esm(() => {
|
|
|
22159
22198
|
|
|
22160
22199
|
// src/minsync/method.ts
|
|
22161
22200
|
import { existsSync as existsSync24, readFileSync as readFileSync22, realpathSync as realpathSync7, statSync as statSync8 } from "node:fs";
|
|
22162
|
-
import { basename as
|
|
22201
|
+
import { basename as basename11, join as join38, normalize } from "node:path";
|
|
22163
22202
|
|
|
22164
22203
|
class MinSyncVectorMethod {
|
|
22165
22204
|
root;
|
|
@@ -22260,7 +22299,7 @@ class MinSyncVectorMethod {
|
|
|
22260
22299
|
const entry = byPath.get(hit.path);
|
|
22261
22300
|
if (!entry || !matchesVirtualPathScope(entry.virtualPath, options.scope))
|
|
22262
22301
|
continue;
|
|
22263
|
-
const chunkDiscriminator = hit.docId ??
|
|
22302
|
+
const chunkDiscriminator = hit.docId ?? basename11(hit.path);
|
|
22264
22303
|
const chunkId = `minsync:${entry.virtualPath}:${chunkDiscriminator}`;
|
|
22265
22304
|
const location = locateChunkInMirror(mirrorTexts, entry, hit.text);
|
|
22266
22305
|
results.push({
|
|
@@ -22360,18 +22399,24 @@ var init_minsync = __esm(() => {
|
|
|
22360
22399
|
|
|
22361
22400
|
// src/mirror/cloud-fingerprints.ts
|
|
22362
22401
|
import { existsSync as existsSync25, readFileSync as readFileSync23 } from "node:fs";
|
|
22363
|
-
import { join as join39 } from "node:path";
|
|
22402
|
+
import { extname as extname5, join as join39 } from "node:path";
|
|
22364
22403
|
function cloudFingerprintStorePath(root) {
|
|
22365
22404
|
return join39(root, ".autorag", CLOUD_FINGERPRINTS_FILE);
|
|
22366
22405
|
}
|
|
22367
22406
|
function emptyCloudFingerprints() {
|
|
22368
|
-
return { version:
|
|
22407
|
+
return { version: 2, entries: {} };
|
|
22369
22408
|
}
|
|
22370
|
-
function
|
|
22409
|
+
function readCloudFingerprint(sourcePath, value, storeVersion) {
|
|
22371
22410
|
if (typeof value !== "object" || value === null)
|
|
22372
|
-
return
|
|
22411
|
+
return;
|
|
22373
22412
|
const record = value;
|
|
22374
|
-
|
|
22413
|
+
const { sourceVersion, exact, modified, revision } = record;
|
|
22414
|
+
if (typeof sourceVersion !== "string" || typeof exact !== "string" || modified !== null && typeof modified !== "string" || revision !== null && typeof revision !== "string") {
|
|
22415
|
+
return;
|
|
22416
|
+
}
|
|
22417
|
+
if (storeVersion !== 2 && (record.chars === 0 || extname5(sourcePath).toLowerCase() === ".hwp"))
|
|
22418
|
+
return;
|
|
22419
|
+
return { sourceVersion, exact, modified, revision };
|
|
22375
22420
|
}
|
|
22376
22421
|
function loadCloudFingerprints(root) {
|
|
22377
22422
|
const path = cloudFingerprintStorePath(root);
|
|
@@ -22384,24 +22429,26 @@ function loadCloudFingerprints(root) {
|
|
|
22384
22429
|
const { entries } = parsed;
|
|
22385
22430
|
if (typeof entries !== "object" || entries === null)
|
|
22386
22431
|
return emptyCloudFingerprints();
|
|
22432
|
+
const storeVersion = "version" in parsed ? parsed.version : undefined;
|
|
22387
22433
|
const valid = {};
|
|
22388
|
-
for (const [sourcePath,
|
|
22389
|
-
|
|
22434
|
+
for (const [sourcePath, value] of Object.entries(entries)) {
|
|
22435
|
+
const fingerprint = readCloudFingerprint(sourcePath, value, storeVersion);
|
|
22436
|
+
if (fingerprint !== undefined)
|
|
22390
22437
|
valid[sourcePath] = fingerprint;
|
|
22391
22438
|
}
|
|
22392
|
-
return { version:
|
|
22439
|
+
return { version: 2, entries: valid };
|
|
22393
22440
|
} catch {
|
|
22394
22441
|
return emptyCloudFingerprints();
|
|
22395
22442
|
}
|
|
22396
22443
|
}
|
|
22397
22444
|
function saveCloudFingerprints(root, entries) {
|
|
22398
|
-
writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version:
|
|
22445
|
+
writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version: 2, entries }, null, 2)}
|
|
22399
22446
|
`);
|
|
22400
22447
|
}
|
|
22401
22448
|
function cloudFingerprintExclusions(store) {
|
|
22402
22449
|
const map = new Map;
|
|
22403
22450
|
for (const [sourcePath, fingerprint] of Object.entries(store.entries)) {
|
|
22404
|
-
map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified
|
|
22451
|
+
map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified });
|
|
22405
22452
|
}
|
|
22406
22453
|
return map;
|
|
22407
22454
|
}
|
|
@@ -22793,7 +22840,7 @@ class Parser {
|
|
|
22793
22840
|
}
|
|
22794
22841
|
|
|
22795
22842
|
// src/parser/csv.ts
|
|
22796
|
-
import { basename as
|
|
22843
|
+
import { basename as basename12, extname as extname6 } from "node:path";
|
|
22797
22844
|
function toCell(text) {
|
|
22798
22845
|
return text.trim() === "" ? undefined : { text, kind: classifyText(text) };
|
|
22799
22846
|
}
|
|
@@ -22924,7 +22971,7 @@ var init_csv = __esm(() => {
|
|
|
22924
22971
|
this.#maxRows = options.maxRows ?? DEFAULT_MAX_ROWS;
|
|
22925
22972
|
}
|
|
22926
22973
|
async parse(input) {
|
|
22927
|
-
const extension =
|
|
22974
|
+
const extension = extname6(input.virtualPath);
|
|
22928
22975
|
const format = extension.slice(1).toLowerCase();
|
|
22929
22976
|
let text = decodeText(input.bytes).replace(/^\uFEFF/, "");
|
|
22930
22977
|
const directive = SEPARATOR_DIRECTIVE.exec(text);
|
|
@@ -22936,10 +22983,10 @@ var init_csv = __esm(() => {
|
|
|
22936
22983
|
{
|
|
22937
22984
|
code: "parser-warning",
|
|
22938
22985
|
severity: "warning",
|
|
22939
|
-
message: `"${
|
|
22986
|
+
message: `"${basename12(input.virtualPath)}": indexed the first ${this.#maxRows} of ${total} rows`
|
|
22940
22987
|
}
|
|
22941
22988
|
] : [];
|
|
22942
|
-
const grid = { name:
|
|
22989
|
+
const grid = { name: basename12(input.virtualPath, extension), rows: rows.map((row) => row.map(toCell)) };
|
|
22943
22990
|
return {
|
|
22944
22991
|
markdown: normalizeMarkdown(renderWorkbook([grid])),
|
|
22945
22992
|
metadata: { parser: this.name, format },
|
|
@@ -23478,7 +23525,7 @@ var init_plain_text = __esm(() => {
|
|
|
23478
23525
|
});
|
|
23479
23526
|
|
|
23480
23527
|
// src/parser/registry.ts
|
|
23481
|
-
import { extname as
|
|
23528
|
+
import { extname as extname7 } from "node:path";
|
|
23482
23529
|
|
|
23483
23530
|
class ParserRegistry {
|
|
23484
23531
|
parsersByExtension = new Map;
|
|
@@ -23497,7 +23544,7 @@ class ParserRegistry {
|
|
|
23497
23544
|
}
|
|
23498
23545
|
}
|
|
23499
23546
|
getForVirtualPath(virtualPath) {
|
|
23500
|
-
return this.parsersByExtension.get(normalizeExtension(
|
|
23547
|
+
return this.parsersByExtension.get(normalizeExtension(extname7(virtualPath)));
|
|
23501
23548
|
}
|
|
23502
23549
|
list() {
|
|
23503
23550
|
return Array.from(new Set(this.parsersByExtension.values()));
|
|
@@ -23510,7 +23557,7 @@ function normalizeExtension(extension) {
|
|
|
23510
23557
|
var init_registry = () => {};
|
|
23511
23558
|
|
|
23512
23559
|
// src/parser/spreadsheet.ts
|
|
23513
|
-
import { extname as
|
|
23560
|
+
import { extname as extname8 } from "node:path";
|
|
23514
23561
|
import * as XLSX from "xlsx";
|
|
23515
23562
|
function assertWorkbookSignature(format, bytes) {
|
|
23516
23563
|
if (startsWith(bytes, ZIP_SIGNATURE) || startsWith(bytes, OLE_SIGNATURE))
|
|
@@ -23609,7 +23656,7 @@ var init_spreadsheet = __esm(() => {
|
|
|
23609
23656
|
this.#maxRowsPerSheet = options.maxRowsPerSheet ?? DEFAULT_MAX_ROWS_PER_SHEET;
|
|
23610
23657
|
}
|
|
23611
23658
|
async parse(input) {
|
|
23612
|
-
const format =
|
|
23659
|
+
const format = extname8(input.virtualPath).slice(1).toLowerCase();
|
|
23613
23660
|
let workbook;
|
|
23614
23661
|
try {
|
|
23615
23662
|
assertWorkbookSignature(format, input.bytes);
|
|
@@ -23883,6 +23930,10 @@ class CloudGate {
|
|
|
23883
23930
|
isPlaceholder(path) {
|
|
23884
23931
|
return this.placeholders.has(path);
|
|
23885
23932
|
}
|
|
23933
|
+
startBatch() {
|
|
23934
|
+
this.downloadedFiles = 0;
|
|
23935
|
+
this.downloadedBytes = 0;
|
|
23936
|
+
}
|
|
23886
23937
|
async decide(path, sizeBytes, directory) {
|
|
23887
23938
|
if (this.probe === undefined || !this.placeholders.has(path))
|
|
23888
23939
|
return LOCAL;
|
|
@@ -23892,7 +23943,7 @@ class CloudGate {
|
|
|
23892
23943
|
const overFiles = this.downloadedFiles >= this.maxFiles;
|
|
23893
23944
|
const overBytes = this.downloadedFiles > 0 && this.downloadedBytes + sizeBytes > this.maxBytes;
|
|
23894
23945
|
if (overFiles || overBytes)
|
|
23895
|
-
return { kind: "
|
|
23946
|
+
return { kind: "next-batch" };
|
|
23896
23947
|
this.downloadedFiles += 1;
|
|
23897
23948
|
this.downloadedBytes += sizeBytes;
|
|
23898
23949
|
return { kind: "download" };
|
|
@@ -23937,7 +23988,7 @@ var init_cloud = __esm(() => {
|
|
|
23937
23988
|
// src/mirror/sync.ts
|
|
23938
23989
|
import { createHash as createHash17 } from "node:crypto";
|
|
23939
23990
|
import { opendir as opendir2, readFile, stat as stat4 } from "node:fs/promises";
|
|
23940
|
-
import { basename as
|
|
23991
|
+
import { basename as basename13, dirname as dirname22, extname as extname9, join as join43, resolve as resolve15, sep as sep9 } from "node:path";
|
|
23941
23992
|
import { setTimeout as sleep4 } from "node:timers/promises";
|
|
23942
23993
|
function errorCode(error) {
|
|
23943
23994
|
if (!(error instanceof Error))
|
|
@@ -24141,195 +24192,221 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
24141
24192
|
});
|
|
24142
24193
|
const pending = new Set(plan.decisions.filter((decision) => decision.action === "process" || decision.action === "retry").map((decision) => decision.key));
|
|
24143
24194
|
const deferred = new Map(plan.decisions.filter((decision) => decision.deferred === true).map((decision) => [decision.key, decision]));
|
|
24144
|
-
|
|
24145
|
-
|
|
24146
|
-
|
|
24147
|
-
const path = pathsByKey.get(decision.key);
|
|
24148
|
-
await session.delete(decision.key);
|
|
24149
|
-
if (path === undefined)
|
|
24150
|
-
continue;
|
|
24151
|
-
if (previous.entries[path])
|
|
24152
|
-
deleted += 1;
|
|
24153
|
-
if (!currentPaths.has(path))
|
|
24154
|
-
diagnostics.push({
|
|
24155
|
-
code: "deleted-mirror",
|
|
24156
|
-
severity: "info",
|
|
24157
|
-
message: "A previously indexed document is gone; its parsed mirror was removed.",
|
|
24158
|
-
source: path
|
|
24159
|
-
});
|
|
24160
|
-
}
|
|
24161
|
-
await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
|
|
24162
|
-
for (const [position, entry] of current.entries()) {
|
|
24163
|
-
options.onProgress?.({
|
|
24164
|
-
processed: position,
|
|
24165
|
-
total: current.length,
|
|
24166
|
-
written,
|
|
24167
|
-
deleted,
|
|
24168
|
-
skipped,
|
|
24169
|
-
currentVirtualPath: entry.virtualPath
|
|
24170
|
-
});
|
|
24171
|
-
if (options.signal?.aborted)
|
|
24172
|
-
throw abortReason(options.signal);
|
|
24173
|
-
const omission = omitted.get(entry.virtualPath);
|
|
24174
|
-
if (omission !== undefined) {
|
|
24175
|
-
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
24176
|
-
if (previous.entries[entry.virtualPath])
|
|
24177
|
-
deleted += 1;
|
|
24178
|
-
await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
|
|
24179
|
-
}
|
|
24180
|
-
skipped += 1;
|
|
24181
|
-
if (omission.diagnostic)
|
|
24182
|
-
diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
|
|
24183
|
-
continue;
|
|
24184
|
-
}
|
|
24185
|
-
const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
|
|
24186
|
-
if (deferredFailure !== undefined) {
|
|
24187
|
-
skipped += 1;
|
|
24188
|
-
const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
|
|
24189
|
-
diagnostics.push({
|
|
24190
|
-
code,
|
|
24191
|
-
severity: "warning",
|
|
24192
|
-
source: entry.virtualPath,
|
|
24193
|
-
message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
|
|
24194
|
-
});
|
|
24195
|
-
continue;
|
|
24196
|
-
}
|
|
24197
|
-
const parser = registry.getForVirtualPath(entry.virtualPath);
|
|
24198
|
-
const outputPath = parsedOutputPath(options.root, entry.virtualPath);
|
|
24199
|
-
const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname22(entry.sourcePath)) : { kind: "local" };
|
|
24200
|
-
if (cloudDecision.kind === "skip") {
|
|
24201
|
-
skipped += 1;
|
|
24202
|
-
diagnostics.push({
|
|
24203
|
-
code: "cloud-placeholder-skipped",
|
|
24204
|
-
severity: "info",
|
|
24205
|
-
message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
|
|
24206
|
-
source: entry.virtualPath,
|
|
24207
|
-
...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
|
|
24208
|
-
});
|
|
24209
|
-
continue;
|
|
24210
|
-
}
|
|
24211
|
-
if (cloudDecision.kind === "defer") {
|
|
24212
|
-
skipped += 1;
|
|
24213
|
-
diagnostics.push({
|
|
24214
|
-
code: "cloud-download-deferred",
|
|
24215
|
-
severity: "info",
|
|
24216
|
-
message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
|
|
24217
|
-
source: entry.virtualPath
|
|
24218
|
-
});
|
|
24219
|
-
continue;
|
|
24220
|
-
}
|
|
24221
|
-
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
24222
|
-
let bytes;
|
|
24223
|
-
try {
|
|
24224
|
-
bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
|
|
24225
|
-
} catch (error) {
|
|
24226
|
-
const failure = sourceReadFailure(error);
|
|
24227
|
-
await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
|
|
24228
|
-
continue;
|
|
24229
|
-
}
|
|
24230
|
-
const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
|
|
24231
|
-
if (tooSmallImage !== undefined) {
|
|
24232
|
-
await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
|
|
24195
|
+
const applyDeleteDecisions = async () => {
|
|
24196
|
+
for (const decision of plan.decisions) {
|
|
24197
|
+
if (decision.action !== "delete")
|
|
24233
24198
|
continue;
|
|
24234
|
-
|
|
24235
|
-
|
|
24236
|
-
|
|
24237
|
-
const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
|
|
24238
|
-
parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
|
|
24239
|
-
} catch (error) {
|
|
24240
|
-
if (options.signal?.aborted)
|
|
24241
|
-
throw abortReason(options.signal);
|
|
24242
|
-
if (error instanceof ParseError)
|
|
24243
|
-
await skipFile(entry, {
|
|
24244
|
-
code: "parser-failed",
|
|
24245
|
-
severity: "warning",
|
|
24246
|
-
message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
|
|
24247
|
-
}, "parser-failed");
|
|
24248
|
-
else
|
|
24249
|
-
await skipFile(entry, {
|
|
24250
|
-
code: "parser-crashed",
|
|
24251
|
-
severity: "warning",
|
|
24252
|
-
message: PARSER_CRASH_MESSAGE,
|
|
24253
|
-
...error instanceof Error ? { reason: error.name } : {}
|
|
24254
|
-
});
|
|
24199
|
+
const path = pathsByKey.get(decision.key);
|
|
24200
|
+
await session.delete(decision.key);
|
|
24201
|
+
if (path === undefined)
|
|
24255
24202
|
continue;
|
|
24256
|
-
|
|
24257
|
-
|
|
24258
|
-
|
|
24259
|
-
|
|
24260
|
-
|
|
24261
|
-
|
|
24203
|
+
if (previous.entries[path])
|
|
24204
|
+
deleted += 1;
|
|
24205
|
+
if (!currentPaths.has(path))
|
|
24206
|
+
diagnostics.push({
|
|
24207
|
+
code: "deleted-mirror",
|
|
24208
|
+
severity: "info",
|
|
24209
|
+
message: "A previously indexed document is gone; its parsed mirror was removed.",
|
|
24210
|
+
source: path
|
|
24211
|
+
});
|
|
24212
|
+
}
|
|
24213
|
+
};
|
|
24214
|
+
const deferDeletions = options.onBatch !== undefined;
|
|
24215
|
+
const batchSize = Math.max(1, Math.floor(options.batchSize ?? DEFAULT_MIRROR_BATCH_SIZE));
|
|
24216
|
+
let batchIndex = 0;
|
|
24217
|
+
let batchStopped = false;
|
|
24218
|
+
const runBatch = async (virtualPaths, final) => {
|
|
24219
|
+
if (options.onBatch === undefined)
|
|
24220
|
+
return;
|
|
24221
|
+
batchIndex += 1;
|
|
24222
|
+
const outcome = await options.onBatch({ index: batchIndex, virtualPaths, final });
|
|
24223
|
+
if (outcome?.stop === true)
|
|
24224
|
+
batchStopped = true;
|
|
24225
|
+
};
|
|
24226
|
+
if (!deferDeletions)
|
|
24227
|
+
await applyDeleteDecisions();
|
|
24228
|
+
await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
|
|
24229
|
+
let batchStart = 0;
|
|
24230
|
+
for (;; ) {
|
|
24231
|
+
const batchLimit = Math.min(batchStart + batchSize, current.length);
|
|
24232
|
+
let batchEnd = batchLimit;
|
|
24233
|
+
const changedBefore = written + deleted;
|
|
24234
|
+
gate.startBatch();
|
|
24235
|
+
for (let position = batchStart;position < batchLimit; position += 1) {
|
|
24236
|
+
const entry = current[position];
|
|
24237
|
+
options.onProgress?.({
|
|
24238
|
+
processed: position,
|
|
24239
|
+
total: current.length,
|
|
24240
|
+
written,
|
|
24241
|
+
deleted,
|
|
24242
|
+
skipped,
|
|
24243
|
+
currentVirtualPath: entry.virtualPath
|
|
24244
|
+
});
|
|
24245
|
+
if (options.signal?.aborted)
|
|
24246
|
+
throw abortReason(options.signal);
|
|
24247
|
+
const omission = omitted.get(entry.virtualPath);
|
|
24248
|
+
if (omission !== undefined) {
|
|
24249
|
+
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
24250
|
+
if (previous.entries[entry.virtualPath])
|
|
24251
|
+
deleted += 1;
|
|
24252
|
+
await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
|
|
24253
|
+
}
|
|
24254
|
+
skipped += 1;
|
|
24255
|
+
if (omission.diagnostic)
|
|
24256
|
+
diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
|
|
24262
24257
|
continue;
|
|
24263
24258
|
}
|
|
24264
|
-
const
|
|
24265
|
-
if (
|
|
24259
|
+
const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
|
|
24260
|
+
if (deferredFailure !== undefined) {
|
|
24266
24261
|
skipped += 1;
|
|
24262
|
+
const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
|
|
24267
24263
|
diagnostics.push({
|
|
24268
|
-
code
|
|
24264
|
+
code,
|
|
24269
24265
|
severity: "warning",
|
|
24270
|
-
|
|
24271
|
-
|
|
24266
|
+
source: entry.virtualPath,
|
|
24267
|
+
message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
|
|
24272
24268
|
});
|
|
24273
|
-
await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
|
|
24274
24269
|
continue;
|
|
24275
24270
|
}
|
|
24276
|
-
const
|
|
24277
|
-
|
|
24278
|
-
|
|
24279
|
-
|
|
24280
|
-
|
|
24281
|
-
|
|
24282
|
-
|
|
24283
|
-
|
|
24284
|
-
|
|
24285
|
-
|
|
24286
|
-
|
|
24287
|
-
...code !== undefined ? { reason: code } : {}
|
|
24271
|
+
const parser = registry.getForVirtualPath(entry.virtualPath);
|
|
24272
|
+
const outputPath = parsedOutputPath(options.root, entry.virtualPath);
|
|
24273
|
+
const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname22(entry.sourcePath)) : { kind: "local" };
|
|
24274
|
+
if (cloudDecision.kind === "skip") {
|
|
24275
|
+
skipped += 1;
|
|
24276
|
+
diagnostics.push({
|
|
24277
|
+
code: "cloud-placeholder-skipped",
|
|
24278
|
+
severity: "info",
|
|
24279
|
+
message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
|
|
24280
|
+
source: entry.virtualPath,
|
|
24281
|
+
...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
|
|
24288
24282
|
});
|
|
24289
24283
|
continue;
|
|
24290
24284
|
}
|
|
24291
|
-
|
|
24292
|
-
|
|
24293
|
-
|
|
24294
|
-
}
|
|
24295
|
-
|
|
24296
|
-
|
|
24297
|
-
|
|
24298
|
-
|
|
24299
|
-
|
|
24300
|
-
|
|
24301
|
-
|
|
24302
|
-
|
|
24303
|
-
|
|
24304
|
-
|
|
24305
|
-
|
|
24306
|
-
|
|
24307
|
-
|
|
24308
|
-
|
|
24309
|
-
|
|
24310
|
-
|
|
24311
|
-
|
|
24312
|
-
|
|
24313
|
-
|
|
24285
|
+
if (cloudDecision.kind === "next-batch") {
|
|
24286
|
+
batchEnd = position;
|
|
24287
|
+
break;
|
|
24288
|
+
}
|
|
24289
|
+
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
24290
|
+
let bytes;
|
|
24291
|
+
try {
|
|
24292
|
+
bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
|
|
24293
|
+
} catch (error) {
|
|
24294
|
+
const failure = sourceReadFailure(error);
|
|
24295
|
+
await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
|
|
24296
|
+
continue;
|
|
24297
|
+
}
|
|
24298
|
+
const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
|
|
24299
|
+
if (tooSmallImage !== undefined) {
|
|
24300
|
+
await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
|
|
24301
|
+
continue;
|
|
24302
|
+
}
|
|
24303
|
+
let parsed;
|
|
24304
|
+
try {
|
|
24305
|
+
const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
|
|
24306
|
+
parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
|
|
24307
|
+
} catch (error) {
|
|
24308
|
+
if (options.signal?.aborted)
|
|
24309
|
+
throw abortReason(options.signal);
|
|
24310
|
+
if (error instanceof ParseError)
|
|
24311
|
+
await skipFile(entry, {
|
|
24312
|
+
code: "parser-failed",
|
|
24314
24313
|
severity: "warning",
|
|
24315
|
-
message:
|
|
24316
|
-
|
|
24314
|
+
message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
|
|
24315
|
+
}, "parser-failed");
|
|
24316
|
+
else
|
|
24317
|
+
await skipFile(entry, {
|
|
24318
|
+
code: "parser-crashed",
|
|
24319
|
+
severity: "warning",
|
|
24320
|
+
message: PARSER_CRASH_MESSAGE,
|
|
24321
|
+
...error instanceof Error ? { reason: error.name } : {}
|
|
24317
24322
|
});
|
|
24318
|
-
|
|
24323
|
+
continue;
|
|
24319
24324
|
}
|
|
24320
|
-
|
|
24321
|
-
|
|
24322
|
-
|
|
24323
|
-
|
|
24325
|
+
let info;
|
|
24326
|
+
try {
|
|
24327
|
+
info = await stat4(entry.sourcePath, { bigint: true });
|
|
24328
|
+
} catch (error) {
|
|
24329
|
+
await skipFile(entry, sourceReadFailure(error));
|
|
24330
|
+
continue;
|
|
24331
|
+
}
|
|
24332
|
+
const unchanged = cloudDecision.kind === "download" ? contentIdentity(info) === entry.contentIdentity : statIdentity(info) === entry.statIdentity;
|
|
24333
|
+
if (!unchanged) {
|
|
24334
|
+
skipped += 1;
|
|
24324
24335
|
diagnostics.push({
|
|
24325
|
-
code: "
|
|
24336
|
+
code: "source-changed",
|
|
24326
24337
|
severity: "warning",
|
|
24327
|
-
message:
|
|
24338
|
+
message: "The source changed while parsing; its last good mirror was kept and it will be retried on the next refresh.",
|
|
24328
24339
|
source: entry.virtualPath
|
|
24329
24340
|
});
|
|
24341
|
+
await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
|
|
24342
|
+
continue;
|
|
24343
|
+
}
|
|
24344
|
+
const markdown = normalizeMarkdown(parsed.markdown);
|
|
24345
|
+
try {
|
|
24346
|
+
await withTransientRetry(() => writeFileAtomic(outputPath, markdown), transientRetries, transientBackoffMs);
|
|
24347
|
+
} catch (error) {
|
|
24348
|
+
const code = errorCode(error);
|
|
24349
|
+
if (code !== undefined && Object.hasOwn(WORKSPACE_ERROR_CODES, code))
|
|
24350
|
+
throw new MirrorWorkspaceError(options.root, error);
|
|
24351
|
+
await skipFile(entry, {
|
|
24352
|
+
code: "mirror-write-failed",
|
|
24353
|
+
severity: "warning",
|
|
24354
|
+
message: error instanceof Error ? error.message : String(error),
|
|
24355
|
+
...code !== undefined ? { reason: code } : {}
|
|
24356
|
+
});
|
|
24357
|
+
continue;
|
|
24358
|
+
}
|
|
24359
|
+
await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
|
|
24360
|
+
path: basename13(outputPath),
|
|
24361
|
+
size: Buffer.byteLength(markdown)
|
|
24362
|
+
});
|
|
24363
|
+
for (const diagnostic of parsed.diagnostics ?? [])
|
|
24364
|
+
diagnostics.push({ ...diagnostic, source: entry.virtualPath });
|
|
24365
|
+
written += 1;
|
|
24366
|
+
if (fingerprintCloud && cloudFilePaths.has(entry.sourcePath)) {
|
|
24367
|
+
const existing = cloudFingerprintEntries[entry.sourcePath];
|
|
24368
|
+
if (existing === undefined || existing.sourceVersion !== entry.sourceVersion) {
|
|
24369
|
+
try {
|
|
24370
|
+
const fingerprintResult = await fingerprintWithDupey(entry.sourcePath, options.cloudFingerprints?.dupey ?? {});
|
|
24371
|
+
cloudFingerprintEntries[entry.sourcePath] = {
|
|
24372
|
+
sourceVersion: entry.sourceVersion,
|
|
24373
|
+
exact: fingerprintResult.exact,
|
|
24374
|
+
modified: fingerprintResult.modified,
|
|
24375
|
+
revision: fingerprintResult.revision
|
|
24376
|
+
};
|
|
24377
|
+
cloudFingerprintsDirty = true;
|
|
24378
|
+
} catch (error) {
|
|
24379
|
+
diagnostics.push({
|
|
24380
|
+
code: "cloud-fingerprint-failed",
|
|
24381
|
+
severity: "warning",
|
|
24382
|
+
message: `The cloud or network file was indexed but its duplicate fingerprint could not be read, so it will not take part in exact-duplicate exclusion: ${error instanceof Error ? error.message : String(error)}`,
|
|
24383
|
+
source: entry.virtualPath
|
|
24384
|
+
});
|
|
24385
|
+
}
|
|
24386
|
+
}
|
|
24387
|
+
}
|
|
24388
|
+
if (cloudDecision.kind === "download" && gate.evictAfterParse) {
|
|
24389
|
+
const released = await gate.release(entry.sourcePath);
|
|
24390
|
+
if (!released.ok) {
|
|
24391
|
+
diagnostics.push({
|
|
24392
|
+
code: "cloud-evict-failed",
|
|
24393
|
+
severity: "warning",
|
|
24394
|
+
message: `The file was indexed but its local copy could not be released: ${released.message}`,
|
|
24395
|
+
source: entry.virtualPath
|
|
24396
|
+
});
|
|
24397
|
+
}
|
|
24330
24398
|
}
|
|
24331
24399
|
}
|
|
24332
24400
|
}
|
|
24401
|
+
const finalBatch = batchEnd >= current.length;
|
|
24402
|
+
if (finalBatch && deferDeletions)
|
|
24403
|
+
await applyDeleteDecisions();
|
|
24404
|
+
if (finalBatch || written + deleted !== changedBefore) {
|
|
24405
|
+
await runBatch(current.slice(batchStart, batchEnd).map((item) => item.virtualPath), finalBatch);
|
|
24406
|
+
}
|
|
24407
|
+
if (finalBatch || batchStopped)
|
|
24408
|
+
break;
|
|
24409
|
+
batchStart = batchEnd;
|
|
24333
24410
|
}
|
|
24334
24411
|
await session.finish();
|
|
24335
24412
|
if (fingerprintCloud) {
|
|
@@ -24419,7 +24496,7 @@ async function detectMirrorStaleness(options) {
|
|
|
24419
24496
|
}
|
|
24420
24497
|
}
|
|
24421
24498
|
function standaloneImageTooSmallMessage(virtualPath, bytes) {
|
|
24422
|
-
if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(
|
|
24499
|
+
if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname9(virtualPath))))
|
|
24423
24500
|
return;
|
|
24424
24501
|
return ocrImageTooSmallMessage(bytes);
|
|
24425
24502
|
}
|
|
@@ -24587,7 +24664,7 @@ async function collectFiles(state, directory, depth) {
|
|
|
24587
24664
|
}
|
|
24588
24665
|
if (!entry.isFile())
|
|
24589
24666
|
continue;
|
|
24590
|
-
const extension = normalizeExtension2(
|
|
24667
|
+
const extension = normalizeExtension2(extname9(entry.name));
|
|
24591
24668
|
if (!state.supportedExtensions.has(extension))
|
|
24592
24669
|
continue;
|
|
24593
24670
|
let fileStat;
|
|
@@ -24621,7 +24698,7 @@ async function collectFiles(state, directory, depth) {
|
|
|
24621
24698
|
warnDirectoryUnreadable(state, directory, code);
|
|
24622
24699
|
}
|
|
24623
24700
|
}
|
|
24624
|
-
var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
|
|
24701
|
+
var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
|
|
24625
24702
|
var init_sync = __esm(() => {
|
|
24626
24703
|
init_cli2();
|
|
24627
24704
|
init_cloud_placeholder();
|
|
@@ -29632,7 +29709,7 @@ var init_automation_tool = __esm(() => {
|
|
|
29632
29709
|
});
|
|
29633
29710
|
|
|
29634
29711
|
// src/agent/citation-links.ts
|
|
29635
|
-
import { basename as
|
|
29712
|
+
import { basename as basename14, isAbsolute as isAbsolute12 } from "node:path";
|
|
29636
29713
|
import { pathToFileURL as pathToFileURL3 } from "node:url";
|
|
29637
29714
|
function openableTarget(source) {
|
|
29638
29715
|
if (/^https?:\/\//iu.test(source)) {
|
|
@@ -29650,7 +29727,7 @@ function linkCitations(markdown, sourceOf, isFile) {
|
|
|
29650
29727
|
for (const marker of answerMarkers(markdown, isFile)) {
|
|
29651
29728
|
if (marker.kind === "number")
|
|
29652
29729
|
continue;
|
|
29653
|
-
const labels = marker.kind === "file" ? [{ label:
|
|
29730
|
+
const labels = marker.kind === "file" ? [{ label: basename14(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
|
|
29654
29731
|
let linked = false;
|
|
29655
29732
|
const parts = labels.map(({ label, source }) => {
|
|
29656
29733
|
const target = source === undefined ? undefined : openableTarget(source);
|
|
@@ -31776,10 +31853,7 @@ function renderScanReport(details) {
|
|
|
31776
31853
|
const lines = [
|
|
31777
31854
|
`dupey scanned ${details.scans.length} configured root(s).`,
|
|
31778
31855
|
`families=${details.familyCount} exactDuplicates=${details.exactDuplicateCount} extractionErrors=${errors}`,
|
|
31779
|
-
"Review exact families before cleanup; near/contains families are not safe deletion evidence."
|
|
31780
|
-
...details.skippedRoots.length === 0 ? [] : [
|
|
31781
|
-
`Not scanned (inside a cloud or network folder; scanning would download or re-read every file): ${details.skippedRoots.join(", ")}. Duplicates there are not reported, and an exact duplicate is only found against files an earlier refresh fingerprinted.`
|
|
31782
|
-
]
|
|
31856
|
+
"Review exact families before cleanup; near/contains families are not safe deletion evidence."
|
|
31783
31857
|
];
|
|
31784
31858
|
let rendered = 0;
|
|
31785
31859
|
let omitted = 0;
|
|
@@ -32757,6 +32831,56 @@ var init_packaged_skill = __esm(() => {
|
|
|
32757
32831
|
PACKAGED_SKILL_ROOTS = ["skills", "internal-skills"];
|
|
32758
32832
|
});
|
|
32759
32833
|
|
|
32834
|
+
// src/agent/pipelined-sync.ts
|
|
32835
|
+
class PipelinedSync {
|
|
32836
|
+
#sync;
|
|
32837
|
+
#covered = [];
|
|
32838
|
+
#dirty = false;
|
|
32839
|
+
#running;
|
|
32840
|
+
#failure;
|
|
32841
|
+
#stopped = false;
|
|
32842
|
+
#runs = 0;
|
|
32843
|
+
constructor(options) {
|
|
32844
|
+
this.#sync = options.sync;
|
|
32845
|
+
}
|
|
32846
|
+
async batch(batch) {
|
|
32847
|
+
this.#raise();
|
|
32848
|
+
this.#covered.push(...batch.virtualPaths);
|
|
32849
|
+
this.#dirty = true;
|
|
32850
|
+
if (!this.#stopped)
|
|
32851
|
+
this.#running ??= this.#loop();
|
|
32852
|
+
if (batch.final)
|
|
32853
|
+
await this.#running;
|
|
32854
|
+
this.#raise();
|
|
32855
|
+
return this.#stopped ? { stop: true } : undefined;
|
|
32856
|
+
}
|
|
32857
|
+
async settle() {
|
|
32858
|
+
this.#stopped = true;
|
|
32859
|
+
await this.#running;
|
|
32860
|
+
}
|
|
32861
|
+
#raise() {
|
|
32862
|
+
if (this.#failure !== undefined)
|
|
32863
|
+
throw this.#failure.error;
|
|
32864
|
+
}
|
|
32865
|
+
async#loop() {
|
|
32866
|
+
try {
|
|
32867
|
+
while (this.#dirty && !this.#stopped) {
|
|
32868
|
+
this.#dirty = false;
|
|
32869
|
+
const virtualPaths = this.#covered;
|
|
32870
|
+
this.#covered = [];
|
|
32871
|
+
this.#runs += 1;
|
|
32872
|
+
const outcome = await this.#sync({ index: this.#runs, virtualPaths });
|
|
32873
|
+
if (outcome?.stop === true)
|
|
32874
|
+
this.#stopped = true;
|
|
32875
|
+
}
|
|
32876
|
+
} catch (error) {
|
|
32877
|
+
this.#failure = { error };
|
|
32878
|
+
} finally {
|
|
32879
|
+
this.#running = undefined;
|
|
32880
|
+
}
|
|
32881
|
+
}
|
|
32882
|
+
}
|
|
32883
|
+
|
|
32760
32884
|
// src/agent/query-decomposition.ts
|
|
32761
32885
|
import { completeSimple as completeSimple2 } from "@earendil-works/pi-ai/compat";
|
|
32762
32886
|
function buildDecompositionPrompt(query) {
|
|
@@ -32996,7 +33120,7 @@ var init_query_routing = __esm(() => {
|
|
|
32996
33120
|
false: "Every part of the question is answered with a concrete, specific, consistent fact (a name, date, number, or definite statement)."
|
|
32997
33121
|
});
|
|
32998
33122
|
BASE_ROUTE_OPTIONS = {
|
|
32999
|
-
local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. When unsure whether intrinsic knowledge suffices, choose local.",
|
|
33123
|
+
local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. A request to find, locate, open, or look up a document, file, slide deck, presentation, paper, note, message, or other material is local even when it names no owner ('find the slides about X', 'where is the Y presentation', 'Z 관련 자료 찾아줘'): such material lives in the user's own files and chats, and a well-known topic is not a reason to answer from general knowledge or the web. When unsure whether intrinsic knowledge suffices, choose local.",
|
|
33000
33124
|
extra: "None of the other options fits and no search of the user's own records is needed first. Either the request is answerable without looking anything up — stable common knowledge, a definition, simple reasoning, or small talk — or it is a task for the assistant's tools that is not file organization and not AutoRAG configuration: committing code, pushing a branch, opening a pull request, running tests or a build, installing a package, running a script, writing or editing code. Programming work is extra even when the code names the user's files or folders ('write a script that lowercases every PNG', 'add this to my parser'): the assistant is asked to produce or change code, not to act on those files now. So is a request to run something (tests, a build, a script, a command) and report the outcome, even when what it runs lives in the user's own repository: running it is a task for the tool, not a search of the user's records. Never extra when the request refers to the user's own life, files, records, chats, calendar, or history (e.g. 'my', 'I', 'our', 'my girlfriend', 'my case', 'my routine'), and never extra when the answer depends on facts that may be recent, specific, or changing (current events, recent releases, prices, versions): search those locally instead. If any other option fits, do not choose extra.",
|
|
33001
33125
|
organize: "The user asks the assistant to DO something to files or folders instead of finding information: move or copy files into folders, tidy or sort a folder, bulk or pattern rename (for example by date or number), rename files according to what their content says, convert files to another format, create folder structures, group files by type or topic, archive, or move to the trash ('tidy this up', 'move these into folders', 'bulk rename these files', 'rename the files to match their content', 'convert these to PDF', 'sort my Downloads folder'). It is about acting on files and folders, never about looking up what a document says or where something is. A question that only asks to find or read a file's content is local, not organize."
|
|
33002
33126
|
};
|
|
@@ -34234,7 +34358,7 @@ var init_page_loader = __esm(() => {
|
|
|
34234
34358
|
});
|
|
34235
34359
|
|
|
34236
34360
|
// src/web/fetch/url-target.ts
|
|
34237
|
-
import { extname as
|
|
34361
|
+
import { extname as extname10 } from "node:path";
|
|
34238
34362
|
function repairCollapsedScheme(value) {
|
|
34239
34363
|
const m = value.match(/^(https?):\/(?!\/)/i);
|
|
34240
34364
|
return m ? `${m[1]}://${value.slice(m[0].length)}` : value;
|
|
@@ -34253,7 +34377,7 @@ function getFilenameExtensionHint(filename) {
|
|
|
34253
34377
|
const lower = filename.toLowerCase();
|
|
34254
34378
|
if (lower.endsWith(".tar.gz"))
|
|
34255
34379
|
return ".tar.gz";
|
|
34256
|
-
return
|
|
34380
|
+
return extname10(filename).toLowerCase();
|
|
34257
34381
|
}
|
|
34258
34382
|
function getExtensionHint(url, contentDisposition) {
|
|
34259
34383
|
if (contentDisposition) {
|
|
@@ -34430,7 +34554,7 @@ var init_convert2 = __esm(() => {
|
|
|
34430
34554
|
});
|
|
34431
34555
|
|
|
34432
34556
|
// src/web/fetch/feeds.ts
|
|
34433
|
-
import { extname as
|
|
34557
|
+
import { extname as extname11 } from "node:path";
|
|
34434
34558
|
import { XMLParser as XMLParser3 } from "fast-xml-parser";
|
|
34435
34559
|
function getHtmlAttribute(tag, attribute) {
|
|
34436
34560
|
const pattern = new RegExp(`\\b${attribute}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'=<>]+))`, "i");
|
|
@@ -34493,7 +34617,7 @@ function extractDocumentLinks(html, baseUrl) {
|
|
|
34493
34617
|
const href = getHtmlAttribute(tag, "href");
|
|
34494
34618
|
if (!href)
|
|
34495
34619
|
continue;
|
|
34496
|
-
const ext =
|
|
34620
|
+
const ext = extname11(href).toLowerCase();
|
|
34497
34621
|
if (!CONVERTIBLE_EXTENSIONS2.has(ext))
|
|
34498
34622
|
continue;
|
|
34499
34623
|
const resolved = href.startsWith("http") ? href : new URL(href, baseUrl).href;
|
|
@@ -36161,7 +36285,7 @@ var init_web_search_tool = __esm(() => {
|
|
|
36161
36285
|
// src/agent/agent.ts
|
|
36162
36286
|
import { randomUUID as randomUUID7 } from "node:crypto";
|
|
36163
36287
|
import { existsSync as existsSync32, watch as fsWatch, mkdirSync as mkdirSync23, realpathSync as realpathSync8, statSync as statSync11, writeFileSync as writeFileSync17 } from "node:fs";
|
|
36164
|
-
import { basename as
|
|
36288
|
+
import { basename as basename15, dirname as dirname27, join as join53, relative as relative8, resolve as resolve17, sep as sep10 } from "node:path";
|
|
36165
36289
|
import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
|
|
36166
36290
|
import { getAgentDir as getAgentDir5, keyText } from "@earendil-works/pi-coding-agent";
|
|
36167
36291
|
function describeDecisionModel(status) {
|
|
@@ -36596,10 +36720,20 @@ class AutoRAGAgent {
|
|
|
36596
36720
|
}
|
|
36597
36721
|
async scanDuplicateDocuments() {
|
|
36598
36722
|
if (this.dupeyOptions === false) {
|
|
36599
|
-
return { scans: [],
|
|
36723
|
+
return { scans: [], familyCount: 0, exactDuplicateCount: 0 };
|
|
36600
36724
|
}
|
|
36601
|
-
const
|
|
36602
|
-
const
|
|
36725
|
+
const cloudRoots = await this.cloudRoots();
|
|
36726
|
+
const scanRoots = [];
|
|
36727
|
+
for (const searchPath of this.searchPaths) {
|
|
36728
|
+
const scope = cloudScope(searchPath, cloudRoots);
|
|
36729
|
+
if (scope.insideCloudRoot)
|
|
36730
|
+
continue;
|
|
36731
|
+
scanRoots.push({
|
|
36732
|
+
path: searchPath,
|
|
36733
|
+
excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path))
|
|
36734
|
+
});
|
|
36735
|
+
}
|
|
36736
|
+
const scans = await Promise.all(scanRoots.map(({ path, excludeDirs }) => scanWithDupey(path, {
|
|
36603
36737
|
...this.dupeyOptions || {},
|
|
36604
36738
|
...excludeDirs.length === 0 ? {} : { excludeDirs }
|
|
36605
36739
|
})));
|
|
@@ -36613,7 +36747,7 @@ class AutoRAGAgent {
|
|
|
36613
36747
|
}
|
|
36614
36748
|
return count + [...hashes.values()].reduce((sum, size) => sum + Math.max(0, size - 1), 0);
|
|
36615
36749
|
}, 0);
|
|
36616
|
-
return { scans,
|
|
36750
|
+
return { scans, familyCount, exactDuplicateCount };
|
|
36617
36751
|
}
|
|
36618
36752
|
loadMemoryContextFor(query) {
|
|
36619
36753
|
if (this.memoryContextCache?.query === query)
|
|
@@ -38266,13 +38400,67 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38266
38400
|
inFlight: true,
|
|
38267
38401
|
lastStartedAt: startedAt
|
|
38268
38402
|
};
|
|
38403
|
+
const syncAbort = new AbortController;
|
|
38404
|
+
const forwardAbort = () => syncAbort.abort(signal?.reason);
|
|
38405
|
+
signal?.addEventListener("abort", forwardAbort, { once: true });
|
|
38406
|
+
let minSyncPipeline;
|
|
38269
38407
|
try {
|
|
38270
38408
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38409
|
+
const batchedMinSync = needsParsed && wants("minsync");
|
|
38410
|
+
const minSyncBatches = [];
|
|
38411
|
+
const minSyncBatchFailures = [];
|
|
38412
|
+
let minSyncSyncedBase = 0;
|
|
38413
|
+
let minSyncTotalBase = 0;
|
|
38414
|
+
minSyncPipeline = batchedMinSync ? new PipelinedSync({
|
|
38415
|
+
sync: async ({ index, virtualPaths }) => {
|
|
38416
|
+
syncAbort.signal.throwIfAborted();
|
|
38417
|
+
let syncTotal = minSyncTotalBase;
|
|
38418
|
+
const result = await this.syncMinSync(force && index === 1, syncAbort.signal, (minsyncProgress) => {
|
|
38419
|
+
syncTotal = minSyncTotalBase + minsyncProgress.total;
|
|
38420
|
+
progress = updateRefreshProgress(progress, {
|
|
38421
|
+
minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: syncTotal }
|
|
38422
|
+
});
|
|
38423
|
+
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38424
|
+
emitProgress({
|
|
38425
|
+
phase: "minsync",
|
|
38426
|
+
processed: minSyncSyncedBase + minsyncProgress.processed,
|
|
38427
|
+
total: syncTotal
|
|
38428
|
+
});
|
|
38429
|
+
});
|
|
38430
|
+
minSyncBatches.push(result);
|
|
38431
|
+
if (result.ok)
|
|
38432
|
+
minSyncBatchFailures.length = 0;
|
|
38433
|
+
else if (result.reason !== "lock-held") {
|
|
38434
|
+
for (const virtualPath of virtualPaths) {
|
|
38435
|
+
minSyncBatchFailures.push({
|
|
38436
|
+
code: "minsync-sync-failed",
|
|
38437
|
+
severity: "warning",
|
|
38438
|
+
message: `This file stayed at its previous index entry: the sync that covered it could not index it and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
|
|
38439
|
+
source: virtualPath
|
|
38440
|
+
});
|
|
38441
|
+
}
|
|
38442
|
+
}
|
|
38443
|
+
minSyncSyncedBase = syncTotal;
|
|
38444
|
+
minSyncTotalBase = syncTotal;
|
|
38445
|
+
return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
|
|
38446
|
+
}
|
|
38447
|
+
}) : undefined;
|
|
38448
|
+
const pipeline = minSyncPipeline;
|
|
38271
38449
|
const summary = needsParsed ? await this.syncParsedMirrors(force, {
|
|
38272
38450
|
...opts?.retrySkipped === true ? { retrySkipped: true } : {},
|
|
38273
38451
|
...signal === undefined ? {} : { signal },
|
|
38452
|
+
...pipeline === undefined ? {} : {
|
|
38453
|
+
onBatch: (batch) => {
|
|
38454
|
+
if (batch.final) {
|
|
38455
|
+
progress = updateRefreshProgress(progress, { phase: "minsync" });
|
|
38456
|
+
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38457
|
+
}
|
|
38458
|
+
return pipeline.batch(batch);
|
|
38459
|
+
}
|
|
38460
|
+
},
|
|
38274
38461
|
onProgress: (parsedProgress) => {
|
|
38275
38462
|
progress = updateRefreshProgress(progress, {
|
|
38463
|
+
phase: "parsed",
|
|
38276
38464
|
sourceFiles: { total: parsedProgress.total },
|
|
38277
38465
|
parsedCounts: {
|
|
38278
38466
|
scanned: parsedProgress.processed,
|
|
@@ -38294,7 +38482,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38294
38482
|
}) : await this.scanMirrorStaleness();
|
|
38295
38483
|
signal?.throwIfAborted();
|
|
38296
38484
|
progress = updateRefreshProgress(progress, {
|
|
38297
|
-
phase: wants("
|
|
38485
|
+
phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
|
|
38298
38486
|
sourceFiles: { total: summary.scanned },
|
|
38299
38487
|
parsedCounts: {
|
|
38300
38488
|
scanned: summary.scanned,
|
|
@@ -38304,24 +38492,12 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38304
38492
|
}
|
|
38305
38493
|
});
|
|
38306
38494
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38307
|
-
const minsync =
|
|
38308
|
-
progress = updateRefreshProgress(progress, {
|
|
38309
|
-
minsync: { synced: minsyncProgress.processed, total: minsyncProgress.total }
|
|
38310
|
-
});
|
|
38311
|
-
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38312
|
-
emitProgress({
|
|
38313
|
-
phase: "minsync",
|
|
38314
|
-
processed: minsyncProgress.processed,
|
|
38315
|
-
total: minsyncProgress.total
|
|
38316
|
-
});
|
|
38317
|
-
}) : undefined;
|
|
38318
|
-
signal?.throwIfAborted();
|
|
38495
|
+
const minsync = batchedMinSync ? aggregateMinSyncBatches(minSyncBatches) : undefined;
|
|
38319
38496
|
if (minsync !== undefined) {
|
|
38320
38497
|
progress = updateRefreshProgress(progress, {
|
|
38321
|
-
phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
|
|
38322
38498
|
minsync: {
|
|
38323
|
-
...progress.minsync,
|
|
38324
38499
|
synced: minsync.synced,
|
|
38500
|
+
total: Math.max(minSyncTotalBase, minsync.synced),
|
|
38325
38501
|
...minsync.filesFailed !== undefined && minsync.filesFailed.length > 0 ? { filesSkipped: minsync.filesFailed.length } : {}
|
|
38326
38502
|
}
|
|
38327
38503
|
});
|
|
@@ -38383,7 +38559,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38383
38559
|
finishedAt: new Date().toISOString()
|
|
38384
38560
|
});
|
|
38385
38561
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38386
|
-
const minsyncDiagnostics = minSyncRefreshDiagnostics(minsync);
|
|
38562
|
+
const minsyncDiagnostics = [...minSyncRefreshDiagnostics(minsync), ...minSyncBatchFailures];
|
|
38387
38563
|
const publicMinsync = minsync ? {
|
|
38388
38564
|
ok: minsync.ok,
|
|
38389
38565
|
synced: minsync.synced,
|
|
@@ -38417,6 +38593,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38417
38593
|
...fsearch !== undefined ? { fsearch } : {}
|
|
38418
38594
|
};
|
|
38419
38595
|
} catch (error) {
|
|
38596
|
+
syncAbort.abort(error);
|
|
38420
38597
|
this.refreshState = {
|
|
38421
38598
|
...this.refreshState,
|
|
38422
38599
|
lastOutcome: "failed",
|
|
@@ -38430,6 +38607,8 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38430
38607
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38431
38608
|
throw error;
|
|
38432
38609
|
} finally {
|
|
38610
|
+
signal?.removeEventListener("abort", forwardAbort);
|
|
38611
|
+
await minSyncPipeline?.settle();
|
|
38433
38612
|
this.refreshState = {
|
|
38434
38613
|
...this.refreshState,
|
|
38435
38614
|
inFlight: false,
|
|
@@ -38651,15 +38830,15 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38651
38830
|
const cloudRoots = await this.cloudRoots();
|
|
38652
38831
|
const fingerprints = cloudFingerprintExclusions(loadCloudFingerprints(this.workspaceProjectRoot));
|
|
38653
38832
|
for (const searchPath of this.searchPaths) {
|
|
38654
|
-
const
|
|
38833
|
+
const scope = cloudScope(searchPath, cloudRoots);
|
|
38655
38834
|
let scan;
|
|
38656
|
-
if (
|
|
38835
|
+
if (scope.insideCloudRoot) {
|
|
38657
38836
|
scan = { dir: searchPath, files: [], families: [], errors: [] };
|
|
38658
38837
|
} else {
|
|
38659
38838
|
try {
|
|
38660
38839
|
scan = await scanWithDupey(searchPath, {
|
|
38661
38840
|
...this.dupeyOptions || {},
|
|
38662
|
-
...
|
|
38841
|
+
...scope.nestedCloudRoots.length === 0 ? {} : { excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path)) }
|
|
38663
38842
|
});
|
|
38664
38843
|
} catch (error) {
|
|
38665
38844
|
if (!(error instanceof DupeyCliError))
|
|
@@ -39188,6 +39367,12 @@ function fsearchRefreshDiagnostics(fsearch) {
|
|
|
39188
39367
|
}
|
|
39189
39368
|
];
|
|
39190
39369
|
}
|
|
39370
|
+
function aggregateMinSyncBatches(batches) {
|
|
39371
|
+
const last = batches.at(-1);
|
|
39372
|
+
if (last === undefined)
|
|
39373
|
+
return;
|
|
39374
|
+
return { ...last, synced: batches.reduce((total, batch) => total + batch.synced, 0) };
|
|
39375
|
+
}
|
|
39191
39376
|
function minSyncRefreshDiagnostics(minsync) {
|
|
39192
39377
|
if (!minsync)
|
|
39193
39378
|
return [];
|
|
@@ -39218,7 +39403,7 @@ function toMinSyncReasonDiagnostic(reason) {
|
|
|
39218
39403
|
function pinExcludedPath(path) {
|
|
39219
39404
|
const resolvedPath = resolve17(path);
|
|
39220
39405
|
try {
|
|
39221
|
-
return join53(realpathSync8(dirname27(resolvedPath)),
|
|
39406
|
+
return join53(realpathSync8(dirname27(resolvedPath)), basename15(resolvedPath));
|
|
39222
39407
|
} catch {
|
|
39223
39408
|
return resolvedPath;
|
|
39224
39409
|
}
|
|
@@ -40207,7 +40392,7 @@ function readPackageVersion() {
|
|
|
40207
40392
|
return "0.0.0";
|
|
40208
40393
|
}
|
|
40209
40394
|
}
|
|
40210
|
-
var
|
|
40395
|
+
var init_version4 = () => {};
|
|
40211
40396
|
|
|
40212
40397
|
// src/cli/commands/init.ts
|
|
40213
40398
|
import { existsSync as existsSync35, readFileSync as readFileSync30 } from "node:fs";
|
|
@@ -42039,20 +42224,18 @@ var init_health = __esm(() => {
|
|
|
42039
42224
|
|
|
42040
42225
|
// src/cli/commands/duplicates.ts
|
|
42041
42226
|
import { resolve as resolve23 } from "node:path";
|
|
42042
|
-
async function runDuplicates(ctx, scanner = (path
|
|
42227
|
+
async function runDuplicates(ctx, scanner = (path) => scanWithDupey(path)) {
|
|
42043
42228
|
try {
|
|
42044
42229
|
const config = resolveConfig({ flags: ctx.flags, cwd: ctx.cwd });
|
|
42045
42230
|
const requested = ctx.positionals[0];
|
|
42046
42231
|
const roots = requested ? [resolve23(ctx.cwd, requested)] : config.searchPaths.map((path) => resolve23(path));
|
|
42047
|
-
const
|
|
42048
|
-
const scans = await Promise.all(plan.targets.map(({ path, excludeDirs }) => scanner(path, excludeDirs)));
|
|
42232
|
+
const scans = await Promise.all(roots.map((root) => scanner(root)));
|
|
42049
42233
|
const exactGroups = scans.flatMap(exactGroupsFromScan);
|
|
42050
42234
|
const extractionErrors = scans.flatMap((scan) => scan.errors);
|
|
42051
42235
|
if (ctx.json) {
|
|
42052
42236
|
ctx.stdout(JSON.stringify({
|
|
42053
42237
|
ok: true,
|
|
42054
42238
|
roots,
|
|
42055
|
-
skippedRoots: plan.skipped,
|
|
42056
42239
|
exactGroups,
|
|
42057
42240
|
families: scans.flatMap((scan) => scan.families),
|
|
42058
42241
|
extractionErrors,
|
|
@@ -42061,14 +42244,11 @@ async function runDuplicates(ctx, scanner = (path, excludeDirs) => scanWithDupey
|
|
|
42061
42244
|
return 0;
|
|
42062
42245
|
}
|
|
42063
42246
|
const lines = [
|
|
42064
|
-
`duplicates: scanned ${
|
|
42247
|
+
`duplicates: scanned ${roots.length} root(s)`,
|
|
42065
42248
|
` exact groups: ${exactGroups.length}`,
|
|
42066
42249
|
` other families: ${scans.reduce((count, scan) => count + scan.families.length, 0)}`,
|
|
42067
42250
|
` extraction errors: ${extractionErrors.length}`
|
|
42068
42251
|
];
|
|
42069
|
-
for (const root of plan.skipped) {
|
|
42070
|
-
lines.push(` not scanned (inside a cloud or network folder; scanning would download or re-read every file): ${root}`);
|
|
42071
|
-
}
|
|
42072
42252
|
for (const [index, group] of exactGroups.entries()) {
|
|
42073
42253
|
lines.push(` exact group ${index + 1}:`);
|
|
42074
42254
|
for (const file of group.files)
|
|
@@ -42096,7 +42276,6 @@ function exactGroupsFromScan(scan) {
|
|
|
42096
42276
|
}
|
|
42097
42277
|
var init_duplicates = __esm(() => {
|
|
42098
42278
|
init_dupey();
|
|
42099
|
-
init_cloud_placeholder();
|
|
42100
42279
|
init_config5();
|
|
42101
42280
|
init_output();
|
|
42102
42281
|
});
|
|
@@ -42352,7 +42531,7 @@ var init_tui = __esm(() => {
|
|
|
42352
42531
|
init_open_target();
|
|
42353
42532
|
init_output();
|
|
42354
42533
|
init_update_check();
|
|
42355
|
-
|
|
42534
|
+
init_version4();
|
|
42356
42535
|
init_whatsapp_extension();
|
|
42357
42536
|
THINKING_LEVELS2 = ["off", "minimal", "low", "medium", "high", "xhigh", "max"];
|
|
42358
42537
|
});
|
|
@@ -42487,7 +42666,7 @@ async function runUpdateCheck(ctx, deps = {}) {
|
|
|
42487
42666
|
}
|
|
42488
42667
|
var init_update_check2 = __esm(() => {
|
|
42489
42668
|
init_update_check();
|
|
42490
|
-
|
|
42669
|
+
init_version4();
|
|
42491
42670
|
});
|
|
42492
42671
|
|
|
42493
42672
|
// src/cli/commands/attachments.ts
|
|
@@ -43610,7 +43789,7 @@ async function runAutomate(ctx) {
|
|
|
43610
43789
|
|
|
43611
43790
|
// src/cli/index.ts
|
|
43612
43791
|
init_output();
|
|
43613
|
-
|
|
43792
|
+
init_version4();
|
|
43614
43793
|
var BOOLEAN_FLAGS = new Set([
|
|
43615
43794
|
"json",
|
|
43616
43795
|
"debug",
|