@autorag/librarian 2.13.6 → 2.13.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/agent.d.ts +4 -4
- package/dist/agent/dupey-tool.d.ts +0 -2
- package/dist/agent/pipelined-sync.d.ts +34 -0
- package/dist/agent/search-documents.d.ts +1 -1
- package/dist/cli/commands/duplicates.d.ts +2 -3
- package/dist/cli/index.js +1521 -1407
- package/dist/core.js +2723 -2603
- package/dist/datasource/skills/factory.d.ts +4 -1
- package/dist/dupey/filter.d.ts +0 -2
- package/dist/dupey/index.d.ts +0 -1
- package/dist/dupey/version.d.ts +17 -0
- package/dist/index.js +2726 -2611
- package/dist/mcp/index.js +2726 -2609
- package/dist/mcp/server.d.ts +0 -3
- package/dist/minsync/installer.d.ts +1 -1
- package/dist/minsync/version.d.ts +3 -2
- package/dist/mirror/cloud-fingerprints.d.ts +6 -5
- package/dist/mirror/cloud.d.ts +9 -6
- package/dist/mirror/sync.d.ts +19 -15
- package/dist/profiles.js +1518 -1398
- package/package.json +1 -1
- package/dist/dupey/scan-plan.d.ts +0 -23
package/dist/cli/index.js
CHANGED
|
@@ -5032,7 +5032,7 @@ function outdatedMinSyncVersion(binaryPath) {
|
|
|
5032
5032
|
}
|
|
5033
5033
|
return version !== undefined && !isSupportedMinSyncVersion(version) ? version : undefined;
|
|
5034
5034
|
}
|
|
5035
|
-
var MINSYNC_MIN_VERSION = "0.
|
|
5035
|
+
var MINSYNC_MIN_VERSION = "0.7.1", VERSION_PROBE_TIMEOUT_MS = 5000, VERSION_LINE, probeCache;
|
|
5036
5036
|
var init_version = __esm(() => {
|
|
5037
5037
|
VERSION_LINE = /^minsync\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
|
|
5038
5038
|
probeCache = new Map;
|
|
@@ -5276,7 +5276,7 @@ function isGitHubAsset(value) {
|
|
|
5276
5276
|
function isRecord2(value) {
|
|
5277
5277
|
return typeof value === "object" && value !== null;
|
|
5278
5278
|
}
|
|
5279
|
-
var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.
|
|
5279
|
+
var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.7.3", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
|
|
5280
5280
|
var init_installer = __esm(() => {
|
|
5281
5281
|
init_atomic_write();
|
|
5282
5282
|
init_process();
|
|
@@ -14567,7 +14567,8 @@ class WombatMethod {
|
|
|
14567
14567
|
return matchesDatasourceScope(datasourceSourcePath("wombat", this.instanceId, hit.chunkId), options.scope);
|
|
14568
14568
|
}).map((hit) => ({
|
|
14569
14569
|
id: `wombat:${this.instanceId}:${hit.chunkId}`,
|
|
14570
|
-
content: hit
|
|
14570
|
+
content: `${sessionLabel(hit)}
|
|
14571
|
+
${hit.snippet}`,
|
|
14571
14572
|
source: datasourceSourcePath("wombat", this.instanceId, hit.chunkId),
|
|
14572
14573
|
score: hit.score,
|
|
14573
14574
|
metadata: {
|
|
@@ -14590,6 +14591,20 @@ class WombatMethod {
|
|
|
14590
14591
|
})).slice(0, topK);
|
|
14591
14592
|
}
|
|
14592
14593
|
}
|
|
14594
|
+
function oneLine(value) {
|
|
14595
|
+
return value.replace(/\s+/gu, " ").trim();
|
|
14596
|
+
}
|
|
14597
|
+
function sessionLabel(hit) {
|
|
14598
|
+
const turns = hit.startTurn === undefined ? undefined : hit.endTurn === undefined || hit.endTurn === hit.startTurn ? `turn ${hit.startTurn}` : `turns ${hit.startTurn}-${hit.endTurn}`;
|
|
14599
|
+
const parts = [
|
|
14600
|
+
oneLine(hit.catalogKey),
|
|
14601
|
+
turns,
|
|
14602
|
+
hit.startedAt === undefined ? undefined : hit.startedAt.slice(0, 10),
|
|
14603
|
+
hit.title.length === 0 ? undefined : oneLine(hit.title),
|
|
14604
|
+
hit.cwd === undefined ? undefined : oneLine(hit.cwd)
|
|
14605
|
+
].filter((part) => part !== undefined && part.length > 0);
|
|
14606
|
+
return `[${parts.join(" | ")}]`;
|
|
14607
|
+
}
|
|
14593
14608
|
var WOMBAT_MODES, METHOD_TYPE;
|
|
14594
14609
|
var init_methods7 = __esm(() => {
|
|
14595
14610
|
init_errors2();
|
|
@@ -14702,7 +14717,7 @@ class WombatSkill {
|
|
|
14702
14717
|
`);
|
|
14703
14718
|
return {
|
|
14704
14719
|
name: "datasource-wombat",
|
|
14705
|
-
description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built.",
|
|
14720
|
+
description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built. Each hit names its session. Load this skill to open that session's full conversation or its original log with the tool calls that the search does not index.",
|
|
14706
14721
|
content: [
|
|
14707
14722
|
"# wombat coding-agent session datasource",
|
|
14708
14723
|
"",
|
|
@@ -14716,10 +14731,10 @@ class WombatSkill {
|
|
|
14716
14731
|
scopes,
|
|
14717
14732
|
"",
|
|
14718
14733
|
"The selectable retrieval modes are keyword (SPLADE), semantic, and hybrid. Scope arguments can only narrow trusted access. Source identifiers remain opaque and hierarchical.",
|
|
14719
|
-
"
|
|
14734
|
+
"Every result starts with a label line, `[<sessionKey> | turns A-B | date | title | project]`, followed by the snippet. `sessionKey` is `<agent>:<session id>`.",
|
|
14720
14735
|
"",
|
|
14721
14736
|
"## Native CLI",
|
|
14722
|
-
"The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the result's
|
|
14737
|
+
"The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the `sessionKey` from the result's label:",
|
|
14723
14738
|
"- `wombat show <sessionKey> [--turn N]`: the full cleaned conversation, or one turn.",
|
|
14724
14739
|
"- `wombat trace <sessionKey> --turn N --context K`: the original session log around that turn, including the tool calls and outputs that the index leaves out.",
|
|
14725
14740
|
"- `wombat chunk context <chunk id> --json`: the turns before and after a result.",
|
|
@@ -14873,6 +14888,7 @@ function common(config, workspaceRoot) {
|
|
|
14873
14888
|
function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
|
|
14874
14889
|
const skills = [];
|
|
14875
14890
|
const unknown = [];
|
|
14891
|
+
const retired = [];
|
|
14876
14892
|
for (const [name, raw] of Object.entries(config ?? {})) {
|
|
14877
14893
|
if (raw === false)
|
|
14878
14894
|
continue;
|
|
@@ -14884,6 +14900,10 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
|
|
|
14884
14900
|
if (entry.enabled === false)
|
|
14885
14901
|
continue;
|
|
14886
14902
|
const templateName = entry.type ?? name;
|
|
14903
|
+
if (Object.hasOwn(RETIRED_TEMPLATES, templateName)) {
|
|
14904
|
+
retired.push(name);
|
|
14905
|
+
continue;
|
|
14906
|
+
}
|
|
14887
14907
|
const builder = Object.hasOwn(BUILDERS, templateName) && datasourceSkillSupported(templateName) ? BUILDERS[templateName] : undefined;
|
|
14888
14908
|
if (builder === undefined) {
|
|
14889
14909
|
unknown.push(name);
|
|
@@ -14907,9 +14927,9 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
|
|
|
14907
14927
|
}
|
|
14908
14928
|
skills.push(typeof entry.description === "string" && entry.description.trim().length > 0 ? new DescribedDatasourceSkill(aliased, entry.description.trim()) : aliased);
|
|
14909
14929
|
}
|
|
14910
|
-
return { skills, unknown };
|
|
14930
|
+
return { skills, unknown, retired };
|
|
14911
14931
|
}
|
|
14912
|
-
var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES;
|
|
14932
|
+
var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES, RETIRED_TEMPLATES;
|
|
14913
14933
|
var init_factory = __esm(() => {
|
|
14914
14934
|
init_aliased_skill();
|
|
14915
14935
|
init_config3();
|
|
@@ -15065,6 +15085,7 @@ var init_factory = __esm(() => {
|
|
|
15065
15085
|
};
|
|
15066
15086
|
PLATFORM_ONLY_SKILLS = { kakao: "darwin" };
|
|
15067
15087
|
BUILTIN_DATASOURCE_SKILL_NAMES = builtinDatasourceSkillNames();
|
|
15088
|
+
RETIRED_TEMPLATES = { spotlight: true };
|
|
15068
15089
|
});
|
|
15069
15090
|
|
|
15070
15091
|
// src/language.ts
|
|
@@ -16297,7 +16318,7 @@ function buildAgentOptions(config, agentDir) {
|
|
|
16297
16318
|
const embedder = createEmbedderFromConfig({ embedder: config.minSync?.embedder });
|
|
16298
16319
|
opts.embedder = embedder;
|
|
16299
16320
|
if (config.datasources !== undefined) {
|
|
16300
|
-
const { skills, unknown } = buildDatasourceSkills(config.datasources, config.workspacePath, {
|
|
16321
|
+
const { skills, unknown, retired } = buildDatasourceSkills(config.datasources, config.workspacePath, {
|
|
16301
16322
|
embedder,
|
|
16302
16323
|
discrawl: async () => discrawlEmbeddingRuntimeFor(await embedder.describeEndpoint()),
|
|
16303
16324
|
cliEndpoint: embeddingEndpointSource(embedder),
|
|
@@ -16310,16 +16331,27 @@ function buildAgentOptions(config, agentDir) {
|
|
|
16310
16331
|
});
|
|
16311
16332
|
if (skills.length > 0)
|
|
16312
16333
|
opts.datasourceSkills = skills;
|
|
16334
|
+
const diagnostics = [];
|
|
16335
|
+
if (retired.length > 0) {
|
|
16336
|
+
const names = retired.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
|
|
16337
|
+
diagnostics.push({
|
|
16338
|
+
code: "retired-datasource-skill",
|
|
16339
|
+
severity: "warning",
|
|
16340
|
+
message: `Datasource config ignored because the Spotlight datasource was removed: ${names.join(", ")}. Delete these entries; use fsearch/mirror search (searchPaths) instead.`,
|
|
16341
|
+
source: "datasources"
|
|
16342
|
+
});
|
|
16343
|
+
}
|
|
16313
16344
|
if (unknown.length > 0) {
|
|
16314
|
-
const
|
|
16315
|
-
|
|
16345
|
+
const names = unknown.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
|
|
16346
|
+
diagnostics.push({
|
|
16316
16347
|
code: "unknown-datasource-skill",
|
|
16317
16348
|
severity: "warning",
|
|
16318
|
-
message: `Unknown datasource skill(s) in config were skipped: ${
|
|
16349
|
+
message: `Unknown datasource skill(s) in config were skipped: ${names.join(", ")}`,
|
|
16319
16350
|
source: "datasources"
|
|
16320
|
-
};
|
|
16321
|
-
opts.startupDiagnostics = [diagnostic];
|
|
16351
|
+
});
|
|
16322
16352
|
}
|
|
16353
|
+
if (diagnostics.length > 0)
|
|
16354
|
+
opts.startupDiagnostics = diagnostics;
|
|
16323
16355
|
}
|
|
16324
16356
|
return opts;
|
|
16325
16357
|
}
|
|
@@ -17832,11 +17864,49 @@ var init_matrix = __esm(() => {
|
|
|
17832
17864
|
};
|
|
17833
17865
|
});
|
|
17834
17866
|
|
|
17867
|
+
// src/dupey/version.ts
|
|
17868
|
+
function parseDupeyVersion(output) {
|
|
17869
|
+
return VERSION_LINE2.exec(output)?.[1];
|
|
17870
|
+
}
|
|
17871
|
+
function isSupportedDupeyVersion(version) {
|
|
17872
|
+
const have = version.split(".").map((part) => Number.parseInt(part, 10));
|
|
17873
|
+
const need = DUPEY_MIN_VERSION.split(".").map((part) => Number.parseInt(part, 10));
|
|
17874
|
+
for (let index = 0;index < need.length; index += 1) {
|
|
17875
|
+
const left = have[index] ?? 0;
|
|
17876
|
+
const right = need[index] ?? 0;
|
|
17877
|
+
if (left !== right)
|
|
17878
|
+
return left > right;
|
|
17879
|
+
}
|
|
17880
|
+
return true;
|
|
17881
|
+
}
|
|
17882
|
+
var DUPEY_MIN_VERSION = "0.2.0", VERSION_LINE2;
|
|
17883
|
+
var init_version2 = __esm(() => {
|
|
17884
|
+
VERSION_LINE2 = /^dupey\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
|
|
17885
|
+
});
|
|
17886
|
+
|
|
17835
17887
|
// src/dupey/cli.ts
|
|
17836
17888
|
import { spawn as spawn14 } from "node:child_process";
|
|
17837
17889
|
import { resolve as resolve9 } from "node:path";
|
|
17838
|
-
function
|
|
17839
|
-
|
|
17890
|
+
function checkDupeyVersion(executable, options) {
|
|
17891
|
+
const cached = versionChecks.get(executable);
|
|
17892
|
+
if (cached !== undefined)
|
|
17893
|
+
return cached;
|
|
17894
|
+
const check = spawnDupey(executable, ["--version"], options).then((stdout) => {
|
|
17895
|
+
const version = parseDupeyVersion(stdout);
|
|
17896
|
+
if (version !== undefined && !isSupportedDupeyVersion(version)) {
|
|
17897
|
+
throw new DupeyCliError(`dupey ${version} at ${executable} is older than the ${DUPEY_MIN_VERSION} AutoRAG requires; run \`cargo install dupey\` to update it.`);
|
|
17898
|
+
}
|
|
17899
|
+
});
|
|
17900
|
+
check.catch(() => versionChecks.delete(executable));
|
|
17901
|
+
versionChecks.set(executable, check);
|
|
17902
|
+
return check;
|
|
17903
|
+
}
|
|
17904
|
+
async function runDupey(args, options) {
|
|
17905
|
+
if (options.run)
|
|
17906
|
+
return options.run(args);
|
|
17907
|
+
const executable = options.executable ?? "dupey";
|
|
17908
|
+
await checkDupeyVersion(executable, options);
|
|
17909
|
+
return spawnDupey(executable, args, options);
|
|
17840
17910
|
}
|
|
17841
17911
|
async function scanWithDupey(dir, options = {}) {
|
|
17842
17912
|
const absoluteDir = resolve9(dir);
|
|
@@ -17970,28 +18040,26 @@ function spawnDupey(executable, args, options) {
|
|
|
17970
18040
|
});
|
|
17971
18041
|
});
|
|
17972
18042
|
}
|
|
17973
|
-
var DupeyCliError, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
|
|
18043
|
+
var DupeyCliError, versionChecks, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
|
|
17974
18044
|
var init_cli2 = __esm(() => {
|
|
17975
18045
|
init_portable_spawn();
|
|
18046
|
+
init_version2();
|
|
17976
18047
|
DupeyCliError = class DupeyCliError extends Error {
|
|
17977
18048
|
constructor(message) {
|
|
17978
18049
|
super(message);
|
|
17979
18050
|
this.name = "DupeyCliError";
|
|
17980
18051
|
}
|
|
17981
18052
|
};
|
|
18053
|
+
versionChecks = new Map;
|
|
17982
18054
|
});
|
|
17983
18055
|
|
|
17984
18056
|
// src/dupey/filter.ts
|
|
17985
18057
|
import { stat as stat2 } from "node:fs/promises";
|
|
17986
18058
|
import { resolve as resolve10 } from "node:path";
|
|
17987
|
-
function extractedNoText(file) {
|
|
17988
|
-
const signals = file.signals;
|
|
17989
|
-
return typeof signals === "object" && signals !== null && "chars" in signals && signals.chars === 0;
|
|
17990
|
-
}
|
|
17991
18059
|
async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerprints) {
|
|
17992
18060
|
const byHash = new Map;
|
|
17993
18061
|
for (const file of scan.files) {
|
|
17994
|
-
if (typeof file.content_hash !== "string" || file.content_hash.length === 0
|
|
18062
|
+
if (typeof file.content_hash !== "string" || file.content_hash.length === 0)
|
|
17995
18063
|
continue;
|
|
17996
18064
|
const path = resolve10(root, file.path);
|
|
17997
18065
|
const group = byHash.get(file.content_hash) ?? [];
|
|
@@ -18000,7 +18068,7 @@ async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerp
|
|
|
18000
18068
|
}
|
|
18001
18069
|
const fingerprintModified = new Map;
|
|
18002
18070
|
for (const [path, fingerprint] of fingerprints ?? []) {
|
|
18003
|
-
if (fingerprint.exact.length === 0
|
|
18071
|
+
if (fingerprint.exact.length === 0)
|
|
18004
18072
|
continue;
|
|
18005
18073
|
fingerprintModified.set(path, fingerprint.modified);
|
|
18006
18074
|
const group = byHash.get(fingerprint.exact) ?? [];
|
|
@@ -18067,1397 +18135,1368 @@ async function rankExactCandidates(paths, dupeyRanking, fingerprintModified, err
|
|
|
18067
18135
|
}
|
|
18068
18136
|
var init_filter = () => {};
|
|
18069
18137
|
|
|
18070
|
-
// src/
|
|
18071
|
-
|
|
18072
|
-
|
|
18073
|
-
|
|
18074
|
-
|
|
18075
|
-
|
|
18076
|
-
|
|
18077
|
-
function
|
|
18078
|
-
const
|
|
18079
|
-
|
|
18080
|
-
}
|
|
18081
|
-
|
|
18082
|
-
|
|
18138
|
+
// src/dupey/index.ts
|
|
18139
|
+
var init_dupey = __esm(() => {
|
|
18140
|
+
init_cli2();
|
|
18141
|
+
init_filter();
|
|
18142
|
+
});
|
|
18143
|
+
|
|
18144
|
+
// src/convert/relate.ts
|
|
18145
|
+
async function convertAndRelate(request, options = {}) {
|
|
18146
|
+
const result = await convertWithFilee(request, options.filee);
|
|
18147
|
+
if (options.dupey === false)
|
|
18148
|
+
return { result, links: [] };
|
|
18149
|
+
const dupeyOptions = options.dupey ?? {};
|
|
18150
|
+
const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
|
|
18151
|
+
const links = await Promise.all(pairs.map(async ({ source, output }) => {
|
|
18152
|
+
try {
|
|
18153
|
+
return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
|
|
18154
|
+
} catch (error) {
|
|
18155
|
+
if (!(error instanceof DupeyCliError))
|
|
18156
|
+
throw error;
|
|
18157
|
+
return { source, output, dupey: { relation: "unavailable", reason: error.message } };
|
|
18158
|
+
}
|
|
18159
|
+
}));
|
|
18160
|
+
return { result, links };
|
|
18083
18161
|
}
|
|
18084
|
-
|
|
18085
|
-
|
|
18162
|
+
var init_relate = __esm(() => {
|
|
18163
|
+
init_dupey();
|
|
18164
|
+
init_cli();
|
|
18165
|
+
});
|
|
18166
|
+
|
|
18167
|
+
// src/convert/index.ts
|
|
18168
|
+
var init_convert = __esm(() => {
|
|
18169
|
+
init_cli();
|
|
18170
|
+
init_installer2();
|
|
18171
|
+
init_matrix();
|
|
18172
|
+
init_relate();
|
|
18173
|
+
});
|
|
18174
|
+
|
|
18175
|
+
// src/datasource/diagnostics.ts
|
|
18176
|
+
function mapDatasourceDiagnostic(ds) {
|
|
18177
|
+
return {
|
|
18178
|
+
code: datasourceCodeToRetrieval(ds.code),
|
|
18179
|
+
severity: ds.severity,
|
|
18180
|
+
message: `[datasource:${ds.code}] ${ds.message}`,
|
|
18181
|
+
source: ds.source ?? (ds.instanceId ? `datasource:${ds.instanceId}` : "datasource")
|
|
18182
|
+
};
|
|
18086
18183
|
}
|
|
18087
|
-
function
|
|
18088
|
-
|
|
18089
|
-
if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
|
|
18090
|
-
return "google-drive";
|
|
18091
|
-
if (lower.includes("onedrive"))
|
|
18092
|
-
return "onedrive";
|
|
18093
|
-
if (lower.includes("dropbox"))
|
|
18094
|
-
return "dropbox";
|
|
18095
|
-
if (lower.includes("mobile documents") || lower.includes("clouddocs"))
|
|
18096
|
-
return "icloud";
|
|
18097
|
-
return "file-provider";
|
|
18184
|
+
function mapDatasourceDiagnostics(diagnostics) {
|
|
18185
|
+
return diagnostics.map(mapDatasourceDiagnostic);
|
|
18098
18186
|
}
|
|
18099
|
-
function
|
|
18100
|
-
|
|
18101
|
-
|
|
18102
|
-
|
|
18103
|
-
|
|
18104
|
-
|
|
18105
|
-
|
|
18106
|
-
|
|
18107
|
-
|
|
18108
|
-
|
|
18109
|
-
|
|
18187
|
+
function datasourceCodeToRetrieval(code) {
|
|
18188
|
+
switch (code) {
|
|
18189
|
+
case "datasource-unavailable":
|
|
18190
|
+
case "datasource-cli-error":
|
|
18191
|
+
case "datasource-empty":
|
|
18192
|
+
case "datasource-rate-limited":
|
|
18193
|
+
case "datasource-auth-error":
|
|
18194
|
+
case "datasource-embedding-egress-rejected":
|
|
18195
|
+
case "datasource-index-failed":
|
|
18196
|
+
case "datasource-index-partial":
|
|
18197
|
+
case "datasource-permission-denied":
|
|
18198
|
+
case "datasource-remote-embedding-rejected":
|
|
18199
|
+
return "retrieval-method-failed";
|
|
18200
|
+
}
|
|
18110
18201
|
}
|
|
18111
|
-
|
|
18112
|
-
|
|
18113
|
-
|
|
18114
|
-
|
|
18115
|
-
|
|
18202
|
+
|
|
18203
|
+
// src/datasource/retry.ts
|
|
18204
|
+
import { setTimeout as delay2 } from "node:timers/promises";
|
|
18205
|
+
function datasourceRetryDelayMs(retry, policy) {
|
|
18206
|
+
const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
|
|
18207
|
+
return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
|
|
18116
18208
|
}
|
|
18117
|
-
function
|
|
18118
|
-
const
|
|
18119
|
-
|
|
18120
|
-
|
|
18121
|
-
|
|
18122
|
-
|
|
18123
|
-
|
|
18209
|
+
async function indexWithRetry(skill, options = {}, signal) {
|
|
18210
|
+
const policy = {
|
|
18211
|
+
maxRetries: options.maxRetries ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxRetries,
|
|
18212
|
+
baseDelayMs: options.baseDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.baseDelayMs,
|
|
18213
|
+
maxDelayMs: options.maxDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxDelayMs
|
|
18214
|
+
};
|
|
18215
|
+
const sleep = options.sleep ?? ((ms) => delay2(ms, undefined, signal === undefined ? undefined : { signal }));
|
|
18216
|
+
let lastError = "";
|
|
18217
|
+
for (let attempt = 1;; attempt += 1) {
|
|
18218
|
+
const result = await runIndex(skill, signal);
|
|
18219
|
+
signal?.throwIfAborted();
|
|
18220
|
+
if (result.ok) {
|
|
18221
|
+
if (attempt === 1)
|
|
18222
|
+
return result;
|
|
18223
|
+
const recovered = {
|
|
18224
|
+
code: "datasource-index-failed",
|
|
18225
|
+
severity: "info",
|
|
18226
|
+
message: `recovered after ${attempt} attempts; last error: ${lastError}`,
|
|
18227
|
+
instanceId: result.instanceId,
|
|
18228
|
+
source: result.skill
|
|
18229
|
+
};
|
|
18230
|
+
return { ...result, attempts: attempt, diagnostics: [...result.diagnostics, recovered] };
|
|
18231
|
+
}
|
|
18232
|
+
lastError = result.message ?? result.error ?? "datasource-index-failed";
|
|
18233
|
+
if (attempt > policy.maxRetries) {
|
|
18234
|
+
if (attempt === 1)
|
|
18235
|
+
return { ...result, attempts: attempt };
|
|
18236
|
+
const suffix = ` (gave up after ${attempt} attempts)`;
|
|
18237
|
+
const last = result.diagnostics.at(-1);
|
|
18238
|
+
return {
|
|
18239
|
+
...result,
|
|
18240
|
+
attempts: attempt,
|
|
18241
|
+
message: `${lastError}${suffix}`,
|
|
18242
|
+
diagnostics: last === undefined ? [
|
|
18243
|
+
{
|
|
18244
|
+
code: result.code ?? "datasource-index-failed",
|
|
18245
|
+
severity: "error",
|
|
18246
|
+
message: `${lastError}${suffix}`,
|
|
18247
|
+
instanceId: result.instanceId,
|
|
18248
|
+
source: result.skill
|
|
18249
|
+
}
|
|
18250
|
+
] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
|
|
18251
|
+
};
|
|
18252
|
+
}
|
|
18253
|
+
await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
|
|
18254
|
+
signal?.throwIfAborted();
|
|
18255
|
+
}
|
|
18124
18256
|
}
|
|
18125
|
-
function
|
|
18257
|
+
async function runIndex(skill, signal) {
|
|
18126
18258
|
try {
|
|
18127
|
-
return
|
|
18128
|
-
} catch {
|
|
18129
|
-
|
|
18259
|
+
return await skill.index(signal === undefined ? undefined : { signal });
|
|
18260
|
+
} catch (error) {
|
|
18261
|
+
signal?.throwIfAborted();
|
|
18262
|
+
const descriptor = skill.describe();
|
|
18263
|
+
const message = error instanceof Error ? error.message : "Datasource indexing failed.";
|
|
18264
|
+
const failed = {
|
|
18265
|
+
ok: false,
|
|
18266
|
+
instanceId: descriptor.instanceId ?? "default",
|
|
18267
|
+
skill: descriptor.name,
|
|
18268
|
+
indexedAt: Date.now(),
|
|
18269
|
+
diagnostics: [
|
|
18270
|
+
{
|
|
18271
|
+
code: "datasource-index-failed",
|
|
18272
|
+
severity: "error",
|
|
18273
|
+
message,
|
|
18274
|
+
source: descriptor.name,
|
|
18275
|
+
instanceId: descriptor.instanceId
|
|
18276
|
+
}
|
|
18277
|
+
],
|
|
18278
|
+
error: "datasource-index-failed",
|
|
18279
|
+
code: "datasource-index-failed",
|
|
18280
|
+
message
|
|
18281
|
+
};
|
|
18282
|
+
return failed;
|
|
18130
18283
|
}
|
|
18131
18284
|
}
|
|
18132
|
-
|
|
18133
|
-
|
|
18134
|
-
|
|
18135
|
-
|
|
18136
|
-
|
|
18137
|
-
|
|
18138
|
-
for (const { raw, canonical } of forms) {
|
|
18139
|
-
if (isPathInside(normalized, raw))
|
|
18140
|
-
return `${canonical}${normalized.slice(raw.length)}`;
|
|
18141
|
-
}
|
|
18142
|
-
return path;
|
|
18285
|
+
var DEFAULT_DATASOURCE_RETRY_POLICY;
|
|
18286
|
+
var init_retry = __esm(() => {
|
|
18287
|
+
DEFAULT_DATASOURCE_RETRY_POLICY = {
|
|
18288
|
+
maxRetries: 2,
|
|
18289
|
+
baseDelayMs: 1000,
|
|
18290
|
+
maxDelayMs: 30000
|
|
18143
18291
|
};
|
|
18292
|
+
});
|
|
18293
|
+
|
|
18294
|
+
// src/everything/bundle.ts
|
|
18295
|
+
import { createHash as createHash11 } from "node:crypto";
|
|
18296
|
+
import { existsSync as existsSync18, mkdirSync as mkdirSync15, readFileSync as readFileSync18, rmSync as rmSync7, writeFileSync as writeFileSync10 } from "node:fs";
|
|
18297
|
+
import { dirname as dirname15, join as join28 } from "node:path";
|
|
18298
|
+
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
18299
|
+
import JSZip from "jszip";
|
|
18300
|
+
function resolveEverythingBundleDir() {
|
|
18301
|
+
let dir = dirname15(fileURLToPath2(import.meta.url));
|
|
18302
|
+
for (let depth = 0;depth < 6; depth += 1) {
|
|
18303
|
+
const candidate = join28(dir, "vendor", "everything");
|
|
18304
|
+
if (existsSync18(join28(candidate, MANIFEST_FILENAME)))
|
|
18305
|
+
return candidate;
|
|
18306
|
+
const parent = dirname15(dir);
|
|
18307
|
+
if (parent === dir)
|
|
18308
|
+
break;
|
|
18309
|
+
dir = parent;
|
|
18310
|
+
}
|
|
18311
|
+
return;
|
|
18144
18312
|
}
|
|
18145
|
-
function
|
|
18146
|
-
|
|
18147
|
-
|
|
18148
|
-
return
|
|
18149
|
-
insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
|
|
18150
|
-
nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
|
|
18151
|
-
};
|
|
18313
|
+
function everythingArch(arch) {
|
|
18314
|
+
if (arch === "x64" || arch === "arm64")
|
|
18315
|
+
return arch;
|
|
18316
|
+
return;
|
|
18152
18317
|
}
|
|
18153
|
-
function
|
|
18154
|
-
|
|
18318
|
+
function everythingBinaryPaths(root, version, arch) {
|
|
18319
|
+
const dir = join28(root, ".autorag", "everything", version, arch);
|
|
18320
|
+
return { everythingPath: join28(dir, "everything.exe"), esPath: join28(dir, "es.exe") };
|
|
18321
|
+
}
|
|
18322
|
+
function loadEverythingBundleManifest(bundleDir) {
|
|
18323
|
+
return JSON.parse(readFileSync18(join28(bundleDir, MANIFEST_FILENAME), "utf8"));
|
|
18324
|
+
}
|
|
18325
|
+
async function ensureEverythingBinaries(options) {
|
|
18326
|
+
const platform = options.platform ?? process.platform;
|
|
18327
|
+
const rawArch = options.arch ?? process.arch;
|
|
18328
|
+
if (platform !== "win32") {
|
|
18329
|
+
return {
|
|
18330
|
+
ok: false,
|
|
18331
|
+
reason: "unsupported-platform",
|
|
18332
|
+
message: `Everything is Windows-only; this host is ${platform}.`
|
|
18333
|
+
};
|
|
18334
|
+
}
|
|
18335
|
+
const arch = everythingArch(rawArch);
|
|
18336
|
+
const bundleDir = options.bundleDir ?? resolveEverythingBundleDir();
|
|
18337
|
+
if (bundleDir === undefined) {
|
|
18338
|
+
return {
|
|
18339
|
+
ok: false,
|
|
18340
|
+
reason: "bundle-missing",
|
|
18341
|
+
message: `Bundled Everything assets (vendor/everything/${MANIFEST_FILENAME}) were not found above ${dirname15(fileURLToPath2(import.meta.url))}.`
|
|
18342
|
+
};
|
|
18343
|
+
}
|
|
18344
|
+
let manifest;
|
|
18155
18345
|
try {
|
|
18156
|
-
|
|
18157
|
-
} catch {
|
|
18158
|
-
return;
|
|
18346
|
+
manifest = options.manifest ?? loadEverythingBundleManifest(bundleDir);
|
|
18347
|
+
} catch (error) {
|
|
18348
|
+
return { ok: false, reason: "bundle-missing", message: errorMessage2(error) };
|
|
18159
18349
|
}
|
|
18160
|
-
|
|
18161
|
-
|
|
18350
|
+
const assets = arch === undefined ? undefined : manifest.architectures[arch];
|
|
18351
|
+
if (arch === undefined || assets === undefined) {
|
|
18352
|
+
return {
|
|
18353
|
+
ok: false,
|
|
18354
|
+
reason: "unsupported-platform",
|
|
18355
|
+
message: `No bundled Everything build for Windows architecture ${rawArch}; bundled: ${Object.keys(manifest.architectures).join(", ")}.`
|
|
18356
|
+
};
|
|
18162
18357
|
}
|
|
18163
|
-
|
|
18164
|
-
|
|
18165
|
-
|
|
18166
|
-
|
|
18167
|
-
|
|
18168
|
-
}
|
|
18169
|
-
|
|
18170
|
-
|
|
18171
|
-
|
|
18172
|
-
|
|
18173
|
-
|
|
18174
|
-
|
|
18175
|
-
|
|
18176
|
-
|
|
18358
|
+
const { everythingPath, esPath } = everythingBinaryPaths(options.root, manifest.everythingVersion, arch);
|
|
18359
|
+
const everythingCached = fileHasSha256(everythingPath, assets.everythingSha256);
|
|
18360
|
+
const esCached = fileHasSha256(esPath, assets.esSha256);
|
|
18361
|
+
if (everythingCached && esCached) {
|
|
18362
|
+
return { ok: true, everythingPath, esPath, source: "cached" };
|
|
18363
|
+
}
|
|
18364
|
+
if ((options.mode ?? "install") === "cached") {
|
|
18365
|
+
const state = everythingCached || esCached ? "are incomplete or do not match their pinned SHA-256 digests" : "are not installed";
|
|
18366
|
+
return {
|
|
18367
|
+
ok: false,
|
|
18368
|
+
reason: "not-installed",
|
|
18369
|
+
message: `Everything ${manifest.everythingVersion} (${arch}) binaries ${state} under ${dirname15(everythingPath)}; run \`autorag refresh\` to (re)install and index them.`
|
|
18370
|
+
};
|
|
18371
|
+
}
|
|
18372
|
+
try {
|
|
18373
|
+
mkdirSync15(dirname15(everythingPath), { recursive: true });
|
|
18374
|
+
const everythingBytes = await extractVerified(join28(bundleDir, assets.everythingArchive), assets.everythingArchiveSha256, assets.everythingMember, assets.everythingSha256);
|
|
18375
|
+
const esBytes = await extractVerified(join28(bundleDir, assets.esArchive), assets.esArchiveSha256, assets.esMember, assets.esSha256);
|
|
18376
|
+
writeAtomic(everythingPath, everythingBytes);
|
|
18377
|
+
writeAtomic(esPath, esBytes);
|
|
18378
|
+
} catch (error) {
|
|
18379
|
+
return { ok: false, reason: "install-failed", message: errorMessage2(error) };
|
|
18380
|
+
}
|
|
18381
|
+
return { ok: true, everythingPath, esPath, source: "installed" };
|
|
18177
18382
|
}
|
|
18178
|
-
function
|
|
18179
|
-
const
|
|
18180
|
-
const
|
|
18181
|
-
if (
|
|
18182
|
-
|
|
18183
|
-
} else if (typeof failure.code === "number") {
|
|
18184
|
-
parts.push(`the helper exited with code ${failure.code}`);
|
|
18383
|
+
async function extractVerified(archivePath, archiveSha256, member, memberSha256) {
|
|
18384
|
+
const archive = readFileSync18(archivePath);
|
|
18385
|
+
const actualArchive = sha256(archive);
|
|
18386
|
+
if (actualArchive !== archiveSha256) {
|
|
18387
|
+
throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
|
|
18185
18388
|
}
|
|
18186
|
-
const
|
|
18187
|
-
|
|
18188
|
-
|
|
18189
|
-
|
|
18190
|
-
|
|
18191
|
-
|
|
18389
|
+
const zip = await JSZip.loadAsync(archive);
|
|
18390
|
+
const entry = zip.file(member);
|
|
18391
|
+
if (entry === null)
|
|
18392
|
+
throw new Error(`${archivePath} does not contain ${member}`);
|
|
18393
|
+
const bytes = await entry.async("nodebuffer");
|
|
18394
|
+
const actualMember = sha256(bytes);
|
|
18395
|
+
if (actualMember !== memberSha256) {
|
|
18396
|
+
throw new Error(`${member} from ${archivePath} SHA-256 ${actualMember} does not match pinned ${memberSha256}`);
|
|
18397
|
+
}
|
|
18398
|
+
return bytes;
|
|
18192
18399
|
}
|
|
18193
|
-
|
|
18194
|
-
|
|
18195
|
-
return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
|
|
18400
|
+
function fileHasSha256(path, expected) {
|
|
18401
|
+
return existsSync18(path) && sha256(readFileSync18(path)) === expected;
|
|
18196
18402
|
}
|
|
18197
|
-
function
|
|
18198
|
-
const
|
|
18199
|
-
|
|
18200
|
-
|
|
18201
|
-
|
|
18202
|
-
|
|
18203
|
-
|
|
18204
|
-
if (open <= 0 || !line.endsWith(")"))
|
|
18205
|
-
continue;
|
|
18206
|
-
const head = line.slice(0, open);
|
|
18207
|
-
const on = head.indexOf(" on ");
|
|
18208
|
-
if (on <= 0)
|
|
18209
|
-
continue;
|
|
18210
|
-
const mountPoint = head.slice(on + " on ".length);
|
|
18211
|
-
if (mountPoint === "")
|
|
18212
|
-
continue;
|
|
18213
|
-
const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
|
|
18214
|
-
mounts.push({
|
|
18215
|
-
device: head.slice(0, on),
|
|
18216
|
-
mountPoint,
|
|
18217
|
-
fileSystem: fields[0] ?? "",
|
|
18218
|
-
options: fields.slice(1)
|
|
18219
|
-
});
|
|
18403
|
+
function writeAtomic(path, bytes) {
|
|
18404
|
+
const temp = `${path}.${process.pid}.tmp`;
|
|
18405
|
+
try {
|
|
18406
|
+
writeFileSync10(temp, bytes);
|
|
18407
|
+
renameWithRetry(temp, path);
|
|
18408
|
+
} finally {
|
|
18409
|
+
rmSync7(temp, { force: true });
|
|
18220
18410
|
}
|
|
18221
|
-
return mounts;
|
|
18222
18411
|
}
|
|
18223
|
-
function
|
|
18224
|
-
|
|
18225
|
-
const seen = new Set;
|
|
18226
|
-
for (const mount of mounts) {
|
|
18227
|
-
if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
|
|
18228
|
-
continue;
|
|
18229
|
-
if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
|
|
18230
|
-
continue;
|
|
18231
|
-
if (seen.has(mount.mountPoint))
|
|
18232
|
-
continue;
|
|
18233
|
-
seen.add(mount.mountPoint);
|
|
18234
|
-
points.push(mount.mountPoint);
|
|
18235
|
-
}
|
|
18236
|
-
return points;
|
|
18412
|
+
function sha256(bytes) {
|
|
18413
|
+
return createHash11("sha256").update(bytes).digest("hex");
|
|
18237
18414
|
}
|
|
18238
|
-
function
|
|
18239
|
-
|
|
18240
|
-
|
|
18241
|
-
|
|
18242
|
-
|
|
18243
|
-
|
|
18244
|
-
|
|
18245
|
-
|
|
18246
|
-
|
|
18247
|
-
|
|
18248
|
-
|
|
18249
|
-
|
|
18250
|
-
|
|
18251
|
-
|
|
18252
|
-
|
|
18253
|
-
}
|
|
18254
|
-
|
|
18415
|
+
function errorMessage2(error) {
|
|
18416
|
+
return error instanceof Error ? error.message : String(error);
|
|
18417
|
+
}
|
|
18418
|
+
var MANIFEST_FILENAME = "manifest.json";
|
|
18419
|
+
var init_bundle = __esm(() => {
|
|
18420
|
+
init_atomic_write();
|
|
18421
|
+
});
|
|
18422
|
+
|
|
18423
|
+
// src/filesystem/path-exclusions.ts
|
|
18424
|
+
import { posix as posix3, win32 as win322 } from "node:path";
|
|
18425
|
+
function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
|
|
18426
|
+
const paths = platform === "win32" ? win322 : posix3;
|
|
18427
|
+
const normalize = (path) => {
|
|
18428
|
+
const absolute = paths.resolve(path);
|
|
18429
|
+
return platform === "win32" ? absolute.toLowerCase() : absolute;
|
|
18430
|
+
};
|
|
18431
|
+
const excluded = excludedPaths.map((path) => {
|
|
18432
|
+
const absolute = normalize(path);
|
|
18433
|
+
return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
|
|
18434
|
+
});
|
|
18435
|
+
return (path) => {
|
|
18436
|
+
const absolute = normalize(path);
|
|
18437
|
+
return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
|
|
18438
|
+
};
|
|
18255
18439
|
}
|
|
18440
|
+
var init_path_exclusions = () => {};
|
|
18256
18441
|
|
|
18257
|
-
|
|
18258
|
-
|
|
18259
|
-
|
|
18260
|
-
|
|
18261
|
-
|
|
18262
|
-
|
|
18263
|
-
|
|
18264
|
-
|
|
18442
|
+
// src/filesystem/shared-exclusive-queue.ts
|
|
18443
|
+
class SharedExclusiveQueue {
|
|
18444
|
+
sharedSinceExclusive = [];
|
|
18445
|
+
lastExclusive = Promise.resolve();
|
|
18446
|
+
shared(task) {
|
|
18447
|
+
const run = this.lastExclusive.then(task);
|
|
18448
|
+
const settled = run.then(() => {
|
|
18449
|
+
return;
|
|
18450
|
+
}, () => {
|
|
18451
|
+
return;
|
|
18452
|
+
});
|
|
18453
|
+
this.sharedSinceExclusive.push(settled);
|
|
18454
|
+
return run;
|
|
18265
18455
|
}
|
|
18266
|
-
|
|
18267
|
-
const
|
|
18268
|
-
const
|
|
18269
|
-
const
|
|
18270
|
-
|
|
18271
|
-
|
|
18272
|
-
|
|
18273
|
-
|
|
18274
|
-
|
|
18275
|
-
|
|
18276
|
-
|
|
18277
|
-
const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
|
|
18278
|
-
for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
|
|
18279
|
-
if (entry.isDirectory())
|
|
18280
|
-
add(join28(cloudStorage, entry.name), "file-provider");
|
|
18281
|
-
}
|
|
18282
|
-
if (await this.tryReadDir(mobileDocuments) !== undefined)
|
|
18283
|
-
add(mobileDocuments, "file-provider");
|
|
18284
|
-
for (const mountPoint of await this.networkMountPoints())
|
|
18285
|
-
add(mountPoint, "network");
|
|
18286
|
-
for (const directory of await this.fileProviderDirectories(searchRoots))
|
|
18287
|
-
add(directory, "file-provider");
|
|
18288
|
-
return roots;
|
|
18456
|
+
exclusive(task) {
|
|
18457
|
+
const earlier = Promise.all([this.lastExclusive, ...this.sharedSinceExclusive]);
|
|
18458
|
+
const run = earlier.then(task);
|
|
18459
|
+
const settled = run.then(() => {
|
|
18460
|
+
return;
|
|
18461
|
+
}, () => {
|
|
18462
|
+
return;
|
|
18463
|
+
});
|
|
18464
|
+
this.sharedSinceExclusive = [];
|
|
18465
|
+
this.lastExclusive = settled;
|
|
18466
|
+
return run;
|
|
18289
18467
|
}
|
|
18290
|
-
|
|
18291
|
-
|
|
18292
|
-
|
|
18293
|
-
|
|
18294
|
-
|
|
18295
|
-
|
|
18296
|
-
|
|
18468
|
+
}
|
|
18469
|
+
|
|
18470
|
+
// src/everything/client.ts
|
|
18471
|
+
import { spawn as spawn15 } from "node:child_process";
|
|
18472
|
+
import { createHash as createHash12 } from "node:crypto";
|
|
18473
|
+
import { mkdirSync as mkdirSync16, writeFileSync as writeFileSync11 } from "node:fs";
|
|
18474
|
+
import { join as join29, resolve as resolve11 } from "node:path";
|
|
18475
|
+
import { setTimeout as sleep2 } from "node:timers/promises";
|
|
18476
|
+
function everythingInstanceName(root) {
|
|
18477
|
+
return `autorag-${createHash12("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
|
|
18478
|
+
}
|
|
18479
|
+
function buildEverythingIni(input) {
|
|
18480
|
+
const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
|
|
18481
|
+
const perFolder = (value) => input.folders.map(() => value).join(",");
|
|
18482
|
+
return [
|
|
18483
|
+
"[Everything]",
|
|
18484
|
+
"app_data=0",
|
|
18485
|
+
"run_as_admin=0",
|
|
18486
|
+
"run_in_background=1",
|
|
18487
|
+
"show_tray_icon=0",
|
|
18488
|
+
"show_in_taskbar=0",
|
|
18489
|
+
"check_for_updates_on_startup=0",
|
|
18490
|
+
"beta_updates=0",
|
|
18491
|
+
"language=1033",
|
|
18492
|
+
"allow_http_server=0",
|
|
18493
|
+
"allow_etp_server=0",
|
|
18494
|
+
"search_history_enabled=0",
|
|
18495
|
+
"run_history_enabled=0",
|
|
18496
|
+
"auto_include_fixed_volumes=0",
|
|
18497
|
+
"auto_include_removable_volumes=0",
|
|
18498
|
+
"auto_include_fixed_refs_volumes=0",
|
|
18499
|
+
"auto_include_removable_refs_volumes=0",
|
|
18500
|
+
"ntfs_volume_paths=",
|
|
18501
|
+
"ntfs_volume_includes=",
|
|
18502
|
+
"refs_volume_paths=",
|
|
18503
|
+
"refs_volume_includes=",
|
|
18504
|
+
"index_size=1",
|
|
18505
|
+
"index_date_modified=1",
|
|
18506
|
+
`folders=${list(input.folders)}`,
|
|
18507
|
+
`folder_monitor_changes=${perFolder("1")}`,
|
|
18508
|
+
`folder_rescan_if_full_list=${perFolder("1")}`,
|
|
18509
|
+
`folder_update_types=${perFolder("0")}`,
|
|
18510
|
+
"exclude_list_enabled=1",
|
|
18511
|
+
`exclude_folders=${list(input.excludeFolders)}`,
|
|
18512
|
+
""
|
|
18513
|
+
].join(`\r
|
|
18514
|
+
`);
|
|
18515
|
+
}
|
|
18516
|
+
function buildEverythingSearchArgs(instance, request, timeoutMs) {
|
|
18517
|
+
const args = [
|
|
18518
|
+
"-argv",
|
|
18519
|
+
"-instance",
|
|
18520
|
+
instance,
|
|
18521
|
+
"-cp",
|
|
18522
|
+
"65001",
|
|
18523
|
+
"-json",
|
|
18524
|
+
"-full-path-and-name",
|
|
18525
|
+
"-size",
|
|
18526
|
+
"-date-modified",
|
|
18527
|
+
"-date-format",
|
|
18528
|
+
"1"
|
|
18529
|
+
];
|
|
18530
|
+
if (timeoutMs !== undefined)
|
|
18531
|
+
args.push("-timeout", String(timeoutMs));
|
|
18532
|
+
if (request.matchCase)
|
|
18533
|
+
args.push("-case");
|
|
18534
|
+
if (request.matchPath)
|
|
18535
|
+
args.push("-match-path");
|
|
18536
|
+
if (request.wholeWord)
|
|
18537
|
+
args.push("-whole-word");
|
|
18538
|
+
if (request.kind === "files")
|
|
18539
|
+
args.push("/a-d");
|
|
18540
|
+
if (request.kind === "folders")
|
|
18541
|
+
args.push("/ad");
|
|
18542
|
+
if (request.path !== undefined)
|
|
18543
|
+
args.push("-path", request.path);
|
|
18544
|
+
if (request.sort !== undefined)
|
|
18545
|
+
args.push("-sort", request.sort);
|
|
18546
|
+
if (request.offset !== undefined && request.offset > 0)
|
|
18547
|
+
args.push("-offset", String(request.offset));
|
|
18548
|
+
args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
|
|
18549
|
+
if (request.regex)
|
|
18550
|
+
args.push("-regex", "--", request.query);
|
|
18551
|
+
else
|
|
18552
|
+
args.push("--", request.query);
|
|
18553
|
+
return args;
|
|
18554
|
+
}
|
|
18555
|
+
function parseEverythingJson(stdout) {
|
|
18556
|
+
const text = stdout.trim();
|
|
18557
|
+
if (text.length === 0)
|
|
18558
|
+
return [];
|
|
18559
|
+
let parsed;
|
|
18560
|
+
try {
|
|
18561
|
+
parsed = JSON.parse(text);
|
|
18562
|
+
} catch {
|
|
18563
|
+
throw new Error(`ES did not return JSON: ${text.slice(0, 2000)}`);
|
|
18297
18564
|
}
|
|
18298
|
-
|
|
18299
|
-
|
|
18300
|
-
|
|
18301
|
-
const
|
|
18302
|
-
|
|
18303
|
-
|
|
18304
|
-
|
|
18305
|
-
|
|
18306
|
-
|
|
18307
|
-
|
|
18308
|
-
|
|
18309
|
-
|
|
18310
|
-
|
|
18311
|
-
|
|
18312
|
-
|
|
18313
|
-
|
|
18314
|
-
|
|
18315
|
-
|
|
18316
|
-
|
|
18317
|
-
|
|
18318
|
-
|
|
18319
|
-
|
|
18320
|
-
|
|
18321
|
-
|
|
18565
|
+
if (!Array.isArray(parsed))
|
|
18566
|
+
throw new Error(`ES did not return a JSON array: ${text.slice(0, 2000)}`);
|
|
18567
|
+
return parsed.map((item) => {
|
|
18568
|
+
const record = item;
|
|
18569
|
+
const filename = String(record.filename ?? "");
|
|
18570
|
+
const isFolder = filename.endsWith("\\") && filename.length > 3;
|
|
18571
|
+
return {
|
|
18572
|
+
path: isFolder ? filename.slice(0, -1) : filename,
|
|
18573
|
+
type: filename.endsWith("\\") ? "folder" : "file",
|
|
18574
|
+
size: typeof record.size === "number" ? record.size : undefined,
|
|
18575
|
+
dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
|
|
18576
|
+
};
|
|
18577
|
+
});
|
|
18578
|
+
}
|
|
18579
|
+
|
|
18580
|
+
class EverythingClient {
|
|
18581
|
+
instanceName;
|
|
18582
|
+
options;
|
|
18583
|
+
platform;
|
|
18584
|
+
run;
|
|
18585
|
+
launch;
|
|
18586
|
+
stateDir;
|
|
18587
|
+
isExcludedPath;
|
|
18588
|
+
binaries;
|
|
18589
|
+
queue = new SharedExclusiveQueue;
|
|
18590
|
+
constructor(options) {
|
|
18591
|
+
this.options = options;
|
|
18592
|
+
this.platform = options.platform ?? process.platform;
|
|
18593
|
+
this.instanceName = everythingInstanceName(options.root);
|
|
18594
|
+
this.run = options.run ?? runProcess;
|
|
18595
|
+
this.launch = options.launch ?? launchDetached;
|
|
18596
|
+
this.stateDir = join29(options.root, ".autorag", "everything");
|
|
18597
|
+
this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
|
|
18322
18598
|
}
|
|
18323
|
-
|
|
18324
|
-
|
|
18325
|
-
const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
|
|
18326
|
-
timeoutMs: 15000,
|
|
18327
|
-
maxBuffer: 8 * 1024 * 1024
|
|
18328
|
-
});
|
|
18329
|
-
return parseXattrValues(stdout, paths);
|
|
18330
|
-
} catch (error) {
|
|
18331
|
-
const failure = error;
|
|
18332
|
-
return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
|
|
18333
|
-
}
|
|
18599
|
+
isSupported() {
|
|
18600
|
+
return this.platform === "win32";
|
|
18334
18601
|
}
|
|
18335
|
-
|
|
18336
|
-
|
|
18337
|
-
|
|
18338
|
-
|
|
18339
|
-
|
|
18340
|
-
|
|
18602
|
+
search(request) {
|
|
18603
|
+
return this.queue.shared(async () => {
|
|
18604
|
+
const binaries = await this.resolveBinaries("cached");
|
|
18605
|
+
if (!binaries.ok)
|
|
18606
|
+
return binaries;
|
|
18607
|
+
const ping = await this.ping(binaries.esPath);
|
|
18608
|
+
if (ping.code !== 0) {
|
|
18609
|
+
return {
|
|
18610
|
+
ok: false,
|
|
18611
|
+
reason: "not-running",
|
|
18612
|
+
message: `Everything instance ${this.instanceName} is not running; run \`autorag refresh\` to start and index it.`
|
|
18613
|
+
};
|
|
18614
|
+
}
|
|
18615
|
+
const es = { ok: true, esPath: binaries.esPath };
|
|
18616
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
18617
|
+
if ((this.options.excludeFolders?.length ?? 0) > 0) {
|
|
18618
|
+
return this.searchWithExclusions(es.esPath, request, timeoutMs);
|
|
18619
|
+
}
|
|
18620
|
+
const args = buildEverythingSearchArgs(this.instanceName, request, timeoutMs);
|
|
18621
|
+
const result = await this.run(es.esPath, args, timeoutMs + 5000);
|
|
18622
|
+
if (result.code !== 0)
|
|
18623
|
+
return this.failure("search-failed", es.esPath, args, result);
|
|
18624
|
+
try {
|
|
18625
|
+
return { ok: true, results: parseEverythingJson(result.stdout) };
|
|
18626
|
+
} catch (error) {
|
|
18627
|
+
return { ok: false, reason: "search-failed", message: error.message };
|
|
18628
|
+
}
|
|
18629
|
+
});
|
|
18341
18630
|
}
|
|
18342
|
-
async
|
|
18343
|
-
const
|
|
18344
|
-
|
|
18345
|
-
|
|
18346
|
-
|
|
18347
|
-
|
|
18348
|
-
|
|
18349
|
-
|
|
18350
|
-
|
|
18351
|
-
|
|
18352
|
-
|
|
18353
|
-
|
|
18354
|
-
|
|
18631
|
+
async searchWithExclusions(esPath, request, timeoutMs) {
|
|
18632
|
+
const deadline = Date.now() + timeoutMs;
|
|
18633
|
+
const offset = request.offset ?? 0;
|
|
18634
|
+
const maxResults = request.maxResults ?? DEFAULT_MAX_RESULTS;
|
|
18635
|
+
const results = [];
|
|
18636
|
+
let skipped = 0;
|
|
18637
|
+
let nativeOffset = 0;
|
|
18638
|
+
const timedOut = () => ({
|
|
18639
|
+
ok: false,
|
|
18640
|
+
reason: "search-failed",
|
|
18641
|
+
message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
|
|
18642
|
+
});
|
|
18643
|
+
for (;; ) {
|
|
18644
|
+
const remainingMs = deadline - Date.now();
|
|
18645
|
+
if (remainingMs <= 0)
|
|
18646
|
+
return timedOut();
|
|
18647
|
+
const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
|
|
18648
|
+
const response = await this.run(esPath, args, remainingMs + 5000);
|
|
18649
|
+
if (response.code !== 0)
|
|
18650
|
+
return this.failure("search-failed", esPath, args, response);
|
|
18651
|
+
if (Date.now() >= deadline)
|
|
18652
|
+
return timedOut();
|
|
18653
|
+
let entries;
|
|
18654
|
+
try {
|
|
18655
|
+
entries = parseEverythingJson(response.stdout);
|
|
18656
|
+
} catch (error) {
|
|
18657
|
+
return { ok: false, reason: "search-failed", message: error.message };
|
|
18658
|
+
}
|
|
18659
|
+
for (const entry of entries) {
|
|
18660
|
+
if (this.isExcludedPath(entry.path))
|
|
18661
|
+
continue;
|
|
18662
|
+
if (skipped < offset) {
|
|
18663
|
+
skipped += 1;
|
|
18664
|
+
continue;
|
|
18665
|
+
}
|
|
18666
|
+
results.push(entry);
|
|
18667
|
+
if (results.length >= maxResults)
|
|
18668
|
+
return { ok: true, results };
|
|
18355
18669
|
}
|
|
18670
|
+
if (entries.length < MAX_FILTER_CANDIDATES)
|
|
18671
|
+
return { ok: true, results };
|
|
18672
|
+
nativeOffset += entries.length;
|
|
18356
18673
|
}
|
|
18357
|
-
return dataless;
|
|
18358
18674
|
}
|
|
18359
|
-
|
|
18360
|
-
|
|
18361
|
-
|
|
18675
|
+
index() {
|
|
18676
|
+
return this.queue.exclusive(async () => {
|
|
18677
|
+
const es = await this.ensureRunning(true);
|
|
18678
|
+
if (!es.ok)
|
|
18679
|
+
return es;
|
|
18680
|
+
const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
|
|
18681
|
+
const saveArgs = ["-instance", this.instanceName, "-save-db"];
|
|
18682
|
+
const saved = await this.run(es.esPath, saveArgs, timeoutMs);
|
|
18683
|
+
if (saved.code !== 0)
|
|
18684
|
+
return this.failure("index-failed", es.esPath, saveArgs, saved);
|
|
18685
|
+
const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
|
|
18686
|
+
const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
|
|
18687
|
+
if (counted.code !== 0)
|
|
18688
|
+
return this.failure("index-failed", es.esPath, countArgs, counted);
|
|
18689
|
+
return {
|
|
18690
|
+
ok: true,
|
|
18691
|
+
indexedFolders: [...this.options.folders],
|
|
18692
|
+
indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
|
|
18693
|
+
};
|
|
18694
|
+
});
|
|
18362
18695
|
}
|
|
18363
|
-
|
|
18364
|
-
return
|
|
18696
|
+
stop() {
|
|
18697
|
+
return this.queue.exclusive(async () => {
|
|
18698
|
+
const binaries = await this.resolveBinaries("cached");
|
|
18699
|
+
if (!binaries.ok)
|
|
18700
|
+
return;
|
|
18701
|
+
await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
18702
|
+
});
|
|
18365
18703
|
}
|
|
18366
|
-
async
|
|
18367
|
-
|
|
18368
|
-
try {
|
|
18369
|
-
stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
|
|
18370
|
-
timeoutMs: 30000
|
|
18371
|
-
})).stdout;
|
|
18372
|
-
} catch (error) {
|
|
18373
|
-
return { ok: false, message: describeReleaseHelperFailure(error) };
|
|
18374
|
-
}
|
|
18375
|
-
const outcome = parseReleaseOutcome(stdout);
|
|
18376
|
-
if (outcome === undefined) {
|
|
18377
|
-
const detail = stdout.trim();
|
|
18704
|
+
async resolveBinaries(mode) {
|
|
18705
|
+
if (!this.isSupported()) {
|
|
18378
18706
|
return {
|
|
18379
18707
|
ok: false,
|
|
18380
|
-
|
|
18708
|
+
reason: "unsupported-platform",
|
|
18709
|
+
message: `Everything is Windows-only; this host is ${this.platform}.`
|
|
18381
18710
|
};
|
|
18382
18711
|
}
|
|
18383
|
-
|
|
18384
|
-
|
|
18385
|
-
}
|
|
18386
|
-
|
|
18387
|
-
|
|
18388
|
-
|
|
18389
|
-
|
|
18390
|
-
`;
|
|
18391
|
-
}
|
|
18392
|
-
function buildWindowsSyncRootsScript() {
|
|
18393
|
-
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
18394
|
-
` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
|
|
18395
|
-
` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
|
|
18396
|
-
`;
|
|
18397
|
-
}
|
|
18398
|
-
function buildWindowsDehydrateScript(path) {
|
|
18399
|
-
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
18400
|
-
` + `$ErrorActionPreference='Stop'
|
|
18401
|
-
` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
|
|
18402
|
-
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
|
|
18403
|
-
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
|
|
18404
|
-
` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
|
|
18405
|
-
` + `'@
|
|
18406
|
-
` + `$h=[IntPtr]::Zero
|
|
18407
|
-
` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
|
|
18408
|
-
` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
|
|
18409
|
-
` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
|
|
18410
|
-
` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
|
|
18411
|
-
` + `[Autorag.CfApi]::CfCloseHandle($h)
|
|
18412
|
-
` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
|
|
18413
|
-
` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
|
|
18414
|
-
`;
|
|
18415
|
-
}
|
|
18416
|
-
function parseWindowsAttributeOutput(stdout, paths) {
|
|
18417
|
-
const lines = stdout.split(/\r?\n/);
|
|
18418
|
-
const attributes = new Map;
|
|
18419
|
-
for (const [index, path] of paths.entries()) {
|
|
18420
|
-
const value = Number.parseInt((lines[index] ?? "").trim(), 10);
|
|
18421
|
-
if (Number.isFinite(value))
|
|
18422
|
-
attributes.set(path, value);
|
|
18423
|
-
}
|
|
18424
|
-
return attributes;
|
|
18425
|
-
}
|
|
18426
|
-
function parseWindowsSyncRoots(stdout) {
|
|
18427
|
-
const roots = [];
|
|
18428
|
-
for (const line of stdout.split(/\r?\n/)) {
|
|
18429
|
-
const tab = line.indexOf("\t");
|
|
18430
|
-
if (tab <= 0)
|
|
18431
|
-
continue;
|
|
18432
|
-
const path = line.slice(0, tab).trim();
|
|
18433
|
-
if (path !== "")
|
|
18434
|
-
roots.push({ path, label: line.slice(tab + 1).trim() });
|
|
18435
|
-
}
|
|
18436
|
-
return roots;
|
|
18437
|
-
}
|
|
18438
|
-
function providerFromWindowsSyncRoot(label) {
|
|
18439
|
-
const lower = label.toLowerCase();
|
|
18440
|
-
if (lower.includes("onedrive") || lower.includes("one drive"))
|
|
18441
|
-
return "onedrive";
|
|
18442
|
-
if (lower.includes("google"))
|
|
18443
|
-
return "google-drive";
|
|
18444
|
-
return "file-provider";
|
|
18445
|
-
}
|
|
18446
|
-
function buildWindowsDiskScript() {
|
|
18447
|
-
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
18448
|
-
` + "Get-CimInstance -ClassName Win32_LogicalDisk | ForEach-Object {" + `$_.DeviceID + [char]9 + $_.DriveType + [char]9 + $_.FileSystem + [char]9 + $_.VolumeName}
|
|
18449
|
-
`;
|
|
18450
|
-
}
|
|
18451
|
-
function parseWindowsLogicalDisks(stdout) {
|
|
18452
|
-
const disks = [];
|
|
18453
|
-
for (const line of stdout.split(/\r?\n/)) {
|
|
18454
|
-
if (line.trim() === "")
|
|
18455
|
-
continue;
|
|
18456
|
-
const [deviceId, driveType, fileSystem, ...rest] = line.split("\t");
|
|
18457
|
-
const type = Number.parseInt((driveType ?? "").trim(), 10);
|
|
18458
|
-
if (!Number.isFinite(type))
|
|
18459
|
-
continue;
|
|
18460
|
-
disks.push({
|
|
18461
|
-
deviceId: (deviceId ?? "").trim(),
|
|
18462
|
-
driveType: type,
|
|
18463
|
-
fileSystem: (fileSystem ?? "").trim(),
|
|
18464
|
-
volumeName: rest.join("\t").trim()
|
|
18465
|
-
});
|
|
18712
|
+
if (this.binaries?.ok)
|
|
18713
|
+
return this.binaries;
|
|
18714
|
+
const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
|
|
18715
|
+
const result = await resolve(mode);
|
|
18716
|
+
if (result.ok)
|
|
18717
|
+
this.binaries = result;
|
|
18718
|
+
return result;
|
|
18466
18719
|
}
|
|
18467
|
-
|
|
18468
|
-
|
|
18469
|
-
|
|
18470
|
-
|
|
18471
|
-
|
|
18472
|
-
|
|
18473
|
-
|
|
18474
|
-
|
|
18475
|
-
|
|
18476
|
-
|
|
18477
|
-
|
|
18478
|
-
|
|
18720
|
+
async ensureRunning(reconfigure) {
|
|
18721
|
+
const binaries = await this.resolveBinaries("install");
|
|
18722
|
+
if (!binaries.ok)
|
|
18723
|
+
return binaries;
|
|
18724
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
18725
|
+
let ping = await this.ping(binaries.esPath);
|
|
18726
|
+
if (ping.code === 0 && !reconfigure)
|
|
18727
|
+
return { ok: true, esPath: binaries.esPath };
|
|
18728
|
+
if (ping.code === 0) {
|
|
18729
|
+
const exitArgs = ["-instance", this.instanceName, "-exit"];
|
|
18730
|
+
const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
|
|
18731
|
+
if (exited.code !== 0)
|
|
18732
|
+
return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
|
|
18479
18733
|
}
|
|
18480
|
-
|
|
18481
|
-
|
|
18482
|
-
|
|
18734
|
+
mkdirSync16(this.stateDir, { recursive: true });
|
|
18735
|
+
const iniPath = join29(this.stateDir, "Everything.ini");
|
|
18736
|
+
writeFileSync11(iniPath, buildEverythingIni({
|
|
18737
|
+
folders: this.options.folders,
|
|
18738
|
+
excludeFolders: [
|
|
18739
|
+
...new Set([
|
|
18740
|
+
join29(this.options.root, ".autorag"),
|
|
18741
|
+
...this.options.folders.map((folder) => join29(folder, ".autorag")),
|
|
18742
|
+
...this.options.excludeFolders ?? []
|
|
18743
|
+
])
|
|
18744
|
+
]
|
|
18745
|
+
}));
|
|
18746
|
+
const launchArgs = [
|
|
18747
|
+
"-instance",
|
|
18748
|
+
this.instanceName,
|
|
18749
|
+
"-config",
|
|
18750
|
+
iniPath,
|
|
18751
|
+
"-db",
|
|
18752
|
+
join29(this.stateDir, "Everything.db"),
|
|
18753
|
+
"-startup"
|
|
18754
|
+
];
|
|
18755
|
+
try {
|
|
18756
|
+
this.launch(binaries.everythingPath, launchArgs);
|
|
18757
|
+
} catch (error) {
|
|
18758
|
+
return {
|
|
18759
|
+
ok: false,
|
|
18760
|
+
reason: "startup-failed",
|
|
18761
|
+
message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
|
|
18762
|
+
};
|
|
18483
18763
|
}
|
|
18484
|
-
|
|
18485
|
-
|
|
18764
|
+
const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
|
|
18765
|
+
const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
18766
|
+
for (;; ) {
|
|
18767
|
+
ping = await this.ping(binaries.esPath);
|
|
18768
|
+
if (ping.code === 0)
|
|
18769
|
+
return { ok: true, esPath: binaries.esPath };
|
|
18770
|
+
if (Date.now() >= deadline)
|
|
18771
|
+
break;
|
|
18772
|
+
await sleep2(pollMs);
|
|
18486
18773
|
}
|
|
18487
|
-
}
|
|
18488
|
-
return roots;
|
|
18489
|
-
}
|
|
18490
|
-
function uncShareRoot(target) {
|
|
18491
|
-
const normalized = target.replaceAll("/", "\\");
|
|
18492
|
-
if (!normalized.startsWith("\\\\") || normalized.startsWith("\\\\?\\") || normalized.startsWith("\\\\.\\"))
|
|
18493
|
-
return;
|
|
18494
|
-
const rest = normalized.slice(2);
|
|
18495
|
-
const separator = rest.indexOf("\\");
|
|
18496
|
-
if (separator <= 0)
|
|
18497
|
-
return;
|
|
18498
|
-
const server = rest.slice(0, separator);
|
|
18499
|
-
const share = rest.slice(separator + 1).split("\\")[0] ?? "";
|
|
18500
|
-
if (server === "" || share === "")
|
|
18501
|
-
return;
|
|
18502
|
-
return `\\\\${server}\\${share}`;
|
|
18503
|
-
}
|
|
18504
|
-
function parseWindowsDehydrateOutput(result) {
|
|
18505
|
-
const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
18506
|
-
if (open === null)
|
|
18507
|
-
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
18508
|
-
if (Number.parseInt(open[1], 16) !== 0) {
|
|
18509
18774
|
return {
|
|
18510
18775
|
ok: false,
|
|
18511
|
-
|
|
18776
|
+
reason: "startup-failed",
|
|
18777
|
+
message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
|
|
18512
18778
|
};
|
|
18513
18779
|
}
|
|
18514
|
-
|
|
18515
|
-
|
|
18516
|
-
return
|
|
18780
|
+
async ping(esPath) {
|
|
18781
|
+
const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
18782
|
+
return result.code === 0 && !VERSION_PATTERN.test(result.stdout.trim()) ? { ...result, code: -1 } : result;
|
|
18517
18783
|
}
|
|
18518
|
-
|
|
18519
|
-
|
|
18520
|
-
|
|
18521
|
-
|
|
18522
|
-
|
|
18523
|
-
|
|
18524
|
-
|
|
18784
|
+
failure(reason, esPath, args, result) {
|
|
18785
|
+
return {
|
|
18786
|
+
ok: false,
|
|
18787
|
+
reason,
|
|
18788
|
+
message: `${esPath} ${args.join(" ")} exit ${result.code}: ${(result.stderr || result.stdout).trim()}`
|
|
18789
|
+
};
|
|
18790
|
+
}
|
|
18791
|
+
}
|
|
18792
|
+
function runProcess(command, args, timeoutMs) {
|
|
18793
|
+
let resolvePromise;
|
|
18794
|
+
const promise = new Promise((resolve) => {
|
|
18795
|
+
resolvePromise = resolve;
|
|
18796
|
+
});
|
|
18797
|
+
const child = spawn15(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
|
|
18798
|
+
const stdout = [];
|
|
18799
|
+
const stderr = [];
|
|
18800
|
+
let bytes = 0;
|
|
18801
|
+
let overflow = false;
|
|
18802
|
+
const collect = (target) => (chunk) => {
|
|
18803
|
+
bytes += chunk.length;
|
|
18804
|
+
if (bytes > MAX_OUTPUT_BYTES) {
|
|
18805
|
+
overflow = true;
|
|
18806
|
+
child.kill();
|
|
18807
|
+
return;
|
|
18808
|
+
}
|
|
18809
|
+
target.push(chunk);
|
|
18525
18810
|
};
|
|
18811
|
+
child.stdout.on("data", collect(stdout));
|
|
18812
|
+
child.stderr.on("data", collect(stderr));
|
|
18813
|
+
const timer = setTimeout(() => child.kill(), timeoutMs);
|
|
18814
|
+
child.on("error", (error) => {
|
|
18815
|
+
clearTimeout(timer);
|
|
18816
|
+
resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
|
|
18817
|
+
});
|
|
18818
|
+
child.on("close", (code, signal) => {
|
|
18819
|
+
clearTimeout(timer);
|
|
18820
|
+
const err = Buffer.concat(stderr).toString("utf8");
|
|
18821
|
+
resolvePromise({
|
|
18822
|
+
code,
|
|
18823
|
+
stdout: Buffer.concat(stdout).toString("utf8"),
|
|
18824
|
+
stderr: overflow ? `${err}
|
|
18825
|
+
output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
|
|
18826
|
+
killed by ${signal} after ${timeoutMs}ms` : err
|
|
18827
|
+
});
|
|
18828
|
+
});
|
|
18829
|
+
return promise;
|
|
18526
18830
|
}
|
|
18527
|
-
function
|
|
18528
|
-
const
|
|
18529
|
-
|
|
18530
|
-
|
|
18531
|
-
const stderr = result.stderr.trim();
|
|
18532
|
-
return stderr !== "" ? stderr : fallback;
|
|
18831
|
+
function launchDetached(command, args) {
|
|
18832
|
+
const child = spawn15(command, args, { detached: true, stdio: "ignore", windowsHide: true });
|
|
18833
|
+
child.on("error", () => {});
|
|
18834
|
+
child.unref();
|
|
18533
18835
|
}
|
|
18836
|
+
var DEFAULT_TIMEOUT_MS8 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
|
|
18837
|
+
var init_client9 = __esm(() => {
|
|
18838
|
+
init_path_exclusions();
|
|
18839
|
+
init_bundle();
|
|
18840
|
+
DEFAULT_INDEX_TIMEOUT_MS3 = 10 * 60000;
|
|
18841
|
+
MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
|
|
18842
|
+
VERSION_PATTERN = /^\d+\.\d+\.\d+\.\d+$/;
|
|
18843
|
+
});
|
|
18534
18844
|
|
|
18535
|
-
|
|
18536
|
-
|
|
18537
|
-
|
|
18538
|
-
|
|
18539
|
-
|
|
18540
|
-
|
|
18541
|
-
|
|
18542
|
-
|
|
18543
|
-
|
|
18544
|
-
|
|
18545
|
-
|
|
18546
|
-
|
|
18547
|
-
|
|
18548
|
-
|
|
18549
|
-
|
|
18550
|
-
|
|
18551
|
-
roots.push({ path: canonical, kind });
|
|
18552
|
-
};
|
|
18553
|
-
for (const root of await this.loadSyncRoots())
|
|
18554
|
-
add(root.path, "cloud-files");
|
|
18555
|
-
for (const root of await this.loadDiskRoots())
|
|
18556
|
-
add(root.path, root.kind);
|
|
18557
|
-
for (const searchRoot of searchRoots) {
|
|
18558
|
-
const share = uncShareRoot(searchRoot);
|
|
18559
|
-
if (share !== undefined)
|
|
18560
|
-
add(share, "network");
|
|
18561
|
-
}
|
|
18562
|
-
return roots;
|
|
18563
|
-
}
|
|
18564
|
-
async datalessAmong(paths) {
|
|
18565
|
-
const dataless = new Set;
|
|
18566
|
-
for (let start = 0;start < paths.length; start += WINDOWS_ATTRIBUTE_BATCH) {
|
|
18567
|
-
const batch = paths.slice(start, start + WINDOWS_ATTRIBUTE_BATCH);
|
|
18568
|
-
const result = await this.run(buildWindowsAttributeScript(batch), 30000);
|
|
18569
|
-
if (!result.ok) {
|
|
18570
|
-
throw new Error(`Could not read Windows cloud file attributes: ${result.stderr.trim() || "powershell.exe failed"}`);
|
|
18571
|
-
}
|
|
18572
|
-
for (const [path, attributes] of parseWindowsAttributeOutput(result.stdout, batch)) {
|
|
18573
|
-
if (datalessFromWindowsAttributes(attributes))
|
|
18574
|
-
dataless.add(path);
|
|
18575
|
-
}
|
|
18576
|
-
}
|
|
18577
|
-
return dataless;
|
|
18578
|
-
}
|
|
18579
|
-
async providerOf(path) {
|
|
18580
|
-
const root = longestSyncRoot(await this.loadSyncRoots(), path);
|
|
18581
|
-
return root === undefined ? cloudProviderFromPath(path) : providerFromWindowsSyncRoot(root.label);
|
|
18582
|
-
}
|
|
18583
|
-
canRelease(provider) {
|
|
18584
|
-
return provider !== undefined;
|
|
18585
|
-
}
|
|
18586
|
-
async evict(path) {
|
|
18587
|
-
return parseWindowsDehydrateOutput(await this.run(buildWindowsDehydrateScript(path), 60000));
|
|
18588
|
-
}
|
|
18589
|
-
loadSyncRoots() {
|
|
18590
|
-
this.syncRoots ??= this.run(buildWindowsSyncRootsScript(), 15000).then((result) => result.ok ? parseWindowsSyncRoots(result.stdout) : [], () => []);
|
|
18591
|
-
return this.syncRoots;
|
|
18592
|
-
}
|
|
18593
|
-
loadDiskRoots() {
|
|
18594
|
-
this.disks ??= this.run(buildWindowsDiskScript(), 30000).then((result) => result.ok ? windowsRootsFromDisks(parseWindowsLogicalDisks(result.stdout)) : [], () => []);
|
|
18595
|
-
return this.disks;
|
|
18596
|
-
}
|
|
18845
|
+
// src/everything/index.ts
|
|
18846
|
+
var init_everything = __esm(() => {
|
|
18847
|
+
init_bundle();
|
|
18848
|
+
init_client9();
|
|
18849
|
+
});
|
|
18850
|
+
|
|
18851
|
+
// src/filesystem/cloud-placeholder.ts
|
|
18852
|
+
import { execFile as execFile2 } from "node:child_process";
|
|
18853
|
+
import { realpathSync as realpathSync3 } from "node:fs";
|
|
18854
|
+
import { lstat, readdir } from "node:fs/promises";
|
|
18855
|
+
import { homedir as homedir6 } from "node:os";
|
|
18856
|
+
import { dirname as dirname16, join as join30 } from "node:path";
|
|
18857
|
+
import { promisify } from "node:util";
|
|
18858
|
+
function homeCloudRoots(home = homedir6()) {
|
|
18859
|
+
const normalizedHome = home.replaceAll("\\", "/").replace(/\/+$/, "");
|
|
18860
|
+
return [`${normalizedHome}/Library/CloudStorage`, `${normalizedHome}/Library/Mobile Documents`];
|
|
18597
18861
|
}
|
|
18598
|
-
function
|
|
18599
|
-
|
|
18600
|
-
let match;
|
|
18601
|
-
let matchLength = -1;
|
|
18602
|
-
for (const root of roots) {
|
|
18603
|
-
const prefix = root.path.replaceAll("/", "\\").replace(/\\+$/u, "").toLowerCase();
|
|
18604
|
-
if (prefix === "" || prefix.length <= matchLength)
|
|
18605
|
-
continue;
|
|
18606
|
-
if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
|
|
18607
|
-
match = root;
|
|
18608
|
-
matchLength = prefix.length;
|
|
18609
|
-
}
|
|
18610
|
-
}
|
|
18611
|
-
return match;
|
|
18862
|
+
function datalessFromDarwinFlags(flags) {
|
|
18863
|
+
return Number.isFinite(flags) && (flags >>> 0 & SF_DATALESS) !== 0;
|
|
18612
18864
|
}
|
|
18613
|
-
function
|
|
18614
|
-
return
|
|
18615
|
-
args: [
|
|
18616
|
-
"-NoProfile",
|
|
18617
|
-
"-NonInteractive",
|
|
18618
|
-
"-Command",
|
|
18619
|
-
`[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; Invoke-Expression $env:${POWERSHELL_SCRIPT_VARIABLE}`
|
|
18620
|
-
],
|
|
18621
|
-
env: { [POWERSHELL_SCRIPT_VARIABLE]: script }
|
|
18622
|
-
};
|
|
18865
|
+
function datalessFromWindowsAttributes(attributes) {
|
|
18866
|
+
return Number.isFinite(attributes) && (attributes & WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) !== 0;
|
|
18623
18867
|
}
|
|
18624
|
-
|
|
18625
|
-
const
|
|
18626
|
-
|
|
18627
|
-
|
|
18628
|
-
|
|
18629
|
-
|
|
18630
|
-
|
|
18631
|
-
|
|
18632
|
-
|
|
18633
|
-
return
|
|
18634
|
-
|
|
18635
|
-
const record = typeof error === "object" && error !== null ? error : {};
|
|
18636
|
-
return {
|
|
18637
|
-
ok: false,
|
|
18638
|
-
stdout: typeof record.stdout === "string" ? record.stdout : "",
|
|
18639
|
-
stderr: typeof record.stderr === "string" ? record.stderr : String(error)
|
|
18640
|
-
};
|
|
18641
|
-
}
|
|
18868
|
+
function cloudProviderFromPath(target) {
|
|
18869
|
+
const lower = target.toLowerCase();
|
|
18870
|
+
if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
|
|
18871
|
+
return "google-drive";
|
|
18872
|
+
if (lower.includes("onedrive"))
|
|
18873
|
+
return "onedrive";
|
|
18874
|
+
if (lower.includes("dropbox"))
|
|
18875
|
+
return "dropbox";
|
|
18876
|
+
if (lower.includes("mobile documents") || lower.includes("clouddocs"))
|
|
18877
|
+
return "icloud";
|
|
18878
|
+
return "file-provider";
|
|
18642
18879
|
}
|
|
18643
|
-
function
|
|
18644
|
-
|
|
18645
|
-
|
|
18646
|
-
|
|
18647
|
-
|
|
18648
|
-
|
|
18880
|
+
function cloudProviderFromDomain(domain) {
|
|
18881
|
+
const lower = domain.toLowerCase();
|
|
18882
|
+
if (lower.includes("clouddocs") || lower.includes("icloud"))
|
|
18883
|
+
return "icloud";
|
|
18884
|
+
if (lower.includes("google"))
|
|
18885
|
+
return "google-drive";
|
|
18886
|
+
if (lower.includes("onedrive"))
|
|
18887
|
+
return "onedrive";
|
|
18888
|
+
if (lower.includes("dropbox"))
|
|
18889
|
+
return "dropbox";
|
|
18890
|
+
return "file-provider";
|
|
18649
18891
|
}
|
|
18650
|
-
|
|
18892
|
+
function normalizeCloudPath(value) {
|
|
18893
|
+
let normalized = value.replaceAll("\\", "/");
|
|
18894
|
+
if (normalized.length > 1)
|
|
18895
|
+
normalized = normalized.replace(/\/+$/u, "");
|
|
18896
|
+
return normalized;
|
|
18897
|
+
}
|
|
18898
|
+
function isPathInside(target, parent) {
|
|
18899
|
+
const normalizedParent = normalizeCloudPath(parent);
|
|
18900
|
+
if (normalizedParent === "")
|
|
18901
|
+
return false;
|
|
18902
|
+
const left = normalizeCloudPath(target);
|
|
18903
|
+
const [a, b] = process.platform === "win32" ? [left.toLowerCase(), normalizedParent.toLowerCase()] : [left, normalizedParent];
|
|
18904
|
+
return a === b || a.startsWith(`${b}/`);
|
|
18905
|
+
}
|
|
18906
|
+
function canonicalPath(value) {
|
|
18651
18907
|
try {
|
|
18652
|
-
|
|
18653
|
-
const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
|
|
18654
|
-
timeout: 2000
|
|
18655
|
-
});
|
|
18656
|
-
const value = stdout.trim();
|
|
18657
|
-
return value.length > 0 ? value : undefined;
|
|
18908
|
+
return realpathSync3(value);
|
|
18658
18909
|
} catch {
|
|
18659
|
-
return;
|
|
18660
|
-
}
|
|
18661
|
-
}
|
|
18662
|
-
async function readFileProviderDomainUpwards(target) {
|
|
18663
|
-
let cursor = target;
|
|
18664
|
-
for (let depth = 0;depth < 12; depth += 1) {
|
|
18665
|
-
const domain = await readFileProviderDomain(cursor);
|
|
18666
|
-
if (domain !== undefined)
|
|
18667
|
-
return domain;
|
|
18668
|
-
const parent = dirname15(cursor);
|
|
18669
|
-
if (parent === cursor)
|
|
18670
|
-
return;
|
|
18671
|
-
cursor = parent;
|
|
18910
|
+
return value;
|
|
18672
18911
|
}
|
|
18673
|
-
return;
|
|
18674
18912
|
}
|
|
18675
|
-
function
|
|
18676
|
-
|
|
18677
|
-
|
|
18678
|
-
|
|
18679
|
-
|
|
18680
|
-
|
|
18681
|
-
|
|
18682
|
-
|
|
18683
|
-
|
|
18684
|
-
|
|
18685
|
-
|
|
18686
|
-
if (released) return JSON.stringify({ ok: true });
|
|
18687
|
-
function resourceValue(key) {
|
|
18688
|
-
var ref = Ref();
|
|
18689
|
-
if (!url.getResourceValueForKeyError(ref, key, null)) return null;
|
|
18690
|
-
var value = ref[0];
|
|
18691
|
-
return value === undefined || value === null ? null : ObjC.unwrap(value);
|
|
18692
|
-
}
|
|
18693
|
-
return JSON.stringify({
|
|
18694
|
-
ok: false,
|
|
18695
|
-
isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
|
|
18696
|
-
isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
|
|
18697
|
-
downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
|
|
18698
|
-
});
|
|
18699
|
-
}`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
|
|
18700
|
-
var init_cloud_placeholder = __esm(() => {
|
|
18701
|
-
execFileAsync = promisify(execFile2);
|
|
18702
|
-
RELEASABLE_DARWIN_PROVIDERS = {
|
|
18703
|
-
icloud: true,
|
|
18704
|
-
"google-drive": true,
|
|
18705
|
-
onedrive: true,
|
|
18706
|
-
dropbox: true
|
|
18913
|
+
function canonicalPathMapper(searchRoots) {
|
|
18914
|
+
const forms = searchRoots.map((root) => ({ raw: normalizeCloudPath(root), canonical: normalizeCloudPath(canonicalPath(root)) })).filter(({ raw, canonical }) => raw !== "" && raw !== canonical).sort((left, right) => right.raw.length - left.raw.length);
|
|
18915
|
+
if (forms.length === 0)
|
|
18916
|
+
return (path) => path;
|
|
18917
|
+
return (path) => {
|
|
18918
|
+
const normalized = normalizeCloudPath(path);
|
|
18919
|
+
for (const { raw, canonical } of forms) {
|
|
18920
|
+
if (isPathInside(normalized, raw))
|
|
18921
|
+
return `${canonical}${normalized.slice(raw.length)}`;
|
|
18922
|
+
}
|
|
18923
|
+
return path;
|
|
18707
18924
|
};
|
|
18708
|
-
|
|
18709
|
-
|
|
18710
|
-
|
|
18711
|
-
|
|
18712
|
-
|
|
18925
|
+
}
|
|
18926
|
+
function cloudScope(searchRoot, cloudRoots) {
|
|
18927
|
+
const canonicalSearchRoot = canonicalPath(searchRoot);
|
|
18928
|
+
const roots = cloudRoots.map((root) => ({ path: canonicalPath(root.path), kind: root.kind }));
|
|
18929
|
+
return {
|
|
18930
|
+
insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
|
|
18931
|
+
nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
|
|
18713
18932
|
};
|
|
18714
|
-
}
|
|
18715
|
-
|
|
18716
|
-
|
|
18717
|
-
import { basename as basename7 } from "node:path";
|
|
18718
|
-
async function listCloudRoots(probe, roots) {
|
|
18719
|
-
if (probe === undefined)
|
|
18720
|
-
return [];
|
|
18933
|
+
}
|
|
18934
|
+
function parseReleaseOutcome(stdout) {
|
|
18935
|
+
let parsed;
|
|
18721
18936
|
try {
|
|
18722
|
-
|
|
18937
|
+
parsed = JSON.parse(stdout.trim());
|
|
18723
18938
|
} catch {
|
|
18724
|
-
return
|
|
18939
|
+
return;
|
|
18725
18940
|
}
|
|
18726
|
-
|
|
18727
|
-
|
|
18728
|
-
const targets = [];
|
|
18729
|
-
const skipped = [];
|
|
18730
|
-
for (const path of roots) {
|
|
18731
|
-
const scope = cloudScope(path, cloudRoots);
|
|
18732
|
-
if (scope.insideCloudRoot) {
|
|
18733
|
-
skipped.push(path);
|
|
18734
|
-
continue;
|
|
18735
|
-
}
|
|
18736
|
-
targets.push({ path, excludeDirs: scope.nestedCloudRoots.map((root) => basename7(root.path)) });
|
|
18941
|
+
if (typeof parsed !== "object" || parsed === null || !("ok" in parsed) || typeof parsed.ok !== "boolean") {
|
|
18942
|
+
return;
|
|
18737
18943
|
}
|
|
18738
|
-
return { targets, skipped };
|
|
18739
|
-
}
|
|
18740
|
-
var init_scan_plan = __esm(() => {
|
|
18741
|
-
init_cloud_placeholder();
|
|
18742
|
-
});
|
|
18743
|
-
|
|
18744
|
-
// src/dupey/index.ts
|
|
18745
|
-
var init_dupey = __esm(() => {
|
|
18746
|
-
init_cli2();
|
|
18747
|
-
init_filter();
|
|
18748
|
-
init_scan_plan();
|
|
18749
|
-
});
|
|
18750
|
-
|
|
18751
|
-
// src/convert/relate.ts
|
|
18752
|
-
async function convertAndRelate(request, options = {}) {
|
|
18753
|
-
const result = await convertWithFilee(request, options.filee);
|
|
18754
|
-
if (options.dupey === false)
|
|
18755
|
-
return { result, links: [] };
|
|
18756
|
-
const dupeyOptions = options.dupey ?? {};
|
|
18757
|
-
const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
|
|
18758
|
-
const links = await Promise.all(pairs.map(async ({ source, output }) => {
|
|
18759
|
-
try {
|
|
18760
|
-
return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
|
|
18761
|
-
} catch (error) {
|
|
18762
|
-
if (!(error instanceof DupeyCliError))
|
|
18763
|
-
throw error;
|
|
18764
|
-
return { source, output, dupey: { relation: "unavailable", reason: error.message } };
|
|
18765
|
-
}
|
|
18766
|
-
}));
|
|
18767
|
-
return { result, links };
|
|
18768
|
-
}
|
|
18769
|
-
var init_relate = __esm(() => {
|
|
18770
|
-
init_dupey();
|
|
18771
|
-
init_cli();
|
|
18772
|
-
});
|
|
18773
|
-
|
|
18774
|
-
// src/convert/index.ts
|
|
18775
|
-
var init_convert = __esm(() => {
|
|
18776
|
-
init_cli();
|
|
18777
|
-
init_installer2();
|
|
18778
|
-
init_matrix();
|
|
18779
|
-
init_relate();
|
|
18780
|
-
});
|
|
18781
|
-
|
|
18782
|
-
// src/datasource/diagnostics.ts
|
|
18783
|
-
function mapDatasourceDiagnostic(ds) {
|
|
18784
18944
|
return {
|
|
18785
|
-
|
|
18786
|
-
|
|
18787
|
-
|
|
18788
|
-
|
|
18945
|
+
ok: parsed.ok,
|
|
18946
|
+
..."isUploaded" in parsed ? { isUploaded: parsed.isUploaded } : {},
|
|
18947
|
+
..."isUploading" in parsed ? { isUploading: parsed.isUploading } : {},
|
|
18948
|
+
..."downloadingStatus" in parsed ? { downloadingStatus: parsed.downloadingStatus } : {}
|
|
18789
18949
|
};
|
|
18790
18950
|
}
|
|
18791
|
-
function
|
|
18792
|
-
|
|
18951
|
+
function describeReleaseRefusal(outcome) {
|
|
18952
|
+
if (outcome.isUploading === true)
|
|
18953
|
+
return "The file is still uploading to the cloud, so it cannot be released yet.";
|
|
18954
|
+
if (outcome.isUploaded === false)
|
|
18955
|
+
return "The file has not finished uploading to the cloud, so it cannot be released yet.";
|
|
18956
|
+
const status = typeof outcome.downloadingStatus === "string" ? outcome.downloadingStatus : "unknown";
|
|
18957
|
+
return `The cloud provider declined to release the local copy (download status ${status}).`;
|
|
18793
18958
|
}
|
|
18794
|
-
function
|
|
18795
|
-
|
|
18796
|
-
|
|
18797
|
-
|
|
18798
|
-
|
|
18799
|
-
|
|
18800
|
-
|
|
18801
|
-
case "datasource-embedding-egress-rejected":
|
|
18802
|
-
case "datasource-index-failed":
|
|
18803
|
-
case "datasource-index-partial":
|
|
18804
|
-
case "datasource-permission-denied":
|
|
18805
|
-
case "datasource-remote-embedding-rejected":
|
|
18806
|
-
return "retrieval-method-failed";
|
|
18959
|
+
function describeReleaseHelperFailure(error) {
|
|
18960
|
+
const failure = error;
|
|
18961
|
+
const parts = [];
|
|
18962
|
+
if (failure.killed === true || typeof failure.signal === "string") {
|
|
18963
|
+
parts.push(`the helper was killed${typeof failure.signal === "string" ? ` by ${failure.signal}` : ""}`);
|
|
18964
|
+
} else if (typeof failure.code === "number") {
|
|
18965
|
+
parts.push(`the helper exited with code ${failure.code}`);
|
|
18807
18966
|
}
|
|
18967
|
+
const stderr = typeof failure.stderr === "string" ? failure.stderr.trim() : "";
|
|
18968
|
+
if (stderr !== "")
|
|
18969
|
+
parts.push(stderr);
|
|
18970
|
+
if (parts.length === 0)
|
|
18971
|
+
parts.push(error instanceof Error ? error.message : String(error));
|
|
18972
|
+
return `osascript could not release the local copy: ${parts.join("; ")}`;
|
|
18808
18973
|
}
|
|
18809
|
-
|
|
18810
|
-
|
|
18811
|
-
|
|
18812
|
-
function datasourceRetryDelayMs(retry, policy) {
|
|
18813
|
-
const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
|
|
18814
|
-
return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
|
|
18974
|
+
async function defaultDarwinReadDir(path) {
|
|
18975
|
+
const entries = await readdir(path, { withFileTypes: true });
|
|
18976
|
+
return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
|
|
18815
18977
|
}
|
|
18816
|
-
|
|
18817
|
-
const
|
|
18818
|
-
|
|
18819
|
-
|
|
18820
|
-
|
|
18821
|
-
|
|
18822
|
-
|
|
18823
|
-
|
|
18824
|
-
|
|
18825
|
-
const
|
|
18826
|
-
|
|
18827
|
-
if (
|
|
18828
|
-
|
|
18829
|
-
|
|
18830
|
-
|
|
18831
|
-
|
|
18832
|
-
|
|
18833
|
-
|
|
18834
|
-
|
|
18835
|
-
|
|
18836
|
-
|
|
18837
|
-
|
|
18838
|
-
}
|
|
18839
|
-
lastError = result.message ?? result.error ?? "datasource-index-failed";
|
|
18840
|
-
if (attempt > policy.maxRetries) {
|
|
18841
|
-
if (attempt === 1)
|
|
18842
|
-
return { ...result, attempts: attempt };
|
|
18843
|
-
const suffix = ` (gave up after ${attempt} attempts)`;
|
|
18844
|
-
const last = result.diagnostics.at(-1);
|
|
18845
|
-
return {
|
|
18846
|
-
...result,
|
|
18847
|
-
attempts: attempt,
|
|
18848
|
-
message: `${lastError}${suffix}`,
|
|
18849
|
-
diagnostics: last === undefined ? [
|
|
18850
|
-
{
|
|
18851
|
-
code: result.code ?? "datasource-index-failed",
|
|
18852
|
-
severity: "error",
|
|
18853
|
-
message: `${lastError}${suffix}`,
|
|
18854
|
-
instanceId: result.instanceId,
|
|
18855
|
-
source: result.skill
|
|
18856
|
-
}
|
|
18857
|
-
] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
|
|
18858
|
-
};
|
|
18859
|
-
}
|
|
18860
|
-
await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
|
|
18861
|
-
signal?.throwIfAborted();
|
|
18978
|
+
function parseDarwinMounts(stdout) {
|
|
18979
|
+
const mounts = [];
|
|
18980
|
+
for (const line of stdout.split(`
|
|
18981
|
+
`)) {
|
|
18982
|
+
if (line.trim() === "")
|
|
18983
|
+
continue;
|
|
18984
|
+
const open = line.lastIndexOf(" (");
|
|
18985
|
+
if (open <= 0 || !line.endsWith(")"))
|
|
18986
|
+
continue;
|
|
18987
|
+
const head = line.slice(0, open);
|
|
18988
|
+
const on = head.indexOf(" on ");
|
|
18989
|
+
if (on <= 0)
|
|
18990
|
+
continue;
|
|
18991
|
+
const mountPoint = head.slice(on + " on ".length);
|
|
18992
|
+
if (mountPoint === "")
|
|
18993
|
+
continue;
|
|
18994
|
+
const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
|
|
18995
|
+
mounts.push({
|
|
18996
|
+
device: head.slice(0, on),
|
|
18997
|
+
mountPoint,
|
|
18998
|
+
fileSystem: fields[0] ?? "",
|
|
18999
|
+
options: fields.slice(1)
|
|
19000
|
+
});
|
|
18862
19001
|
}
|
|
19002
|
+
return mounts;
|
|
18863
19003
|
}
|
|
18864
|
-
|
|
18865
|
-
|
|
18866
|
-
|
|
18867
|
-
|
|
18868
|
-
|
|
18869
|
-
|
|
18870
|
-
|
|
18871
|
-
|
|
18872
|
-
|
|
18873
|
-
|
|
18874
|
-
|
|
18875
|
-
|
|
18876
|
-
diagnostics: [
|
|
18877
|
-
{
|
|
18878
|
-
code: "datasource-index-failed",
|
|
18879
|
-
severity: "error",
|
|
18880
|
-
message,
|
|
18881
|
-
source: descriptor.name,
|
|
18882
|
-
instanceId: descriptor.instanceId
|
|
18883
|
-
}
|
|
18884
|
-
],
|
|
18885
|
-
error: "datasource-index-failed",
|
|
18886
|
-
code: "datasource-index-failed",
|
|
18887
|
-
message
|
|
18888
|
-
};
|
|
18889
|
-
return failed;
|
|
19004
|
+
function networkMountPoints(mounts) {
|
|
19005
|
+
const points = [];
|
|
19006
|
+
const seen = new Set;
|
|
19007
|
+
for (const mount of mounts) {
|
|
19008
|
+
if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
|
|
19009
|
+
continue;
|
|
19010
|
+
if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
|
|
19011
|
+
continue;
|
|
19012
|
+
if (seen.has(mount.mountPoint))
|
|
19013
|
+
continue;
|
|
19014
|
+
seen.add(mount.mountPoint);
|
|
19015
|
+
points.push(mount.mountPoint);
|
|
18890
19016
|
}
|
|
19017
|
+
return points;
|
|
18891
19018
|
}
|
|
18892
|
-
|
|
18893
|
-
|
|
18894
|
-
|
|
18895
|
-
|
|
18896
|
-
|
|
18897
|
-
|
|
18898
|
-
|
|
18899
|
-
|
|
18900
|
-
|
|
18901
|
-
|
|
18902
|
-
|
|
18903
|
-
|
|
18904
|
-
|
|
18905
|
-
|
|
18906
|
-
|
|
18907
|
-
function resolveEverythingBundleDir() {
|
|
18908
|
-
let dir = dirname16(fileURLToPath2(import.meta.url));
|
|
18909
|
-
for (let depth = 0;depth < 6; depth += 1) {
|
|
18910
|
-
const candidate = join29(dir, "vendor", "everything");
|
|
18911
|
-
if (existsSync18(join29(candidate, MANIFEST_FILENAME)))
|
|
18912
|
-
return candidate;
|
|
18913
|
-
const parent = dirname16(dir);
|
|
18914
|
-
if (parent === dir)
|
|
18915
|
-
break;
|
|
18916
|
-
dir = parent;
|
|
19019
|
+
function parseXattrValues(stdout, candidates) {
|
|
19020
|
+
const values = new Map;
|
|
19021
|
+
if (candidates.length === 0)
|
|
19022
|
+
return values;
|
|
19023
|
+
const ordered = [...candidates].sort((left, right) => right.length - left.length);
|
|
19024
|
+
for (const line of stdout.split(`
|
|
19025
|
+
`)) {
|
|
19026
|
+
if (line === "")
|
|
19027
|
+
continue;
|
|
19028
|
+
for (const candidate of ordered) {
|
|
19029
|
+
if (line.startsWith(`${candidate}: `)) {
|
|
19030
|
+
values.set(candidate, line.slice(candidate.length + 2));
|
|
19031
|
+
break;
|
|
19032
|
+
}
|
|
19033
|
+
}
|
|
18917
19034
|
}
|
|
18918
|
-
return;
|
|
18919
|
-
}
|
|
18920
|
-
function everythingArch(arch) {
|
|
18921
|
-
if (arch === "x64" || arch === "arm64")
|
|
18922
|
-
return arch;
|
|
18923
|
-
return;
|
|
18924
|
-
}
|
|
18925
|
-
function everythingBinaryPaths(root, version, arch) {
|
|
18926
|
-
const dir = join29(root, ".autorag", "everything", version, arch);
|
|
18927
|
-
return { everythingPath: join29(dir, "everything.exe"), esPath: join29(dir, "es.exe") };
|
|
18928
|
-
}
|
|
18929
|
-
function loadEverythingBundleManifest(bundleDir) {
|
|
18930
|
-
return JSON.parse(readFileSync18(join29(bundleDir, MANIFEST_FILENAME), "utf8"));
|
|
19035
|
+
return values;
|
|
18931
19036
|
}
|
|
18932
|
-
|
|
18933
|
-
|
|
18934
|
-
|
|
18935
|
-
|
|
18936
|
-
|
|
18937
|
-
|
|
18938
|
-
|
|
18939
|
-
|
|
18940
|
-
|
|
19037
|
+
|
|
19038
|
+
class DarwinCloudProbe {
|
|
19039
|
+
exec;
|
|
19040
|
+
readDir;
|
|
19041
|
+
home;
|
|
19042
|
+
constructor(options = {}) {
|
|
19043
|
+
this.exec = options.exec ?? defaultCloudExec;
|
|
19044
|
+
this.readDir = options.readDir ?? defaultDarwinReadDir;
|
|
19045
|
+
this.home = options.home ?? homedir6();
|
|
18941
19046
|
}
|
|
18942
|
-
|
|
18943
|
-
|
|
18944
|
-
|
|
18945
|
-
|
|
18946
|
-
|
|
18947
|
-
|
|
18948
|
-
|
|
19047
|
+
async cloudRoots(searchRoots = []) {
|
|
19048
|
+
const roots = [];
|
|
19049
|
+
const seen = new Set;
|
|
19050
|
+
const add = (path, kind) => {
|
|
19051
|
+
const canonical = canonicalPath(path);
|
|
19052
|
+
const key = normalizeCloudPath(canonical);
|
|
19053
|
+
if (key === "" || seen.has(key))
|
|
19054
|
+
return;
|
|
19055
|
+
seen.add(key);
|
|
19056
|
+
roots.push({ path: canonical, kind });
|
|
18949
19057
|
};
|
|
19058
|
+
const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
|
|
19059
|
+
for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
|
|
19060
|
+
if (entry.isDirectory())
|
|
19061
|
+
add(join30(cloudStorage, entry.name), "file-provider");
|
|
19062
|
+
}
|
|
19063
|
+
if (await this.tryReadDir(mobileDocuments) !== undefined)
|
|
19064
|
+
add(mobileDocuments, "file-provider");
|
|
19065
|
+
for (const mountPoint of await this.networkMountPoints())
|
|
19066
|
+
add(mountPoint, "network");
|
|
19067
|
+
for (const directory of await this.fileProviderDirectories(searchRoots))
|
|
19068
|
+
add(directory, "file-provider");
|
|
19069
|
+
return roots;
|
|
18950
19070
|
}
|
|
18951
|
-
|
|
18952
|
-
|
|
18953
|
-
|
|
18954
|
-
|
|
18955
|
-
|
|
19071
|
+
async networkMountPoints() {
|
|
19072
|
+
try {
|
|
19073
|
+
const { stdout } = await this.exec("/sbin/mount", [], { timeoutMs: 1e4, maxBuffer: 4 * 1024 * 1024 });
|
|
19074
|
+
return networkMountPoints(parseDarwinMounts(stdout));
|
|
19075
|
+
} catch {
|
|
19076
|
+
return [];
|
|
19077
|
+
}
|
|
18956
19078
|
}
|
|
18957
|
-
|
|
18958
|
-
|
|
18959
|
-
|
|
18960
|
-
|
|
18961
|
-
|
|
18962
|
-
|
|
18963
|
-
|
|
19079
|
+
async fileProviderDirectories(searchRoots) {
|
|
19080
|
+
if (searchRoots.length === 0)
|
|
19081
|
+
return [];
|
|
19082
|
+
const candidates = new Set;
|
|
19083
|
+
for (const searchRoot of searchRoots) {
|
|
19084
|
+
const canonical = canonicalPath(searchRoot);
|
|
19085
|
+
for (let cursor = canonical;; cursor = dirname16(cursor)) {
|
|
19086
|
+
candidates.add(cursor);
|
|
19087
|
+
const parent = dirname16(cursor);
|
|
19088
|
+
if (parent === cursor)
|
|
19089
|
+
break;
|
|
19090
|
+
}
|
|
19091
|
+
for (const entry of await this.tryReadDir(canonical) ?? []) {
|
|
19092
|
+
if (entry.isDirectory())
|
|
19093
|
+
candidates.add(join30(canonical, entry.name));
|
|
19094
|
+
}
|
|
19095
|
+
}
|
|
19096
|
+
const paths = [...candidates];
|
|
19097
|
+
const managed = [];
|
|
19098
|
+
for (let start = 0;start < paths.length; start += XATTR_BATCH) {
|
|
19099
|
+
for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
|
|
19100
|
+
managed.push(path);
|
|
19101
|
+
}
|
|
19102
|
+
return managed;
|
|
18964
19103
|
}
|
|
18965
|
-
|
|
18966
|
-
|
|
18967
|
-
|
|
18968
|
-
|
|
18969
|
-
|
|
19104
|
+
async xattrValues(paths) {
|
|
19105
|
+
try {
|
|
19106
|
+
const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
|
|
19107
|
+
timeoutMs: 15000,
|
|
19108
|
+
maxBuffer: 8 * 1024 * 1024
|
|
19109
|
+
});
|
|
19110
|
+
return parseXattrValues(stdout, paths);
|
|
19111
|
+
} catch (error) {
|
|
19112
|
+
const failure = error;
|
|
19113
|
+
return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
|
|
19114
|
+
}
|
|
18970
19115
|
}
|
|
18971
|
-
|
|
18972
|
-
|
|
18973
|
-
|
|
18974
|
-
|
|
18975
|
-
|
|
18976
|
-
|
|
18977
|
-
};
|
|
19116
|
+
async tryReadDir(path) {
|
|
19117
|
+
try {
|
|
19118
|
+
return await this.readDir(path);
|
|
19119
|
+
} catch {
|
|
19120
|
+
return;
|
|
19121
|
+
}
|
|
18978
19122
|
}
|
|
18979
|
-
|
|
18980
|
-
|
|
18981
|
-
|
|
18982
|
-
|
|
18983
|
-
|
|
18984
|
-
|
|
18985
|
-
|
|
18986
|
-
|
|
19123
|
+
async datalessAmong(paths) {
|
|
19124
|
+
const dataless = new Set;
|
|
19125
|
+
for (let start = 0;start < paths.length; start += STAT_BATCH) {
|
|
19126
|
+
const batch = paths.slice(start, start + STAT_BATCH);
|
|
19127
|
+
const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
|
|
19128
|
+
timeoutMs: 30000,
|
|
19129
|
+
maxBuffer: 4 * 1024 * 1024
|
|
19130
|
+
});
|
|
19131
|
+
const flags = stdout.split(`
|
|
19132
|
+
`).filter(Boolean);
|
|
19133
|
+
for (const [index, path] of batch.entries()) {
|
|
19134
|
+
if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
|
|
19135
|
+
dataless.add(path);
|
|
19136
|
+
}
|
|
19137
|
+
}
|
|
19138
|
+
return dataless;
|
|
18987
19139
|
}
|
|
18988
|
-
|
|
18989
|
-
|
|
18990
|
-
|
|
18991
|
-
const archive = readFileSync18(archivePath);
|
|
18992
|
-
const actualArchive = sha256(archive);
|
|
18993
|
-
if (actualArchive !== archiveSha256) {
|
|
18994
|
-
throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
|
|
19140
|
+
async providerOf(path) {
|
|
19141
|
+
const domain = await readFileProviderDomainUpwards(path);
|
|
19142
|
+
return domain === undefined ? undefined : cloudProviderFromDomain(domain);
|
|
18995
19143
|
}
|
|
18996
|
-
|
|
18997
|
-
|
|
18998
|
-
|
|
18999
|
-
|
|
19000
|
-
|
|
19001
|
-
|
|
19002
|
-
|
|
19003
|
-
|
|
19144
|
+
canRelease(provider) {
|
|
19145
|
+
return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
|
|
19146
|
+
}
|
|
19147
|
+
async evict(path) {
|
|
19148
|
+
let stdout;
|
|
19149
|
+
try {
|
|
19150
|
+
stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
|
|
19151
|
+
timeoutMs: 30000
|
|
19152
|
+
})).stdout;
|
|
19153
|
+
} catch (error) {
|
|
19154
|
+
return { ok: false, message: describeReleaseHelperFailure(error) };
|
|
19155
|
+
}
|
|
19156
|
+
const outcome = parseReleaseOutcome(stdout);
|
|
19157
|
+
if (outcome === undefined) {
|
|
19158
|
+
const detail = stdout.trim();
|
|
19159
|
+
return {
|
|
19160
|
+
ok: false,
|
|
19161
|
+
message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
|
|
19162
|
+
};
|
|
19163
|
+
}
|
|
19164
|
+
return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
|
|
19004
19165
|
}
|
|
19005
|
-
return bytes;
|
|
19006
19166
|
}
|
|
19007
|
-
function
|
|
19008
|
-
return
|
|
19167
|
+
function buildWindowsAttributeScript(paths) {
|
|
19168
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19169
|
+
` + `$ErrorActionPreference='SilentlyContinue'
|
|
19170
|
+
` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
|
|
19171
|
+
`;
|
|
19009
19172
|
}
|
|
19010
|
-
function
|
|
19011
|
-
|
|
19012
|
-
|
|
19013
|
-
|
|
19014
|
-
|
|
19015
|
-
|
|
19016
|
-
|
|
19173
|
+
function buildWindowsSyncRootsScript() {
|
|
19174
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19175
|
+
` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
|
|
19176
|
+
` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
|
|
19177
|
+
`;
|
|
19178
|
+
}
|
|
19179
|
+
function buildWindowsDehydrateScript(path) {
|
|
19180
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19181
|
+
` + `$ErrorActionPreference='Stop'
|
|
19182
|
+
` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
|
|
19183
|
+
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
|
|
19184
|
+
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
|
|
19185
|
+
` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
|
|
19186
|
+
` + `'@
|
|
19187
|
+
` + `$h=[IntPtr]::Zero
|
|
19188
|
+
` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
|
|
19189
|
+
` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
|
|
19190
|
+
` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
|
|
19191
|
+
` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
|
|
19192
|
+
` + `[Autorag.CfApi]::CfCloseHandle($h)
|
|
19193
|
+
` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
|
|
19194
|
+
` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
|
|
19195
|
+
`;
|
|
19196
|
+
}
|
|
19197
|
+
function parseWindowsAttributeOutput(stdout, paths) {
|
|
19198
|
+
const lines = stdout.split(/\r?\n/);
|
|
19199
|
+
const attributes = new Map;
|
|
19200
|
+
for (const [index, path] of paths.entries()) {
|
|
19201
|
+
const value = Number.parseInt((lines[index] ?? "").trim(), 10);
|
|
19202
|
+
if (Number.isFinite(value))
|
|
19203
|
+
attributes.set(path, value);
|
|
19017
19204
|
}
|
|
19205
|
+
return attributes;
|
|
19018
19206
|
}
|
|
19019
|
-
function
|
|
19020
|
-
|
|
19207
|
+
function parseWindowsSyncRoots(stdout) {
|
|
19208
|
+
const roots = [];
|
|
19209
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
19210
|
+
const tab = line.indexOf("\t");
|
|
19211
|
+
if (tab <= 0)
|
|
19212
|
+
continue;
|
|
19213
|
+
const path = line.slice(0, tab).trim();
|
|
19214
|
+
if (path !== "")
|
|
19215
|
+
roots.push({ path, label: line.slice(tab + 1).trim() });
|
|
19216
|
+
}
|
|
19217
|
+
return roots;
|
|
19021
19218
|
}
|
|
19022
|
-
function
|
|
19023
|
-
|
|
19219
|
+
function providerFromWindowsSyncRoot(label) {
|
|
19220
|
+
const lower = label.toLowerCase();
|
|
19221
|
+
if (lower.includes("onedrive") || lower.includes("one drive"))
|
|
19222
|
+
return "onedrive";
|
|
19223
|
+
if (lower.includes("google"))
|
|
19224
|
+
return "google-drive";
|
|
19225
|
+
return "file-provider";
|
|
19024
19226
|
}
|
|
19025
|
-
|
|
19026
|
-
|
|
19027
|
-
|
|
19028
|
-
|
|
19029
|
-
|
|
19030
|
-
// src/filesystem/path-exclusions.ts
|
|
19031
|
-
import { posix as posix3, win32 as win322 } from "node:path";
|
|
19032
|
-
function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
|
|
19033
|
-
const paths = platform === "win32" ? win322 : posix3;
|
|
19034
|
-
const normalize = (path) => {
|
|
19035
|
-
const absolute = paths.resolve(path);
|
|
19036
|
-
return platform === "win32" ? absolute.toLowerCase() : absolute;
|
|
19037
|
-
};
|
|
19038
|
-
const excluded = excludedPaths.map((path) => {
|
|
19039
|
-
const absolute = normalize(path);
|
|
19040
|
-
return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
|
|
19041
|
-
});
|
|
19042
|
-
return (path) => {
|
|
19043
|
-
const absolute = normalize(path);
|
|
19044
|
-
return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
|
|
19045
|
-
};
|
|
19227
|
+
function buildWindowsDiskScript() {
|
|
19228
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19229
|
+
` + "Get-CimInstance -ClassName Win32_LogicalDisk | ForEach-Object {" + `$_.DeviceID + [char]9 + $_.DriveType + [char]9 + $_.FileSystem + [char]9 + $_.VolumeName}
|
|
19230
|
+
`;
|
|
19046
19231
|
}
|
|
19047
|
-
|
|
19048
|
-
|
|
19049
|
-
|
|
19050
|
-
|
|
19051
|
-
|
|
19052
|
-
|
|
19053
|
-
|
|
19054
|
-
|
|
19055
|
-
|
|
19056
|
-
|
|
19057
|
-
|
|
19058
|
-
|
|
19059
|
-
|
|
19060
|
-
|
|
19061
|
-
return run;
|
|
19062
|
-
}
|
|
19063
|
-
exclusive(task) {
|
|
19064
|
-
const earlier = Promise.all([this.lastExclusive, ...this.sharedSinceExclusive]);
|
|
19065
|
-
const run = earlier.then(task);
|
|
19066
|
-
const settled = run.then(() => {
|
|
19067
|
-
return;
|
|
19068
|
-
}, () => {
|
|
19069
|
-
return;
|
|
19232
|
+
function parseWindowsLogicalDisks(stdout) {
|
|
19233
|
+
const disks = [];
|
|
19234
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
19235
|
+
if (line.trim() === "")
|
|
19236
|
+
continue;
|
|
19237
|
+
const [deviceId, driveType, fileSystem, ...rest] = line.split("\t");
|
|
19238
|
+
const type = Number.parseInt((driveType ?? "").trim(), 10);
|
|
19239
|
+
if (!Number.isFinite(type))
|
|
19240
|
+
continue;
|
|
19241
|
+
disks.push({
|
|
19242
|
+
deviceId: (deviceId ?? "").trim(),
|
|
19243
|
+
driveType: type,
|
|
19244
|
+
fileSystem: (fileSystem ?? "").trim(),
|
|
19245
|
+
volumeName: rest.join("\t").trim()
|
|
19070
19246
|
});
|
|
19071
|
-
this.sharedSinceExclusive = [];
|
|
19072
|
-
this.lastExclusive = settled;
|
|
19073
|
-
return run;
|
|
19074
19247
|
}
|
|
19248
|
+
return disks;
|
|
19075
19249
|
}
|
|
19076
|
-
|
|
19077
|
-
|
|
19078
|
-
|
|
19079
|
-
|
|
19080
|
-
|
|
19081
|
-
|
|
19082
|
-
|
|
19083
|
-
|
|
19084
|
-
|
|
19085
|
-
|
|
19086
|
-
|
|
19087
|
-
|
|
19088
|
-
|
|
19089
|
-
|
|
19090
|
-
|
|
19091
|
-
"
|
|
19092
|
-
|
|
19093
|
-
|
|
19094
|
-
|
|
19095
|
-
|
|
19096
|
-
"check_for_updates_on_startup=0",
|
|
19097
|
-
"beta_updates=0",
|
|
19098
|
-
"language=1033",
|
|
19099
|
-
"allow_http_server=0",
|
|
19100
|
-
"allow_etp_server=0",
|
|
19101
|
-
"search_history_enabled=0",
|
|
19102
|
-
"run_history_enabled=0",
|
|
19103
|
-
"auto_include_fixed_volumes=0",
|
|
19104
|
-
"auto_include_removable_volumes=0",
|
|
19105
|
-
"auto_include_fixed_refs_volumes=0",
|
|
19106
|
-
"auto_include_removable_refs_volumes=0",
|
|
19107
|
-
"ntfs_volume_paths=",
|
|
19108
|
-
"ntfs_volume_includes=",
|
|
19109
|
-
"refs_volume_paths=",
|
|
19110
|
-
"refs_volume_includes=",
|
|
19111
|
-
"index_size=1",
|
|
19112
|
-
"index_date_modified=1",
|
|
19113
|
-
`folders=${list(input.folders)}`,
|
|
19114
|
-
`folder_monitor_changes=${perFolder("1")}`,
|
|
19115
|
-
`folder_rescan_if_full_list=${perFolder("1")}`,
|
|
19116
|
-
`folder_update_types=${perFolder("0")}`,
|
|
19117
|
-
"exclude_list_enabled=1",
|
|
19118
|
-
`exclude_folders=${list(input.excludeFolders)}`,
|
|
19119
|
-
""
|
|
19120
|
-
].join(`\r
|
|
19121
|
-
`);
|
|
19250
|
+
function windowsRootsFromDisks(disks) {
|
|
19251
|
+
const roots = [];
|
|
19252
|
+
for (const disk of disks) {
|
|
19253
|
+
if (disk.deviceId === "")
|
|
19254
|
+
continue;
|
|
19255
|
+
const path = `${disk.deviceId}\\`;
|
|
19256
|
+
const fileSystem = disk.fileSystem.toLowerCase();
|
|
19257
|
+
if (disk.driveType === WINDOWS_DRIVE_TYPE_NETWORK) {
|
|
19258
|
+
roots.push({ path, kind: "network" });
|
|
19259
|
+
continue;
|
|
19260
|
+
}
|
|
19261
|
+
if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
|
|
19262
|
+
roots.push({ path, kind: "network" });
|
|
19263
|
+
continue;
|
|
19264
|
+
}
|
|
19265
|
+
if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
|
|
19266
|
+
roots.push({ path, kind: "cloud-files" });
|
|
19267
|
+
}
|
|
19268
|
+
}
|
|
19269
|
+
return roots;
|
|
19122
19270
|
}
|
|
19123
|
-
function
|
|
19124
|
-
const
|
|
19125
|
-
|
|
19126
|
-
|
|
19127
|
-
|
|
19128
|
-
|
|
19129
|
-
|
|
19130
|
-
|
|
19131
|
-
|
|
19132
|
-
|
|
19133
|
-
|
|
19134
|
-
|
|
19135
|
-
|
|
19136
|
-
];
|
|
19137
|
-
if (timeoutMs !== undefined)
|
|
19138
|
-
args.push("-timeout", String(timeoutMs));
|
|
19139
|
-
if (request.matchCase)
|
|
19140
|
-
args.push("-case");
|
|
19141
|
-
if (request.matchPath)
|
|
19142
|
-
args.push("-match-path");
|
|
19143
|
-
if (request.wholeWord)
|
|
19144
|
-
args.push("-whole-word");
|
|
19145
|
-
if (request.kind === "files")
|
|
19146
|
-
args.push("/a-d");
|
|
19147
|
-
if (request.kind === "folders")
|
|
19148
|
-
args.push("/ad");
|
|
19149
|
-
if (request.path !== undefined)
|
|
19150
|
-
args.push("-path", request.path);
|
|
19151
|
-
if (request.sort !== undefined)
|
|
19152
|
-
args.push("-sort", request.sort);
|
|
19153
|
-
if (request.offset !== undefined && request.offset > 0)
|
|
19154
|
-
args.push("-offset", String(request.offset));
|
|
19155
|
-
args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
|
|
19156
|
-
if (request.regex)
|
|
19157
|
-
args.push("-regex", "--", request.query);
|
|
19158
|
-
else
|
|
19159
|
-
args.push("--", request.query);
|
|
19160
|
-
return args;
|
|
19271
|
+
function uncShareRoot(target) {
|
|
19272
|
+
const normalized = target.replaceAll("/", "\\");
|
|
19273
|
+
if (!normalized.startsWith("\\\\") || normalized.startsWith("\\\\?\\") || normalized.startsWith("\\\\.\\"))
|
|
19274
|
+
return;
|
|
19275
|
+
const rest = normalized.slice(2);
|
|
19276
|
+
const separator = rest.indexOf("\\");
|
|
19277
|
+
if (separator <= 0)
|
|
19278
|
+
return;
|
|
19279
|
+
const server = rest.slice(0, separator);
|
|
19280
|
+
const share = rest.slice(separator + 1).split("\\")[0] ?? "";
|
|
19281
|
+
if (server === "" || share === "")
|
|
19282
|
+
return;
|
|
19283
|
+
return `\\\\${server}\\${share}`;
|
|
19161
19284
|
}
|
|
19162
|
-
function
|
|
19163
|
-
const
|
|
19164
|
-
if (
|
|
19165
|
-
return
|
|
19166
|
-
|
|
19167
|
-
try {
|
|
19168
|
-
parsed = JSON.parse(text);
|
|
19169
|
-
} catch {
|
|
19170
|
-
throw new Error(`ES did not return JSON: ${text.slice(0, 2000)}`);
|
|
19171
|
-
}
|
|
19172
|
-
if (!Array.isArray(parsed))
|
|
19173
|
-
throw new Error(`ES did not return a JSON array: ${text.slice(0, 2000)}`);
|
|
19174
|
-
return parsed.map((item) => {
|
|
19175
|
-
const record = item;
|
|
19176
|
-
const filename = String(record.filename ?? "");
|
|
19177
|
-
const isFolder = filename.endsWith("\\") && filename.length > 3;
|
|
19285
|
+
function parseWindowsDehydrateOutput(result) {
|
|
19286
|
+
const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
19287
|
+
if (open === null)
|
|
19288
|
+
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
19289
|
+
if (Number.parseInt(open[1], 16) !== 0) {
|
|
19178
19290
|
return {
|
|
19179
|
-
|
|
19180
|
-
|
|
19181
|
-
size: typeof record.size === "number" ? record.size : undefined,
|
|
19182
|
-
dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
|
|
19291
|
+
ok: false,
|
|
19292
|
+
message: windowsFailureMessage(result, `could not open the file for release (0x${open[1]})`)
|
|
19183
19293
|
};
|
|
19184
|
-
}
|
|
19294
|
+
}
|
|
19295
|
+
const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
19296
|
+
if (dehydrate === null) {
|
|
19297
|
+
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
19298
|
+
}
|
|
19299
|
+
const hresult = dehydrate[1];
|
|
19300
|
+
if (Number.parseInt(hresult, 16) === 0)
|
|
19301
|
+
return { ok: true };
|
|
19302
|
+
const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
|
|
19303
|
+
return {
|
|
19304
|
+
ok: false,
|
|
19305
|
+
message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
|
|
19306
|
+
};
|
|
19307
|
+
}
|
|
19308
|
+
function windowsFailureMessage(result, fallback) {
|
|
19309
|
+
const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
|
|
19310
|
+
if (printed !== undefined && printed !== "")
|
|
19311
|
+
return printed;
|
|
19312
|
+
const stderr = result.stderr.trim();
|
|
19313
|
+
return stderr !== "" ? stderr : fallback;
|
|
19185
19314
|
}
|
|
19186
19315
|
|
|
19187
|
-
class
|
|
19188
|
-
instanceName;
|
|
19189
|
-
options;
|
|
19190
|
-
platform;
|
|
19316
|
+
class WindowsCloudProbe {
|
|
19191
19317
|
run;
|
|
19192
|
-
|
|
19193
|
-
|
|
19194
|
-
|
|
19195
|
-
|
|
19196
|
-
queue = new SharedExclusiveQueue;
|
|
19197
|
-
constructor(options) {
|
|
19198
|
-
this.options = options;
|
|
19199
|
-
this.platform = options.platform ?? process.platform;
|
|
19200
|
-
this.instanceName = everythingInstanceName(options.root);
|
|
19201
|
-
this.run = options.run ?? runProcess;
|
|
19202
|
-
this.launch = options.launch ?? launchDetached;
|
|
19203
|
-
this.stateDir = join30(options.root, ".autorag", "everything");
|
|
19204
|
-
this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
|
|
19205
|
-
}
|
|
19206
|
-
isSupported() {
|
|
19207
|
-
return this.platform === "win32";
|
|
19318
|
+
syncRoots;
|
|
19319
|
+
disks;
|
|
19320
|
+
constructor(options = {}) {
|
|
19321
|
+
this.run = options.run ?? runPowerShell;
|
|
19208
19322
|
}
|
|
19209
|
-
|
|
19210
|
-
|
|
19211
|
-
|
|
19212
|
-
|
|
19213
|
-
|
|
19214
|
-
const
|
|
19215
|
-
if (
|
|
19216
|
-
return
|
|
19217
|
-
|
|
19218
|
-
|
|
19219
|
-
|
|
19220
|
-
|
|
19221
|
-
|
|
19222
|
-
|
|
19223
|
-
|
|
19224
|
-
|
|
19225
|
-
|
|
19226
|
-
|
|
19227
|
-
|
|
19228
|
-
|
|
19229
|
-
|
|
19230
|
-
return this.failure("search-failed", es.esPath, args, result);
|
|
19231
|
-
try {
|
|
19232
|
-
return { ok: true, results: parseEverythingJson(result.stdout) };
|
|
19233
|
-
} catch (error) {
|
|
19234
|
-
return { ok: false, reason: "search-failed", message: error.message };
|
|
19235
|
-
}
|
|
19236
|
-
});
|
|
19323
|
+
async cloudRoots(searchRoots = []) {
|
|
19324
|
+
const roots = [];
|
|
19325
|
+
const seen = new Set;
|
|
19326
|
+
const add = (path, kind) => {
|
|
19327
|
+
const canonical = canonicalPath(path);
|
|
19328
|
+
const key = normalizeCloudPath(canonical).toLowerCase();
|
|
19329
|
+
if (key === "" || seen.has(key))
|
|
19330
|
+
return;
|
|
19331
|
+
seen.add(key);
|
|
19332
|
+
roots.push({ path: canonical, kind });
|
|
19333
|
+
};
|
|
19334
|
+
for (const root of await this.loadSyncRoots())
|
|
19335
|
+
add(root.path, "cloud-files");
|
|
19336
|
+
for (const root of await this.loadDiskRoots())
|
|
19337
|
+
add(root.path, root.kind);
|
|
19338
|
+
for (const searchRoot of searchRoots) {
|
|
19339
|
+
const share = uncShareRoot(searchRoot);
|
|
19340
|
+
if (share !== undefined)
|
|
19341
|
+
add(share, "network");
|
|
19342
|
+
}
|
|
19343
|
+
return roots;
|
|
19237
19344
|
}
|
|
19238
|
-
async
|
|
19239
|
-
const
|
|
19240
|
-
|
|
19241
|
-
|
|
19242
|
-
|
|
19243
|
-
|
|
19244
|
-
|
|
19245
|
-
const timedOut = () => ({
|
|
19246
|
-
ok: false,
|
|
19247
|
-
reason: "search-failed",
|
|
19248
|
-
message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
|
|
19249
|
-
});
|
|
19250
|
-
for (;; ) {
|
|
19251
|
-
const remainingMs = deadline - Date.now();
|
|
19252
|
-
if (remainingMs <= 0)
|
|
19253
|
-
return timedOut();
|
|
19254
|
-
const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
|
|
19255
|
-
const response = await this.run(esPath, args, remainingMs + 5000);
|
|
19256
|
-
if (response.code !== 0)
|
|
19257
|
-
return this.failure("search-failed", esPath, args, response);
|
|
19258
|
-
if (Date.now() >= deadline)
|
|
19259
|
-
return timedOut();
|
|
19260
|
-
let entries;
|
|
19261
|
-
try {
|
|
19262
|
-
entries = parseEverythingJson(response.stdout);
|
|
19263
|
-
} catch (error) {
|
|
19264
|
-
return { ok: false, reason: "search-failed", message: error.message };
|
|
19345
|
+
async datalessAmong(paths) {
|
|
19346
|
+
const dataless = new Set;
|
|
19347
|
+
for (let start = 0;start < paths.length; start += WINDOWS_ATTRIBUTE_BATCH) {
|
|
19348
|
+
const batch = paths.slice(start, start + WINDOWS_ATTRIBUTE_BATCH);
|
|
19349
|
+
const result = await this.run(buildWindowsAttributeScript(batch), 30000);
|
|
19350
|
+
if (!result.ok) {
|
|
19351
|
+
throw new Error(`Could not read Windows cloud file attributes: ${result.stderr.trim() || "powershell.exe failed"}`);
|
|
19265
19352
|
}
|
|
19266
|
-
for (const
|
|
19267
|
-
if (
|
|
19268
|
-
|
|
19269
|
-
if (skipped < offset) {
|
|
19270
|
-
skipped += 1;
|
|
19271
|
-
continue;
|
|
19272
|
-
}
|
|
19273
|
-
results.push(entry);
|
|
19274
|
-
if (results.length >= maxResults)
|
|
19275
|
-
return { ok: true, results };
|
|
19353
|
+
for (const [path, attributes] of parseWindowsAttributeOutput(result.stdout, batch)) {
|
|
19354
|
+
if (datalessFromWindowsAttributes(attributes))
|
|
19355
|
+
dataless.add(path);
|
|
19276
19356
|
}
|
|
19277
|
-
if (entries.length < MAX_FILTER_CANDIDATES)
|
|
19278
|
-
return { ok: true, results };
|
|
19279
|
-
nativeOffset += entries.length;
|
|
19280
19357
|
}
|
|
19358
|
+
return dataless;
|
|
19281
19359
|
}
|
|
19282
|
-
|
|
19283
|
-
|
|
19284
|
-
|
|
19285
|
-
if (!es.ok)
|
|
19286
|
-
return es;
|
|
19287
|
-
const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
|
|
19288
|
-
const saveArgs = ["-instance", this.instanceName, "-save-db"];
|
|
19289
|
-
const saved = await this.run(es.esPath, saveArgs, timeoutMs);
|
|
19290
|
-
if (saved.code !== 0)
|
|
19291
|
-
return this.failure("index-failed", es.esPath, saveArgs, saved);
|
|
19292
|
-
const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
|
|
19293
|
-
const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
|
|
19294
|
-
if (counted.code !== 0)
|
|
19295
|
-
return this.failure("index-failed", es.esPath, countArgs, counted);
|
|
19296
|
-
return {
|
|
19297
|
-
ok: true,
|
|
19298
|
-
indexedFolders: [...this.options.folders],
|
|
19299
|
-
indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
|
|
19300
|
-
};
|
|
19301
|
-
});
|
|
19360
|
+
async providerOf(path) {
|
|
19361
|
+
const root = longestSyncRoot(await this.loadSyncRoots(), path);
|
|
19362
|
+
return root === undefined ? cloudProviderFromPath(path) : providerFromWindowsSyncRoot(root.label);
|
|
19302
19363
|
}
|
|
19303
|
-
|
|
19304
|
-
return
|
|
19305
|
-
const binaries = await this.resolveBinaries("cached");
|
|
19306
|
-
if (!binaries.ok)
|
|
19307
|
-
return;
|
|
19308
|
-
await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
19309
|
-
});
|
|
19364
|
+
canRelease(provider) {
|
|
19365
|
+
return provider !== undefined;
|
|
19310
19366
|
}
|
|
19311
|
-
async
|
|
19312
|
-
|
|
19313
|
-
return {
|
|
19314
|
-
ok: false,
|
|
19315
|
-
reason: "unsupported-platform",
|
|
19316
|
-
message: `Everything is Windows-only; this host is ${this.platform}.`
|
|
19317
|
-
};
|
|
19318
|
-
}
|
|
19319
|
-
if (this.binaries?.ok)
|
|
19320
|
-
return this.binaries;
|
|
19321
|
-
const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
|
|
19322
|
-
const result = await resolve(mode);
|
|
19323
|
-
if (result.ok)
|
|
19324
|
-
this.binaries = result;
|
|
19325
|
-
return result;
|
|
19367
|
+
async evict(path) {
|
|
19368
|
+
return parseWindowsDehydrateOutput(await this.run(buildWindowsDehydrateScript(path), 60000));
|
|
19326
19369
|
}
|
|
19327
|
-
|
|
19328
|
-
|
|
19329
|
-
|
|
19330
|
-
return binaries;
|
|
19331
|
-
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
19332
|
-
let ping = await this.ping(binaries.esPath);
|
|
19333
|
-
if (ping.code === 0 && !reconfigure)
|
|
19334
|
-
return { ok: true, esPath: binaries.esPath };
|
|
19335
|
-
if (ping.code === 0) {
|
|
19336
|
-
const exitArgs = ["-instance", this.instanceName, "-exit"];
|
|
19337
|
-
const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
|
|
19338
|
-
if (exited.code !== 0)
|
|
19339
|
-
return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
|
|
19340
|
-
}
|
|
19341
|
-
mkdirSync16(this.stateDir, { recursive: true });
|
|
19342
|
-
const iniPath = join30(this.stateDir, "Everything.ini");
|
|
19343
|
-
writeFileSync11(iniPath, buildEverythingIni({
|
|
19344
|
-
folders: this.options.folders,
|
|
19345
|
-
excludeFolders: [
|
|
19346
|
-
...new Set([
|
|
19347
|
-
join30(this.options.root, ".autorag"),
|
|
19348
|
-
...this.options.folders.map((folder) => join30(folder, ".autorag")),
|
|
19349
|
-
...this.options.excludeFolders ?? []
|
|
19350
|
-
])
|
|
19351
|
-
]
|
|
19352
|
-
}));
|
|
19353
|
-
const launchArgs = [
|
|
19354
|
-
"-instance",
|
|
19355
|
-
this.instanceName,
|
|
19356
|
-
"-config",
|
|
19357
|
-
iniPath,
|
|
19358
|
-
"-db",
|
|
19359
|
-
join30(this.stateDir, "Everything.db"),
|
|
19360
|
-
"-startup"
|
|
19361
|
-
];
|
|
19362
|
-
try {
|
|
19363
|
-
this.launch(binaries.everythingPath, launchArgs);
|
|
19364
|
-
} catch (error) {
|
|
19365
|
-
return {
|
|
19366
|
-
ok: false,
|
|
19367
|
-
reason: "startup-failed",
|
|
19368
|
-
message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
|
|
19369
|
-
};
|
|
19370
|
-
}
|
|
19371
|
-
const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
|
|
19372
|
-
const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
19373
|
-
for (;; ) {
|
|
19374
|
-
ping = await this.ping(binaries.esPath);
|
|
19375
|
-
if (ping.code === 0)
|
|
19376
|
-
return { ok: true, esPath: binaries.esPath };
|
|
19377
|
-
if (Date.now() >= deadline)
|
|
19378
|
-
break;
|
|
19379
|
-
await sleep2(pollMs);
|
|
19380
|
-
}
|
|
19381
|
-
return {
|
|
19382
|
-
ok: false,
|
|
19383
|
-
reason: "startup-failed",
|
|
19384
|
-
message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
|
|
19385
|
-
};
|
|
19370
|
+
loadSyncRoots() {
|
|
19371
|
+
this.syncRoots ??= this.run(buildWindowsSyncRootsScript(), 15000).then((result) => result.ok ? parseWindowsSyncRoots(result.stdout) : [], () => []);
|
|
19372
|
+
return this.syncRoots;
|
|
19386
19373
|
}
|
|
19387
|
-
|
|
19388
|
-
|
|
19389
|
-
return
|
|
19374
|
+
loadDiskRoots() {
|
|
19375
|
+
this.disks ??= this.run(buildWindowsDiskScript(), 30000).then((result) => result.ok ? windowsRootsFromDisks(parseWindowsLogicalDisks(result.stdout)) : [], () => []);
|
|
19376
|
+
return this.disks;
|
|
19390
19377
|
}
|
|
19391
|
-
|
|
19378
|
+
}
|
|
19379
|
+
function longestSyncRoot(roots, target) {
|
|
19380
|
+
const normalized = target.replaceAll("/", "\\").toLowerCase();
|
|
19381
|
+
let match;
|
|
19382
|
+
let matchLength = -1;
|
|
19383
|
+
for (const root of roots) {
|
|
19384
|
+
const prefix = root.path.replaceAll("/", "\\").replace(/\\+$/u, "").toLowerCase();
|
|
19385
|
+
if (prefix === "" || prefix.length <= matchLength)
|
|
19386
|
+
continue;
|
|
19387
|
+
if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
|
|
19388
|
+
match = root;
|
|
19389
|
+
matchLength = prefix.length;
|
|
19390
|
+
}
|
|
19391
|
+
}
|
|
19392
|
+
return match;
|
|
19393
|
+
}
|
|
19394
|
+
function powerShellLaunch(script) {
|
|
19395
|
+
return {
|
|
19396
|
+
args: [
|
|
19397
|
+
"-NoProfile",
|
|
19398
|
+
"-NonInteractive",
|
|
19399
|
+
"-Command",
|
|
19400
|
+
`[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; Invoke-Expression $env:${POWERSHELL_SCRIPT_VARIABLE}`
|
|
19401
|
+
],
|
|
19402
|
+
env: { [POWERSHELL_SCRIPT_VARIABLE]: script }
|
|
19403
|
+
};
|
|
19404
|
+
}
|
|
19405
|
+
async function runPowerShell(script, timeoutMs) {
|
|
19406
|
+
const launch = powerShellLaunch(script);
|
|
19407
|
+
try {
|
|
19408
|
+
const { stdout, stderr } = await execFileAsync("powershell.exe", [...launch.args], {
|
|
19409
|
+
timeout: timeoutMs,
|
|
19410
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
19411
|
+
windowsHide: true,
|
|
19412
|
+
env: { ...process.env, ...launch.env }
|
|
19413
|
+
});
|
|
19414
|
+
return { ok: true, stdout, stderr };
|
|
19415
|
+
} catch (error) {
|
|
19416
|
+
const record = typeof error === "object" && error !== null ? error : {};
|
|
19392
19417
|
return {
|
|
19393
19418
|
ok: false,
|
|
19394
|
-
|
|
19395
|
-
|
|
19419
|
+
stdout: typeof record.stdout === "string" ? record.stdout : "",
|
|
19420
|
+
stderr: typeof record.stderr === "string" ? record.stderr : String(error)
|
|
19396
19421
|
};
|
|
19397
19422
|
}
|
|
19398
19423
|
}
|
|
19399
|
-
function
|
|
19400
|
-
|
|
19401
|
-
|
|
19402
|
-
|
|
19403
|
-
|
|
19404
|
-
|
|
19405
|
-
|
|
19406
|
-
|
|
19407
|
-
|
|
19408
|
-
|
|
19409
|
-
|
|
19410
|
-
|
|
19411
|
-
if (bytes > MAX_OUTPUT_BYTES) {
|
|
19412
|
-
overflow = true;
|
|
19413
|
-
child.kill();
|
|
19414
|
-
return;
|
|
19415
|
-
}
|
|
19416
|
-
target.push(chunk);
|
|
19417
|
-
};
|
|
19418
|
-
child.stdout.on("data", collect(stdout));
|
|
19419
|
-
child.stderr.on("data", collect(stderr));
|
|
19420
|
-
const timer = setTimeout(() => child.kill(), timeoutMs);
|
|
19421
|
-
child.on("error", (error) => {
|
|
19422
|
-
clearTimeout(timer);
|
|
19423
|
-
resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
|
|
19424
|
-
});
|
|
19425
|
-
child.on("close", (code, signal) => {
|
|
19426
|
-
clearTimeout(timer);
|
|
19427
|
-
const err = Buffer.concat(stderr).toString("utf8");
|
|
19428
|
-
resolvePromise({
|
|
19429
|
-
code,
|
|
19430
|
-
stdout: Buffer.concat(stdout).toString("utf8"),
|
|
19431
|
-
stderr: overflow ? `${err}
|
|
19432
|
-
output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
|
|
19433
|
-
killed by ${signal} after ${timeoutMs}ms` : err
|
|
19424
|
+
function defaultCloudProbe() {
|
|
19425
|
+
if (process.platform === "darwin")
|
|
19426
|
+
return new DarwinCloudProbe;
|
|
19427
|
+
if (process.platform === "win32")
|
|
19428
|
+
return new WindowsCloudProbe;
|
|
19429
|
+
return;
|
|
19430
|
+
}
|
|
19431
|
+
async function readFileProviderDomain(target) {
|
|
19432
|
+
try {
|
|
19433
|
+
await lstat(target);
|
|
19434
|
+
const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
|
|
19435
|
+
timeout: 2000
|
|
19434
19436
|
});
|
|
19435
|
-
|
|
19436
|
-
|
|
19437
|
+
const value = stdout.trim();
|
|
19438
|
+
return value.length > 0 ? value : undefined;
|
|
19439
|
+
} catch {
|
|
19440
|
+
return;
|
|
19441
|
+
}
|
|
19437
19442
|
}
|
|
19438
|
-
function
|
|
19439
|
-
|
|
19440
|
-
|
|
19441
|
-
|
|
19443
|
+
async function readFileProviderDomainUpwards(target) {
|
|
19444
|
+
let cursor = target;
|
|
19445
|
+
for (let depth = 0;depth < 12; depth += 1) {
|
|
19446
|
+
const domain = await readFileProviderDomain(cursor);
|
|
19447
|
+
if (domain !== undefined)
|
|
19448
|
+
return domain;
|
|
19449
|
+
const parent = dirname16(cursor);
|
|
19450
|
+
if (parent === cursor)
|
|
19451
|
+
return;
|
|
19452
|
+
cursor = parent;
|
|
19453
|
+
}
|
|
19454
|
+
return;
|
|
19442
19455
|
}
|
|
19443
|
-
|
|
19444
|
-
|
|
19445
|
-
|
|
19446
|
-
|
|
19447
|
-
|
|
19448
|
-
|
|
19449
|
-
|
|
19450
|
-
|
|
19451
|
-
|
|
19452
|
-
|
|
19453
|
-
var
|
|
19454
|
-
|
|
19455
|
-
|
|
19456
|
+
function quotePowerShell(value) {
|
|
19457
|
+
return `'${value.replace(/['\u2018\u2019\u201A\u201B]/gu, (quote) => quote + quote)}'`;
|
|
19458
|
+
}
|
|
19459
|
+
var execFileAsync, SF_DATALESS = 1073741824, FILE_PROVIDER_XATTR = "com.apple.file-provider-domain-id", WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 4194304, STAT_BATCH = 200, defaultCloudExec = (file, args, options) => execFileAsync(file, [...args], {
|
|
19460
|
+
timeout: options?.timeoutMs ?? 30000,
|
|
19461
|
+
...options?.maxBuffer === undefined ? {} : { maxBuffer: options.maxBuffer }
|
|
19462
|
+
}), JXA_RELEASE_PROGRAM = `ObjC.import("Foundation");
|
|
19463
|
+
function run(argv) {
|
|
19464
|
+
var url = $.NSURL.fileURLWithPath(argv[0]);
|
|
19465
|
+
var errorRef = Ref();
|
|
19466
|
+
var released = $.NSFileManager.defaultManager.evictUbiquitousItemAtURLError(url, errorRef);
|
|
19467
|
+
if (released) return JSON.stringify({ ok: true });
|
|
19468
|
+
function resourceValue(key) {
|
|
19469
|
+
var ref = Ref();
|
|
19470
|
+
if (!url.getResourceValueForKeyError(ref, key, null)) return null;
|
|
19471
|
+
var value = ref[0];
|
|
19472
|
+
return value === undefined || value === null ? null : ObjC.unwrap(value);
|
|
19473
|
+
}
|
|
19474
|
+
return JSON.stringify({
|
|
19475
|
+
ok: false,
|
|
19476
|
+
isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
|
|
19477
|
+
isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
|
|
19478
|
+
downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
|
|
19479
|
+
});
|
|
19480
|
+
}`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
|
|
19481
|
+
var init_cloud_placeholder = __esm(() => {
|
|
19482
|
+
execFileAsync = promisify(execFile2);
|
|
19483
|
+
RELEASABLE_DARWIN_PROVIDERS = {
|
|
19484
|
+
icloud: true,
|
|
19485
|
+
"google-drive": true,
|
|
19486
|
+
onedrive: true,
|
|
19487
|
+
dropbox: true
|
|
19488
|
+
};
|
|
19489
|
+
DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
|
|
19490
|
+
WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
|
|
19491
|
+
WINDOWS_DEHYDRATE_REFUSALS = {
|
|
19492
|
+
"80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
|
|
19493
|
+
"80070188": "The file is pinned to stay on this device, so it is not released."
|
|
19494
|
+
};
|
|
19456
19495
|
});
|
|
19457
19496
|
|
|
19458
19497
|
// src/fsearch/walk.ts
|
|
19459
19498
|
import { readdir as readdir2, stat as stat3 } from "node:fs/promises";
|
|
19460
|
-
import { basename as
|
|
19499
|
+
import { basename as basename7, join as join31, sep as sep5 } from "node:path";
|
|
19461
19500
|
function compare(a, b) {
|
|
19462
19501
|
return a < b ? -1 : a > b ? 1 : 0;
|
|
19463
19502
|
}
|
|
@@ -19468,11 +19507,11 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
19468
19507
|
const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_DEADLINE_MS);
|
|
19469
19508
|
const matches = request.regex ? (() => {
|
|
19470
19509
|
const pattern = new RegExp(request.query, request.matchCase ? "" : "i");
|
|
19471
|
-
return (path) => pattern.test(request.matchPath ? path :
|
|
19510
|
+
return (path) => pattern.test(request.matchPath ? path : basename7(path));
|
|
19472
19511
|
})() : (() => {
|
|
19473
19512
|
const needle = request.matchCase ? request.query : request.query.toLowerCase();
|
|
19474
19513
|
return (path) => {
|
|
19475
|
-
const haystack = request.matchPath ? path :
|
|
19514
|
+
const haystack = request.matchPath ? path : basename7(path);
|
|
19476
19515
|
return (request.matchCase ? haystack : haystack.toLowerCase()).includes(needle);
|
|
19477
19516
|
};
|
|
19478
19517
|
})();
|
|
@@ -19527,7 +19566,7 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
19527
19566
|
continue;
|
|
19528
19567
|
collected.push({
|
|
19529
19568
|
path,
|
|
19530
|
-
name:
|
|
19569
|
+
name: basename7(path),
|
|
19531
19570
|
type: info.isDirectory() ? "folder" : "file",
|
|
19532
19571
|
size: info.isDirectory() ? undefined : info.size,
|
|
19533
19572
|
dateModified: info.mtime.toISOString()
|
|
@@ -19552,7 +19591,7 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
19552
19591
|
continue;
|
|
19553
19592
|
collected.push({
|
|
19554
19593
|
path: folder,
|
|
19555
|
-
name:
|
|
19594
|
+
name: basename7(folder),
|
|
19556
19595
|
type: "folder",
|
|
19557
19596
|
size: undefined,
|
|
19558
19597
|
dateModified: info.mtime.toISOString()
|
|
@@ -19585,7 +19624,7 @@ var init_walk = __esm(() => {
|
|
|
19585
19624
|
import { spawn as spawn16 } from "node:child_process";
|
|
19586
19625
|
import { createHash as createHash13 } from "node:crypto";
|
|
19587
19626
|
import { existsSync as existsSync19, mkdirSync as mkdirSync17, readFileSync as readFileSync19, realpathSync as realpathSync4, rmSync as rmSync8, writeFileSync as writeFileSync12 } from "node:fs";
|
|
19588
|
-
import { basename as
|
|
19627
|
+
import { basename as basename8, join as join32, resolve as resolve12, sep as sep6 } from "node:path";
|
|
19589
19628
|
import { setTimeout as sleep3 } from "node:timers/promises";
|
|
19590
19629
|
function fsearchSocketPath(dbPath) {
|
|
19591
19630
|
let canonical;
|
|
@@ -19649,7 +19688,7 @@ function parseFSearchJsonLines(stdout) {
|
|
|
19649
19688
|
const path = String(record.path ?? "");
|
|
19650
19689
|
entries.push({
|
|
19651
19690
|
path,
|
|
19652
|
-
name:
|
|
19691
|
+
name: basename8(path),
|
|
19653
19692
|
type: record.type === "folder" ? "folder" : "file",
|
|
19654
19693
|
size: typeof record.size === "number" ? record.size : undefined,
|
|
19655
19694
|
dateModified: typeof record.mtime === "number" ? new Date(record.mtime * 1000).toISOString() : undefined
|
|
@@ -20067,7 +20106,7 @@ var init_diagnostics = __esm(() => {
|
|
|
20067
20106
|
});
|
|
20068
20107
|
|
|
20069
20108
|
// src/jikji/retry-command.ts
|
|
20070
|
-
import { basename as
|
|
20109
|
+
import { basename as basename9 } from "node:path";
|
|
20071
20110
|
function shellWords(command) {
|
|
20072
20111
|
const words = [];
|
|
20073
20112
|
let current = "";
|
|
@@ -20134,7 +20173,7 @@ function parseJikjiRetryCommand(command, envelopeProof) {
|
|
|
20134
20173
|
const words = shellWords(command);
|
|
20135
20174
|
if (words === undefined || words.length < 4)
|
|
20136
20175
|
return;
|
|
20137
|
-
const program =
|
|
20176
|
+
const program = basename9(words[0]).replace(/\.exe$/iu, "");
|
|
20138
20177
|
if (program !== "jikji" || words[1] !== "discover")
|
|
20139
20178
|
return;
|
|
20140
20179
|
const root = words[2];
|
|
@@ -20542,7 +20581,7 @@ class SerialQueue {
|
|
|
20542
20581
|
|
|
20543
20582
|
// src/jikji/version.ts
|
|
20544
20583
|
function parseJikjiVersion(output) {
|
|
20545
|
-
return
|
|
20584
|
+
return VERSION_LINE3.exec(output)?.[1];
|
|
20546
20585
|
}
|
|
20547
20586
|
function parts(version) {
|
|
20548
20587
|
return version.split(".").map((part) => Number.parseInt(part, 10));
|
|
@@ -20558,9 +20597,9 @@ function isSupportedJikjiVersion(version) {
|
|
|
20558
20597
|
}
|
|
20559
20598
|
return true;
|
|
20560
20599
|
}
|
|
20561
|
-
var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1",
|
|
20562
|
-
var
|
|
20563
|
-
|
|
20600
|
+
var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1", VERSION_LINE3;
|
|
20601
|
+
var init_version3 = __esm(() => {
|
|
20602
|
+
VERSION_LINE3 = /^jikji(?:-cli)?\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
|
|
20564
20603
|
});
|
|
20565
20604
|
|
|
20566
20605
|
// src/jikji/client.ts
|
|
@@ -20928,7 +20967,7 @@ var init_client11 = __esm(() => {
|
|
|
20928
20967
|
init_answer_pack();
|
|
20929
20968
|
init_installer3();
|
|
20930
20969
|
init_prepare_budget();
|
|
20931
|
-
|
|
20970
|
+
init_version3();
|
|
20932
20971
|
ABORTED_BEFORE_START = { ok: false, reason: "aborted", stdout: "", stderr: "", code: null };
|
|
20933
20972
|
});
|
|
20934
20973
|
|
|
@@ -22013,7 +22052,7 @@ import {
|
|
|
22013
22052
|
rmSync as rmSync10,
|
|
22014
22053
|
writeFileSync as writeFileSync14
|
|
22015
22054
|
} from "node:fs";
|
|
22016
|
-
import { basename as
|
|
22055
|
+
import { basename as basename10, dirname as dirname19, isAbsolute as isAbsolute9, join as join37, relative as relative7, sep as sep8 } from "node:path";
|
|
22017
22056
|
function syncMinSyncWorkspace(root, options = {}) {
|
|
22018
22057
|
const workspacePath = options.workspacePath ?? minSyncWorkspaceRoot(root);
|
|
22019
22058
|
const filesRoot = join37(workspacePath, "files");
|
|
@@ -22054,7 +22093,7 @@ function syncMinSyncWorkspace(root, options = {}) {
|
|
|
22054
22093
|
return { workspacePath, entries, excluded };
|
|
22055
22094
|
}
|
|
22056
22095
|
function stagingStatePath(workspacePath) {
|
|
22057
|
-
return join37(dirname19(workspacePath), `.${
|
|
22096
|
+
return join37(dirname19(workspacePath), `.${basename10(workspacePath)}-autorag-staging.json`);
|
|
22058
22097
|
}
|
|
22059
22098
|
function loadStagingState2(workspacePath) {
|
|
22060
22099
|
const path = stagingStatePath(workspacePath);
|
|
@@ -22105,7 +22144,7 @@ function ensureManagedDirectory(filesRoot, directory) {
|
|
|
22105
22144
|
}
|
|
22106
22145
|
}
|
|
22107
22146
|
function copyFileAtomically(source, destination) {
|
|
22108
|
-
const temporaryPath = join37(dirname19(destination), `.${
|
|
22147
|
+
const temporaryPath = join37(dirname19(destination), `.${basename10(destination)}.${randomUUID6()}.tmp`);
|
|
22109
22148
|
try {
|
|
22110
22149
|
copyFileSync4(source, temporaryPath);
|
|
22111
22150
|
renameWithRetry(temporaryPath, destination);
|
|
@@ -22159,7 +22198,7 @@ var init_workspace = __esm(() => {
|
|
|
22159
22198
|
|
|
22160
22199
|
// src/minsync/method.ts
|
|
22161
22200
|
import { existsSync as existsSync24, readFileSync as readFileSync22, realpathSync as realpathSync7, statSync as statSync8 } from "node:fs";
|
|
22162
|
-
import { basename as
|
|
22201
|
+
import { basename as basename11, join as join38, normalize } from "node:path";
|
|
22163
22202
|
|
|
22164
22203
|
class MinSyncVectorMethod {
|
|
22165
22204
|
root;
|
|
@@ -22260,7 +22299,7 @@ class MinSyncVectorMethod {
|
|
|
22260
22299
|
const entry = byPath.get(hit.path);
|
|
22261
22300
|
if (!entry || !matchesVirtualPathScope(entry.virtualPath, options.scope))
|
|
22262
22301
|
continue;
|
|
22263
|
-
const chunkDiscriminator = hit.docId ??
|
|
22302
|
+
const chunkDiscriminator = hit.docId ?? basename11(hit.path);
|
|
22264
22303
|
const chunkId = `minsync:${entry.virtualPath}:${chunkDiscriminator}`;
|
|
22265
22304
|
const location = locateChunkInMirror(mirrorTexts, entry, hit.text);
|
|
22266
22305
|
results.push({
|
|
@@ -22360,18 +22399,24 @@ var init_minsync = __esm(() => {
|
|
|
22360
22399
|
|
|
22361
22400
|
// src/mirror/cloud-fingerprints.ts
|
|
22362
22401
|
import { existsSync as existsSync25, readFileSync as readFileSync23 } from "node:fs";
|
|
22363
|
-
import { join as join39 } from "node:path";
|
|
22402
|
+
import { extname as extname5, join as join39 } from "node:path";
|
|
22364
22403
|
function cloudFingerprintStorePath(root) {
|
|
22365
22404
|
return join39(root, ".autorag", CLOUD_FINGERPRINTS_FILE);
|
|
22366
22405
|
}
|
|
22367
22406
|
function emptyCloudFingerprints() {
|
|
22368
|
-
return { version:
|
|
22407
|
+
return { version: 2, entries: {} };
|
|
22369
22408
|
}
|
|
22370
|
-
function
|
|
22409
|
+
function readCloudFingerprint(sourcePath, value, storeVersion) {
|
|
22371
22410
|
if (typeof value !== "object" || value === null)
|
|
22372
|
-
return
|
|
22411
|
+
return;
|
|
22373
22412
|
const record = value;
|
|
22374
|
-
|
|
22413
|
+
const { sourceVersion, exact, modified, revision } = record;
|
|
22414
|
+
if (typeof sourceVersion !== "string" || typeof exact !== "string" || modified !== null && typeof modified !== "string" || revision !== null && typeof revision !== "string") {
|
|
22415
|
+
return;
|
|
22416
|
+
}
|
|
22417
|
+
if (storeVersion !== 2 && (record.chars === 0 || extname5(sourcePath).toLowerCase() === ".hwp"))
|
|
22418
|
+
return;
|
|
22419
|
+
return { sourceVersion, exact, modified, revision };
|
|
22375
22420
|
}
|
|
22376
22421
|
function loadCloudFingerprints(root) {
|
|
22377
22422
|
const path = cloudFingerprintStorePath(root);
|
|
@@ -22384,24 +22429,26 @@ function loadCloudFingerprints(root) {
|
|
|
22384
22429
|
const { entries } = parsed;
|
|
22385
22430
|
if (typeof entries !== "object" || entries === null)
|
|
22386
22431
|
return emptyCloudFingerprints();
|
|
22432
|
+
const storeVersion = "version" in parsed ? parsed.version : undefined;
|
|
22387
22433
|
const valid = {};
|
|
22388
|
-
for (const [sourcePath,
|
|
22389
|
-
|
|
22434
|
+
for (const [sourcePath, value] of Object.entries(entries)) {
|
|
22435
|
+
const fingerprint = readCloudFingerprint(sourcePath, value, storeVersion);
|
|
22436
|
+
if (fingerprint !== undefined)
|
|
22390
22437
|
valid[sourcePath] = fingerprint;
|
|
22391
22438
|
}
|
|
22392
|
-
return { version:
|
|
22439
|
+
return { version: 2, entries: valid };
|
|
22393
22440
|
} catch {
|
|
22394
22441
|
return emptyCloudFingerprints();
|
|
22395
22442
|
}
|
|
22396
22443
|
}
|
|
22397
22444
|
function saveCloudFingerprints(root, entries) {
|
|
22398
|
-
writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version:
|
|
22445
|
+
writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version: 2, entries }, null, 2)}
|
|
22399
22446
|
`);
|
|
22400
22447
|
}
|
|
22401
22448
|
function cloudFingerprintExclusions(store) {
|
|
22402
22449
|
const map = new Map;
|
|
22403
22450
|
for (const [sourcePath, fingerprint] of Object.entries(store.entries)) {
|
|
22404
|
-
map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified
|
|
22451
|
+
map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified });
|
|
22405
22452
|
}
|
|
22406
22453
|
return map;
|
|
22407
22454
|
}
|
|
@@ -22793,7 +22840,7 @@ class Parser {
|
|
|
22793
22840
|
}
|
|
22794
22841
|
|
|
22795
22842
|
// src/parser/csv.ts
|
|
22796
|
-
import { basename as
|
|
22843
|
+
import { basename as basename12, extname as extname6 } from "node:path";
|
|
22797
22844
|
function toCell(text) {
|
|
22798
22845
|
return text.trim() === "" ? undefined : { text, kind: classifyText(text) };
|
|
22799
22846
|
}
|
|
@@ -22924,7 +22971,7 @@ var init_csv = __esm(() => {
|
|
|
22924
22971
|
this.#maxRows = options.maxRows ?? DEFAULT_MAX_ROWS;
|
|
22925
22972
|
}
|
|
22926
22973
|
async parse(input) {
|
|
22927
|
-
const extension =
|
|
22974
|
+
const extension = extname6(input.virtualPath);
|
|
22928
22975
|
const format = extension.slice(1).toLowerCase();
|
|
22929
22976
|
let text = decodeText(input.bytes).replace(/^\uFEFF/, "");
|
|
22930
22977
|
const directive = SEPARATOR_DIRECTIVE.exec(text);
|
|
@@ -22936,10 +22983,10 @@ var init_csv = __esm(() => {
|
|
|
22936
22983
|
{
|
|
22937
22984
|
code: "parser-warning",
|
|
22938
22985
|
severity: "warning",
|
|
22939
|
-
message: `"${
|
|
22986
|
+
message: `"${basename12(input.virtualPath)}": indexed the first ${this.#maxRows} of ${total} rows`
|
|
22940
22987
|
}
|
|
22941
22988
|
] : [];
|
|
22942
|
-
const grid = { name:
|
|
22989
|
+
const grid = { name: basename12(input.virtualPath, extension), rows: rows.map((row) => row.map(toCell)) };
|
|
22943
22990
|
return {
|
|
22944
22991
|
markdown: normalizeMarkdown(renderWorkbook([grid])),
|
|
22945
22992
|
metadata: { parser: this.name, format },
|
|
@@ -23478,7 +23525,7 @@ var init_plain_text = __esm(() => {
|
|
|
23478
23525
|
});
|
|
23479
23526
|
|
|
23480
23527
|
// src/parser/registry.ts
|
|
23481
|
-
import { extname as
|
|
23528
|
+
import { extname as extname7 } from "node:path";
|
|
23482
23529
|
|
|
23483
23530
|
class ParserRegistry {
|
|
23484
23531
|
parsersByExtension = new Map;
|
|
@@ -23497,7 +23544,7 @@ class ParserRegistry {
|
|
|
23497
23544
|
}
|
|
23498
23545
|
}
|
|
23499
23546
|
getForVirtualPath(virtualPath) {
|
|
23500
|
-
return this.parsersByExtension.get(normalizeExtension(
|
|
23547
|
+
return this.parsersByExtension.get(normalizeExtension(extname7(virtualPath)));
|
|
23501
23548
|
}
|
|
23502
23549
|
list() {
|
|
23503
23550
|
return Array.from(new Set(this.parsersByExtension.values()));
|
|
@@ -23510,7 +23557,7 @@ function normalizeExtension(extension) {
|
|
|
23510
23557
|
var init_registry = () => {};
|
|
23511
23558
|
|
|
23512
23559
|
// src/parser/spreadsheet.ts
|
|
23513
|
-
import { extname as
|
|
23560
|
+
import { extname as extname8 } from "node:path";
|
|
23514
23561
|
import * as XLSX from "xlsx";
|
|
23515
23562
|
function assertWorkbookSignature(format, bytes) {
|
|
23516
23563
|
if (startsWith(bytes, ZIP_SIGNATURE) || startsWith(bytes, OLE_SIGNATURE))
|
|
@@ -23609,7 +23656,7 @@ var init_spreadsheet = __esm(() => {
|
|
|
23609
23656
|
this.#maxRowsPerSheet = options.maxRowsPerSheet ?? DEFAULT_MAX_ROWS_PER_SHEET;
|
|
23610
23657
|
}
|
|
23611
23658
|
async parse(input) {
|
|
23612
|
-
const format =
|
|
23659
|
+
const format = extname8(input.virtualPath).slice(1).toLowerCase();
|
|
23613
23660
|
let workbook;
|
|
23614
23661
|
try {
|
|
23615
23662
|
assertWorkbookSignature(format, input.bytes);
|
|
@@ -23883,6 +23930,10 @@ class CloudGate {
|
|
|
23883
23930
|
isPlaceholder(path) {
|
|
23884
23931
|
return this.placeholders.has(path);
|
|
23885
23932
|
}
|
|
23933
|
+
startBatch() {
|
|
23934
|
+
this.downloadedFiles = 0;
|
|
23935
|
+
this.downloadedBytes = 0;
|
|
23936
|
+
}
|
|
23886
23937
|
async decide(path, sizeBytes, directory) {
|
|
23887
23938
|
if (this.probe === undefined || !this.placeholders.has(path))
|
|
23888
23939
|
return LOCAL;
|
|
@@ -23892,7 +23943,7 @@ class CloudGate {
|
|
|
23892
23943
|
const overFiles = this.downloadedFiles >= this.maxFiles;
|
|
23893
23944
|
const overBytes = this.downloadedFiles > 0 && this.downloadedBytes + sizeBytes > this.maxBytes;
|
|
23894
23945
|
if (overFiles || overBytes)
|
|
23895
|
-
return { kind: "
|
|
23946
|
+
return { kind: "next-batch" };
|
|
23896
23947
|
this.downloadedFiles += 1;
|
|
23897
23948
|
this.downloadedBytes += sizeBytes;
|
|
23898
23949
|
return { kind: "download" };
|
|
@@ -23937,7 +23988,7 @@ var init_cloud = __esm(() => {
|
|
|
23937
23988
|
// src/mirror/sync.ts
|
|
23938
23989
|
import { createHash as createHash17 } from "node:crypto";
|
|
23939
23990
|
import { opendir as opendir2, readFile, stat as stat4 } from "node:fs/promises";
|
|
23940
|
-
import { basename as
|
|
23991
|
+
import { basename as basename13, dirname as dirname22, extname as extname9, join as join43, resolve as resolve15, sep as sep9 } from "node:path";
|
|
23941
23992
|
import { setTimeout as sleep4 } from "node:timers/promises";
|
|
23942
23993
|
function errorCode(error) {
|
|
23943
23994
|
if (!(error instanceof Error))
|
|
@@ -24162,25 +24213,26 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
24162
24213
|
};
|
|
24163
24214
|
const deferDeletions = options.onBatch !== undefined;
|
|
24164
24215
|
const batchSize = Math.max(1, Math.floor(options.batchSize ?? DEFAULT_MIRROR_BATCH_SIZE));
|
|
24165
|
-
const batchCount = Math.max(1, Math.ceil(current.length / batchSize));
|
|
24166
24216
|
let batchIndex = 0;
|
|
24167
24217
|
let batchStopped = false;
|
|
24168
24218
|
const runBatch = async (virtualPaths, final) => {
|
|
24169
24219
|
if (options.onBatch === undefined)
|
|
24170
24220
|
return;
|
|
24171
24221
|
batchIndex += 1;
|
|
24172
|
-
const outcome = await options.onBatch({ index: batchIndex,
|
|
24222
|
+
const outcome = await options.onBatch({ index: batchIndex, virtualPaths, final });
|
|
24173
24223
|
if (outcome?.stop === true)
|
|
24174
24224
|
batchStopped = true;
|
|
24175
24225
|
};
|
|
24176
24226
|
if (!deferDeletions)
|
|
24177
24227
|
await applyDeleteDecisions();
|
|
24178
24228
|
await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
|
|
24179
|
-
|
|
24180
|
-
|
|
24181
|
-
const
|
|
24229
|
+
let batchStart = 0;
|
|
24230
|
+
for (;; ) {
|
|
24231
|
+
const batchLimit = Math.min(batchStart + batchSize, current.length);
|
|
24232
|
+
let batchEnd = batchLimit;
|
|
24182
24233
|
const changedBefore = written + deleted;
|
|
24183
|
-
|
|
24234
|
+
gate.startBatch();
|
|
24235
|
+
for (let position = batchStart;position < batchLimit; position += 1) {
|
|
24184
24236
|
const entry = current[position];
|
|
24185
24237
|
options.onProgress?.({
|
|
24186
24238
|
processed: position,
|
|
@@ -24230,15 +24282,9 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
24230
24282
|
});
|
|
24231
24283
|
continue;
|
|
24232
24284
|
}
|
|
24233
|
-
if (cloudDecision.kind === "
|
|
24234
|
-
|
|
24235
|
-
|
|
24236
|
-
code: "cloud-download-deferred",
|
|
24237
|
-
severity: "info",
|
|
24238
|
-
message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
|
|
24239
|
-
source: entry.virtualPath
|
|
24240
|
-
});
|
|
24241
|
-
continue;
|
|
24285
|
+
if (cloudDecision.kind === "next-batch") {
|
|
24286
|
+
batchEnd = position;
|
|
24287
|
+
break;
|
|
24242
24288
|
}
|
|
24243
24289
|
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
24244
24290
|
let bytes;
|
|
@@ -24311,7 +24357,7 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
24311
24357
|
continue;
|
|
24312
24358
|
}
|
|
24313
24359
|
await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
|
|
24314
|
-
path:
|
|
24360
|
+
path: basename13(outputPath),
|
|
24315
24361
|
size: Buffer.byteLength(markdown)
|
|
24316
24362
|
});
|
|
24317
24363
|
for (const diagnostic of parsed.diagnostics ?? [])
|
|
@@ -24325,7 +24371,6 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
24325
24371
|
cloudFingerprintEntries[entry.sourcePath] = {
|
|
24326
24372
|
sourceVersion: entry.sourceVersion,
|
|
24327
24373
|
exact: fingerprintResult.exact,
|
|
24328
|
-
chars: fingerprintResult.chars,
|
|
24329
24374
|
modified: fingerprintResult.modified,
|
|
24330
24375
|
revision: fingerprintResult.revision
|
|
24331
24376
|
};
|
|
@@ -24353,13 +24398,15 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
24353
24398
|
}
|
|
24354
24399
|
}
|
|
24355
24400
|
}
|
|
24401
|
+
const finalBatch = batchEnd >= current.length;
|
|
24356
24402
|
if (finalBatch && deferDeletions)
|
|
24357
24403
|
await applyDeleteDecisions();
|
|
24358
24404
|
if (finalBatch || written + deleted !== changedBefore) {
|
|
24359
24405
|
await runBatch(current.slice(batchStart, batchEnd).map((item) => item.virtualPath), finalBatch);
|
|
24360
24406
|
}
|
|
24361
|
-
if (batchStopped)
|
|
24407
|
+
if (finalBatch || batchStopped)
|
|
24362
24408
|
break;
|
|
24409
|
+
batchStart = batchEnd;
|
|
24363
24410
|
}
|
|
24364
24411
|
await session.finish();
|
|
24365
24412
|
if (fingerprintCloud) {
|
|
@@ -24449,7 +24496,7 @@ async function detectMirrorStaleness(options) {
|
|
|
24449
24496
|
}
|
|
24450
24497
|
}
|
|
24451
24498
|
function standaloneImageTooSmallMessage(virtualPath, bytes) {
|
|
24452
|
-
if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(
|
|
24499
|
+
if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname9(virtualPath))))
|
|
24453
24500
|
return;
|
|
24454
24501
|
return ocrImageTooSmallMessage(bytes);
|
|
24455
24502
|
}
|
|
@@ -24617,7 +24664,7 @@ async function collectFiles(state, directory, depth) {
|
|
|
24617
24664
|
}
|
|
24618
24665
|
if (!entry.isFile())
|
|
24619
24666
|
continue;
|
|
24620
|
-
const extension = normalizeExtension2(
|
|
24667
|
+
const extension = normalizeExtension2(extname9(entry.name));
|
|
24621
24668
|
if (!state.supportedExtensions.has(extension))
|
|
24622
24669
|
continue;
|
|
24623
24670
|
let fileStat;
|
|
@@ -29662,7 +29709,7 @@ var init_automation_tool = __esm(() => {
|
|
|
29662
29709
|
});
|
|
29663
29710
|
|
|
29664
29711
|
// src/agent/citation-links.ts
|
|
29665
|
-
import { basename as
|
|
29712
|
+
import { basename as basename14, isAbsolute as isAbsolute12 } from "node:path";
|
|
29666
29713
|
import { pathToFileURL as pathToFileURL3 } from "node:url";
|
|
29667
29714
|
function openableTarget(source) {
|
|
29668
29715
|
if (/^https?:\/\//iu.test(source)) {
|
|
@@ -29680,7 +29727,7 @@ function linkCitations(markdown, sourceOf, isFile) {
|
|
|
29680
29727
|
for (const marker of answerMarkers(markdown, isFile)) {
|
|
29681
29728
|
if (marker.kind === "number")
|
|
29682
29729
|
continue;
|
|
29683
|
-
const labels = marker.kind === "file" ? [{ label:
|
|
29730
|
+
const labels = marker.kind === "file" ? [{ label: basename14(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
|
|
29684
29731
|
let linked = false;
|
|
29685
29732
|
const parts = labels.map(({ label, source }) => {
|
|
29686
29733
|
const target = source === undefined ? undefined : openableTarget(source);
|
|
@@ -31806,10 +31853,7 @@ function renderScanReport(details) {
|
|
|
31806
31853
|
const lines = [
|
|
31807
31854
|
`dupey scanned ${details.scans.length} configured root(s).`,
|
|
31808
31855
|
`families=${details.familyCount} exactDuplicates=${details.exactDuplicateCount} extractionErrors=${errors}`,
|
|
31809
|
-
"Review exact families before cleanup; near/contains families are not safe deletion evidence."
|
|
31810
|
-
...details.skippedRoots.length === 0 ? [] : [
|
|
31811
|
-
`Not scanned (inside a cloud or network folder; scanning would download or re-read every file): ${details.skippedRoots.join(", ")}. Duplicates there are not reported, and an exact duplicate is only found against files an earlier refresh fingerprinted.`
|
|
31812
|
-
]
|
|
31856
|
+
"Review exact families before cleanup; near/contains families are not safe deletion evidence."
|
|
31813
31857
|
];
|
|
31814
31858
|
let rendered = 0;
|
|
31815
31859
|
let omitted = 0;
|
|
@@ -32787,6 +32831,56 @@ var init_packaged_skill = __esm(() => {
|
|
|
32787
32831
|
PACKAGED_SKILL_ROOTS = ["skills", "internal-skills"];
|
|
32788
32832
|
});
|
|
32789
32833
|
|
|
32834
|
+
// src/agent/pipelined-sync.ts
|
|
32835
|
+
class PipelinedSync {
|
|
32836
|
+
#sync;
|
|
32837
|
+
#covered = [];
|
|
32838
|
+
#dirty = false;
|
|
32839
|
+
#running;
|
|
32840
|
+
#failure;
|
|
32841
|
+
#stopped = false;
|
|
32842
|
+
#runs = 0;
|
|
32843
|
+
constructor(options) {
|
|
32844
|
+
this.#sync = options.sync;
|
|
32845
|
+
}
|
|
32846
|
+
async batch(batch) {
|
|
32847
|
+
this.#raise();
|
|
32848
|
+
this.#covered.push(...batch.virtualPaths);
|
|
32849
|
+
this.#dirty = true;
|
|
32850
|
+
if (!this.#stopped)
|
|
32851
|
+
this.#running ??= this.#loop();
|
|
32852
|
+
if (batch.final)
|
|
32853
|
+
await this.#running;
|
|
32854
|
+
this.#raise();
|
|
32855
|
+
return this.#stopped ? { stop: true } : undefined;
|
|
32856
|
+
}
|
|
32857
|
+
async settle() {
|
|
32858
|
+
this.#stopped = true;
|
|
32859
|
+
await this.#running;
|
|
32860
|
+
}
|
|
32861
|
+
#raise() {
|
|
32862
|
+
if (this.#failure !== undefined)
|
|
32863
|
+
throw this.#failure.error;
|
|
32864
|
+
}
|
|
32865
|
+
async#loop() {
|
|
32866
|
+
try {
|
|
32867
|
+
while (this.#dirty && !this.#stopped) {
|
|
32868
|
+
this.#dirty = false;
|
|
32869
|
+
const virtualPaths = this.#covered;
|
|
32870
|
+
this.#covered = [];
|
|
32871
|
+
this.#runs += 1;
|
|
32872
|
+
const outcome = await this.#sync({ index: this.#runs, virtualPaths });
|
|
32873
|
+
if (outcome?.stop === true)
|
|
32874
|
+
this.#stopped = true;
|
|
32875
|
+
}
|
|
32876
|
+
} catch (error) {
|
|
32877
|
+
this.#failure = { error };
|
|
32878
|
+
} finally {
|
|
32879
|
+
this.#running = undefined;
|
|
32880
|
+
}
|
|
32881
|
+
}
|
|
32882
|
+
}
|
|
32883
|
+
|
|
32790
32884
|
// src/agent/query-decomposition.ts
|
|
32791
32885
|
import { completeSimple as completeSimple2 } from "@earendil-works/pi-ai/compat";
|
|
32792
32886
|
function buildDecompositionPrompt(query) {
|
|
@@ -33026,7 +33120,7 @@ var init_query_routing = __esm(() => {
|
|
|
33026
33120
|
false: "Every part of the question is answered with a concrete, specific, consistent fact (a name, date, number, or definite statement)."
|
|
33027
33121
|
});
|
|
33028
33122
|
BASE_ROUTE_OPTIONS = {
|
|
33029
|
-
local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. When unsure whether intrinsic knowledge suffices, choose local.",
|
|
33123
|
+
local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. A request to find, locate, open, or look up a document, file, slide deck, presentation, paper, note, message, or other material is local even when it names no owner ('find the slides about X', 'where is the Y presentation', 'Z 관련 자료 찾아줘'): such material lives in the user's own files and chats, and a well-known topic is not a reason to answer from general knowledge or the web. When unsure whether intrinsic knowledge suffices, choose local.",
|
|
33030
33124
|
extra: "None of the other options fits and no search of the user's own records is needed first. Either the request is answerable without looking anything up — stable common knowledge, a definition, simple reasoning, or small talk — or it is a task for the assistant's tools that is not file organization and not AutoRAG configuration: committing code, pushing a branch, opening a pull request, running tests or a build, installing a package, running a script, writing or editing code. Programming work is extra even when the code names the user's files or folders ('write a script that lowercases every PNG', 'add this to my parser'): the assistant is asked to produce or change code, not to act on those files now. So is a request to run something (tests, a build, a script, a command) and report the outcome, even when what it runs lives in the user's own repository: running it is a task for the tool, not a search of the user's records. Never extra when the request refers to the user's own life, files, records, chats, calendar, or history (e.g. 'my', 'I', 'our', 'my girlfriend', 'my case', 'my routine'), and never extra when the answer depends on facts that may be recent, specific, or changing (current events, recent releases, prices, versions): search those locally instead. If any other option fits, do not choose extra.",
|
|
33031
33125
|
organize: "The user asks the assistant to DO something to files or folders instead of finding information: move or copy files into folders, tidy or sort a folder, bulk or pattern rename (for example by date or number), rename files according to what their content says, convert files to another format, create folder structures, group files by type or topic, archive, or move to the trash ('tidy this up', 'move these into folders', 'bulk rename these files', 'rename the files to match their content', 'convert these to PDF', 'sort my Downloads folder'). It is about acting on files and folders, never about looking up what a document says or where something is. A question that only asks to find or read a file's content is local, not organize."
|
|
33032
33126
|
};
|
|
@@ -34264,7 +34358,7 @@ var init_page_loader = __esm(() => {
|
|
|
34264
34358
|
});
|
|
34265
34359
|
|
|
34266
34360
|
// src/web/fetch/url-target.ts
|
|
34267
|
-
import { extname as
|
|
34361
|
+
import { extname as extname10 } from "node:path";
|
|
34268
34362
|
function repairCollapsedScheme(value) {
|
|
34269
34363
|
const m = value.match(/^(https?):\/(?!\/)/i);
|
|
34270
34364
|
return m ? `${m[1]}://${value.slice(m[0].length)}` : value;
|
|
@@ -34283,7 +34377,7 @@ function getFilenameExtensionHint(filename) {
|
|
|
34283
34377
|
const lower = filename.toLowerCase();
|
|
34284
34378
|
if (lower.endsWith(".tar.gz"))
|
|
34285
34379
|
return ".tar.gz";
|
|
34286
|
-
return
|
|
34380
|
+
return extname10(filename).toLowerCase();
|
|
34287
34381
|
}
|
|
34288
34382
|
function getExtensionHint(url, contentDisposition) {
|
|
34289
34383
|
if (contentDisposition) {
|
|
@@ -34460,7 +34554,7 @@ var init_convert2 = __esm(() => {
|
|
|
34460
34554
|
});
|
|
34461
34555
|
|
|
34462
34556
|
// src/web/fetch/feeds.ts
|
|
34463
|
-
import { extname as
|
|
34557
|
+
import { extname as extname11 } from "node:path";
|
|
34464
34558
|
import { XMLParser as XMLParser3 } from "fast-xml-parser";
|
|
34465
34559
|
function getHtmlAttribute(tag, attribute) {
|
|
34466
34560
|
const pattern = new RegExp(`\\b${attribute}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'=<>]+))`, "i");
|
|
@@ -34523,7 +34617,7 @@ function extractDocumentLinks(html, baseUrl) {
|
|
|
34523
34617
|
const href = getHtmlAttribute(tag, "href");
|
|
34524
34618
|
if (!href)
|
|
34525
34619
|
continue;
|
|
34526
|
-
const ext =
|
|
34620
|
+
const ext = extname11(href).toLowerCase();
|
|
34527
34621
|
if (!CONVERTIBLE_EXTENSIONS2.has(ext))
|
|
34528
34622
|
continue;
|
|
34529
34623
|
const resolved = href.startsWith("http") ? href : new URL(href, baseUrl).href;
|
|
@@ -36191,7 +36285,7 @@ var init_web_search_tool = __esm(() => {
|
|
|
36191
36285
|
// src/agent/agent.ts
|
|
36192
36286
|
import { randomUUID as randomUUID7 } from "node:crypto";
|
|
36193
36287
|
import { existsSync as existsSync32, watch as fsWatch, mkdirSync as mkdirSync23, realpathSync as realpathSync8, statSync as statSync11, writeFileSync as writeFileSync17 } from "node:fs";
|
|
36194
|
-
import { basename as
|
|
36288
|
+
import { basename as basename15, dirname as dirname27, join as join53, relative as relative8, resolve as resolve17, sep as sep10 } from "node:path";
|
|
36195
36289
|
import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
|
|
36196
36290
|
import { getAgentDir as getAgentDir5, keyText } from "@earendil-works/pi-coding-agent";
|
|
36197
36291
|
function describeDecisionModel(status) {
|
|
@@ -36626,10 +36720,20 @@ class AutoRAGAgent {
|
|
|
36626
36720
|
}
|
|
36627
36721
|
async scanDuplicateDocuments() {
|
|
36628
36722
|
if (this.dupeyOptions === false) {
|
|
36629
|
-
return { scans: [],
|
|
36723
|
+
return { scans: [], familyCount: 0, exactDuplicateCount: 0 };
|
|
36724
|
+
}
|
|
36725
|
+
const cloudRoots = await this.cloudRoots();
|
|
36726
|
+
const scanRoots = [];
|
|
36727
|
+
for (const searchPath of this.searchPaths) {
|
|
36728
|
+
const scope = cloudScope(searchPath, cloudRoots);
|
|
36729
|
+
if (scope.insideCloudRoot)
|
|
36730
|
+
continue;
|
|
36731
|
+
scanRoots.push({
|
|
36732
|
+
path: searchPath,
|
|
36733
|
+
excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path))
|
|
36734
|
+
});
|
|
36630
36735
|
}
|
|
36631
|
-
const {
|
|
36632
|
-
const scans = await Promise.all(targets.map(({ path, excludeDirs }) => scanWithDupey(path, {
|
|
36736
|
+
const scans = await Promise.all(scanRoots.map(({ path, excludeDirs }) => scanWithDupey(path, {
|
|
36633
36737
|
...this.dupeyOptions || {},
|
|
36634
36738
|
...excludeDirs.length === 0 ? {} : { excludeDirs }
|
|
36635
36739
|
})));
|
|
@@ -36643,7 +36747,7 @@ class AutoRAGAgent {
|
|
|
36643
36747
|
}
|
|
36644
36748
|
return count + [...hashes.values()].reduce((sum, size) => sum + Math.max(0, size - 1), 0);
|
|
36645
36749
|
}, 0);
|
|
36646
|
-
return { scans,
|
|
36750
|
+
return { scans, familyCount, exactDuplicateCount };
|
|
36647
36751
|
}
|
|
36648
36752
|
loadMemoryContextFor(query) {
|
|
36649
36753
|
if (this.memoryContextCache?.query === query)
|
|
@@ -38296,6 +38400,10 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38296
38400
|
inFlight: true,
|
|
38297
38401
|
lastStartedAt: startedAt
|
|
38298
38402
|
};
|
|
38403
|
+
const syncAbort = new AbortController;
|
|
38404
|
+
const forwardAbort = () => syncAbort.abort(signal?.reason);
|
|
38405
|
+
signal?.addEventListener("abort", forwardAbort, { once: true });
|
|
38406
|
+
let minSyncPipeline;
|
|
38299
38407
|
try {
|
|
38300
38408
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38301
38409
|
const batchedMinSync = needsParsed && wants("minsync");
|
|
@@ -38303,44 +38411,53 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38303
38411
|
const minSyncBatchFailures = [];
|
|
38304
38412
|
let minSyncSyncedBase = 0;
|
|
38305
38413
|
let minSyncTotalBase = 0;
|
|
38306
|
-
|
|
38307
|
-
|
|
38308
|
-
|
|
38309
|
-
|
|
38310
|
-
|
|
38311
|
-
|
|
38312
|
-
|
|
38313
|
-
|
|
38314
|
-
|
|
38315
|
-
|
|
38316
|
-
|
|
38317
|
-
|
|
38318
|
-
|
|
38319
|
-
|
|
38320
|
-
total: batchTotal
|
|
38321
|
-
});
|
|
38322
|
-
});
|
|
38323
|
-
minSyncBatches.push(result);
|
|
38324
|
-
if (result.ok)
|
|
38325
|
-
minSyncBatchFailures.length = 0;
|
|
38326
|
-
else if (result.reason !== "lock-held") {
|
|
38327
|
-
for (const virtualPath of batch.virtualPaths) {
|
|
38328
|
-
minSyncBatchFailures.push({
|
|
38329
|
-
code: "minsync-sync-failed",
|
|
38330
|
-
severity: "warning",
|
|
38331
|
-
message: `This file stayed at its previous index entry: its batch could not be indexed and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
|
|
38332
|
-
source: virtualPath
|
|
38414
|
+
minSyncPipeline = batchedMinSync ? new PipelinedSync({
|
|
38415
|
+
sync: async ({ index, virtualPaths }) => {
|
|
38416
|
+
syncAbort.signal.throwIfAborted();
|
|
38417
|
+
let syncTotal = minSyncTotalBase;
|
|
38418
|
+
const result = await this.syncMinSync(force && index === 1, syncAbort.signal, (minsyncProgress) => {
|
|
38419
|
+
syncTotal = minSyncTotalBase + minsyncProgress.total;
|
|
38420
|
+
progress = updateRefreshProgress(progress, {
|
|
38421
|
+
minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: syncTotal }
|
|
38422
|
+
});
|
|
38423
|
+
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38424
|
+
emitProgress({
|
|
38425
|
+
phase: "minsync",
|
|
38426
|
+
processed: minSyncSyncedBase + minsyncProgress.processed,
|
|
38427
|
+
total: syncTotal
|
|
38333
38428
|
});
|
|
38429
|
+
});
|
|
38430
|
+
minSyncBatches.push(result);
|
|
38431
|
+
if (result.ok)
|
|
38432
|
+
minSyncBatchFailures.length = 0;
|
|
38433
|
+
else if (result.reason !== "lock-held") {
|
|
38434
|
+
for (const virtualPath of virtualPaths) {
|
|
38435
|
+
minSyncBatchFailures.push({
|
|
38436
|
+
code: "minsync-sync-failed",
|
|
38437
|
+
severity: "warning",
|
|
38438
|
+
message: `This file stayed at its previous index entry: the sync that covered it could not index it and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
|
|
38439
|
+
source: virtualPath
|
|
38440
|
+
});
|
|
38441
|
+
}
|
|
38334
38442
|
}
|
|
38443
|
+
minSyncSyncedBase = syncTotal;
|
|
38444
|
+
minSyncTotalBase = syncTotal;
|
|
38445
|
+
return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
|
|
38335
38446
|
}
|
|
38336
|
-
|
|
38337
|
-
|
|
38338
|
-
return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
|
|
38339
|
-
};
|
|
38447
|
+
}) : undefined;
|
|
38448
|
+
const pipeline = minSyncPipeline;
|
|
38340
38449
|
const summary = needsParsed ? await this.syncParsedMirrors(force, {
|
|
38341
38450
|
...opts?.retrySkipped === true ? { retrySkipped: true } : {},
|
|
38342
38451
|
...signal === undefined ? {} : { signal },
|
|
38343
|
-
...
|
|
38452
|
+
...pipeline === undefined ? {} : {
|
|
38453
|
+
onBatch: (batch) => {
|
|
38454
|
+
if (batch.final) {
|
|
38455
|
+
progress = updateRefreshProgress(progress, { phase: "minsync" });
|
|
38456
|
+
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38457
|
+
}
|
|
38458
|
+
return pipeline.batch(batch);
|
|
38459
|
+
}
|
|
38460
|
+
},
|
|
38344
38461
|
onProgress: (parsedProgress) => {
|
|
38345
38462
|
progress = updateRefreshProgress(progress, {
|
|
38346
38463
|
phase: "parsed",
|
|
@@ -38476,6 +38593,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38476
38593
|
...fsearch !== undefined ? { fsearch } : {}
|
|
38477
38594
|
};
|
|
38478
38595
|
} catch (error) {
|
|
38596
|
+
syncAbort.abort(error);
|
|
38479
38597
|
this.refreshState = {
|
|
38480
38598
|
...this.refreshState,
|
|
38481
38599
|
lastOutcome: "failed",
|
|
@@ -38489,6 +38607,8 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38489
38607
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38490
38608
|
throw error;
|
|
38491
38609
|
} finally {
|
|
38610
|
+
signal?.removeEventListener("abort", forwardAbort);
|
|
38611
|
+
await minSyncPipeline?.settle();
|
|
38492
38612
|
this.refreshState = {
|
|
38493
38613
|
...this.refreshState,
|
|
38494
38614
|
inFlight: false,
|
|
@@ -38710,15 +38830,15 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38710
38830
|
const cloudRoots = await this.cloudRoots();
|
|
38711
38831
|
const fingerprints = cloudFingerprintExclusions(loadCloudFingerprints(this.workspaceProjectRoot));
|
|
38712
38832
|
for (const searchPath of this.searchPaths) {
|
|
38713
|
-
const
|
|
38833
|
+
const scope = cloudScope(searchPath, cloudRoots);
|
|
38714
38834
|
let scan;
|
|
38715
|
-
if (
|
|
38835
|
+
if (scope.insideCloudRoot) {
|
|
38716
38836
|
scan = { dir: searchPath, files: [], families: [], errors: [] };
|
|
38717
38837
|
} else {
|
|
38718
38838
|
try {
|
|
38719
38839
|
scan = await scanWithDupey(searchPath, {
|
|
38720
38840
|
...this.dupeyOptions || {},
|
|
38721
|
-
...
|
|
38841
|
+
...scope.nestedCloudRoots.length === 0 ? {} : { excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path)) }
|
|
38722
38842
|
});
|
|
38723
38843
|
} catch (error) {
|
|
38724
38844
|
if (!(error instanceof DupeyCliError))
|
|
@@ -39283,7 +39403,7 @@ function toMinSyncReasonDiagnostic(reason) {
|
|
|
39283
39403
|
function pinExcludedPath(path) {
|
|
39284
39404
|
const resolvedPath = resolve17(path);
|
|
39285
39405
|
try {
|
|
39286
|
-
return join53(realpathSync8(dirname27(resolvedPath)),
|
|
39406
|
+
return join53(realpathSync8(dirname27(resolvedPath)), basename15(resolvedPath));
|
|
39287
39407
|
} catch {
|
|
39288
39408
|
return resolvedPath;
|
|
39289
39409
|
}
|
|
@@ -40272,7 +40392,7 @@ function readPackageVersion() {
|
|
|
40272
40392
|
return "0.0.0";
|
|
40273
40393
|
}
|
|
40274
40394
|
}
|
|
40275
|
-
var
|
|
40395
|
+
var init_version4 = () => {};
|
|
40276
40396
|
|
|
40277
40397
|
// src/cli/commands/init.ts
|
|
40278
40398
|
import { existsSync as existsSync35, readFileSync as readFileSync30 } from "node:fs";
|
|
@@ -42104,20 +42224,18 @@ var init_health = __esm(() => {
|
|
|
42104
42224
|
|
|
42105
42225
|
// src/cli/commands/duplicates.ts
|
|
42106
42226
|
import { resolve as resolve23 } from "node:path";
|
|
42107
|
-
async function runDuplicates(ctx, scanner = (path
|
|
42227
|
+
async function runDuplicates(ctx, scanner = (path) => scanWithDupey(path)) {
|
|
42108
42228
|
try {
|
|
42109
42229
|
const config = resolveConfig({ flags: ctx.flags, cwd: ctx.cwd });
|
|
42110
42230
|
const requested = ctx.positionals[0];
|
|
42111
42231
|
const roots = requested ? [resolve23(ctx.cwd, requested)] : config.searchPaths.map((path) => resolve23(path));
|
|
42112
|
-
const
|
|
42113
|
-
const scans = await Promise.all(plan.targets.map(({ path, excludeDirs }) => scanner(path, excludeDirs)));
|
|
42232
|
+
const scans = await Promise.all(roots.map((root) => scanner(root)));
|
|
42114
42233
|
const exactGroups = scans.flatMap(exactGroupsFromScan);
|
|
42115
42234
|
const extractionErrors = scans.flatMap((scan) => scan.errors);
|
|
42116
42235
|
if (ctx.json) {
|
|
42117
42236
|
ctx.stdout(JSON.stringify({
|
|
42118
42237
|
ok: true,
|
|
42119
42238
|
roots,
|
|
42120
|
-
skippedRoots: plan.skipped,
|
|
42121
42239
|
exactGroups,
|
|
42122
42240
|
families: scans.flatMap((scan) => scan.families),
|
|
42123
42241
|
extractionErrors,
|
|
@@ -42126,14 +42244,11 @@ async function runDuplicates(ctx, scanner = (path, excludeDirs) => scanWithDupey
|
|
|
42126
42244
|
return 0;
|
|
42127
42245
|
}
|
|
42128
42246
|
const lines = [
|
|
42129
|
-
`duplicates: scanned ${
|
|
42247
|
+
`duplicates: scanned ${roots.length} root(s)`,
|
|
42130
42248
|
` exact groups: ${exactGroups.length}`,
|
|
42131
42249
|
` other families: ${scans.reduce((count, scan) => count + scan.families.length, 0)}`,
|
|
42132
42250
|
` extraction errors: ${extractionErrors.length}`
|
|
42133
42251
|
];
|
|
42134
|
-
for (const root of plan.skipped) {
|
|
42135
|
-
lines.push(` not scanned (inside a cloud or network folder; scanning would download or re-read every file): ${root}`);
|
|
42136
|
-
}
|
|
42137
42252
|
for (const [index, group] of exactGroups.entries()) {
|
|
42138
42253
|
lines.push(` exact group ${index + 1}:`);
|
|
42139
42254
|
for (const file of group.files)
|
|
@@ -42161,7 +42276,6 @@ function exactGroupsFromScan(scan) {
|
|
|
42161
42276
|
}
|
|
42162
42277
|
var init_duplicates = __esm(() => {
|
|
42163
42278
|
init_dupey();
|
|
42164
|
-
init_cloud_placeholder();
|
|
42165
42279
|
init_config5();
|
|
42166
42280
|
init_output();
|
|
42167
42281
|
});
|
|
@@ -42417,7 +42531,7 @@ var init_tui = __esm(() => {
|
|
|
42417
42531
|
init_open_target();
|
|
42418
42532
|
init_output();
|
|
42419
42533
|
init_update_check();
|
|
42420
|
-
|
|
42534
|
+
init_version4();
|
|
42421
42535
|
init_whatsapp_extension();
|
|
42422
42536
|
THINKING_LEVELS2 = ["off", "minimal", "low", "medium", "high", "xhigh", "max"];
|
|
42423
42537
|
});
|
|
@@ -42552,7 +42666,7 @@ async function runUpdateCheck(ctx, deps = {}) {
|
|
|
42552
42666
|
}
|
|
42553
42667
|
var init_update_check2 = __esm(() => {
|
|
42554
42668
|
init_update_check();
|
|
42555
|
-
|
|
42669
|
+
init_version4();
|
|
42556
42670
|
});
|
|
42557
42671
|
|
|
42558
42672
|
// src/cli/commands/attachments.ts
|
|
@@ -43675,7 +43789,7 @@ async function runAutomate(ctx) {
|
|
|
43675
43789
|
|
|
43676
43790
|
// src/cli/index.ts
|
|
43677
43791
|
init_output();
|
|
43678
|
-
|
|
43792
|
+
init_version4();
|
|
43679
43793
|
var BOOLEAN_FLAGS = new Set([
|
|
43680
43794
|
"json",
|
|
43681
43795
|
"debug",
|