@autorag/librarian 2.13.6 → 2.13.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/agent.d.ts +4 -4
- package/dist/agent/dupey-tool.d.ts +0 -2
- package/dist/agent/pipelined-sync.d.ts +34 -0
- package/dist/agent/search-documents.d.ts +1 -1
- package/dist/cli/commands/duplicates.d.ts +2 -3
- package/dist/cli/index.js +1521 -1407
- package/dist/core.js +2723 -2603
- package/dist/datasource/skills/factory.d.ts +4 -1
- package/dist/dupey/filter.d.ts +0 -2
- package/dist/dupey/index.d.ts +0 -1
- package/dist/dupey/version.d.ts +17 -0
- package/dist/index.js +2726 -2611
- package/dist/mcp/index.js +2726 -2609
- package/dist/mcp/server.d.ts +0 -3
- package/dist/minsync/installer.d.ts +1 -1
- package/dist/minsync/version.d.ts +3 -2
- package/dist/mirror/cloud-fingerprints.d.ts +6 -5
- package/dist/mirror/cloud.d.ts +9 -6
- package/dist/mirror/sync.d.ts +19 -15
- package/dist/profiles.js +1518 -1398
- package/package.json +1 -1
- package/dist/dupey/scan-plan.d.ts +0 -23
package/dist/profiles.js
CHANGED
|
@@ -3627,7 +3627,7 @@ function outdatedMinSyncVersion(binaryPath) {
|
|
|
3627
3627
|
}
|
|
3628
3628
|
return version !== undefined && !isSupportedMinSyncVersion(version) ? version : undefined;
|
|
3629
3629
|
}
|
|
3630
|
-
var MINSYNC_MIN_VERSION = "0.
|
|
3630
|
+
var MINSYNC_MIN_VERSION = "0.7.1", VERSION_PROBE_TIMEOUT_MS = 5000, VERSION_LINE, probeCache;
|
|
3631
3631
|
var init_version = __esm(() => {
|
|
3632
3632
|
VERSION_LINE = /^minsync\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
|
|
3633
3633
|
probeCache = new Map;
|
|
@@ -3871,7 +3871,7 @@ function isGitHubAsset(value) {
|
|
|
3871
3871
|
function isRecord2(value) {
|
|
3872
3872
|
return typeof value === "object" && value !== null;
|
|
3873
3873
|
}
|
|
3874
|
-
var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.
|
|
3874
|
+
var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.7.3", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
|
|
3875
3875
|
var init_installer = __esm(() => {
|
|
3876
3876
|
init_atomic_write();
|
|
3877
3877
|
init_process();
|
|
@@ -13749,7 +13749,8 @@ class WombatMethod {
|
|
|
13749
13749
|
return matchesDatasourceScope(datasourceSourcePath("wombat", this.instanceId, hit.chunkId), options.scope);
|
|
13750
13750
|
}).map((hit) => ({
|
|
13751
13751
|
id: `wombat:${this.instanceId}:${hit.chunkId}`,
|
|
13752
|
-
content: hit
|
|
13752
|
+
content: `${sessionLabel(hit)}
|
|
13753
|
+
${hit.snippet}`,
|
|
13753
13754
|
source: datasourceSourcePath("wombat", this.instanceId, hit.chunkId),
|
|
13754
13755
|
score: hit.score,
|
|
13755
13756
|
metadata: {
|
|
@@ -13772,6 +13773,20 @@ class WombatMethod {
|
|
|
13772
13773
|
})).slice(0, topK);
|
|
13773
13774
|
}
|
|
13774
13775
|
}
|
|
13776
|
+
function oneLine(value) {
|
|
13777
|
+
return value.replace(/\s+/gu, " ").trim();
|
|
13778
|
+
}
|
|
13779
|
+
function sessionLabel(hit) {
|
|
13780
|
+
const turns = hit.startTurn === undefined ? undefined : hit.endTurn === undefined || hit.endTurn === hit.startTurn ? `turn ${hit.startTurn}` : `turns ${hit.startTurn}-${hit.endTurn}`;
|
|
13781
|
+
const parts = [
|
|
13782
|
+
oneLine(hit.catalogKey),
|
|
13783
|
+
turns,
|
|
13784
|
+
hit.startedAt === undefined ? undefined : hit.startedAt.slice(0, 10),
|
|
13785
|
+
hit.title.length === 0 ? undefined : oneLine(hit.title),
|
|
13786
|
+
hit.cwd === undefined ? undefined : oneLine(hit.cwd)
|
|
13787
|
+
].filter((part) => part !== undefined && part.length > 0);
|
|
13788
|
+
return `[${parts.join(" | ")}]`;
|
|
13789
|
+
}
|
|
13775
13790
|
var WOMBAT_MODES, METHOD_TYPE;
|
|
13776
13791
|
var init_methods7 = __esm(() => {
|
|
13777
13792
|
init_errors2();
|
|
@@ -13884,7 +13899,7 @@ class WombatSkill {
|
|
|
13884
13899
|
`);
|
|
13885
13900
|
return {
|
|
13886
13901
|
name: "datasource-wombat",
|
|
13887
|
-
description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built.",
|
|
13902
|
+
description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built. Each hit names its session. Load this skill to open that session's full conversation or its original log with the tool calls that the search does not index.",
|
|
13888
13903
|
content: [
|
|
13889
13904
|
"# wombat coding-agent session datasource",
|
|
13890
13905
|
"",
|
|
@@ -13898,10 +13913,10 @@ class WombatSkill {
|
|
|
13898
13913
|
scopes,
|
|
13899
13914
|
"",
|
|
13900
13915
|
"The selectable retrieval modes are keyword (SPLADE), semantic, and hybrid. Scope arguments can only narrow trusted access. Source identifiers remain opaque and hierarchical.",
|
|
13901
|
-
"
|
|
13916
|
+
"Every result starts with a label line, `[<sessionKey> | turns A-B | date | title | project]`, followed by the snippet. `sessionKey` is `<agent>:<session id>`.",
|
|
13902
13917
|
"",
|
|
13903
13918
|
"## Native CLI",
|
|
13904
|
-
"The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the result's
|
|
13919
|
+
"The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the `sessionKey` from the result's label:",
|
|
13905
13920
|
"- `wombat show <sessionKey> [--turn N]`: the full cleaned conversation, or one turn.",
|
|
13906
13921
|
"- `wombat trace <sessionKey> --turn N --context K`: the original session log around that turn, including the tool calls and outputs that the index leaves out.",
|
|
13907
13922
|
"- `wombat chunk context <chunk id> --json`: the turns before and after a result.",
|
|
@@ -14055,6 +14070,7 @@ function common(config, workspaceRoot) {
|
|
|
14055
14070
|
function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
|
|
14056
14071
|
const skills = [];
|
|
14057
14072
|
const unknown = [];
|
|
14073
|
+
const retired = [];
|
|
14058
14074
|
for (const [name, raw] of Object.entries(config ?? {})) {
|
|
14059
14075
|
if (raw === false)
|
|
14060
14076
|
continue;
|
|
@@ -14066,6 +14082,10 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
|
|
|
14066
14082
|
if (entry.enabled === false)
|
|
14067
14083
|
continue;
|
|
14068
14084
|
const templateName = entry.type ?? name;
|
|
14085
|
+
if (Object.hasOwn(RETIRED_TEMPLATES, templateName)) {
|
|
14086
|
+
retired.push(name);
|
|
14087
|
+
continue;
|
|
14088
|
+
}
|
|
14069
14089
|
const builder = Object.hasOwn(BUILDERS, templateName) && datasourceSkillSupported(templateName) ? BUILDERS[templateName] : undefined;
|
|
14070
14090
|
if (builder === undefined) {
|
|
14071
14091
|
unknown.push(name);
|
|
@@ -14089,9 +14109,9 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
|
|
|
14089
14109
|
}
|
|
14090
14110
|
skills.push(typeof entry.description === "string" && entry.description.trim().length > 0 ? new DescribedDatasourceSkill(aliased, entry.description.trim()) : aliased);
|
|
14091
14111
|
}
|
|
14092
|
-
return { skills, unknown };
|
|
14112
|
+
return { skills, unknown, retired };
|
|
14093
14113
|
}
|
|
14094
|
-
var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES;
|
|
14114
|
+
var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES, RETIRED_TEMPLATES;
|
|
14095
14115
|
var init_factory = __esm(() => {
|
|
14096
14116
|
init_aliased_skill();
|
|
14097
14117
|
init_config2();
|
|
@@ -14247,6 +14267,7 @@ var init_factory = __esm(() => {
|
|
|
14247
14267
|
};
|
|
14248
14268
|
PLATFORM_ONLY_SKILLS = { kakao: "darwin" };
|
|
14249
14269
|
BUILTIN_DATASOURCE_SKILL_NAMES = builtinDatasourceSkillNames();
|
|
14270
|
+
RETIRED_TEMPLATES = { spotlight: true };
|
|
14250
14271
|
});
|
|
14251
14272
|
|
|
14252
14273
|
// src/embedding/http-embedder.ts
|
|
@@ -15833,7 +15854,7 @@ function buildAgentOptions(config, agentDir) {
|
|
|
15833
15854
|
const embedder = createEmbedderFromConfig({ embedder: config.minSync?.embedder });
|
|
15834
15855
|
opts.embedder = embedder;
|
|
15835
15856
|
if (config.datasources !== undefined) {
|
|
15836
|
-
const { skills, unknown } = buildDatasourceSkills(config.datasources, config.workspacePath, {
|
|
15857
|
+
const { skills, unknown, retired } = buildDatasourceSkills(config.datasources, config.workspacePath, {
|
|
15837
15858
|
embedder,
|
|
15838
15859
|
discrawl: async () => discrawlEmbeddingRuntimeFor(await embedder.describeEndpoint()),
|
|
15839
15860
|
cliEndpoint: embeddingEndpointSource(embedder),
|
|
@@ -15846,16 +15867,27 @@ function buildAgentOptions(config, agentDir) {
|
|
|
15846
15867
|
});
|
|
15847
15868
|
if (skills.length > 0)
|
|
15848
15869
|
opts.datasourceSkills = skills;
|
|
15870
|
+
const diagnostics = [];
|
|
15871
|
+
if (retired.length > 0) {
|
|
15872
|
+
const names = retired.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
|
|
15873
|
+
diagnostics.push({
|
|
15874
|
+
code: "retired-datasource-skill",
|
|
15875
|
+
severity: "warning",
|
|
15876
|
+
message: `Datasource config ignored because the Spotlight datasource was removed: ${names.join(", ")}. Delete these entries; use fsearch/mirror search (searchPaths) instead.`,
|
|
15877
|
+
source: "datasources"
|
|
15878
|
+
});
|
|
15879
|
+
}
|
|
15849
15880
|
if (unknown.length > 0) {
|
|
15850
|
-
const
|
|
15851
|
-
|
|
15881
|
+
const names = unknown.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
|
|
15882
|
+
diagnostics.push({
|
|
15852
15883
|
code: "unknown-datasource-skill",
|
|
15853
15884
|
severity: "warning",
|
|
15854
|
-
message: `Unknown datasource skill(s) in config were skipped: ${
|
|
15885
|
+
message: `Unknown datasource skill(s) in config were skipped: ${names.join(", ")}`,
|
|
15855
15886
|
source: "datasources"
|
|
15856
|
-
};
|
|
15857
|
-
opts.startupDiagnostics = [diagnostic];
|
|
15887
|
+
});
|
|
15858
15888
|
}
|
|
15889
|
+
if (diagnostics.length > 0)
|
|
15890
|
+
opts.startupDiagnostics = diagnostics;
|
|
15859
15891
|
}
|
|
15860
15892
|
return opts;
|
|
15861
15893
|
}
|
|
@@ -17680,11 +17712,49 @@ var init_matrix = __esm(() => {
|
|
|
17680
17712
|
};
|
|
17681
17713
|
});
|
|
17682
17714
|
|
|
17715
|
+
// src/dupey/version.ts
|
|
17716
|
+
function parseDupeyVersion(output) {
|
|
17717
|
+
return VERSION_LINE2.exec(output)?.[1];
|
|
17718
|
+
}
|
|
17719
|
+
function isSupportedDupeyVersion(version) {
|
|
17720
|
+
const have = version.split(".").map((part) => Number.parseInt(part, 10));
|
|
17721
|
+
const need = DUPEY_MIN_VERSION.split(".").map((part) => Number.parseInt(part, 10));
|
|
17722
|
+
for (let index = 0;index < need.length; index += 1) {
|
|
17723
|
+
const left = have[index] ?? 0;
|
|
17724
|
+
const right = need[index] ?? 0;
|
|
17725
|
+
if (left !== right)
|
|
17726
|
+
return left > right;
|
|
17727
|
+
}
|
|
17728
|
+
return true;
|
|
17729
|
+
}
|
|
17730
|
+
var DUPEY_MIN_VERSION = "0.2.0", VERSION_LINE2;
|
|
17731
|
+
var init_version2 = __esm(() => {
|
|
17732
|
+
VERSION_LINE2 = /^dupey\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
|
|
17733
|
+
});
|
|
17734
|
+
|
|
17683
17735
|
// src/dupey/cli.ts
|
|
17684
17736
|
import { spawn as spawn13 } from "node:child_process";
|
|
17685
17737
|
import { resolve as resolve9 } from "node:path";
|
|
17686
|
-
function
|
|
17687
|
-
|
|
17738
|
+
function checkDupeyVersion(executable, options) {
|
|
17739
|
+
const cached = versionChecks.get(executable);
|
|
17740
|
+
if (cached !== undefined)
|
|
17741
|
+
return cached;
|
|
17742
|
+
const check = spawnDupey(executable, ["--version"], options).then((stdout) => {
|
|
17743
|
+
const version = parseDupeyVersion(stdout);
|
|
17744
|
+
if (version !== undefined && !isSupportedDupeyVersion(version)) {
|
|
17745
|
+
throw new DupeyCliError(`dupey ${version} at ${executable} is older than the ${DUPEY_MIN_VERSION} AutoRAG requires; run \`cargo install dupey\` to update it.`);
|
|
17746
|
+
}
|
|
17747
|
+
});
|
|
17748
|
+
check.catch(() => versionChecks.delete(executable));
|
|
17749
|
+
versionChecks.set(executable, check);
|
|
17750
|
+
return check;
|
|
17751
|
+
}
|
|
17752
|
+
async function runDupey(args, options) {
|
|
17753
|
+
if (options.run)
|
|
17754
|
+
return options.run(args);
|
|
17755
|
+
const executable = options.executable ?? "dupey";
|
|
17756
|
+
await checkDupeyVersion(executable, options);
|
|
17757
|
+
return spawnDupey(executable, args, options);
|
|
17688
17758
|
}
|
|
17689
17759
|
async function scanWithDupey(dir, options = {}) {
|
|
17690
17760
|
const absoluteDir = resolve9(dir);
|
|
@@ -17818,28 +17888,26 @@ function spawnDupey(executable, args, options) {
|
|
|
17818
17888
|
});
|
|
17819
17889
|
});
|
|
17820
17890
|
}
|
|
17821
|
-
var DupeyCliError, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
|
|
17891
|
+
var DupeyCliError, versionChecks, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
|
|
17822
17892
|
var init_cli2 = __esm(() => {
|
|
17823
17893
|
init_portable_spawn();
|
|
17894
|
+
init_version2();
|
|
17824
17895
|
DupeyCliError = class DupeyCliError extends Error {
|
|
17825
17896
|
constructor(message) {
|
|
17826
17897
|
super(message);
|
|
17827
17898
|
this.name = "DupeyCliError";
|
|
17828
17899
|
}
|
|
17829
17900
|
};
|
|
17901
|
+
versionChecks = new Map;
|
|
17830
17902
|
});
|
|
17831
17903
|
|
|
17832
17904
|
// src/dupey/filter.ts
|
|
17833
17905
|
import { stat as stat2 } from "node:fs/promises";
|
|
17834
17906
|
import { resolve as resolve10 } from "node:path";
|
|
17835
|
-
function extractedNoText(file) {
|
|
17836
|
-
const signals = file.signals;
|
|
17837
|
-
return typeof signals === "object" && signals !== null && "chars" in signals && signals.chars === 0;
|
|
17838
|
-
}
|
|
17839
17907
|
async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerprints) {
|
|
17840
17908
|
const byHash = new Map;
|
|
17841
17909
|
for (const file of scan.files) {
|
|
17842
|
-
if (typeof file.content_hash !== "string" || file.content_hash.length === 0
|
|
17910
|
+
if (typeof file.content_hash !== "string" || file.content_hash.length === 0)
|
|
17843
17911
|
continue;
|
|
17844
17912
|
const path = resolve10(root, file.path);
|
|
17845
17913
|
const group = byHash.get(file.content_hash) ?? [];
|
|
@@ -17848,7 +17916,7 @@ async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerp
|
|
|
17848
17916
|
}
|
|
17849
17917
|
const fingerprintModified = new Map;
|
|
17850
17918
|
for (const [path, fingerprint] of fingerprints ?? []) {
|
|
17851
|
-
if (fingerprint.exact.length === 0
|
|
17919
|
+
if (fingerprint.exact.length === 0)
|
|
17852
17920
|
continue;
|
|
17853
17921
|
fingerprintModified.set(path, fingerprint.modified);
|
|
17854
17922
|
const group = byHash.get(fingerprint.exact) ?? [];
|
|
@@ -17915,1397 +17983,1368 @@ async function rankExactCandidates(paths, dupeyRanking, fingerprintModified, err
|
|
|
17915
17983
|
}
|
|
17916
17984
|
var init_filter = () => {};
|
|
17917
17985
|
|
|
17918
|
-
// src/
|
|
17919
|
-
|
|
17920
|
-
|
|
17921
|
-
|
|
17922
|
-
|
|
17923
|
-
|
|
17924
|
-
|
|
17925
|
-
function
|
|
17926
|
-
const
|
|
17927
|
-
|
|
17928
|
-
}
|
|
17929
|
-
|
|
17930
|
-
|
|
17986
|
+
// src/dupey/index.ts
|
|
17987
|
+
var init_dupey = __esm(() => {
|
|
17988
|
+
init_cli2();
|
|
17989
|
+
init_filter();
|
|
17990
|
+
});
|
|
17991
|
+
|
|
17992
|
+
// src/convert/relate.ts
|
|
17993
|
+
async function convertAndRelate(request, options = {}) {
|
|
17994
|
+
const result = await convertWithFilee(request, options.filee);
|
|
17995
|
+
if (options.dupey === false)
|
|
17996
|
+
return { result, links: [] };
|
|
17997
|
+
const dupeyOptions = options.dupey ?? {};
|
|
17998
|
+
const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
|
|
17999
|
+
const links = await Promise.all(pairs.map(async ({ source, output }) => {
|
|
18000
|
+
try {
|
|
18001
|
+
return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
|
|
18002
|
+
} catch (error) {
|
|
18003
|
+
if (!(error instanceof DupeyCliError))
|
|
18004
|
+
throw error;
|
|
18005
|
+
return { source, output, dupey: { relation: "unavailable", reason: error.message } };
|
|
18006
|
+
}
|
|
18007
|
+
}));
|
|
18008
|
+
return { result, links };
|
|
17931
18009
|
}
|
|
17932
|
-
|
|
17933
|
-
|
|
18010
|
+
var init_relate = __esm(() => {
|
|
18011
|
+
init_dupey();
|
|
18012
|
+
init_cli();
|
|
18013
|
+
});
|
|
18014
|
+
|
|
18015
|
+
// src/convert/index.ts
|
|
18016
|
+
var init_convert = __esm(() => {
|
|
18017
|
+
init_cli();
|
|
18018
|
+
init_installer2();
|
|
18019
|
+
init_matrix();
|
|
18020
|
+
init_relate();
|
|
18021
|
+
});
|
|
18022
|
+
|
|
18023
|
+
// src/datasource/diagnostics.ts
|
|
18024
|
+
function mapDatasourceDiagnostic(ds) {
|
|
18025
|
+
return {
|
|
18026
|
+
code: datasourceCodeToRetrieval(ds.code),
|
|
18027
|
+
severity: ds.severity,
|
|
18028
|
+
message: `[datasource:${ds.code}] ${ds.message}`,
|
|
18029
|
+
source: ds.source ?? (ds.instanceId ? `datasource:${ds.instanceId}` : "datasource")
|
|
18030
|
+
};
|
|
17934
18031
|
}
|
|
17935
|
-
function
|
|
17936
|
-
|
|
17937
|
-
if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
|
|
17938
|
-
return "google-drive";
|
|
17939
|
-
if (lower.includes("onedrive"))
|
|
17940
|
-
return "onedrive";
|
|
17941
|
-
if (lower.includes("dropbox"))
|
|
17942
|
-
return "dropbox";
|
|
17943
|
-
if (lower.includes("mobile documents") || lower.includes("clouddocs"))
|
|
17944
|
-
return "icloud";
|
|
17945
|
-
return "file-provider";
|
|
18032
|
+
function mapDatasourceDiagnostics(diagnostics) {
|
|
18033
|
+
return diagnostics.map(mapDatasourceDiagnostic);
|
|
17946
18034
|
}
|
|
17947
|
-
function
|
|
17948
|
-
|
|
17949
|
-
|
|
17950
|
-
|
|
17951
|
-
|
|
17952
|
-
|
|
17953
|
-
|
|
17954
|
-
|
|
17955
|
-
|
|
17956
|
-
|
|
17957
|
-
|
|
18035
|
+
function datasourceCodeToRetrieval(code) {
|
|
18036
|
+
switch (code) {
|
|
18037
|
+
case "datasource-unavailable":
|
|
18038
|
+
case "datasource-cli-error":
|
|
18039
|
+
case "datasource-empty":
|
|
18040
|
+
case "datasource-rate-limited":
|
|
18041
|
+
case "datasource-auth-error":
|
|
18042
|
+
case "datasource-embedding-egress-rejected":
|
|
18043
|
+
case "datasource-index-failed":
|
|
18044
|
+
case "datasource-index-partial":
|
|
18045
|
+
case "datasource-permission-denied":
|
|
18046
|
+
case "datasource-remote-embedding-rejected":
|
|
18047
|
+
return "retrieval-method-failed";
|
|
18048
|
+
}
|
|
17958
18049
|
}
|
|
17959
|
-
|
|
17960
|
-
|
|
17961
|
-
|
|
17962
|
-
|
|
17963
|
-
|
|
18050
|
+
|
|
18051
|
+
// src/datasource/retry.ts
|
|
18052
|
+
import { setTimeout as delay2 } from "node:timers/promises";
|
|
18053
|
+
function datasourceRetryDelayMs(retry, policy) {
|
|
18054
|
+
const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
|
|
18055
|
+
return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
|
|
17964
18056
|
}
|
|
17965
|
-
function
|
|
17966
|
-
const
|
|
17967
|
-
|
|
17968
|
-
|
|
17969
|
-
|
|
17970
|
-
|
|
17971
|
-
|
|
18057
|
+
async function indexWithRetry(skill, options = {}, signal) {
|
|
18058
|
+
const policy = {
|
|
18059
|
+
maxRetries: options.maxRetries ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxRetries,
|
|
18060
|
+
baseDelayMs: options.baseDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.baseDelayMs,
|
|
18061
|
+
maxDelayMs: options.maxDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxDelayMs
|
|
18062
|
+
};
|
|
18063
|
+
const sleep = options.sleep ?? ((ms) => delay2(ms, undefined, signal === undefined ? undefined : { signal }));
|
|
18064
|
+
let lastError = "";
|
|
18065
|
+
for (let attempt = 1;; attempt += 1) {
|
|
18066
|
+
const result = await runIndex(skill, signal);
|
|
18067
|
+
signal?.throwIfAborted();
|
|
18068
|
+
if (result.ok) {
|
|
18069
|
+
if (attempt === 1)
|
|
18070
|
+
return result;
|
|
18071
|
+
const recovered = {
|
|
18072
|
+
code: "datasource-index-failed",
|
|
18073
|
+
severity: "info",
|
|
18074
|
+
message: `recovered after ${attempt} attempts; last error: ${lastError}`,
|
|
18075
|
+
instanceId: result.instanceId,
|
|
18076
|
+
source: result.skill
|
|
18077
|
+
};
|
|
18078
|
+
return { ...result, attempts: attempt, diagnostics: [...result.diagnostics, recovered] };
|
|
18079
|
+
}
|
|
18080
|
+
lastError = result.message ?? result.error ?? "datasource-index-failed";
|
|
18081
|
+
if (attempt > policy.maxRetries) {
|
|
18082
|
+
if (attempt === 1)
|
|
18083
|
+
return { ...result, attempts: attempt };
|
|
18084
|
+
const suffix = ` (gave up after ${attempt} attempts)`;
|
|
18085
|
+
const last = result.diagnostics.at(-1);
|
|
18086
|
+
return {
|
|
18087
|
+
...result,
|
|
18088
|
+
attempts: attempt,
|
|
18089
|
+
message: `${lastError}${suffix}`,
|
|
18090
|
+
diagnostics: last === undefined ? [
|
|
18091
|
+
{
|
|
18092
|
+
code: result.code ?? "datasource-index-failed",
|
|
18093
|
+
severity: "error",
|
|
18094
|
+
message: `${lastError}${suffix}`,
|
|
18095
|
+
instanceId: result.instanceId,
|
|
18096
|
+
source: result.skill
|
|
18097
|
+
}
|
|
18098
|
+
] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
|
|
18099
|
+
};
|
|
18100
|
+
}
|
|
18101
|
+
await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
|
|
18102
|
+
signal?.throwIfAborted();
|
|
18103
|
+
}
|
|
17972
18104
|
}
|
|
17973
|
-
function
|
|
18105
|
+
async function runIndex(skill, signal) {
|
|
17974
18106
|
try {
|
|
17975
|
-
return
|
|
17976
|
-
} catch {
|
|
17977
|
-
|
|
18107
|
+
return await skill.index(signal === undefined ? undefined : { signal });
|
|
18108
|
+
} catch (error) {
|
|
18109
|
+
signal?.throwIfAborted();
|
|
18110
|
+
const descriptor = skill.describe();
|
|
18111
|
+
const message = error instanceof Error ? error.message : "Datasource indexing failed.";
|
|
18112
|
+
const failed = {
|
|
18113
|
+
ok: false,
|
|
18114
|
+
instanceId: descriptor.instanceId ?? "default",
|
|
18115
|
+
skill: descriptor.name,
|
|
18116
|
+
indexedAt: Date.now(),
|
|
18117
|
+
diagnostics: [
|
|
18118
|
+
{
|
|
18119
|
+
code: "datasource-index-failed",
|
|
18120
|
+
severity: "error",
|
|
18121
|
+
message,
|
|
18122
|
+
source: descriptor.name,
|
|
18123
|
+
instanceId: descriptor.instanceId
|
|
18124
|
+
}
|
|
18125
|
+
],
|
|
18126
|
+
error: "datasource-index-failed",
|
|
18127
|
+
code: "datasource-index-failed",
|
|
18128
|
+
message
|
|
18129
|
+
};
|
|
18130
|
+
return failed;
|
|
17978
18131
|
}
|
|
17979
18132
|
}
|
|
17980
|
-
|
|
17981
|
-
|
|
17982
|
-
|
|
17983
|
-
|
|
17984
|
-
|
|
17985
|
-
|
|
17986
|
-
for (const { raw, canonical } of forms) {
|
|
17987
|
-
if (isPathInside(normalized, raw))
|
|
17988
|
-
return `${canonical}${normalized.slice(raw.length)}`;
|
|
17989
|
-
}
|
|
17990
|
-
return path;
|
|
18133
|
+
var DEFAULT_DATASOURCE_RETRY_POLICY;
|
|
18134
|
+
var init_retry = __esm(() => {
|
|
18135
|
+
DEFAULT_DATASOURCE_RETRY_POLICY = {
|
|
18136
|
+
maxRetries: 2,
|
|
18137
|
+
baseDelayMs: 1000,
|
|
18138
|
+
maxDelayMs: 30000
|
|
17991
18139
|
};
|
|
18140
|
+
});
|
|
18141
|
+
|
|
18142
|
+
// src/everything/bundle.ts
|
|
18143
|
+
import { createHash as createHash11 } from "node:crypto";
|
|
18144
|
+
import { existsSync as existsSync17, mkdirSync as mkdirSync14, readFileSync as readFileSync17, rmSync as rmSync7, writeFileSync as writeFileSync10 } from "node:fs";
|
|
18145
|
+
import { dirname as dirname14, join as join27 } from "node:path";
|
|
18146
|
+
import { fileURLToPath } from "node:url";
|
|
18147
|
+
import JSZip from "jszip";
|
|
18148
|
+
function resolveEverythingBundleDir() {
|
|
18149
|
+
let dir = dirname14(fileURLToPath(import.meta.url));
|
|
18150
|
+
for (let depth = 0;depth < 6; depth += 1) {
|
|
18151
|
+
const candidate = join27(dir, "vendor", "everything");
|
|
18152
|
+
if (existsSync17(join27(candidate, MANIFEST_FILENAME)))
|
|
18153
|
+
return candidate;
|
|
18154
|
+
const parent = dirname14(dir);
|
|
18155
|
+
if (parent === dir)
|
|
18156
|
+
break;
|
|
18157
|
+
dir = parent;
|
|
18158
|
+
}
|
|
18159
|
+
return;
|
|
17992
18160
|
}
|
|
17993
|
-
function
|
|
17994
|
-
|
|
17995
|
-
|
|
17996
|
-
return
|
|
17997
|
-
insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
|
|
17998
|
-
nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
|
|
17999
|
-
};
|
|
18161
|
+
function everythingArch(arch) {
|
|
18162
|
+
if (arch === "x64" || arch === "arm64")
|
|
18163
|
+
return arch;
|
|
18164
|
+
return;
|
|
18000
18165
|
}
|
|
18001
|
-
function
|
|
18002
|
-
|
|
18166
|
+
function everythingBinaryPaths(root, version, arch) {
|
|
18167
|
+
const dir = join27(root, ".autorag", "everything", version, arch);
|
|
18168
|
+
return { everythingPath: join27(dir, "everything.exe"), esPath: join27(dir, "es.exe") };
|
|
18169
|
+
}
|
|
18170
|
+
function loadEverythingBundleManifest(bundleDir) {
|
|
18171
|
+
return JSON.parse(readFileSync17(join27(bundleDir, MANIFEST_FILENAME), "utf8"));
|
|
18172
|
+
}
|
|
18173
|
+
async function ensureEverythingBinaries(options) {
|
|
18174
|
+
const platform = options.platform ?? process.platform;
|
|
18175
|
+
const rawArch = options.arch ?? process.arch;
|
|
18176
|
+
if (platform !== "win32") {
|
|
18177
|
+
return {
|
|
18178
|
+
ok: false,
|
|
18179
|
+
reason: "unsupported-platform",
|
|
18180
|
+
message: `Everything is Windows-only; this host is ${platform}.`
|
|
18181
|
+
};
|
|
18182
|
+
}
|
|
18183
|
+
const arch = everythingArch(rawArch);
|
|
18184
|
+
const bundleDir = options.bundleDir ?? resolveEverythingBundleDir();
|
|
18185
|
+
if (bundleDir === undefined) {
|
|
18186
|
+
return {
|
|
18187
|
+
ok: false,
|
|
18188
|
+
reason: "bundle-missing",
|
|
18189
|
+
message: `Bundled Everything assets (vendor/everything/${MANIFEST_FILENAME}) were not found above ${dirname14(fileURLToPath(import.meta.url))}.`
|
|
18190
|
+
};
|
|
18191
|
+
}
|
|
18192
|
+
let manifest;
|
|
18003
18193
|
try {
|
|
18004
|
-
|
|
18005
|
-
} catch {
|
|
18006
|
-
return;
|
|
18194
|
+
manifest = options.manifest ?? loadEverythingBundleManifest(bundleDir);
|
|
18195
|
+
} catch (error) {
|
|
18196
|
+
return { ok: false, reason: "bundle-missing", message: errorMessage2(error) };
|
|
18007
18197
|
}
|
|
18008
|
-
|
|
18009
|
-
|
|
18198
|
+
const assets = arch === undefined ? undefined : manifest.architectures[arch];
|
|
18199
|
+
if (arch === undefined || assets === undefined) {
|
|
18200
|
+
return {
|
|
18201
|
+
ok: false,
|
|
18202
|
+
reason: "unsupported-platform",
|
|
18203
|
+
message: `No bundled Everything build for Windows architecture ${rawArch}; bundled: ${Object.keys(manifest.architectures).join(", ")}.`
|
|
18204
|
+
};
|
|
18010
18205
|
}
|
|
18011
|
-
|
|
18012
|
-
|
|
18013
|
-
|
|
18014
|
-
|
|
18015
|
-
|
|
18016
|
-
}
|
|
18017
|
-
|
|
18018
|
-
|
|
18019
|
-
|
|
18020
|
-
|
|
18021
|
-
|
|
18022
|
-
|
|
18023
|
-
|
|
18024
|
-
|
|
18206
|
+
const { everythingPath, esPath } = everythingBinaryPaths(options.root, manifest.everythingVersion, arch);
|
|
18207
|
+
const everythingCached = fileHasSha256(everythingPath, assets.everythingSha256);
|
|
18208
|
+
const esCached = fileHasSha256(esPath, assets.esSha256);
|
|
18209
|
+
if (everythingCached && esCached) {
|
|
18210
|
+
return { ok: true, everythingPath, esPath, source: "cached" };
|
|
18211
|
+
}
|
|
18212
|
+
if ((options.mode ?? "install") === "cached") {
|
|
18213
|
+
const state = everythingCached || esCached ? "are incomplete or do not match their pinned SHA-256 digests" : "are not installed";
|
|
18214
|
+
return {
|
|
18215
|
+
ok: false,
|
|
18216
|
+
reason: "not-installed",
|
|
18217
|
+
message: `Everything ${manifest.everythingVersion} (${arch}) binaries ${state} under ${dirname14(everythingPath)}; run \`autorag refresh\` to (re)install and index them.`
|
|
18218
|
+
};
|
|
18219
|
+
}
|
|
18220
|
+
try {
|
|
18221
|
+
mkdirSync14(dirname14(everythingPath), { recursive: true });
|
|
18222
|
+
const everythingBytes = await extractVerified(join27(bundleDir, assets.everythingArchive), assets.everythingArchiveSha256, assets.everythingMember, assets.everythingSha256);
|
|
18223
|
+
const esBytes = await extractVerified(join27(bundleDir, assets.esArchive), assets.esArchiveSha256, assets.esMember, assets.esSha256);
|
|
18224
|
+
writeAtomic(everythingPath, everythingBytes);
|
|
18225
|
+
writeAtomic(esPath, esBytes);
|
|
18226
|
+
} catch (error) {
|
|
18227
|
+
return { ok: false, reason: "install-failed", message: errorMessage2(error) };
|
|
18228
|
+
}
|
|
18229
|
+
return { ok: true, everythingPath, esPath, source: "installed" };
|
|
18025
18230
|
}
|
|
18026
|
-
function
|
|
18027
|
-
const
|
|
18028
|
-
const
|
|
18029
|
-
if (
|
|
18030
|
-
|
|
18031
|
-
} else if (typeof failure.code === "number") {
|
|
18032
|
-
parts.push(`the helper exited with code ${failure.code}`);
|
|
18231
|
+
async function extractVerified(archivePath, archiveSha256, member, memberSha256) {
|
|
18232
|
+
const archive = readFileSync17(archivePath);
|
|
18233
|
+
const actualArchive = sha256(archive);
|
|
18234
|
+
if (actualArchive !== archiveSha256) {
|
|
18235
|
+
throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
|
|
18033
18236
|
}
|
|
18034
|
-
const
|
|
18035
|
-
|
|
18036
|
-
|
|
18037
|
-
|
|
18038
|
-
|
|
18039
|
-
|
|
18237
|
+
const zip = await JSZip.loadAsync(archive);
|
|
18238
|
+
const entry = zip.file(member);
|
|
18239
|
+
if (entry === null)
|
|
18240
|
+
throw new Error(`${archivePath} does not contain ${member}`);
|
|
18241
|
+
const bytes = await entry.async("nodebuffer");
|
|
18242
|
+
const actualMember = sha256(bytes);
|
|
18243
|
+
if (actualMember !== memberSha256) {
|
|
18244
|
+
throw new Error(`${member} from ${archivePath} SHA-256 ${actualMember} does not match pinned ${memberSha256}`);
|
|
18245
|
+
}
|
|
18246
|
+
return bytes;
|
|
18040
18247
|
}
|
|
18041
|
-
|
|
18042
|
-
|
|
18043
|
-
return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
|
|
18248
|
+
function fileHasSha256(path, expected) {
|
|
18249
|
+
return existsSync17(path) && sha256(readFileSync17(path)) === expected;
|
|
18044
18250
|
}
|
|
18045
|
-
function
|
|
18046
|
-
const
|
|
18047
|
-
|
|
18048
|
-
|
|
18049
|
-
|
|
18050
|
-
|
|
18051
|
-
|
|
18052
|
-
if (open <= 0 || !line.endsWith(")"))
|
|
18053
|
-
continue;
|
|
18054
|
-
const head = line.slice(0, open);
|
|
18055
|
-
const on = head.indexOf(" on ");
|
|
18056
|
-
if (on <= 0)
|
|
18057
|
-
continue;
|
|
18058
|
-
const mountPoint = head.slice(on + " on ".length);
|
|
18059
|
-
if (mountPoint === "")
|
|
18060
|
-
continue;
|
|
18061
|
-
const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
|
|
18062
|
-
mounts.push({
|
|
18063
|
-
device: head.slice(0, on),
|
|
18064
|
-
mountPoint,
|
|
18065
|
-
fileSystem: fields[0] ?? "",
|
|
18066
|
-
options: fields.slice(1)
|
|
18067
|
-
});
|
|
18251
|
+
function writeAtomic(path, bytes) {
|
|
18252
|
+
const temp = `${path}.${process.pid}.tmp`;
|
|
18253
|
+
try {
|
|
18254
|
+
writeFileSync10(temp, bytes);
|
|
18255
|
+
renameWithRetry(temp, path);
|
|
18256
|
+
} finally {
|
|
18257
|
+
rmSync7(temp, { force: true });
|
|
18068
18258
|
}
|
|
18069
|
-
return mounts;
|
|
18070
18259
|
}
|
|
18071
|
-
function
|
|
18072
|
-
|
|
18073
|
-
const seen = new Set;
|
|
18074
|
-
for (const mount of mounts) {
|
|
18075
|
-
if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
|
|
18076
|
-
continue;
|
|
18077
|
-
if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
|
|
18078
|
-
continue;
|
|
18079
|
-
if (seen.has(mount.mountPoint))
|
|
18080
|
-
continue;
|
|
18081
|
-
seen.add(mount.mountPoint);
|
|
18082
|
-
points.push(mount.mountPoint);
|
|
18083
|
-
}
|
|
18084
|
-
return points;
|
|
18260
|
+
function sha256(bytes) {
|
|
18261
|
+
return createHash11("sha256").update(bytes).digest("hex");
|
|
18085
18262
|
}
|
|
18086
|
-
function
|
|
18087
|
-
|
|
18088
|
-
|
|
18089
|
-
|
|
18090
|
-
|
|
18091
|
-
|
|
18092
|
-
|
|
18093
|
-
|
|
18094
|
-
|
|
18095
|
-
|
|
18096
|
-
|
|
18097
|
-
|
|
18098
|
-
|
|
18099
|
-
|
|
18100
|
-
|
|
18101
|
-
}
|
|
18102
|
-
|
|
18263
|
+
function errorMessage2(error) {
|
|
18264
|
+
return error instanceof Error ? error.message : String(error);
|
|
18265
|
+
}
|
|
18266
|
+
var MANIFEST_FILENAME = "manifest.json";
|
|
18267
|
+
var init_bundle = __esm(() => {
|
|
18268
|
+
init_atomic_write();
|
|
18269
|
+
});
|
|
18270
|
+
|
|
18271
|
+
// src/filesystem/path-exclusions.ts
|
|
18272
|
+
import { posix as posix3, win32 as win322 } from "node:path";
|
|
18273
|
+
function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
|
|
18274
|
+
const paths = platform === "win32" ? win322 : posix3;
|
|
18275
|
+
const normalize = (path) => {
|
|
18276
|
+
const absolute = paths.resolve(path);
|
|
18277
|
+
return platform === "win32" ? absolute.toLowerCase() : absolute;
|
|
18278
|
+
};
|
|
18279
|
+
const excluded = excludedPaths.map((path) => {
|
|
18280
|
+
const absolute = normalize(path);
|
|
18281
|
+
return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
|
|
18282
|
+
});
|
|
18283
|
+
return (path) => {
|
|
18284
|
+
const absolute = normalize(path);
|
|
18285
|
+
return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
|
|
18286
|
+
};
|
|
18103
18287
|
}
|
|
18288
|
+
var init_path_exclusions = () => {};
|
|
18104
18289
|
|
|
18105
|
-
|
|
18106
|
-
|
|
18107
|
-
|
|
18108
|
-
|
|
18109
|
-
|
|
18110
|
-
|
|
18111
|
-
|
|
18112
|
-
|
|
18113
|
-
|
|
18114
|
-
|
|
18115
|
-
|
|
18116
|
-
|
|
18117
|
-
|
|
18118
|
-
const canonical = canonicalPath(path);
|
|
18119
|
-
const key = normalizeCloudPath(canonical);
|
|
18120
|
-
if (key === "" || seen.has(key))
|
|
18121
|
-
return;
|
|
18122
|
-
seen.add(key);
|
|
18123
|
-
roots.push({ path: canonical, kind });
|
|
18124
|
-
};
|
|
18125
|
-
const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
|
|
18126
|
-
for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
|
|
18127
|
-
if (entry.isDirectory())
|
|
18128
|
-
add(join27(cloudStorage, entry.name), "file-provider");
|
|
18129
|
-
}
|
|
18130
|
-
if (await this.tryReadDir(mobileDocuments) !== undefined)
|
|
18131
|
-
add(mobileDocuments, "file-provider");
|
|
18132
|
-
for (const mountPoint of await this.networkMountPoints())
|
|
18133
|
-
add(mountPoint, "network");
|
|
18134
|
-
for (const directory of await this.fileProviderDirectories(searchRoots))
|
|
18135
|
-
add(directory, "file-provider");
|
|
18136
|
-
return roots;
|
|
18290
|
+
// src/filesystem/shared-exclusive-queue.ts
|
|
18291
|
+
class SharedExclusiveQueue {
|
|
18292
|
+
sharedSinceExclusive = [];
|
|
18293
|
+
lastExclusive = Promise.resolve();
|
|
18294
|
+
shared(task) {
|
|
18295
|
+
const run = this.lastExclusive.then(task);
|
|
18296
|
+
const settled = run.then(() => {
|
|
18297
|
+
return;
|
|
18298
|
+
}, () => {
|
|
18299
|
+
return;
|
|
18300
|
+
});
|
|
18301
|
+
this.sharedSinceExclusive.push(settled);
|
|
18302
|
+
return run;
|
|
18137
18303
|
}
|
|
18138
|
-
|
|
18139
|
-
|
|
18140
|
-
|
|
18141
|
-
|
|
18142
|
-
} catch {
|
|
18143
|
-
return [];
|
|
18144
|
-
}
|
|
18145
|
-
}
|
|
18146
|
-
async fileProviderDirectories(searchRoots) {
|
|
18147
|
-
if (searchRoots.length === 0)
|
|
18148
|
-
return [];
|
|
18149
|
-
const candidates = new Set;
|
|
18150
|
-
for (const searchRoot of searchRoots) {
|
|
18151
|
-
const canonical = canonicalPath(searchRoot);
|
|
18152
|
-
for (let cursor = canonical;; cursor = dirname14(cursor)) {
|
|
18153
|
-
candidates.add(cursor);
|
|
18154
|
-
const parent = dirname14(cursor);
|
|
18155
|
-
if (parent === cursor)
|
|
18156
|
-
break;
|
|
18157
|
-
}
|
|
18158
|
-
for (const entry of await this.tryReadDir(canonical) ?? []) {
|
|
18159
|
-
if (entry.isDirectory())
|
|
18160
|
-
candidates.add(join27(canonical, entry.name));
|
|
18161
|
-
}
|
|
18162
|
-
}
|
|
18163
|
-
const paths = [...candidates];
|
|
18164
|
-
const managed = [];
|
|
18165
|
-
for (let start = 0;start < paths.length; start += XATTR_BATCH) {
|
|
18166
|
-
for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
|
|
18167
|
-
managed.push(path);
|
|
18168
|
-
}
|
|
18169
|
-
return managed;
|
|
18170
|
-
}
|
|
18171
|
-
async xattrValues(paths) {
|
|
18172
|
-
try {
|
|
18173
|
-
const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
|
|
18174
|
-
timeoutMs: 15000,
|
|
18175
|
-
maxBuffer: 8 * 1024 * 1024
|
|
18176
|
-
});
|
|
18177
|
-
return parseXattrValues(stdout, paths);
|
|
18178
|
-
} catch (error) {
|
|
18179
|
-
const failure = error;
|
|
18180
|
-
return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
|
|
18181
|
-
}
|
|
18182
|
-
}
|
|
18183
|
-
async tryReadDir(path) {
|
|
18184
|
-
try {
|
|
18185
|
-
return await this.readDir(path);
|
|
18186
|
-
} catch {
|
|
18304
|
+
exclusive(task) {
|
|
18305
|
+
const earlier = Promise.all([this.lastExclusive, ...this.sharedSinceExclusive]);
|
|
18306
|
+
const run = earlier.then(task);
|
|
18307
|
+
const settled = run.then(() => {
|
|
18187
18308
|
return;
|
|
18188
|
-
}
|
|
18189
|
-
|
|
18190
|
-
|
|
18191
|
-
|
|
18192
|
-
|
|
18193
|
-
|
|
18194
|
-
const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
|
|
18195
|
-
timeoutMs: 30000,
|
|
18196
|
-
maxBuffer: 4 * 1024 * 1024
|
|
18197
|
-
});
|
|
18198
|
-
const flags = stdout.split(`
|
|
18199
|
-
`).filter(Boolean);
|
|
18200
|
-
for (const [index, path] of batch.entries()) {
|
|
18201
|
-
if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
|
|
18202
|
-
dataless.add(path);
|
|
18203
|
-
}
|
|
18204
|
-
}
|
|
18205
|
-
return dataless;
|
|
18206
|
-
}
|
|
18207
|
-
async providerOf(path) {
|
|
18208
|
-
const domain = await readFileProviderDomainUpwards(path);
|
|
18209
|
-
return domain === undefined ? undefined : cloudProviderFromDomain(domain);
|
|
18210
|
-
}
|
|
18211
|
-
canRelease(provider) {
|
|
18212
|
-
return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
|
|
18213
|
-
}
|
|
18214
|
-
async evict(path) {
|
|
18215
|
-
let stdout;
|
|
18216
|
-
try {
|
|
18217
|
-
stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
|
|
18218
|
-
timeoutMs: 30000
|
|
18219
|
-
})).stdout;
|
|
18220
|
-
} catch (error) {
|
|
18221
|
-
return { ok: false, message: describeReleaseHelperFailure(error) };
|
|
18222
|
-
}
|
|
18223
|
-
const outcome = parseReleaseOutcome(stdout);
|
|
18224
|
-
if (outcome === undefined) {
|
|
18225
|
-
const detail = stdout.trim();
|
|
18226
|
-
return {
|
|
18227
|
-
ok: false,
|
|
18228
|
-
message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
|
|
18229
|
-
};
|
|
18230
|
-
}
|
|
18231
|
-
return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
|
|
18232
|
-
}
|
|
18233
|
-
}
|
|
18234
|
-
function buildWindowsAttributeScript(paths) {
|
|
18235
|
-
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
18236
|
-
` + `$ErrorActionPreference='SilentlyContinue'
|
|
18237
|
-
` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
|
|
18238
|
-
`;
|
|
18239
|
-
}
|
|
18240
|
-
function buildWindowsSyncRootsScript() {
|
|
18241
|
-
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
18242
|
-
` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
|
|
18243
|
-
` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
|
|
18244
|
-
`;
|
|
18245
|
-
}
|
|
18246
|
-
function buildWindowsDehydrateScript(path) {
|
|
18247
|
-
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
18248
|
-
` + `$ErrorActionPreference='Stop'
|
|
18249
|
-
` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
|
|
18250
|
-
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
|
|
18251
|
-
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
|
|
18252
|
-
` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
|
|
18253
|
-
` + `'@
|
|
18254
|
-
` + `$h=[IntPtr]::Zero
|
|
18255
|
-
` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
|
|
18256
|
-
` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
|
|
18257
|
-
` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
|
|
18258
|
-
` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
|
|
18259
|
-
` + `[Autorag.CfApi]::CfCloseHandle($h)
|
|
18260
|
-
` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
|
|
18261
|
-
` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
|
|
18262
|
-
`;
|
|
18263
|
-
}
|
|
18264
|
-
function parseWindowsAttributeOutput(stdout, paths) {
|
|
18265
|
-
const lines = stdout.split(/\r?\n/);
|
|
18266
|
-
const attributes = new Map;
|
|
18267
|
-
for (const [index, path] of paths.entries()) {
|
|
18268
|
-
const value = Number.parseInt((lines[index] ?? "").trim(), 10);
|
|
18269
|
-
if (Number.isFinite(value))
|
|
18270
|
-
attributes.set(path, value);
|
|
18271
|
-
}
|
|
18272
|
-
return attributes;
|
|
18273
|
-
}
|
|
18274
|
-
function parseWindowsSyncRoots(stdout) {
|
|
18275
|
-
const roots = [];
|
|
18276
|
-
for (const line of stdout.split(/\r?\n/)) {
|
|
18277
|
-
const tab = line.indexOf("\t");
|
|
18278
|
-
if (tab <= 0)
|
|
18279
|
-
continue;
|
|
18280
|
-
const path = line.slice(0, tab).trim();
|
|
18281
|
-
if (path !== "")
|
|
18282
|
-
roots.push({ path, label: line.slice(tab + 1).trim() });
|
|
18309
|
+
}, () => {
|
|
18310
|
+
return;
|
|
18311
|
+
});
|
|
18312
|
+
this.sharedSinceExclusive = [];
|
|
18313
|
+
this.lastExclusive = settled;
|
|
18314
|
+
return run;
|
|
18283
18315
|
}
|
|
18284
|
-
return roots;
|
|
18285
18316
|
}
|
|
18286
|
-
|
|
18287
|
-
|
|
18288
|
-
|
|
18289
|
-
|
|
18290
|
-
|
|
18291
|
-
|
|
18292
|
-
|
|
18317
|
+
|
|
18318
|
+
// src/everything/client.ts
|
|
18319
|
+
import { spawn as spawn14 } from "node:child_process";
|
|
18320
|
+
import { createHash as createHash12 } from "node:crypto";
|
|
18321
|
+
import { mkdirSync as mkdirSync15, writeFileSync as writeFileSync11 } from "node:fs";
|
|
18322
|
+
import { join as join28, resolve as resolve11 } from "node:path";
|
|
18323
|
+
import { setTimeout as sleep2 } from "node:timers/promises";
|
|
18324
|
+
function everythingInstanceName(root) {
|
|
18325
|
+
return `autorag-${createHash12("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
|
|
18293
18326
|
}
|
|
18294
|
-
function
|
|
18295
|
-
|
|
18296
|
-
|
|
18297
|
-
|
|
18327
|
+
function buildEverythingIni(input) {
|
|
18328
|
+
const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
|
|
18329
|
+
const perFolder = (value) => input.folders.map(() => value).join(",");
|
|
18330
|
+
return [
|
|
18331
|
+
"[Everything]",
|
|
18332
|
+
"app_data=0",
|
|
18333
|
+
"run_as_admin=0",
|
|
18334
|
+
"run_in_background=1",
|
|
18335
|
+
"show_tray_icon=0",
|
|
18336
|
+
"show_in_taskbar=0",
|
|
18337
|
+
"check_for_updates_on_startup=0",
|
|
18338
|
+
"beta_updates=0",
|
|
18339
|
+
"language=1033",
|
|
18340
|
+
"allow_http_server=0",
|
|
18341
|
+
"allow_etp_server=0",
|
|
18342
|
+
"search_history_enabled=0",
|
|
18343
|
+
"run_history_enabled=0",
|
|
18344
|
+
"auto_include_fixed_volumes=0",
|
|
18345
|
+
"auto_include_removable_volumes=0",
|
|
18346
|
+
"auto_include_fixed_refs_volumes=0",
|
|
18347
|
+
"auto_include_removable_refs_volumes=0",
|
|
18348
|
+
"ntfs_volume_paths=",
|
|
18349
|
+
"ntfs_volume_includes=",
|
|
18350
|
+
"refs_volume_paths=",
|
|
18351
|
+
"refs_volume_includes=",
|
|
18352
|
+
"index_size=1",
|
|
18353
|
+
"index_date_modified=1",
|
|
18354
|
+
`folders=${list(input.folders)}`,
|
|
18355
|
+
`folder_monitor_changes=${perFolder("1")}`,
|
|
18356
|
+
`folder_rescan_if_full_list=${perFolder("1")}`,
|
|
18357
|
+
`folder_update_types=${perFolder("0")}`,
|
|
18358
|
+
"exclude_list_enabled=1",
|
|
18359
|
+
`exclude_folders=${list(input.excludeFolders)}`,
|
|
18360
|
+
""
|
|
18361
|
+
].join(`\r
|
|
18362
|
+
`);
|
|
18298
18363
|
}
|
|
18299
|
-
function
|
|
18300
|
-
const
|
|
18301
|
-
|
|
18302
|
-
|
|
18303
|
-
|
|
18304
|
-
|
|
18305
|
-
|
|
18306
|
-
|
|
18307
|
-
|
|
18308
|
-
|
|
18309
|
-
|
|
18310
|
-
|
|
18311
|
-
|
|
18312
|
-
|
|
18313
|
-
|
|
18314
|
-
|
|
18315
|
-
|
|
18364
|
+
function buildEverythingSearchArgs(instance, request, timeoutMs) {
|
|
18365
|
+
const args = [
|
|
18366
|
+
"-argv",
|
|
18367
|
+
"-instance",
|
|
18368
|
+
instance,
|
|
18369
|
+
"-cp",
|
|
18370
|
+
"65001",
|
|
18371
|
+
"-json",
|
|
18372
|
+
"-full-path-and-name",
|
|
18373
|
+
"-size",
|
|
18374
|
+
"-date-modified",
|
|
18375
|
+
"-date-format",
|
|
18376
|
+
"1"
|
|
18377
|
+
];
|
|
18378
|
+
if (timeoutMs !== undefined)
|
|
18379
|
+
args.push("-timeout", String(timeoutMs));
|
|
18380
|
+
if (request.matchCase)
|
|
18381
|
+
args.push("-case");
|
|
18382
|
+
if (request.matchPath)
|
|
18383
|
+
args.push("-match-path");
|
|
18384
|
+
if (request.wholeWord)
|
|
18385
|
+
args.push("-whole-word");
|
|
18386
|
+
if (request.kind === "files")
|
|
18387
|
+
args.push("/a-d");
|
|
18388
|
+
if (request.kind === "folders")
|
|
18389
|
+
args.push("/ad");
|
|
18390
|
+
if (request.path !== undefined)
|
|
18391
|
+
args.push("-path", request.path);
|
|
18392
|
+
if (request.sort !== undefined)
|
|
18393
|
+
args.push("-sort", request.sort);
|
|
18394
|
+
if (request.offset !== undefined && request.offset > 0)
|
|
18395
|
+
args.push("-offset", String(request.offset));
|
|
18396
|
+
args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
|
|
18397
|
+
if (request.regex)
|
|
18398
|
+
args.push("-regex", "--", request.query);
|
|
18399
|
+
else
|
|
18400
|
+
args.push("--", request.query);
|
|
18401
|
+
return args;
|
|
18316
18402
|
}
|
|
18317
|
-
function
|
|
18318
|
-
const
|
|
18319
|
-
|
|
18320
|
-
|
|
18321
|
-
|
|
18322
|
-
|
|
18323
|
-
|
|
18324
|
-
|
|
18325
|
-
|
|
18326
|
-
continue;
|
|
18327
|
-
}
|
|
18328
|
-
if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
|
|
18329
|
-
roots.push({ path, kind: "network" });
|
|
18330
|
-
continue;
|
|
18331
|
-
}
|
|
18332
|
-
if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
|
|
18333
|
-
roots.push({ path, kind: "cloud-files" });
|
|
18334
|
-
}
|
|
18403
|
+
function parseEverythingJson(stdout) {
|
|
18404
|
+
const text = stdout.trim();
|
|
18405
|
+
if (text.length === 0)
|
|
18406
|
+
return [];
|
|
18407
|
+
let parsed;
|
|
18408
|
+
try {
|
|
18409
|
+
parsed = JSON.parse(text);
|
|
18410
|
+
} catch {
|
|
18411
|
+
throw new Error(`ES did not return JSON: ${text.slice(0, 2000)}`);
|
|
18335
18412
|
}
|
|
18336
|
-
|
|
18337
|
-
}
|
|
18338
|
-
|
|
18339
|
-
|
|
18340
|
-
|
|
18341
|
-
|
|
18342
|
-
const rest = normalized.slice(2);
|
|
18343
|
-
const separator = rest.indexOf("\\");
|
|
18344
|
-
if (separator <= 0)
|
|
18345
|
-
return;
|
|
18346
|
-
const server = rest.slice(0, separator);
|
|
18347
|
-
const share = rest.slice(separator + 1).split("\\")[0] ?? "";
|
|
18348
|
-
if (server === "" || share === "")
|
|
18349
|
-
return;
|
|
18350
|
-
return `\\\\${server}\\${share}`;
|
|
18351
|
-
}
|
|
18352
|
-
function parseWindowsDehydrateOutput(result) {
|
|
18353
|
-
const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
18354
|
-
if (open === null)
|
|
18355
|
-
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
18356
|
-
if (Number.parseInt(open[1], 16) !== 0) {
|
|
18413
|
+
if (!Array.isArray(parsed))
|
|
18414
|
+
throw new Error(`ES did not return a JSON array: ${text.slice(0, 2000)}`);
|
|
18415
|
+
return parsed.map((item) => {
|
|
18416
|
+
const record = item;
|
|
18417
|
+
const filename = String(record.filename ?? "");
|
|
18418
|
+
const isFolder = filename.endsWith("\\") && filename.length > 3;
|
|
18357
18419
|
return {
|
|
18358
|
-
|
|
18359
|
-
|
|
18420
|
+
path: isFolder ? filename.slice(0, -1) : filename,
|
|
18421
|
+
type: filename.endsWith("\\") ? "folder" : "file",
|
|
18422
|
+
size: typeof record.size === "number" ? record.size : undefined,
|
|
18423
|
+
dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
|
|
18360
18424
|
};
|
|
18361
|
-
}
|
|
18362
|
-
const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
18363
|
-
if (dehydrate === null) {
|
|
18364
|
-
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
18365
|
-
}
|
|
18366
|
-
const hresult = dehydrate[1];
|
|
18367
|
-
if (Number.parseInt(hresult, 16) === 0)
|
|
18368
|
-
return { ok: true };
|
|
18369
|
-
const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
|
|
18370
|
-
return {
|
|
18371
|
-
ok: false,
|
|
18372
|
-
message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
|
|
18373
|
-
};
|
|
18374
|
-
}
|
|
18375
|
-
function windowsFailureMessage(result, fallback) {
|
|
18376
|
-
const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
|
|
18377
|
-
if (printed !== undefined && printed !== "")
|
|
18378
|
-
return printed;
|
|
18379
|
-
const stderr = result.stderr.trim();
|
|
18380
|
-
return stderr !== "" ? stderr : fallback;
|
|
18425
|
+
});
|
|
18381
18426
|
}
|
|
18382
18427
|
|
|
18383
|
-
class
|
|
18428
|
+
class EverythingClient {
|
|
18429
|
+
instanceName;
|
|
18430
|
+
options;
|
|
18431
|
+
platform;
|
|
18384
18432
|
run;
|
|
18385
|
-
|
|
18386
|
-
|
|
18387
|
-
|
|
18388
|
-
|
|
18433
|
+
launch;
|
|
18434
|
+
stateDir;
|
|
18435
|
+
isExcludedPath;
|
|
18436
|
+
binaries;
|
|
18437
|
+
queue = new SharedExclusiveQueue;
|
|
18438
|
+
constructor(options) {
|
|
18439
|
+
this.options = options;
|
|
18440
|
+
this.platform = options.platform ?? process.platform;
|
|
18441
|
+
this.instanceName = everythingInstanceName(options.root);
|
|
18442
|
+
this.run = options.run ?? runProcess;
|
|
18443
|
+
this.launch = options.launch ?? launchDetached;
|
|
18444
|
+
this.stateDir = join28(options.root, ".autorag", "everything");
|
|
18445
|
+
this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
|
|
18389
18446
|
}
|
|
18390
|
-
|
|
18391
|
-
|
|
18392
|
-
const seen = new Set;
|
|
18393
|
-
const add = (path, kind) => {
|
|
18394
|
-
const canonical = canonicalPath(path);
|
|
18395
|
-
const key = normalizeCloudPath(canonical).toLowerCase();
|
|
18396
|
-
if (key === "" || seen.has(key))
|
|
18397
|
-
return;
|
|
18398
|
-
seen.add(key);
|
|
18399
|
-
roots.push({ path: canonical, kind });
|
|
18400
|
-
};
|
|
18401
|
-
for (const root of await this.loadSyncRoots())
|
|
18402
|
-
add(root.path, "cloud-files");
|
|
18403
|
-
for (const root of await this.loadDiskRoots())
|
|
18404
|
-
add(root.path, root.kind);
|
|
18405
|
-
for (const searchRoot of searchRoots) {
|
|
18406
|
-
const share = uncShareRoot(searchRoot);
|
|
18407
|
-
if (share !== undefined)
|
|
18408
|
-
add(share, "network");
|
|
18409
|
-
}
|
|
18410
|
-
return roots;
|
|
18447
|
+
isSupported() {
|
|
18448
|
+
return this.platform === "win32";
|
|
18411
18449
|
}
|
|
18412
|
-
|
|
18413
|
-
|
|
18414
|
-
|
|
18415
|
-
|
|
18416
|
-
|
|
18417
|
-
|
|
18418
|
-
|
|
18450
|
+
search(request) {
|
|
18451
|
+
return this.queue.shared(async () => {
|
|
18452
|
+
const binaries = await this.resolveBinaries("cached");
|
|
18453
|
+
if (!binaries.ok)
|
|
18454
|
+
return binaries;
|
|
18455
|
+
const ping = await this.ping(binaries.esPath);
|
|
18456
|
+
if (ping.code !== 0) {
|
|
18457
|
+
return {
|
|
18458
|
+
ok: false,
|
|
18459
|
+
reason: "not-running",
|
|
18460
|
+
message: `Everything instance ${this.instanceName} is not running; run \`autorag refresh\` to start and index it.`
|
|
18461
|
+
};
|
|
18419
18462
|
}
|
|
18420
|
-
|
|
18421
|
-
|
|
18422
|
-
|
|
18463
|
+
const es = { ok: true, esPath: binaries.esPath };
|
|
18464
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
18465
|
+
if ((this.options.excludeFolders?.length ?? 0) > 0) {
|
|
18466
|
+
return this.searchWithExclusions(es.esPath, request, timeoutMs);
|
|
18423
18467
|
}
|
|
18424
|
-
|
|
18425
|
-
|
|
18426
|
-
|
|
18427
|
-
|
|
18428
|
-
|
|
18429
|
-
|
|
18430
|
-
|
|
18431
|
-
|
|
18432
|
-
|
|
18468
|
+
const args = buildEverythingSearchArgs(this.instanceName, request, timeoutMs);
|
|
18469
|
+
const result = await this.run(es.esPath, args, timeoutMs + 5000);
|
|
18470
|
+
if (result.code !== 0)
|
|
18471
|
+
return this.failure("search-failed", es.esPath, args, result);
|
|
18472
|
+
try {
|
|
18473
|
+
return { ok: true, results: parseEverythingJson(result.stdout) };
|
|
18474
|
+
} catch (error) {
|
|
18475
|
+
return { ok: false, reason: "search-failed", message: error.message };
|
|
18476
|
+
}
|
|
18477
|
+
});
|
|
18433
18478
|
}
|
|
18434
|
-
async
|
|
18435
|
-
|
|
18479
|
+
async searchWithExclusions(esPath, request, timeoutMs) {
|
|
18480
|
+
const deadline = Date.now() + timeoutMs;
|
|
18481
|
+
const offset = request.offset ?? 0;
|
|
18482
|
+
const maxResults = request.maxResults ?? DEFAULT_MAX_RESULTS;
|
|
18483
|
+
const results = [];
|
|
18484
|
+
let skipped = 0;
|
|
18485
|
+
let nativeOffset = 0;
|
|
18486
|
+
const timedOut = () => ({
|
|
18487
|
+
ok: false,
|
|
18488
|
+
reason: "search-failed",
|
|
18489
|
+
message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
|
|
18490
|
+
});
|
|
18491
|
+
for (;; ) {
|
|
18492
|
+
const remainingMs = deadline - Date.now();
|
|
18493
|
+
if (remainingMs <= 0)
|
|
18494
|
+
return timedOut();
|
|
18495
|
+
const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
|
|
18496
|
+
const response = await this.run(esPath, args, remainingMs + 5000);
|
|
18497
|
+
if (response.code !== 0)
|
|
18498
|
+
return this.failure("search-failed", esPath, args, response);
|
|
18499
|
+
if (Date.now() >= deadline)
|
|
18500
|
+
return timedOut();
|
|
18501
|
+
let entries;
|
|
18502
|
+
try {
|
|
18503
|
+
entries = parseEverythingJson(response.stdout);
|
|
18504
|
+
} catch (error) {
|
|
18505
|
+
return { ok: false, reason: "search-failed", message: error.message };
|
|
18506
|
+
}
|
|
18507
|
+
for (const entry of entries) {
|
|
18508
|
+
if (this.isExcludedPath(entry.path))
|
|
18509
|
+
continue;
|
|
18510
|
+
if (skipped < offset) {
|
|
18511
|
+
skipped += 1;
|
|
18512
|
+
continue;
|
|
18513
|
+
}
|
|
18514
|
+
results.push(entry);
|
|
18515
|
+
if (results.length >= maxResults)
|
|
18516
|
+
return { ok: true, results };
|
|
18517
|
+
}
|
|
18518
|
+
if (entries.length < MAX_FILTER_CANDIDATES)
|
|
18519
|
+
return { ok: true, results };
|
|
18520
|
+
nativeOffset += entries.length;
|
|
18521
|
+
}
|
|
18436
18522
|
}
|
|
18437
|
-
|
|
18438
|
-
|
|
18439
|
-
|
|
18523
|
+
index() {
|
|
18524
|
+
return this.queue.exclusive(async () => {
|
|
18525
|
+
const es = await this.ensureRunning(true);
|
|
18526
|
+
if (!es.ok)
|
|
18527
|
+
return es;
|
|
18528
|
+
const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
|
|
18529
|
+
const saveArgs = ["-instance", this.instanceName, "-save-db"];
|
|
18530
|
+
const saved = await this.run(es.esPath, saveArgs, timeoutMs);
|
|
18531
|
+
if (saved.code !== 0)
|
|
18532
|
+
return this.failure("index-failed", es.esPath, saveArgs, saved);
|
|
18533
|
+
const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
|
|
18534
|
+
const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
|
|
18535
|
+
if (counted.code !== 0)
|
|
18536
|
+
return this.failure("index-failed", es.esPath, countArgs, counted);
|
|
18537
|
+
return {
|
|
18538
|
+
ok: true,
|
|
18539
|
+
indexedFolders: [...this.options.folders],
|
|
18540
|
+
indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
|
|
18541
|
+
};
|
|
18542
|
+
});
|
|
18440
18543
|
}
|
|
18441
|
-
|
|
18442
|
-
|
|
18443
|
-
|
|
18544
|
+
stop() {
|
|
18545
|
+
return this.queue.exclusive(async () => {
|
|
18546
|
+
const binaries = await this.resolveBinaries("cached");
|
|
18547
|
+
if (!binaries.ok)
|
|
18548
|
+
return;
|
|
18549
|
+
await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
18550
|
+
});
|
|
18444
18551
|
}
|
|
18445
|
-
|
|
18446
|
-
|
|
18447
|
-
|
|
18448
|
-
|
|
18449
|
-
|
|
18450
|
-
|
|
18451
|
-
|
|
18452
|
-
if (prefix === "" || prefix.length <= matchLength)
|
|
18453
|
-
continue;
|
|
18454
|
-
if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
|
|
18455
|
-
match = root;
|
|
18456
|
-
matchLength = prefix.length;
|
|
18552
|
+
async resolveBinaries(mode) {
|
|
18553
|
+
if (!this.isSupported()) {
|
|
18554
|
+
return {
|
|
18555
|
+
ok: false,
|
|
18556
|
+
reason: "unsupported-platform",
|
|
18557
|
+
message: `Everything is Windows-only; this host is ${this.platform}.`
|
|
18558
|
+
};
|
|
18457
18559
|
}
|
|
18560
|
+
if (this.binaries?.ok)
|
|
18561
|
+
return this.binaries;
|
|
18562
|
+
const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
|
|
18563
|
+
const result = await resolve(mode);
|
|
18564
|
+
if (result.ok)
|
|
18565
|
+
this.binaries = result;
|
|
18566
|
+
return result;
|
|
18458
18567
|
}
|
|
18459
|
-
|
|
18460
|
-
|
|
18461
|
-
|
|
18462
|
-
|
|
18463
|
-
|
|
18464
|
-
|
|
18465
|
-
|
|
18466
|
-
|
|
18467
|
-
|
|
18468
|
-
|
|
18469
|
-
|
|
18470
|
-
|
|
18471
|
-
|
|
18472
|
-
|
|
18473
|
-
|
|
18474
|
-
|
|
18475
|
-
|
|
18476
|
-
|
|
18477
|
-
|
|
18478
|
-
|
|
18479
|
-
|
|
18480
|
-
|
|
18481
|
-
|
|
18482
|
-
|
|
18483
|
-
|
|
18568
|
+
async ensureRunning(reconfigure) {
|
|
18569
|
+
const binaries = await this.resolveBinaries("install");
|
|
18570
|
+
if (!binaries.ok)
|
|
18571
|
+
return binaries;
|
|
18572
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
18573
|
+
let ping = await this.ping(binaries.esPath);
|
|
18574
|
+
if (ping.code === 0 && !reconfigure)
|
|
18575
|
+
return { ok: true, esPath: binaries.esPath };
|
|
18576
|
+
if (ping.code === 0) {
|
|
18577
|
+
const exitArgs = ["-instance", this.instanceName, "-exit"];
|
|
18578
|
+
const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
|
|
18579
|
+
if (exited.code !== 0)
|
|
18580
|
+
return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
|
|
18581
|
+
}
|
|
18582
|
+
mkdirSync15(this.stateDir, { recursive: true });
|
|
18583
|
+
const iniPath = join28(this.stateDir, "Everything.ini");
|
|
18584
|
+
writeFileSync11(iniPath, buildEverythingIni({
|
|
18585
|
+
folders: this.options.folders,
|
|
18586
|
+
excludeFolders: [
|
|
18587
|
+
...new Set([
|
|
18588
|
+
join28(this.options.root, ".autorag"),
|
|
18589
|
+
...this.options.folders.map((folder) => join28(folder, ".autorag")),
|
|
18590
|
+
...this.options.excludeFolders ?? []
|
|
18591
|
+
])
|
|
18592
|
+
]
|
|
18593
|
+
}));
|
|
18594
|
+
const launchArgs = [
|
|
18595
|
+
"-instance",
|
|
18596
|
+
this.instanceName,
|
|
18597
|
+
"-config",
|
|
18598
|
+
iniPath,
|
|
18599
|
+
"-db",
|
|
18600
|
+
join28(this.stateDir, "Everything.db"),
|
|
18601
|
+
"-startup"
|
|
18602
|
+
];
|
|
18603
|
+
try {
|
|
18604
|
+
this.launch(binaries.everythingPath, launchArgs);
|
|
18605
|
+
} catch (error) {
|
|
18606
|
+
return {
|
|
18607
|
+
ok: false,
|
|
18608
|
+
reason: "startup-failed",
|
|
18609
|
+
message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
|
|
18610
|
+
};
|
|
18611
|
+
}
|
|
18612
|
+
const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
|
|
18613
|
+
const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
18614
|
+
for (;; ) {
|
|
18615
|
+
ping = await this.ping(binaries.esPath);
|
|
18616
|
+
if (ping.code === 0)
|
|
18617
|
+
return { ok: true, esPath: binaries.esPath };
|
|
18618
|
+
if (Date.now() >= deadline)
|
|
18619
|
+
break;
|
|
18620
|
+
await sleep2(pollMs);
|
|
18621
|
+
}
|
|
18484
18622
|
return {
|
|
18485
18623
|
ok: false,
|
|
18486
|
-
|
|
18487
|
-
|
|
18624
|
+
reason: "startup-failed",
|
|
18625
|
+
message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
|
|
18488
18626
|
};
|
|
18489
18627
|
}
|
|
18490
|
-
|
|
18491
|
-
|
|
18492
|
-
|
|
18493
|
-
return new DarwinCloudProbe;
|
|
18494
|
-
if (process.platform === "win32")
|
|
18495
|
-
return new WindowsCloudProbe;
|
|
18496
|
-
return;
|
|
18497
|
-
}
|
|
18498
|
-
async function readFileProviderDomain(target) {
|
|
18499
|
-
try {
|
|
18500
|
-
await lstat(target);
|
|
18501
|
-
const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
|
|
18502
|
-
timeout: 2000
|
|
18503
|
-
});
|
|
18504
|
-
const value = stdout.trim();
|
|
18505
|
-
return value.length > 0 ? value : undefined;
|
|
18506
|
-
} catch {
|
|
18507
|
-
return;
|
|
18628
|
+
async ping(esPath) {
|
|
18629
|
+
const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
18630
|
+
return result.code === 0 && !VERSION_PATTERN.test(result.stdout.trim()) ? { ...result, code: -1 } : result;
|
|
18508
18631
|
}
|
|
18509
|
-
|
|
18510
|
-
|
|
18511
|
-
|
|
18512
|
-
|
|
18513
|
-
|
|
18514
|
-
|
|
18515
|
-
return domain;
|
|
18516
|
-
const parent = dirname14(cursor);
|
|
18517
|
-
if (parent === cursor)
|
|
18518
|
-
return;
|
|
18519
|
-
cursor = parent;
|
|
18632
|
+
failure(reason, esPath, args, result) {
|
|
18633
|
+
return {
|
|
18634
|
+
ok: false,
|
|
18635
|
+
reason,
|
|
18636
|
+
message: `${esPath} ${args.join(" ")} exit ${result.code}: ${(result.stderr || result.stdout).trim()}`
|
|
18637
|
+
};
|
|
18520
18638
|
}
|
|
18521
|
-
return;
|
|
18522
|
-
}
|
|
18523
|
-
function quotePowerShell(value) {
|
|
18524
|
-
return `'${value.replace(/['\u2018\u2019\u201A\u201B]/gu, (quote) => quote + quote)}'`;
|
|
18525
18639
|
}
|
|
18526
|
-
|
|
18527
|
-
|
|
18528
|
-
|
|
18529
|
-
|
|
18530
|
-
|
|
18531
|
-
|
|
18532
|
-
|
|
18533
|
-
|
|
18534
|
-
|
|
18535
|
-
|
|
18536
|
-
|
|
18537
|
-
|
|
18538
|
-
|
|
18539
|
-
|
|
18540
|
-
|
|
18541
|
-
|
|
18542
|
-
|
|
18543
|
-
|
|
18544
|
-
isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
|
|
18545
|
-
downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
|
|
18546
|
-
});
|
|
18547
|
-
}`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
|
|
18548
|
-
var init_cloud_placeholder = __esm(() => {
|
|
18549
|
-
execFileAsync = promisify(execFile2);
|
|
18550
|
-
RELEASABLE_DARWIN_PROVIDERS = {
|
|
18551
|
-
icloud: true,
|
|
18552
|
-
"google-drive": true,
|
|
18553
|
-
onedrive: true,
|
|
18554
|
-
dropbox: true
|
|
18555
|
-
};
|
|
18556
|
-
DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
|
|
18557
|
-
WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
|
|
18558
|
-
WINDOWS_DEHYDRATE_REFUSALS = {
|
|
18559
|
-
"80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
|
|
18560
|
-
"80070188": "The file is pinned to stay on this device, so it is not released."
|
|
18640
|
+
function runProcess(command, args, timeoutMs) {
|
|
18641
|
+
let resolvePromise;
|
|
18642
|
+
const promise = new Promise((resolve) => {
|
|
18643
|
+
resolvePromise = resolve;
|
|
18644
|
+
});
|
|
18645
|
+
const child = spawn14(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
|
|
18646
|
+
const stdout = [];
|
|
18647
|
+
const stderr = [];
|
|
18648
|
+
let bytes = 0;
|
|
18649
|
+
let overflow = false;
|
|
18650
|
+
const collect = (target) => (chunk) => {
|
|
18651
|
+
bytes += chunk.length;
|
|
18652
|
+
if (bytes > MAX_OUTPUT_BYTES) {
|
|
18653
|
+
overflow = true;
|
|
18654
|
+
child.kill();
|
|
18655
|
+
return;
|
|
18656
|
+
}
|
|
18657
|
+
target.push(chunk);
|
|
18561
18658
|
};
|
|
18562
|
-
|
|
18563
|
-
|
|
18564
|
-
|
|
18565
|
-
|
|
18566
|
-
|
|
18567
|
-
|
|
18568
|
-
|
|
18569
|
-
|
|
18570
|
-
|
|
18571
|
-
|
|
18572
|
-
|
|
18573
|
-
|
|
18659
|
+
child.stdout.on("data", collect(stdout));
|
|
18660
|
+
child.stderr.on("data", collect(stderr));
|
|
18661
|
+
const timer = setTimeout(() => child.kill(), timeoutMs);
|
|
18662
|
+
child.on("error", (error) => {
|
|
18663
|
+
clearTimeout(timer);
|
|
18664
|
+
resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
|
|
18665
|
+
});
|
|
18666
|
+
child.on("close", (code, signal) => {
|
|
18667
|
+
clearTimeout(timer);
|
|
18668
|
+
const err = Buffer.concat(stderr).toString("utf8");
|
|
18669
|
+
resolvePromise({
|
|
18670
|
+
code,
|
|
18671
|
+
stdout: Buffer.concat(stdout).toString("utf8"),
|
|
18672
|
+
stderr: overflow ? `${err}
|
|
18673
|
+
output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
|
|
18674
|
+
killed by ${signal} after ${timeoutMs}ms` : err
|
|
18675
|
+
});
|
|
18676
|
+
});
|
|
18677
|
+
return promise;
|
|
18574
18678
|
}
|
|
18575
|
-
function
|
|
18576
|
-
const
|
|
18577
|
-
|
|
18578
|
-
|
|
18579
|
-
const scope = cloudScope(path, cloudRoots);
|
|
18580
|
-
if (scope.insideCloudRoot) {
|
|
18581
|
-
skipped.push(path);
|
|
18582
|
-
continue;
|
|
18583
|
-
}
|
|
18584
|
-
targets.push({ path, excludeDirs: scope.nestedCloudRoots.map((root) => basename7(root.path)) });
|
|
18585
|
-
}
|
|
18586
|
-
return { targets, skipped };
|
|
18679
|
+
function launchDetached(command, args) {
|
|
18680
|
+
const child = spawn14(command, args, { detached: true, stdio: "ignore", windowsHide: true });
|
|
18681
|
+
child.on("error", () => {});
|
|
18682
|
+
child.unref();
|
|
18587
18683
|
}
|
|
18588
|
-
var
|
|
18589
|
-
|
|
18684
|
+
var DEFAULT_TIMEOUT_MS8 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
|
|
18685
|
+
var init_client9 = __esm(() => {
|
|
18686
|
+
init_path_exclusions();
|
|
18687
|
+
init_bundle();
|
|
18688
|
+
DEFAULT_INDEX_TIMEOUT_MS3 = 10 * 60000;
|
|
18689
|
+
MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
|
|
18690
|
+
VERSION_PATTERN = /^\d+\.\d+\.\d+\.\d+$/;
|
|
18590
18691
|
});
|
|
18591
18692
|
|
|
18592
|
-
// src/
|
|
18593
|
-
var
|
|
18594
|
-
|
|
18595
|
-
|
|
18596
|
-
init_scan_plan();
|
|
18693
|
+
// src/everything/index.ts
|
|
18694
|
+
var init_everything = __esm(() => {
|
|
18695
|
+
init_bundle();
|
|
18696
|
+
init_client9();
|
|
18597
18697
|
});
|
|
18598
18698
|
|
|
18599
|
-
// src/
|
|
18600
|
-
|
|
18601
|
-
|
|
18602
|
-
|
|
18603
|
-
|
|
18604
|
-
|
|
18605
|
-
|
|
18606
|
-
|
|
18607
|
-
|
|
18608
|
-
|
|
18609
|
-
} catch (error) {
|
|
18610
|
-
if (!(error instanceof DupeyCliError))
|
|
18611
|
-
throw error;
|
|
18612
|
-
return { source, output, dupey: { relation: "unavailable", reason: error.message } };
|
|
18613
|
-
}
|
|
18614
|
-
}));
|
|
18615
|
-
return { result, links };
|
|
18699
|
+
// src/filesystem/cloud-placeholder.ts
|
|
18700
|
+
import { execFile as execFile2 } from "node:child_process";
|
|
18701
|
+
import { realpathSync as realpathSync3 } from "node:fs";
|
|
18702
|
+
import { lstat, readdir } from "node:fs/promises";
|
|
18703
|
+
import { homedir as homedir6 } from "node:os";
|
|
18704
|
+
import { dirname as dirname15, join as join29 } from "node:path";
|
|
18705
|
+
import { promisify } from "node:util";
|
|
18706
|
+
function homeCloudRoots(home = homedir6()) {
|
|
18707
|
+
const normalizedHome = home.replaceAll("\\", "/").replace(/\/+$/, "");
|
|
18708
|
+
return [`${normalizedHome}/Library/CloudStorage`, `${normalizedHome}/Library/Mobile Documents`];
|
|
18616
18709
|
}
|
|
18617
|
-
|
|
18618
|
-
|
|
18619
|
-
init_cli();
|
|
18620
|
-
});
|
|
18621
|
-
|
|
18622
|
-
// src/convert/index.ts
|
|
18623
|
-
var init_convert = __esm(() => {
|
|
18624
|
-
init_cli();
|
|
18625
|
-
init_installer2();
|
|
18626
|
-
init_matrix();
|
|
18627
|
-
init_relate();
|
|
18628
|
-
});
|
|
18629
|
-
|
|
18630
|
-
// src/datasource/diagnostics.ts
|
|
18631
|
-
function mapDatasourceDiagnostic(ds) {
|
|
18632
|
-
return {
|
|
18633
|
-
code: datasourceCodeToRetrieval(ds.code),
|
|
18634
|
-
severity: ds.severity,
|
|
18635
|
-
message: `[datasource:${ds.code}] ${ds.message}`,
|
|
18636
|
-
source: ds.source ?? (ds.instanceId ? `datasource:${ds.instanceId}` : "datasource")
|
|
18637
|
-
};
|
|
18710
|
+
function datalessFromDarwinFlags(flags) {
|
|
18711
|
+
return Number.isFinite(flags) && (flags >>> 0 & SF_DATALESS) !== 0;
|
|
18638
18712
|
}
|
|
18639
|
-
function
|
|
18640
|
-
return
|
|
18713
|
+
function datalessFromWindowsAttributes(attributes) {
|
|
18714
|
+
return Number.isFinite(attributes) && (attributes & WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) !== 0;
|
|
18641
18715
|
}
|
|
18642
|
-
function
|
|
18643
|
-
|
|
18644
|
-
|
|
18645
|
-
|
|
18646
|
-
|
|
18647
|
-
|
|
18648
|
-
|
|
18649
|
-
|
|
18650
|
-
|
|
18651
|
-
|
|
18652
|
-
|
|
18653
|
-
|
|
18654
|
-
|
|
18716
|
+
function cloudProviderFromPath(target) {
|
|
18717
|
+
const lower = target.toLowerCase();
|
|
18718
|
+
if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
|
|
18719
|
+
return "google-drive";
|
|
18720
|
+
if (lower.includes("onedrive"))
|
|
18721
|
+
return "onedrive";
|
|
18722
|
+
if (lower.includes("dropbox"))
|
|
18723
|
+
return "dropbox";
|
|
18724
|
+
if (lower.includes("mobile documents") || lower.includes("clouddocs"))
|
|
18725
|
+
return "icloud";
|
|
18726
|
+
return "file-provider";
|
|
18727
|
+
}
|
|
18728
|
+
function cloudProviderFromDomain(domain) {
|
|
18729
|
+
const lower = domain.toLowerCase();
|
|
18730
|
+
if (lower.includes("clouddocs") || lower.includes("icloud"))
|
|
18731
|
+
return "icloud";
|
|
18732
|
+
if (lower.includes("google"))
|
|
18733
|
+
return "google-drive";
|
|
18734
|
+
if (lower.includes("onedrive"))
|
|
18735
|
+
return "onedrive";
|
|
18736
|
+
if (lower.includes("dropbox"))
|
|
18737
|
+
return "dropbox";
|
|
18738
|
+
return "file-provider";
|
|
18739
|
+
}
|
|
18740
|
+
function normalizeCloudPath(value) {
|
|
18741
|
+
let normalized = value.replaceAll("\\", "/");
|
|
18742
|
+
if (normalized.length > 1)
|
|
18743
|
+
normalized = normalized.replace(/\/+$/u, "");
|
|
18744
|
+
return normalized;
|
|
18745
|
+
}
|
|
18746
|
+
function isPathInside(target, parent) {
|
|
18747
|
+
const normalizedParent = normalizeCloudPath(parent);
|
|
18748
|
+
if (normalizedParent === "")
|
|
18749
|
+
return false;
|
|
18750
|
+
const left = normalizeCloudPath(target);
|
|
18751
|
+
const [a, b] = process.platform === "win32" ? [left.toLowerCase(), normalizedParent.toLowerCase()] : [left, normalizedParent];
|
|
18752
|
+
return a === b || a.startsWith(`${b}/`);
|
|
18753
|
+
}
|
|
18754
|
+
function canonicalPath(value) {
|
|
18755
|
+
try {
|
|
18756
|
+
return realpathSync3(value);
|
|
18757
|
+
} catch {
|
|
18758
|
+
return value;
|
|
18655
18759
|
}
|
|
18656
18760
|
}
|
|
18657
|
-
|
|
18658
|
-
|
|
18659
|
-
|
|
18660
|
-
|
|
18661
|
-
|
|
18662
|
-
|
|
18761
|
+
function canonicalPathMapper(searchRoots) {
|
|
18762
|
+
const forms = searchRoots.map((root) => ({ raw: normalizeCloudPath(root), canonical: normalizeCloudPath(canonicalPath(root)) })).filter(({ raw, canonical }) => raw !== "" && raw !== canonical).sort((left, right) => right.raw.length - left.raw.length);
|
|
18763
|
+
if (forms.length === 0)
|
|
18764
|
+
return (path) => path;
|
|
18765
|
+
return (path) => {
|
|
18766
|
+
const normalized = normalizeCloudPath(path);
|
|
18767
|
+
for (const { raw, canonical } of forms) {
|
|
18768
|
+
if (isPathInside(normalized, raw))
|
|
18769
|
+
return `${canonical}${normalized.slice(raw.length)}`;
|
|
18770
|
+
}
|
|
18771
|
+
return path;
|
|
18772
|
+
};
|
|
18663
18773
|
}
|
|
18664
|
-
|
|
18665
|
-
const
|
|
18666
|
-
|
|
18667
|
-
|
|
18668
|
-
|
|
18774
|
+
function cloudScope(searchRoot, cloudRoots) {
|
|
18775
|
+
const canonicalSearchRoot = canonicalPath(searchRoot);
|
|
18776
|
+
const roots = cloudRoots.map((root) => ({ path: canonicalPath(root.path), kind: root.kind }));
|
|
18777
|
+
return {
|
|
18778
|
+
insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
|
|
18779
|
+
nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
|
|
18669
18780
|
};
|
|
18670
|
-
const sleep = options.sleep ?? ((ms) => delay2(ms, undefined, signal === undefined ? undefined : { signal }));
|
|
18671
|
-
let lastError = "";
|
|
18672
|
-
for (let attempt = 1;; attempt += 1) {
|
|
18673
|
-
const result = await runIndex(skill, signal);
|
|
18674
|
-
signal?.throwIfAborted();
|
|
18675
|
-
if (result.ok) {
|
|
18676
|
-
if (attempt === 1)
|
|
18677
|
-
return result;
|
|
18678
|
-
const recovered = {
|
|
18679
|
-
code: "datasource-index-failed",
|
|
18680
|
-
severity: "info",
|
|
18681
|
-
message: `recovered after ${attempt} attempts; last error: ${lastError}`,
|
|
18682
|
-
instanceId: result.instanceId,
|
|
18683
|
-
source: result.skill
|
|
18684
|
-
};
|
|
18685
|
-
return { ...result, attempts: attempt, diagnostics: [...result.diagnostics, recovered] };
|
|
18686
|
-
}
|
|
18687
|
-
lastError = result.message ?? result.error ?? "datasource-index-failed";
|
|
18688
|
-
if (attempt > policy.maxRetries) {
|
|
18689
|
-
if (attempt === 1)
|
|
18690
|
-
return { ...result, attempts: attempt };
|
|
18691
|
-
const suffix = ` (gave up after ${attempt} attempts)`;
|
|
18692
|
-
const last = result.diagnostics.at(-1);
|
|
18693
|
-
return {
|
|
18694
|
-
...result,
|
|
18695
|
-
attempts: attempt,
|
|
18696
|
-
message: `${lastError}${suffix}`,
|
|
18697
|
-
diagnostics: last === undefined ? [
|
|
18698
|
-
{
|
|
18699
|
-
code: result.code ?? "datasource-index-failed",
|
|
18700
|
-
severity: "error",
|
|
18701
|
-
message: `${lastError}${suffix}`,
|
|
18702
|
-
instanceId: result.instanceId,
|
|
18703
|
-
source: result.skill
|
|
18704
|
-
}
|
|
18705
|
-
] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
|
|
18706
|
-
};
|
|
18707
|
-
}
|
|
18708
|
-
await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
|
|
18709
|
-
signal?.throwIfAborted();
|
|
18710
|
-
}
|
|
18711
18781
|
}
|
|
18712
|
-
|
|
18782
|
+
function parseReleaseOutcome(stdout) {
|
|
18783
|
+
let parsed;
|
|
18713
18784
|
try {
|
|
18714
|
-
|
|
18715
|
-
} catch
|
|
18716
|
-
|
|
18717
|
-
|
|
18718
|
-
|
|
18719
|
-
|
|
18720
|
-
|
|
18721
|
-
|
|
18722
|
-
|
|
18723
|
-
|
|
18724
|
-
|
|
18725
|
-
|
|
18726
|
-
|
|
18727
|
-
|
|
18728
|
-
|
|
18729
|
-
|
|
18730
|
-
|
|
18731
|
-
|
|
18732
|
-
|
|
18733
|
-
|
|
18734
|
-
|
|
18735
|
-
|
|
18736
|
-
|
|
18737
|
-
|
|
18785
|
+
parsed = JSON.parse(stdout.trim());
|
|
18786
|
+
} catch {
|
|
18787
|
+
return;
|
|
18788
|
+
}
|
|
18789
|
+
if (typeof parsed !== "object" || parsed === null || !("ok" in parsed) || typeof parsed.ok !== "boolean") {
|
|
18790
|
+
return;
|
|
18791
|
+
}
|
|
18792
|
+
return {
|
|
18793
|
+
ok: parsed.ok,
|
|
18794
|
+
..."isUploaded" in parsed ? { isUploaded: parsed.isUploaded } : {},
|
|
18795
|
+
..."isUploading" in parsed ? { isUploading: parsed.isUploading } : {},
|
|
18796
|
+
..."downloadingStatus" in parsed ? { downloadingStatus: parsed.downloadingStatus } : {}
|
|
18797
|
+
};
|
|
18798
|
+
}
|
|
18799
|
+
function describeReleaseRefusal(outcome) {
|
|
18800
|
+
if (outcome.isUploading === true)
|
|
18801
|
+
return "The file is still uploading to the cloud, so it cannot be released yet.";
|
|
18802
|
+
if (outcome.isUploaded === false)
|
|
18803
|
+
return "The file has not finished uploading to the cloud, so it cannot be released yet.";
|
|
18804
|
+
const status = typeof outcome.downloadingStatus === "string" ? outcome.downloadingStatus : "unknown";
|
|
18805
|
+
return `The cloud provider declined to release the local copy (download status ${status}).`;
|
|
18806
|
+
}
|
|
18807
|
+
function describeReleaseHelperFailure(error) {
|
|
18808
|
+
const failure = error;
|
|
18809
|
+
const parts = [];
|
|
18810
|
+
if (failure.killed === true || typeof failure.signal === "string") {
|
|
18811
|
+
parts.push(`the helper was killed${typeof failure.signal === "string" ? ` by ${failure.signal}` : ""}`);
|
|
18812
|
+
} else if (typeof failure.code === "number") {
|
|
18813
|
+
parts.push(`the helper exited with code ${failure.code}`);
|
|
18738
18814
|
}
|
|
18815
|
+
const stderr = typeof failure.stderr === "string" ? failure.stderr.trim() : "";
|
|
18816
|
+
if (stderr !== "")
|
|
18817
|
+
parts.push(stderr);
|
|
18818
|
+
if (parts.length === 0)
|
|
18819
|
+
parts.push(error instanceof Error ? error.message : String(error));
|
|
18820
|
+
return `osascript could not release the local copy: ${parts.join("; ")}`;
|
|
18739
18821
|
}
|
|
18740
|
-
|
|
18741
|
-
|
|
18742
|
-
|
|
18743
|
-
maxRetries: 2,
|
|
18744
|
-
baseDelayMs: 1000,
|
|
18745
|
-
maxDelayMs: 30000
|
|
18746
|
-
};
|
|
18747
|
-
});
|
|
18748
|
-
|
|
18749
|
-
// src/everything/bundle.ts
|
|
18750
|
-
import { createHash as createHash11 } from "node:crypto";
|
|
18751
|
-
import { existsSync as existsSync17, mkdirSync as mkdirSync14, readFileSync as readFileSync17, rmSync as rmSync7, writeFileSync as writeFileSync10 } from "node:fs";
|
|
18752
|
-
import { dirname as dirname15, join as join28 } from "node:path";
|
|
18753
|
-
import { fileURLToPath } from "node:url";
|
|
18754
|
-
import JSZip from "jszip";
|
|
18755
|
-
function resolveEverythingBundleDir() {
|
|
18756
|
-
let dir = dirname15(fileURLToPath(import.meta.url));
|
|
18757
|
-
for (let depth = 0;depth < 6; depth += 1) {
|
|
18758
|
-
const candidate = join28(dir, "vendor", "everything");
|
|
18759
|
-
if (existsSync17(join28(candidate, MANIFEST_FILENAME)))
|
|
18760
|
-
return candidate;
|
|
18761
|
-
const parent = dirname15(dir);
|
|
18762
|
-
if (parent === dir)
|
|
18763
|
-
break;
|
|
18764
|
-
dir = parent;
|
|
18765
|
-
}
|
|
18766
|
-
return;
|
|
18822
|
+
async function defaultDarwinReadDir(path) {
|
|
18823
|
+
const entries = await readdir(path, { withFileTypes: true });
|
|
18824
|
+
return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
|
|
18767
18825
|
}
|
|
18768
|
-
function
|
|
18769
|
-
|
|
18770
|
-
|
|
18771
|
-
|
|
18826
|
+
function parseDarwinMounts(stdout) {
|
|
18827
|
+
const mounts = [];
|
|
18828
|
+
for (const line of stdout.split(`
|
|
18829
|
+
`)) {
|
|
18830
|
+
if (line.trim() === "")
|
|
18831
|
+
continue;
|
|
18832
|
+
const open = line.lastIndexOf(" (");
|
|
18833
|
+
if (open <= 0 || !line.endsWith(")"))
|
|
18834
|
+
continue;
|
|
18835
|
+
const head = line.slice(0, open);
|
|
18836
|
+
const on = head.indexOf(" on ");
|
|
18837
|
+
if (on <= 0)
|
|
18838
|
+
continue;
|
|
18839
|
+
const mountPoint = head.slice(on + " on ".length);
|
|
18840
|
+
if (mountPoint === "")
|
|
18841
|
+
continue;
|
|
18842
|
+
const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
|
|
18843
|
+
mounts.push({
|
|
18844
|
+
device: head.slice(0, on),
|
|
18845
|
+
mountPoint,
|
|
18846
|
+
fileSystem: fields[0] ?? "",
|
|
18847
|
+
options: fields.slice(1)
|
|
18848
|
+
});
|
|
18849
|
+
}
|
|
18850
|
+
return mounts;
|
|
18772
18851
|
}
|
|
18773
|
-
function
|
|
18774
|
-
const
|
|
18775
|
-
|
|
18852
|
+
function networkMountPoints(mounts) {
|
|
18853
|
+
const points = [];
|
|
18854
|
+
const seen = new Set;
|
|
18855
|
+
for (const mount of mounts) {
|
|
18856
|
+
if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
|
|
18857
|
+
continue;
|
|
18858
|
+
if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
|
|
18859
|
+
continue;
|
|
18860
|
+
if (seen.has(mount.mountPoint))
|
|
18861
|
+
continue;
|
|
18862
|
+
seen.add(mount.mountPoint);
|
|
18863
|
+
points.push(mount.mountPoint);
|
|
18864
|
+
}
|
|
18865
|
+
return points;
|
|
18776
18866
|
}
|
|
18777
|
-
function
|
|
18778
|
-
|
|
18867
|
+
function parseXattrValues(stdout, candidates) {
|
|
18868
|
+
const values = new Map;
|
|
18869
|
+
if (candidates.length === 0)
|
|
18870
|
+
return values;
|
|
18871
|
+
const ordered = [...candidates].sort((left, right) => right.length - left.length);
|
|
18872
|
+
for (const line of stdout.split(`
|
|
18873
|
+
`)) {
|
|
18874
|
+
if (line === "")
|
|
18875
|
+
continue;
|
|
18876
|
+
for (const candidate of ordered) {
|
|
18877
|
+
if (line.startsWith(`${candidate}: `)) {
|
|
18878
|
+
values.set(candidate, line.slice(candidate.length + 2));
|
|
18879
|
+
break;
|
|
18880
|
+
}
|
|
18881
|
+
}
|
|
18882
|
+
}
|
|
18883
|
+
return values;
|
|
18779
18884
|
}
|
|
18780
|
-
|
|
18781
|
-
|
|
18782
|
-
|
|
18783
|
-
|
|
18784
|
-
|
|
18785
|
-
|
|
18786
|
-
|
|
18787
|
-
|
|
18788
|
-
|
|
18885
|
+
|
|
18886
|
+
class DarwinCloudProbe {
|
|
18887
|
+
exec;
|
|
18888
|
+
readDir;
|
|
18889
|
+
home;
|
|
18890
|
+
constructor(options = {}) {
|
|
18891
|
+
this.exec = options.exec ?? defaultCloudExec;
|
|
18892
|
+
this.readDir = options.readDir ?? defaultDarwinReadDir;
|
|
18893
|
+
this.home = options.home ?? homedir6();
|
|
18789
18894
|
}
|
|
18790
|
-
|
|
18791
|
-
|
|
18792
|
-
|
|
18793
|
-
|
|
18794
|
-
|
|
18795
|
-
|
|
18796
|
-
|
|
18895
|
+
async cloudRoots(searchRoots = []) {
|
|
18896
|
+
const roots = [];
|
|
18897
|
+
const seen = new Set;
|
|
18898
|
+
const add = (path, kind) => {
|
|
18899
|
+
const canonical = canonicalPath(path);
|
|
18900
|
+
const key = normalizeCloudPath(canonical);
|
|
18901
|
+
if (key === "" || seen.has(key))
|
|
18902
|
+
return;
|
|
18903
|
+
seen.add(key);
|
|
18904
|
+
roots.push({ path: canonical, kind });
|
|
18797
18905
|
};
|
|
18906
|
+
const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
|
|
18907
|
+
for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
|
|
18908
|
+
if (entry.isDirectory())
|
|
18909
|
+
add(join29(cloudStorage, entry.name), "file-provider");
|
|
18910
|
+
}
|
|
18911
|
+
if (await this.tryReadDir(mobileDocuments) !== undefined)
|
|
18912
|
+
add(mobileDocuments, "file-provider");
|
|
18913
|
+
for (const mountPoint of await this.networkMountPoints())
|
|
18914
|
+
add(mountPoint, "network");
|
|
18915
|
+
for (const directory of await this.fileProviderDirectories(searchRoots))
|
|
18916
|
+
add(directory, "file-provider");
|
|
18917
|
+
return roots;
|
|
18798
18918
|
}
|
|
18799
|
-
|
|
18800
|
-
|
|
18801
|
-
|
|
18802
|
-
|
|
18803
|
-
|
|
18919
|
+
async networkMountPoints() {
|
|
18920
|
+
try {
|
|
18921
|
+
const { stdout } = await this.exec("/sbin/mount", [], { timeoutMs: 1e4, maxBuffer: 4 * 1024 * 1024 });
|
|
18922
|
+
return networkMountPoints(parseDarwinMounts(stdout));
|
|
18923
|
+
} catch {
|
|
18924
|
+
return [];
|
|
18925
|
+
}
|
|
18804
18926
|
}
|
|
18805
|
-
|
|
18806
|
-
|
|
18807
|
-
|
|
18808
|
-
|
|
18809
|
-
|
|
18810
|
-
|
|
18811
|
-
|
|
18927
|
+
async fileProviderDirectories(searchRoots) {
|
|
18928
|
+
if (searchRoots.length === 0)
|
|
18929
|
+
return [];
|
|
18930
|
+
const candidates = new Set;
|
|
18931
|
+
for (const searchRoot of searchRoots) {
|
|
18932
|
+
const canonical = canonicalPath(searchRoot);
|
|
18933
|
+
for (let cursor = canonical;; cursor = dirname15(cursor)) {
|
|
18934
|
+
candidates.add(cursor);
|
|
18935
|
+
const parent = dirname15(cursor);
|
|
18936
|
+
if (parent === cursor)
|
|
18937
|
+
break;
|
|
18938
|
+
}
|
|
18939
|
+
for (const entry of await this.tryReadDir(canonical) ?? []) {
|
|
18940
|
+
if (entry.isDirectory())
|
|
18941
|
+
candidates.add(join29(canonical, entry.name));
|
|
18942
|
+
}
|
|
18943
|
+
}
|
|
18944
|
+
const paths = [...candidates];
|
|
18945
|
+
const managed = [];
|
|
18946
|
+
for (let start = 0;start < paths.length; start += XATTR_BATCH) {
|
|
18947
|
+
for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
|
|
18948
|
+
managed.push(path);
|
|
18949
|
+
}
|
|
18950
|
+
return managed;
|
|
18812
18951
|
}
|
|
18813
|
-
|
|
18814
|
-
|
|
18815
|
-
|
|
18816
|
-
|
|
18817
|
-
|
|
18952
|
+
async xattrValues(paths) {
|
|
18953
|
+
try {
|
|
18954
|
+
const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
|
|
18955
|
+
timeoutMs: 15000,
|
|
18956
|
+
maxBuffer: 8 * 1024 * 1024
|
|
18957
|
+
});
|
|
18958
|
+
return parseXattrValues(stdout, paths);
|
|
18959
|
+
} catch (error) {
|
|
18960
|
+
const failure = error;
|
|
18961
|
+
return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
|
|
18962
|
+
}
|
|
18818
18963
|
}
|
|
18819
|
-
|
|
18820
|
-
|
|
18821
|
-
|
|
18822
|
-
|
|
18823
|
-
|
|
18824
|
-
|
|
18825
|
-
};
|
|
18964
|
+
async tryReadDir(path) {
|
|
18965
|
+
try {
|
|
18966
|
+
return await this.readDir(path);
|
|
18967
|
+
} catch {
|
|
18968
|
+
return;
|
|
18969
|
+
}
|
|
18826
18970
|
}
|
|
18827
|
-
|
|
18828
|
-
|
|
18829
|
-
|
|
18830
|
-
|
|
18831
|
-
|
|
18832
|
-
|
|
18833
|
-
|
|
18834
|
-
|
|
18971
|
+
async datalessAmong(paths) {
|
|
18972
|
+
const dataless = new Set;
|
|
18973
|
+
for (let start = 0;start < paths.length; start += STAT_BATCH) {
|
|
18974
|
+
const batch = paths.slice(start, start + STAT_BATCH);
|
|
18975
|
+
const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
|
|
18976
|
+
timeoutMs: 30000,
|
|
18977
|
+
maxBuffer: 4 * 1024 * 1024
|
|
18978
|
+
});
|
|
18979
|
+
const flags = stdout.split(`
|
|
18980
|
+
`).filter(Boolean);
|
|
18981
|
+
for (const [index, path] of batch.entries()) {
|
|
18982
|
+
if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
|
|
18983
|
+
dataless.add(path);
|
|
18984
|
+
}
|
|
18985
|
+
}
|
|
18986
|
+
return dataless;
|
|
18835
18987
|
}
|
|
18836
|
-
|
|
18837
|
-
|
|
18838
|
-
|
|
18839
|
-
const archive = readFileSync17(archivePath);
|
|
18840
|
-
const actualArchive = sha256(archive);
|
|
18841
|
-
if (actualArchive !== archiveSha256) {
|
|
18842
|
-
throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
|
|
18988
|
+
async providerOf(path) {
|
|
18989
|
+
const domain = await readFileProviderDomainUpwards(path);
|
|
18990
|
+
return domain === undefined ? undefined : cloudProviderFromDomain(domain);
|
|
18843
18991
|
}
|
|
18844
|
-
|
|
18845
|
-
|
|
18846
|
-
|
|
18847
|
-
|
|
18848
|
-
|
|
18849
|
-
|
|
18850
|
-
|
|
18851
|
-
|
|
18992
|
+
canRelease(provider) {
|
|
18993
|
+
return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
|
|
18994
|
+
}
|
|
18995
|
+
async evict(path) {
|
|
18996
|
+
let stdout;
|
|
18997
|
+
try {
|
|
18998
|
+
stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
|
|
18999
|
+
timeoutMs: 30000
|
|
19000
|
+
})).stdout;
|
|
19001
|
+
} catch (error) {
|
|
19002
|
+
return { ok: false, message: describeReleaseHelperFailure(error) };
|
|
19003
|
+
}
|
|
19004
|
+
const outcome = parseReleaseOutcome(stdout);
|
|
19005
|
+
if (outcome === undefined) {
|
|
19006
|
+
const detail = stdout.trim();
|
|
19007
|
+
return {
|
|
19008
|
+
ok: false,
|
|
19009
|
+
message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
|
|
19010
|
+
};
|
|
19011
|
+
}
|
|
19012
|
+
return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
|
|
18852
19013
|
}
|
|
18853
|
-
return bytes;
|
|
18854
19014
|
}
|
|
18855
|
-
function
|
|
18856
|
-
return
|
|
19015
|
+
function buildWindowsAttributeScript(paths) {
|
|
19016
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19017
|
+
` + `$ErrorActionPreference='SilentlyContinue'
|
|
19018
|
+
` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
|
|
19019
|
+
`;
|
|
18857
19020
|
}
|
|
18858
|
-
function
|
|
18859
|
-
|
|
18860
|
-
|
|
18861
|
-
|
|
18862
|
-
|
|
18863
|
-
|
|
18864
|
-
|
|
19021
|
+
function buildWindowsSyncRootsScript() {
|
|
19022
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19023
|
+
` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
|
|
19024
|
+
` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
|
|
19025
|
+
`;
|
|
19026
|
+
}
|
|
19027
|
+
function buildWindowsDehydrateScript(path) {
|
|
19028
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19029
|
+
` + `$ErrorActionPreference='Stop'
|
|
19030
|
+
` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
|
|
19031
|
+
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
|
|
19032
|
+
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
|
|
19033
|
+
` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
|
|
19034
|
+
` + `'@
|
|
19035
|
+
` + `$h=[IntPtr]::Zero
|
|
19036
|
+
` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
|
|
19037
|
+
` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
|
|
19038
|
+
` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
|
|
19039
|
+
` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
|
|
19040
|
+
` + `[Autorag.CfApi]::CfCloseHandle($h)
|
|
19041
|
+
` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
|
|
19042
|
+
` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
|
|
19043
|
+
`;
|
|
19044
|
+
}
|
|
19045
|
+
function parseWindowsAttributeOutput(stdout, paths) {
|
|
19046
|
+
const lines = stdout.split(/\r?\n/);
|
|
19047
|
+
const attributes = new Map;
|
|
19048
|
+
for (const [index, path] of paths.entries()) {
|
|
19049
|
+
const value = Number.parseInt((lines[index] ?? "").trim(), 10);
|
|
19050
|
+
if (Number.isFinite(value))
|
|
19051
|
+
attributes.set(path, value);
|
|
18865
19052
|
}
|
|
19053
|
+
return attributes;
|
|
18866
19054
|
}
|
|
18867
|
-
function
|
|
18868
|
-
|
|
19055
|
+
function parseWindowsSyncRoots(stdout) {
|
|
19056
|
+
const roots = [];
|
|
19057
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
19058
|
+
const tab = line.indexOf("\t");
|
|
19059
|
+
if (tab <= 0)
|
|
19060
|
+
continue;
|
|
19061
|
+
const path = line.slice(0, tab).trim();
|
|
19062
|
+
if (path !== "")
|
|
19063
|
+
roots.push({ path, label: line.slice(tab + 1).trim() });
|
|
19064
|
+
}
|
|
19065
|
+
return roots;
|
|
18869
19066
|
}
|
|
18870
|
-
function
|
|
18871
|
-
|
|
19067
|
+
function providerFromWindowsSyncRoot(label) {
|
|
19068
|
+
const lower = label.toLowerCase();
|
|
19069
|
+
if (lower.includes("onedrive") || lower.includes("one drive"))
|
|
19070
|
+
return "onedrive";
|
|
19071
|
+
if (lower.includes("google"))
|
|
19072
|
+
return "google-drive";
|
|
19073
|
+
return "file-provider";
|
|
18872
19074
|
}
|
|
18873
|
-
|
|
18874
|
-
|
|
18875
|
-
|
|
18876
|
-
|
|
18877
|
-
|
|
18878
|
-
// src/filesystem/path-exclusions.ts
|
|
18879
|
-
import { posix as posix3, win32 as win322 } from "node:path";
|
|
18880
|
-
function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
|
|
18881
|
-
const paths = platform === "win32" ? win322 : posix3;
|
|
18882
|
-
const normalize = (path) => {
|
|
18883
|
-
const absolute = paths.resolve(path);
|
|
18884
|
-
return platform === "win32" ? absolute.toLowerCase() : absolute;
|
|
18885
|
-
};
|
|
18886
|
-
const excluded = excludedPaths.map((path) => {
|
|
18887
|
-
const absolute = normalize(path);
|
|
18888
|
-
return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
|
|
18889
|
-
});
|
|
18890
|
-
return (path) => {
|
|
18891
|
-
const absolute = normalize(path);
|
|
18892
|
-
return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
|
|
18893
|
-
};
|
|
19075
|
+
function buildWindowsDiskScript() {
|
|
19076
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19077
|
+
` + "Get-CimInstance -ClassName Win32_LogicalDisk | ForEach-Object {" + `$_.DeviceID + [char]9 + $_.DriveType + [char]9 + $_.FileSystem + [char]9 + $_.VolumeName}
|
|
19078
|
+
`;
|
|
18894
19079
|
}
|
|
18895
|
-
|
|
18896
|
-
|
|
18897
|
-
|
|
18898
|
-
|
|
18899
|
-
|
|
18900
|
-
|
|
18901
|
-
|
|
18902
|
-
|
|
18903
|
-
|
|
18904
|
-
|
|
18905
|
-
|
|
18906
|
-
|
|
18907
|
-
|
|
18908
|
-
|
|
18909
|
-
return run;
|
|
18910
|
-
}
|
|
18911
|
-
exclusive(task) {
|
|
18912
|
-
const earlier = Promise.all([this.lastExclusive, ...this.sharedSinceExclusive]);
|
|
18913
|
-
const run = earlier.then(task);
|
|
18914
|
-
const settled = run.then(() => {
|
|
18915
|
-
return;
|
|
18916
|
-
}, () => {
|
|
18917
|
-
return;
|
|
19080
|
+
function parseWindowsLogicalDisks(stdout) {
|
|
19081
|
+
const disks = [];
|
|
19082
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
19083
|
+
if (line.trim() === "")
|
|
19084
|
+
continue;
|
|
19085
|
+
const [deviceId, driveType, fileSystem, ...rest] = line.split("\t");
|
|
19086
|
+
const type = Number.parseInt((driveType ?? "").trim(), 10);
|
|
19087
|
+
if (!Number.isFinite(type))
|
|
19088
|
+
continue;
|
|
19089
|
+
disks.push({
|
|
19090
|
+
deviceId: (deviceId ?? "").trim(),
|
|
19091
|
+
driveType: type,
|
|
19092
|
+
fileSystem: (fileSystem ?? "").trim(),
|
|
19093
|
+
volumeName: rest.join("\t").trim()
|
|
18918
19094
|
});
|
|
18919
|
-
this.sharedSinceExclusive = [];
|
|
18920
|
-
this.lastExclusive = settled;
|
|
18921
|
-
return run;
|
|
18922
19095
|
}
|
|
19096
|
+
return disks;
|
|
18923
19097
|
}
|
|
18924
|
-
|
|
18925
|
-
|
|
18926
|
-
|
|
18927
|
-
|
|
18928
|
-
|
|
18929
|
-
|
|
18930
|
-
|
|
18931
|
-
|
|
18932
|
-
|
|
18933
|
-
|
|
18934
|
-
|
|
18935
|
-
|
|
18936
|
-
|
|
18937
|
-
|
|
18938
|
-
|
|
18939
|
-
"
|
|
18940
|
-
|
|
18941
|
-
|
|
18942
|
-
|
|
18943
|
-
|
|
18944
|
-
"check_for_updates_on_startup=0",
|
|
18945
|
-
"beta_updates=0",
|
|
18946
|
-
"language=1033",
|
|
18947
|
-
"allow_http_server=0",
|
|
18948
|
-
"allow_etp_server=0",
|
|
18949
|
-
"search_history_enabled=0",
|
|
18950
|
-
"run_history_enabled=0",
|
|
18951
|
-
"auto_include_fixed_volumes=0",
|
|
18952
|
-
"auto_include_removable_volumes=0",
|
|
18953
|
-
"auto_include_fixed_refs_volumes=0",
|
|
18954
|
-
"auto_include_removable_refs_volumes=0",
|
|
18955
|
-
"ntfs_volume_paths=",
|
|
18956
|
-
"ntfs_volume_includes=",
|
|
18957
|
-
"refs_volume_paths=",
|
|
18958
|
-
"refs_volume_includes=",
|
|
18959
|
-
"index_size=1",
|
|
18960
|
-
"index_date_modified=1",
|
|
18961
|
-
`folders=${list(input.folders)}`,
|
|
18962
|
-
`folder_monitor_changes=${perFolder("1")}`,
|
|
18963
|
-
`folder_rescan_if_full_list=${perFolder("1")}`,
|
|
18964
|
-
`folder_update_types=${perFolder("0")}`,
|
|
18965
|
-
"exclude_list_enabled=1",
|
|
18966
|
-
`exclude_folders=${list(input.excludeFolders)}`,
|
|
18967
|
-
""
|
|
18968
|
-
].join(`\r
|
|
18969
|
-
`);
|
|
19098
|
+
function windowsRootsFromDisks(disks) {
|
|
19099
|
+
const roots = [];
|
|
19100
|
+
for (const disk of disks) {
|
|
19101
|
+
if (disk.deviceId === "")
|
|
19102
|
+
continue;
|
|
19103
|
+
const path = `${disk.deviceId}\\`;
|
|
19104
|
+
const fileSystem = disk.fileSystem.toLowerCase();
|
|
19105
|
+
if (disk.driveType === WINDOWS_DRIVE_TYPE_NETWORK) {
|
|
19106
|
+
roots.push({ path, kind: "network" });
|
|
19107
|
+
continue;
|
|
19108
|
+
}
|
|
19109
|
+
if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
|
|
19110
|
+
roots.push({ path, kind: "network" });
|
|
19111
|
+
continue;
|
|
19112
|
+
}
|
|
19113
|
+
if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
|
|
19114
|
+
roots.push({ path, kind: "cloud-files" });
|
|
19115
|
+
}
|
|
19116
|
+
}
|
|
19117
|
+
return roots;
|
|
18970
19118
|
}
|
|
18971
|
-
function
|
|
18972
|
-
const
|
|
18973
|
-
|
|
18974
|
-
|
|
18975
|
-
|
|
18976
|
-
|
|
18977
|
-
|
|
18978
|
-
|
|
18979
|
-
|
|
18980
|
-
|
|
18981
|
-
|
|
18982
|
-
|
|
18983
|
-
|
|
18984
|
-
];
|
|
18985
|
-
if (timeoutMs !== undefined)
|
|
18986
|
-
args.push("-timeout", String(timeoutMs));
|
|
18987
|
-
if (request.matchCase)
|
|
18988
|
-
args.push("-case");
|
|
18989
|
-
if (request.matchPath)
|
|
18990
|
-
args.push("-match-path");
|
|
18991
|
-
if (request.wholeWord)
|
|
18992
|
-
args.push("-whole-word");
|
|
18993
|
-
if (request.kind === "files")
|
|
18994
|
-
args.push("/a-d");
|
|
18995
|
-
if (request.kind === "folders")
|
|
18996
|
-
args.push("/ad");
|
|
18997
|
-
if (request.path !== undefined)
|
|
18998
|
-
args.push("-path", request.path);
|
|
18999
|
-
if (request.sort !== undefined)
|
|
19000
|
-
args.push("-sort", request.sort);
|
|
19001
|
-
if (request.offset !== undefined && request.offset > 0)
|
|
19002
|
-
args.push("-offset", String(request.offset));
|
|
19003
|
-
args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
|
|
19004
|
-
if (request.regex)
|
|
19005
|
-
args.push("-regex", "--", request.query);
|
|
19006
|
-
else
|
|
19007
|
-
args.push("--", request.query);
|
|
19008
|
-
return args;
|
|
19119
|
+
function uncShareRoot(target) {
|
|
19120
|
+
const normalized = target.replaceAll("/", "\\");
|
|
19121
|
+
if (!normalized.startsWith("\\\\") || normalized.startsWith("\\\\?\\") || normalized.startsWith("\\\\.\\"))
|
|
19122
|
+
return;
|
|
19123
|
+
const rest = normalized.slice(2);
|
|
19124
|
+
const separator = rest.indexOf("\\");
|
|
19125
|
+
if (separator <= 0)
|
|
19126
|
+
return;
|
|
19127
|
+
const server = rest.slice(0, separator);
|
|
19128
|
+
const share = rest.slice(separator + 1).split("\\")[0] ?? "";
|
|
19129
|
+
if (server === "" || share === "")
|
|
19130
|
+
return;
|
|
19131
|
+
return `\\\\${server}\\${share}`;
|
|
19009
19132
|
}
|
|
19010
|
-
function
|
|
19011
|
-
const
|
|
19012
|
-
if (
|
|
19013
|
-
return
|
|
19014
|
-
|
|
19015
|
-
try {
|
|
19016
|
-
parsed = JSON.parse(text);
|
|
19017
|
-
} catch {
|
|
19018
|
-
throw new Error(`ES did not return JSON: ${text.slice(0, 2000)}`);
|
|
19019
|
-
}
|
|
19020
|
-
if (!Array.isArray(parsed))
|
|
19021
|
-
throw new Error(`ES did not return a JSON array: ${text.slice(0, 2000)}`);
|
|
19022
|
-
return parsed.map((item) => {
|
|
19023
|
-
const record = item;
|
|
19024
|
-
const filename = String(record.filename ?? "");
|
|
19025
|
-
const isFolder = filename.endsWith("\\") && filename.length > 3;
|
|
19133
|
+
function parseWindowsDehydrateOutput(result) {
|
|
19134
|
+
const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
19135
|
+
if (open === null)
|
|
19136
|
+
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
19137
|
+
if (Number.parseInt(open[1], 16) !== 0) {
|
|
19026
19138
|
return {
|
|
19027
|
-
|
|
19028
|
-
|
|
19029
|
-
size: typeof record.size === "number" ? record.size : undefined,
|
|
19030
|
-
dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
|
|
19139
|
+
ok: false,
|
|
19140
|
+
message: windowsFailureMessage(result, `could not open the file for release (0x${open[1]})`)
|
|
19031
19141
|
};
|
|
19032
|
-
}
|
|
19142
|
+
}
|
|
19143
|
+
const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
19144
|
+
if (dehydrate === null) {
|
|
19145
|
+
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
19146
|
+
}
|
|
19147
|
+
const hresult = dehydrate[1];
|
|
19148
|
+
if (Number.parseInt(hresult, 16) === 0)
|
|
19149
|
+
return { ok: true };
|
|
19150
|
+
const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
|
|
19151
|
+
return {
|
|
19152
|
+
ok: false,
|
|
19153
|
+
message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
|
|
19154
|
+
};
|
|
19155
|
+
}
|
|
19156
|
+
function windowsFailureMessage(result, fallback) {
|
|
19157
|
+
const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
|
|
19158
|
+
if (printed !== undefined && printed !== "")
|
|
19159
|
+
return printed;
|
|
19160
|
+
const stderr = result.stderr.trim();
|
|
19161
|
+
return stderr !== "" ? stderr : fallback;
|
|
19033
19162
|
}
|
|
19034
19163
|
|
|
19035
|
-
class
|
|
19036
|
-
instanceName;
|
|
19037
|
-
options;
|
|
19038
|
-
platform;
|
|
19164
|
+
class WindowsCloudProbe {
|
|
19039
19165
|
run;
|
|
19040
|
-
|
|
19041
|
-
|
|
19042
|
-
|
|
19043
|
-
|
|
19044
|
-
queue = new SharedExclusiveQueue;
|
|
19045
|
-
constructor(options) {
|
|
19046
|
-
this.options = options;
|
|
19047
|
-
this.platform = options.platform ?? process.platform;
|
|
19048
|
-
this.instanceName = everythingInstanceName(options.root);
|
|
19049
|
-
this.run = options.run ?? runProcess;
|
|
19050
|
-
this.launch = options.launch ?? launchDetached;
|
|
19051
|
-
this.stateDir = join29(options.root, ".autorag", "everything");
|
|
19052
|
-
this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
|
|
19053
|
-
}
|
|
19054
|
-
isSupported() {
|
|
19055
|
-
return this.platform === "win32";
|
|
19166
|
+
syncRoots;
|
|
19167
|
+
disks;
|
|
19168
|
+
constructor(options = {}) {
|
|
19169
|
+
this.run = options.run ?? runPowerShell;
|
|
19056
19170
|
}
|
|
19057
|
-
|
|
19058
|
-
|
|
19059
|
-
|
|
19060
|
-
|
|
19061
|
-
|
|
19062
|
-
const
|
|
19063
|
-
if (
|
|
19064
|
-
return
|
|
19065
|
-
|
|
19066
|
-
|
|
19067
|
-
|
|
19068
|
-
|
|
19069
|
-
|
|
19070
|
-
|
|
19071
|
-
|
|
19072
|
-
|
|
19073
|
-
|
|
19074
|
-
|
|
19075
|
-
|
|
19076
|
-
|
|
19077
|
-
|
|
19078
|
-
return this.failure("search-failed", es.esPath, args, result);
|
|
19079
|
-
try {
|
|
19080
|
-
return { ok: true, results: parseEverythingJson(result.stdout) };
|
|
19081
|
-
} catch (error) {
|
|
19082
|
-
return { ok: false, reason: "search-failed", message: error.message };
|
|
19083
|
-
}
|
|
19084
|
-
});
|
|
19171
|
+
async cloudRoots(searchRoots = []) {
|
|
19172
|
+
const roots = [];
|
|
19173
|
+
const seen = new Set;
|
|
19174
|
+
const add = (path, kind) => {
|
|
19175
|
+
const canonical = canonicalPath(path);
|
|
19176
|
+
const key = normalizeCloudPath(canonical).toLowerCase();
|
|
19177
|
+
if (key === "" || seen.has(key))
|
|
19178
|
+
return;
|
|
19179
|
+
seen.add(key);
|
|
19180
|
+
roots.push({ path: canonical, kind });
|
|
19181
|
+
};
|
|
19182
|
+
for (const root of await this.loadSyncRoots())
|
|
19183
|
+
add(root.path, "cloud-files");
|
|
19184
|
+
for (const root of await this.loadDiskRoots())
|
|
19185
|
+
add(root.path, root.kind);
|
|
19186
|
+
for (const searchRoot of searchRoots) {
|
|
19187
|
+
const share = uncShareRoot(searchRoot);
|
|
19188
|
+
if (share !== undefined)
|
|
19189
|
+
add(share, "network");
|
|
19190
|
+
}
|
|
19191
|
+
return roots;
|
|
19085
19192
|
}
|
|
19086
|
-
async
|
|
19087
|
-
const
|
|
19088
|
-
|
|
19089
|
-
|
|
19090
|
-
|
|
19091
|
-
|
|
19092
|
-
|
|
19093
|
-
const timedOut = () => ({
|
|
19094
|
-
ok: false,
|
|
19095
|
-
reason: "search-failed",
|
|
19096
|
-
message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
|
|
19097
|
-
});
|
|
19098
|
-
for (;; ) {
|
|
19099
|
-
const remainingMs = deadline - Date.now();
|
|
19100
|
-
if (remainingMs <= 0)
|
|
19101
|
-
return timedOut();
|
|
19102
|
-
const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
|
|
19103
|
-
const response = await this.run(esPath, args, remainingMs + 5000);
|
|
19104
|
-
if (response.code !== 0)
|
|
19105
|
-
return this.failure("search-failed", esPath, args, response);
|
|
19106
|
-
if (Date.now() >= deadline)
|
|
19107
|
-
return timedOut();
|
|
19108
|
-
let entries;
|
|
19109
|
-
try {
|
|
19110
|
-
entries = parseEverythingJson(response.stdout);
|
|
19111
|
-
} catch (error) {
|
|
19112
|
-
return { ok: false, reason: "search-failed", message: error.message };
|
|
19193
|
+
async datalessAmong(paths) {
|
|
19194
|
+
const dataless = new Set;
|
|
19195
|
+
for (let start = 0;start < paths.length; start += WINDOWS_ATTRIBUTE_BATCH) {
|
|
19196
|
+
const batch = paths.slice(start, start + WINDOWS_ATTRIBUTE_BATCH);
|
|
19197
|
+
const result = await this.run(buildWindowsAttributeScript(batch), 30000);
|
|
19198
|
+
if (!result.ok) {
|
|
19199
|
+
throw new Error(`Could not read Windows cloud file attributes: ${result.stderr.trim() || "powershell.exe failed"}`);
|
|
19113
19200
|
}
|
|
19114
|
-
for (const
|
|
19115
|
-
if (
|
|
19116
|
-
|
|
19117
|
-
if (skipped < offset) {
|
|
19118
|
-
skipped += 1;
|
|
19119
|
-
continue;
|
|
19120
|
-
}
|
|
19121
|
-
results.push(entry);
|
|
19122
|
-
if (results.length >= maxResults)
|
|
19123
|
-
return { ok: true, results };
|
|
19201
|
+
for (const [path, attributes] of parseWindowsAttributeOutput(result.stdout, batch)) {
|
|
19202
|
+
if (datalessFromWindowsAttributes(attributes))
|
|
19203
|
+
dataless.add(path);
|
|
19124
19204
|
}
|
|
19125
|
-
if (entries.length < MAX_FILTER_CANDIDATES)
|
|
19126
|
-
return { ok: true, results };
|
|
19127
|
-
nativeOffset += entries.length;
|
|
19128
19205
|
}
|
|
19206
|
+
return dataless;
|
|
19129
19207
|
}
|
|
19130
|
-
|
|
19131
|
-
|
|
19132
|
-
|
|
19133
|
-
if (!es.ok)
|
|
19134
|
-
return es;
|
|
19135
|
-
const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
|
|
19136
|
-
const saveArgs = ["-instance", this.instanceName, "-save-db"];
|
|
19137
|
-
const saved = await this.run(es.esPath, saveArgs, timeoutMs);
|
|
19138
|
-
if (saved.code !== 0)
|
|
19139
|
-
return this.failure("index-failed", es.esPath, saveArgs, saved);
|
|
19140
|
-
const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
|
|
19141
|
-
const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
|
|
19142
|
-
if (counted.code !== 0)
|
|
19143
|
-
return this.failure("index-failed", es.esPath, countArgs, counted);
|
|
19144
|
-
return {
|
|
19145
|
-
ok: true,
|
|
19146
|
-
indexedFolders: [...this.options.folders],
|
|
19147
|
-
indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
|
|
19148
|
-
};
|
|
19149
|
-
});
|
|
19208
|
+
async providerOf(path) {
|
|
19209
|
+
const root = longestSyncRoot(await this.loadSyncRoots(), path);
|
|
19210
|
+
return root === undefined ? cloudProviderFromPath(path) : providerFromWindowsSyncRoot(root.label);
|
|
19150
19211
|
}
|
|
19151
|
-
|
|
19152
|
-
return
|
|
19153
|
-
const binaries = await this.resolveBinaries("cached");
|
|
19154
|
-
if (!binaries.ok)
|
|
19155
|
-
return;
|
|
19156
|
-
await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
19157
|
-
});
|
|
19212
|
+
canRelease(provider) {
|
|
19213
|
+
return provider !== undefined;
|
|
19158
19214
|
}
|
|
19159
|
-
async
|
|
19160
|
-
|
|
19161
|
-
return {
|
|
19162
|
-
ok: false,
|
|
19163
|
-
reason: "unsupported-platform",
|
|
19164
|
-
message: `Everything is Windows-only; this host is ${this.platform}.`
|
|
19165
|
-
};
|
|
19166
|
-
}
|
|
19167
|
-
if (this.binaries?.ok)
|
|
19168
|
-
return this.binaries;
|
|
19169
|
-
const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
|
|
19170
|
-
const result = await resolve(mode);
|
|
19171
|
-
if (result.ok)
|
|
19172
|
-
this.binaries = result;
|
|
19173
|
-
return result;
|
|
19215
|
+
async evict(path) {
|
|
19216
|
+
return parseWindowsDehydrateOutput(await this.run(buildWindowsDehydrateScript(path), 60000));
|
|
19174
19217
|
}
|
|
19175
|
-
|
|
19176
|
-
|
|
19177
|
-
|
|
19178
|
-
return binaries;
|
|
19179
|
-
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
19180
|
-
let ping = await this.ping(binaries.esPath);
|
|
19181
|
-
if (ping.code === 0 && !reconfigure)
|
|
19182
|
-
return { ok: true, esPath: binaries.esPath };
|
|
19183
|
-
if (ping.code === 0) {
|
|
19184
|
-
const exitArgs = ["-instance", this.instanceName, "-exit"];
|
|
19185
|
-
const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
|
|
19186
|
-
if (exited.code !== 0)
|
|
19187
|
-
return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
|
|
19188
|
-
}
|
|
19189
|
-
mkdirSync15(this.stateDir, { recursive: true });
|
|
19190
|
-
const iniPath = join29(this.stateDir, "Everything.ini");
|
|
19191
|
-
writeFileSync11(iniPath, buildEverythingIni({
|
|
19192
|
-
folders: this.options.folders,
|
|
19193
|
-
excludeFolders: [
|
|
19194
|
-
...new Set([
|
|
19195
|
-
join29(this.options.root, ".autorag"),
|
|
19196
|
-
...this.options.folders.map((folder) => join29(folder, ".autorag")),
|
|
19197
|
-
...this.options.excludeFolders ?? []
|
|
19198
|
-
])
|
|
19199
|
-
]
|
|
19200
|
-
}));
|
|
19201
|
-
const launchArgs = [
|
|
19202
|
-
"-instance",
|
|
19203
|
-
this.instanceName,
|
|
19204
|
-
"-config",
|
|
19205
|
-
iniPath,
|
|
19206
|
-
"-db",
|
|
19207
|
-
join29(this.stateDir, "Everything.db"),
|
|
19208
|
-
"-startup"
|
|
19209
|
-
];
|
|
19210
|
-
try {
|
|
19211
|
-
this.launch(binaries.everythingPath, launchArgs);
|
|
19212
|
-
} catch (error) {
|
|
19213
|
-
return {
|
|
19214
|
-
ok: false,
|
|
19215
|
-
reason: "startup-failed",
|
|
19216
|
-
message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
|
|
19217
|
-
};
|
|
19218
|
-
}
|
|
19219
|
-
const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
|
|
19220
|
-
const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
19221
|
-
for (;; ) {
|
|
19222
|
-
ping = await this.ping(binaries.esPath);
|
|
19223
|
-
if (ping.code === 0)
|
|
19224
|
-
return { ok: true, esPath: binaries.esPath };
|
|
19225
|
-
if (Date.now() >= deadline)
|
|
19226
|
-
break;
|
|
19227
|
-
await sleep2(pollMs);
|
|
19228
|
-
}
|
|
19229
|
-
return {
|
|
19230
|
-
ok: false,
|
|
19231
|
-
reason: "startup-failed",
|
|
19232
|
-
message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
|
|
19233
|
-
};
|
|
19218
|
+
loadSyncRoots() {
|
|
19219
|
+
this.syncRoots ??= this.run(buildWindowsSyncRootsScript(), 15000).then((result) => result.ok ? parseWindowsSyncRoots(result.stdout) : [], () => []);
|
|
19220
|
+
return this.syncRoots;
|
|
19234
19221
|
}
|
|
19235
|
-
|
|
19236
|
-
|
|
19237
|
-
return
|
|
19222
|
+
loadDiskRoots() {
|
|
19223
|
+
this.disks ??= this.run(buildWindowsDiskScript(), 30000).then((result) => result.ok ? windowsRootsFromDisks(parseWindowsLogicalDisks(result.stdout)) : [], () => []);
|
|
19224
|
+
return this.disks;
|
|
19238
19225
|
}
|
|
19239
|
-
|
|
19226
|
+
}
|
|
19227
|
+
function longestSyncRoot(roots, target) {
|
|
19228
|
+
const normalized = target.replaceAll("/", "\\").toLowerCase();
|
|
19229
|
+
let match;
|
|
19230
|
+
let matchLength = -1;
|
|
19231
|
+
for (const root of roots) {
|
|
19232
|
+
const prefix = root.path.replaceAll("/", "\\").replace(/\\+$/u, "").toLowerCase();
|
|
19233
|
+
if (prefix === "" || prefix.length <= matchLength)
|
|
19234
|
+
continue;
|
|
19235
|
+
if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
|
|
19236
|
+
match = root;
|
|
19237
|
+
matchLength = prefix.length;
|
|
19238
|
+
}
|
|
19239
|
+
}
|
|
19240
|
+
return match;
|
|
19241
|
+
}
|
|
19242
|
+
function powerShellLaunch(script) {
|
|
19243
|
+
return {
|
|
19244
|
+
args: [
|
|
19245
|
+
"-NoProfile",
|
|
19246
|
+
"-NonInteractive",
|
|
19247
|
+
"-Command",
|
|
19248
|
+
`[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; Invoke-Expression $env:${POWERSHELL_SCRIPT_VARIABLE}`
|
|
19249
|
+
],
|
|
19250
|
+
env: { [POWERSHELL_SCRIPT_VARIABLE]: script }
|
|
19251
|
+
};
|
|
19252
|
+
}
|
|
19253
|
+
async function runPowerShell(script, timeoutMs) {
|
|
19254
|
+
const launch = powerShellLaunch(script);
|
|
19255
|
+
try {
|
|
19256
|
+
const { stdout, stderr } = await execFileAsync("powershell.exe", [...launch.args], {
|
|
19257
|
+
timeout: timeoutMs,
|
|
19258
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
19259
|
+
windowsHide: true,
|
|
19260
|
+
env: { ...process.env, ...launch.env }
|
|
19261
|
+
});
|
|
19262
|
+
return { ok: true, stdout, stderr };
|
|
19263
|
+
} catch (error) {
|
|
19264
|
+
const record = typeof error === "object" && error !== null ? error : {};
|
|
19240
19265
|
return {
|
|
19241
19266
|
ok: false,
|
|
19242
|
-
|
|
19243
|
-
|
|
19267
|
+
stdout: typeof record.stdout === "string" ? record.stdout : "",
|
|
19268
|
+
stderr: typeof record.stderr === "string" ? record.stderr : String(error)
|
|
19244
19269
|
};
|
|
19245
19270
|
}
|
|
19246
19271
|
}
|
|
19247
|
-
function
|
|
19248
|
-
|
|
19249
|
-
|
|
19250
|
-
|
|
19251
|
-
|
|
19252
|
-
|
|
19253
|
-
|
|
19254
|
-
|
|
19255
|
-
|
|
19256
|
-
|
|
19257
|
-
|
|
19258
|
-
|
|
19259
|
-
if (bytes > MAX_OUTPUT_BYTES) {
|
|
19260
|
-
overflow = true;
|
|
19261
|
-
child.kill();
|
|
19262
|
-
return;
|
|
19263
|
-
}
|
|
19264
|
-
target.push(chunk);
|
|
19265
|
-
};
|
|
19266
|
-
child.stdout.on("data", collect(stdout));
|
|
19267
|
-
child.stderr.on("data", collect(stderr));
|
|
19268
|
-
const timer = setTimeout(() => child.kill(), timeoutMs);
|
|
19269
|
-
child.on("error", (error) => {
|
|
19270
|
-
clearTimeout(timer);
|
|
19271
|
-
resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
|
|
19272
|
-
});
|
|
19273
|
-
child.on("close", (code, signal) => {
|
|
19274
|
-
clearTimeout(timer);
|
|
19275
|
-
const err = Buffer.concat(stderr).toString("utf8");
|
|
19276
|
-
resolvePromise({
|
|
19277
|
-
code,
|
|
19278
|
-
stdout: Buffer.concat(stdout).toString("utf8"),
|
|
19279
|
-
stderr: overflow ? `${err}
|
|
19280
|
-
output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
|
|
19281
|
-
killed by ${signal} after ${timeoutMs}ms` : err
|
|
19272
|
+
function defaultCloudProbe() {
|
|
19273
|
+
if (process.platform === "darwin")
|
|
19274
|
+
return new DarwinCloudProbe;
|
|
19275
|
+
if (process.platform === "win32")
|
|
19276
|
+
return new WindowsCloudProbe;
|
|
19277
|
+
return;
|
|
19278
|
+
}
|
|
19279
|
+
async function readFileProviderDomain(target) {
|
|
19280
|
+
try {
|
|
19281
|
+
await lstat(target);
|
|
19282
|
+
const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
|
|
19283
|
+
timeout: 2000
|
|
19282
19284
|
});
|
|
19283
|
-
|
|
19284
|
-
|
|
19285
|
+
const value = stdout.trim();
|
|
19286
|
+
return value.length > 0 ? value : undefined;
|
|
19287
|
+
} catch {
|
|
19288
|
+
return;
|
|
19289
|
+
}
|
|
19285
19290
|
}
|
|
19286
|
-
function
|
|
19287
|
-
|
|
19288
|
-
|
|
19289
|
-
|
|
19291
|
+
async function readFileProviderDomainUpwards(target) {
|
|
19292
|
+
let cursor = target;
|
|
19293
|
+
for (let depth = 0;depth < 12; depth += 1) {
|
|
19294
|
+
const domain = await readFileProviderDomain(cursor);
|
|
19295
|
+
if (domain !== undefined)
|
|
19296
|
+
return domain;
|
|
19297
|
+
const parent = dirname15(cursor);
|
|
19298
|
+
if (parent === cursor)
|
|
19299
|
+
return;
|
|
19300
|
+
cursor = parent;
|
|
19301
|
+
}
|
|
19302
|
+
return;
|
|
19290
19303
|
}
|
|
19291
|
-
|
|
19292
|
-
|
|
19293
|
-
|
|
19294
|
-
|
|
19295
|
-
|
|
19296
|
-
|
|
19297
|
-
|
|
19298
|
-
|
|
19299
|
-
|
|
19300
|
-
|
|
19301
|
-
var
|
|
19302
|
-
|
|
19303
|
-
|
|
19304
|
+
function quotePowerShell(value) {
|
|
19305
|
+
return `'${value.replace(/['\u2018\u2019\u201A\u201B]/gu, (quote) => quote + quote)}'`;
|
|
19306
|
+
}
|
|
19307
|
+
var execFileAsync, SF_DATALESS = 1073741824, FILE_PROVIDER_XATTR = "com.apple.file-provider-domain-id", WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 4194304, STAT_BATCH = 200, defaultCloudExec = (file, args, options) => execFileAsync(file, [...args], {
|
|
19308
|
+
timeout: options?.timeoutMs ?? 30000,
|
|
19309
|
+
...options?.maxBuffer === undefined ? {} : { maxBuffer: options.maxBuffer }
|
|
19310
|
+
}), JXA_RELEASE_PROGRAM = `ObjC.import("Foundation");
|
|
19311
|
+
function run(argv) {
|
|
19312
|
+
var url = $.NSURL.fileURLWithPath(argv[0]);
|
|
19313
|
+
var errorRef = Ref();
|
|
19314
|
+
var released = $.NSFileManager.defaultManager.evictUbiquitousItemAtURLError(url, errorRef);
|
|
19315
|
+
if (released) return JSON.stringify({ ok: true });
|
|
19316
|
+
function resourceValue(key) {
|
|
19317
|
+
var ref = Ref();
|
|
19318
|
+
if (!url.getResourceValueForKeyError(ref, key, null)) return null;
|
|
19319
|
+
var value = ref[0];
|
|
19320
|
+
return value === undefined || value === null ? null : ObjC.unwrap(value);
|
|
19321
|
+
}
|
|
19322
|
+
return JSON.stringify({
|
|
19323
|
+
ok: false,
|
|
19324
|
+
isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
|
|
19325
|
+
isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
|
|
19326
|
+
downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
|
|
19327
|
+
});
|
|
19328
|
+
}`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
|
|
19329
|
+
var init_cloud_placeholder = __esm(() => {
|
|
19330
|
+
execFileAsync = promisify(execFile2);
|
|
19331
|
+
RELEASABLE_DARWIN_PROVIDERS = {
|
|
19332
|
+
icloud: true,
|
|
19333
|
+
"google-drive": true,
|
|
19334
|
+
onedrive: true,
|
|
19335
|
+
dropbox: true
|
|
19336
|
+
};
|
|
19337
|
+
DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
|
|
19338
|
+
WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
|
|
19339
|
+
WINDOWS_DEHYDRATE_REFUSALS = {
|
|
19340
|
+
"80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
|
|
19341
|
+
"80070188": "The file is pinned to stay on this device, so it is not released."
|
|
19342
|
+
};
|
|
19304
19343
|
});
|
|
19305
19344
|
|
|
19306
19345
|
// src/fsearch/walk.ts
|
|
19307
19346
|
import { readdir as readdir2, stat as stat3 } from "node:fs/promises";
|
|
19308
|
-
import { basename as
|
|
19347
|
+
import { basename as basename7, join as join30, sep as sep5 } from "node:path";
|
|
19309
19348
|
function compare(a, b) {
|
|
19310
19349
|
return a < b ? -1 : a > b ? 1 : 0;
|
|
19311
19350
|
}
|
|
@@ -19316,11 +19355,11 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
19316
19355
|
const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_DEADLINE_MS);
|
|
19317
19356
|
const matches = request.regex ? (() => {
|
|
19318
19357
|
const pattern = new RegExp(request.query, request.matchCase ? "" : "i");
|
|
19319
|
-
return (path) => pattern.test(request.matchPath ? path :
|
|
19358
|
+
return (path) => pattern.test(request.matchPath ? path : basename7(path));
|
|
19320
19359
|
})() : (() => {
|
|
19321
19360
|
const needle = request.matchCase ? request.query : request.query.toLowerCase();
|
|
19322
19361
|
return (path) => {
|
|
19323
|
-
const haystack = request.matchPath ? path :
|
|
19362
|
+
const haystack = request.matchPath ? path : basename7(path);
|
|
19324
19363
|
return (request.matchCase ? haystack : haystack.toLowerCase()).includes(needle);
|
|
19325
19364
|
};
|
|
19326
19365
|
})();
|
|
@@ -19375,7 +19414,7 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
19375
19414
|
continue;
|
|
19376
19415
|
collected.push({
|
|
19377
19416
|
path,
|
|
19378
|
-
name:
|
|
19417
|
+
name: basename7(path),
|
|
19379
19418
|
type: info.isDirectory() ? "folder" : "file",
|
|
19380
19419
|
size: info.isDirectory() ? undefined : info.size,
|
|
19381
19420
|
dateModified: info.mtime.toISOString()
|
|
@@ -19400,7 +19439,7 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
19400
19439
|
continue;
|
|
19401
19440
|
collected.push({
|
|
19402
19441
|
path: folder,
|
|
19403
|
-
name:
|
|
19442
|
+
name: basename7(folder),
|
|
19404
19443
|
type: "folder",
|
|
19405
19444
|
size: undefined,
|
|
19406
19445
|
dateModified: info.mtime.toISOString()
|
|
@@ -19433,7 +19472,7 @@ var init_walk = __esm(() => {
|
|
|
19433
19472
|
import { spawn as spawn15 } from "node:child_process";
|
|
19434
19473
|
import { createHash as createHash13 } from "node:crypto";
|
|
19435
19474
|
import { existsSync as existsSync18, mkdirSync as mkdirSync16, readFileSync as readFileSync18, realpathSync as realpathSync4, rmSync as rmSync8, writeFileSync as writeFileSync12 } from "node:fs";
|
|
19436
|
-
import { basename as
|
|
19475
|
+
import { basename as basename8, join as join31, resolve as resolve12, sep as sep6 } from "node:path";
|
|
19437
19476
|
import { setTimeout as sleep3 } from "node:timers/promises";
|
|
19438
19477
|
function fsearchSocketPath(dbPath) {
|
|
19439
19478
|
let canonical;
|
|
@@ -19497,7 +19536,7 @@ function parseFSearchJsonLines(stdout) {
|
|
|
19497
19536
|
const path = String(record.path ?? "");
|
|
19498
19537
|
entries.push({
|
|
19499
19538
|
path,
|
|
19500
|
-
name:
|
|
19539
|
+
name: basename8(path),
|
|
19501
19540
|
type: record.type === "folder" ? "folder" : "file",
|
|
19502
19541
|
size: typeof record.size === "number" ? record.size : undefined,
|
|
19503
19542
|
dateModified: typeof record.mtime === "number" ? new Date(record.mtime * 1000).toISOString() : undefined
|
|
@@ -19915,7 +19954,7 @@ var init_diagnostics = __esm(() => {
|
|
|
19915
19954
|
});
|
|
19916
19955
|
|
|
19917
19956
|
// src/jikji/retry-command.ts
|
|
19918
|
-
import { basename as
|
|
19957
|
+
import { basename as basename9 } from "node:path";
|
|
19919
19958
|
function shellWords(command) {
|
|
19920
19959
|
const words = [];
|
|
19921
19960
|
let current = "";
|
|
@@ -19982,7 +20021,7 @@ function parseJikjiRetryCommand(command, envelopeProof) {
|
|
|
19982
20021
|
const words = shellWords(command);
|
|
19983
20022
|
if (words === undefined || words.length < 4)
|
|
19984
20023
|
return;
|
|
19985
|
-
const program =
|
|
20024
|
+
const program = basename9(words[0]).replace(/\.exe$/iu, "");
|
|
19986
20025
|
if (program !== "jikji" || words[1] !== "discover")
|
|
19987
20026
|
return;
|
|
19988
20027
|
const root = words[2];
|
|
@@ -20390,7 +20429,7 @@ class SerialQueue {
|
|
|
20390
20429
|
|
|
20391
20430
|
// src/jikji/version.ts
|
|
20392
20431
|
function parseJikjiVersion(output) {
|
|
20393
|
-
return
|
|
20432
|
+
return VERSION_LINE3.exec(output)?.[1];
|
|
20394
20433
|
}
|
|
20395
20434
|
function parts(version) {
|
|
20396
20435
|
return version.split(".").map((part) => Number.parseInt(part, 10));
|
|
@@ -20406,9 +20445,9 @@ function isSupportedJikjiVersion(version) {
|
|
|
20406
20445
|
}
|
|
20407
20446
|
return true;
|
|
20408
20447
|
}
|
|
20409
|
-
var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1",
|
|
20410
|
-
var
|
|
20411
|
-
|
|
20448
|
+
var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1", VERSION_LINE3;
|
|
20449
|
+
var init_version3 = __esm(() => {
|
|
20450
|
+
VERSION_LINE3 = /^jikji(?:-cli)?\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
|
|
20412
20451
|
});
|
|
20413
20452
|
|
|
20414
20453
|
// src/jikji/client.ts
|
|
@@ -20776,7 +20815,7 @@ var init_client11 = __esm(() => {
|
|
|
20776
20815
|
init_answer_pack();
|
|
20777
20816
|
init_installer3();
|
|
20778
20817
|
init_prepare_budget();
|
|
20779
|
-
|
|
20818
|
+
init_version3();
|
|
20780
20819
|
ABORTED_BEFORE_START = { ok: false, reason: "aborted", stdout: "", stderr: "", code: null };
|
|
20781
20820
|
});
|
|
20782
20821
|
|
|
@@ -21861,7 +21900,7 @@ import {
|
|
|
21861
21900
|
rmSync as rmSync10,
|
|
21862
21901
|
writeFileSync as writeFileSync14
|
|
21863
21902
|
} from "node:fs";
|
|
21864
|
-
import { basename as
|
|
21903
|
+
import { basename as basename10, dirname as dirname18, isAbsolute as isAbsolute9, join as join36, relative as relative7, sep as sep8 } from "node:path";
|
|
21865
21904
|
function syncMinSyncWorkspace(root, options = {}) {
|
|
21866
21905
|
const workspacePath = options.workspacePath ?? minSyncWorkspaceRoot(root);
|
|
21867
21906
|
const filesRoot = join36(workspacePath, "files");
|
|
@@ -21902,7 +21941,7 @@ function syncMinSyncWorkspace(root, options = {}) {
|
|
|
21902
21941
|
return { workspacePath, entries, excluded };
|
|
21903
21942
|
}
|
|
21904
21943
|
function stagingStatePath(workspacePath) {
|
|
21905
|
-
return join36(dirname18(workspacePath), `.${
|
|
21944
|
+
return join36(dirname18(workspacePath), `.${basename10(workspacePath)}-autorag-staging.json`);
|
|
21906
21945
|
}
|
|
21907
21946
|
function loadStagingState2(workspacePath) {
|
|
21908
21947
|
const path = stagingStatePath(workspacePath);
|
|
@@ -21953,7 +21992,7 @@ function ensureManagedDirectory(filesRoot, directory) {
|
|
|
21953
21992
|
}
|
|
21954
21993
|
}
|
|
21955
21994
|
function copyFileAtomically(source, destination) {
|
|
21956
|
-
const temporaryPath = join36(dirname18(destination), `.${
|
|
21995
|
+
const temporaryPath = join36(dirname18(destination), `.${basename10(destination)}.${randomUUID6()}.tmp`);
|
|
21957
21996
|
try {
|
|
21958
21997
|
copyFileSync4(source, temporaryPath);
|
|
21959
21998
|
renameWithRetry(temporaryPath, destination);
|
|
@@ -22007,7 +22046,7 @@ var init_workspace = __esm(() => {
|
|
|
22007
22046
|
|
|
22008
22047
|
// src/minsync/method.ts
|
|
22009
22048
|
import { existsSync as existsSync23, readFileSync as readFileSync21, realpathSync as realpathSync7, statSync as statSync7 } from "node:fs";
|
|
22010
|
-
import { basename as
|
|
22049
|
+
import { basename as basename11, join as join37, normalize } from "node:path";
|
|
22011
22050
|
|
|
22012
22051
|
class MinSyncVectorMethod {
|
|
22013
22052
|
root;
|
|
@@ -22108,7 +22147,7 @@ class MinSyncVectorMethod {
|
|
|
22108
22147
|
const entry = byPath.get(hit.path);
|
|
22109
22148
|
if (!entry || !matchesVirtualPathScope(entry.virtualPath, options.scope))
|
|
22110
22149
|
continue;
|
|
22111
|
-
const chunkDiscriminator = hit.docId ??
|
|
22150
|
+
const chunkDiscriminator = hit.docId ?? basename11(hit.path);
|
|
22112
22151
|
const chunkId = `minsync:${entry.virtualPath}:${chunkDiscriminator}`;
|
|
22113
22152
|
const location = locateChunkInMirror(mirrorTexts, entry, hit.text);
|
|
22114
22153
|
results.push({
|
|
@@ -22208,18 +22247,24 @@ var init_minsync = __esm(() => {
|
|
|
22208
22247
|
|
|
22209
22248
|
// src/mirror/cloud-fingerprints.ts
|
|
22210
22249
|
import { existsSync as existsSync24, readFileSync as readFileSync22 } from "node:fs";
|
|
22211
|
-
import { join as join38 } from "node:path";
|
|
22250
|
+
import { extname as extname5, join as join38 } from "node:path";
|
|
22212
22251
|
function cloudFingerprintStorePath(root) {
|
|
22213
22252
|
return join38(root, ".autorag", CLOUD_FINGERPRINTS_FILE);
|
|
22214
22253
|
}
|
|
22215
22254
|
function emptyCloudFingerprints() {
|
|
22216
|
-
return { version:
|
|
22255
|
+
return { version: 2, entries: {} };
|
|
22217
22256
|
}
|
|
22218
|
-
function
|
|
22257
|
+
function readCloudFingerprint(sourcePath, value, storeVersion) {
|
|
22219
22258
|
if (typeof value !== "object" || value === null)
|
|
22220
|
-
return
|
|
22259
|
+
return;
|
|
22221
22260
|
const record = value;
|
|
22222
|
-
|
|
22261
|
+
const { sourceVersion, exact, modified, revision } = record;
|
|
22262
|
+
if (typeof sourceVersion !== "string" || typeof exact !== "string" || modified !== null && typeof modified !== "string" || revision !== null && typeof revision !== "string") {
|
|
22263
|
+
return;
|
|
22264
|
+
}
|
|
22265
|
+
if (storeVersion !== 2 && (record.chars === 0 || extname5(sourcePath).toLowerCase() === ".hwp"))
|
|
22266
|
+
return;
|
|
22267
|
+
return { sourceVersion, exact, modified, revision };
|
|
22223
22268
|
}
|
|
22224
22269
|
function loadCloudFingerprints(root) {
|
|
22225
22270
|
const path = cloudFingerprintStorePath(root);
|
|
@@ -22232,24 +22277,26 @@ function loadCloudFingerprints(root) {
|
|
|
22232
22277
|
const { entries } = parsed;
|
|
22233
22278
|
if (typeof entries !== "object" || entries === null)
|
|
22234
22279
|
return emptyCloudFingerprints();
|
|
22280
|
+
const storeVersion = "version" in parsed ? parsed.version : undefined;
|
|
22235
22281
|
const valid = {};
|
|
22236
|
-
for (const [sourcePath,
|
|
22237
|
-
|
|
22282
|
+
for (const [sourcePath, value] of Object.entries(entries)) {
|
|
22283
|
+
const fingerprint = readCloudFingerprint(sourcePath, value, storeVersion);
|
|
22284
|
+
if (fingerprint !== undefined)
|
|
22238
22285
|
valid[sourcePath] = fingerprint;
|
|
22239
22286
|
}
|
|
22240
|
-
return { version:
|
|
22287
|
+
return { version: 2, entries: valid };
|
|
22241
22288
|
} catch {
|
|
22242
22289
|
return emptyCloudFingerprints();
|
|
22243
22290
|
}
|
|
22244
22291
|
}
|
|
22245
22292
|
function saveCloudFingerprints(root, entries) {
|
|
22246
|
-
writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version:
|
|
22293
|
+
writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version: 2, entries }, null, 2)}
|
|
22247
22294
|
`);
|
|
22248
22295
|
}
|
|
22249
22296
|
function cloudFingerprintExclusions(store) {
|
|
22250
22297
|
const map = new Map;
|
|
22251
22298
|
for (const [sourcePath, fingerprint] of Object.entries(store.entries)) {
|
|
22252
|
-
map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified
|
|
22299
|
+
map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified });
|
|
22253
22300
|
}
|
|
22254
22301
|
return map;
|
|
22255
22302
|
}
|
|
@@ -22641,7 +22688,7 @@ class Parser {
|
|
|
22641
22688
|
}
|
|
22642
22689
|
|
|
22643
22690
|
// src/parser/csv.ts
|
|
22644
|
-
import { basename as
|
|
22691
|
+
import { basename as basename12, extname as extname6 } from "node:path";
|
|
22645
22692
|
function toCell(text) {
|
|
22646
22693
|
return text.trim() === "" ? undefined : { text, kind: classifyText(text) };
|
|
22647
22694
|
}
|
|
@@ -22772,7 +22819,7 @@ var init_csv = __esm(() => {
|
|
|
22772
22819
|
this.#maxRows = options.maxRows ?? DEFAULT_MAX_ROWS;
|
|
22773
22820
|
}
|
|
22774
22821
|
async parse(input) {
|
|
22775
|
-
const extension =
|
|
22822
|
+
const extension = extname6(input.virtualPath);
|
|
22776
22823
|
const format = extension.slice(1).toLowerCase();
|
|
22777
22824
|
let text = decodeText(input.bytes).replace(/^\uFEFF/, "");
|
|
22778
22825
|
const directive = SEPARATOR_DIRECTIVE.exec(text);
|
|
@@ -22784,10 +22831,10 @@ var init_csv = __esm(() => {
|
|
|
22784
22831
|
{
|
|
22785
22832
|
code: "parser-warning",
|
|
22786
22833
|
severity: "warning",
|
|
22787
|
-
message: `"${
|
|
22834
|
+
message: `"${basename12(input.virtualPath)}": indexed the first ${this.#maxRows} of ${total} rows`
|
|
22788
22835
|
}
|
|
22789
22836
|
] : [];
|
|
22790
|
-
const grid = { name:
|
|
22837
|
+
const grid = { name: basename12(input.virtualPath, extension), rows: rows.map((row) => row.map(toCell)) };
|
|
22791
22838
|
return {
|
|
22792
22839
|
markdown: normalizeMarkdown(renderWorkbook([grid])),
|
|
22793
22840
|
metadata: { parser: this.name, format },
|
|
@@ -23326,7 +23373,7 @@ var init_plain_text = __esm(() => {
|
|
|
23326
23373
|
});
|
|
23327
23374
|
|
|
23328
23375
|
// src/parser/registry.ts
|
|
23329
|
-
import { extname as
|
|
23376
|
+
import { extname as extname7 } from "node:path";
|
|
23330
23377
|
|
|
23331
23378
|
class ParserRegistry {
|
|
23332
23379
|
parsersByExtension = new Map;
|
|
@@ -23345,7 +23392,7 @@ class ParserRegistry {
|
|
|
23345
23392
|
}
|
|
23346
23393
|
}
|
|
23347
23394
|
getForVirtualPath(virtualPath) {
|
|
23348
|
-
return this.parsersByExtension.get(normalizeExtension(
|
|
23395
|
+
return this.parsersByExtension.get(normalizeExtension(extname7(virtualPath)));
|
|
23349
23396
|
}
|
|
23350
23397
|
list() {
|
|
23351
23398
|
return Array.from(new Set(this.parsersByExtension.values()));
|
|
@@ -23358,7 +23405,7 @@ function normalizeExtension(extension) {
|
|
|
23358
23405
|
var init_registry = () => {};
|
|
23359
23406
|
|
|
23360
23407
|
// src/parser/spreadsheet.ts
|
|
23361
|
-
import { extname as
|
|
23408
|
+
import { extname as extname8 } from "node:path";
|
|
23362
23409
|
import * as XLSX from "xlsx";
|
|
23363
23410
|
function assertWorkbookSignature(format, bytes) {
|
|
23364
23411
|
if (startsWith(bytes, ZIP_SIGNATURE) || startsWith(bytes, OLE_SIGNATURE))
|
|
@@ -23457,7 +23504,7 @@ var init_spreadsheet = __esm(() => {
|
|
|
23457
23504
|
this.#maxRowsPerSheet = options.maxRowsPerSheet ?? DEFAULT_MAX_ROWS_PER_SHEET;
|
|
23458
23505
|
}
|
|
23459
23506
|
async parse(input) {
|
|
23460
|
-
const format =
|
|
23507
|
+
const format = extname8(input.virtualPath).slice(1).toLowerCase();
|
|
23461
23508
|
let workbook;
|
|
23462
23509
|
try {
|
|
23463
23510
|
assertWorkbookSignature(format, input.bytes);
|
|
@@ -23731,6 +23778,10 @@ class CloudGate {
|
|
|
23731
23778
|
isPlaceholder(path) {
|
|
23732
23779
|
return this.placeholders.has(path);
|
|
23733
23780
|
}
|
|
23781
|
+
startBatch() {
|
|
23782
|
+
this.downloadedFiles = 0;
|
|
23783
|
+
this.downloadedBytes = 0;
|
|
23784
|
+
}
|
|
23734
23785
|
async decide(path, sizeBytes, directory) {
|
|
23735
23786
|
if (this.probe === undefined || !this.placeholders.has(path))
|
|
23736
23787
|
return LOCAL;
|
|
@@ -23740,7 +23791,7 @@ class CloudGate {
|
|
|
23740
23791
|
const overFiles = this.downloadedFiles >= this.maxFiles;
|
|
23741
23792
|
const overBytes = this.downloadedFiles > 0 && this.downloadedBytes + sizeBytes > this.maxBytes;
|
|
23742
23793
|
if (overFiles || overBytes)
|
|
23743
|
-
return { kind: "
|
|
23794
|
+
return { kind: "next-batch" };
|
|
23744
23795
|
this.downloadedFiles += 1;
|
|
23745
23796
|
this.downloadedBytes += sizeBytes;
|
|
23746
23797
|
return { kind: "download" };
|
|
@@ -23785,7 +23836,7 @@ var init_cloud = __esm(() => {
|
|
|
23785
23836
|
// src/mirror/sync.ts
|
|
23786
23837
|
import { createHash as createHash17 } from "node:crypto";
|
|
23787
23838
|
import { opendir as opendir2, readFile, stat as stat4 } from "node:fs/promises";
|
|
23788
|
-
import { basename as
|
|
23839
|
+
import { basename as basename13, dirname as dirname21, extname as extname9, join as join42, resolve as resolve15, sep as sep9 } from "node:path";
|
|
23789
23840
|
import { setTimeout as sleep4 } from "node:timers/promises";
|
|
23790
23841
|
function errorCode(error) {
|
|
23791
23842
|
if (!(error instanceof Error))
|
|
@@ -24010,25 +24061,26 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
24010
24061
|
};
|
|
24011
24062
|
const deferDeletions = options.onBatch !== undefined;
|
|
24012
24063
|
const batchSize = Math.max(1, Math.floor(options.batchSize ?? DEFAULT_MIRROR_BATCH_SIZE));
|
|
24013
|
-
const batchCount = Math.max(1, Math.ceil(current.length / batchSize));
|
|
24014
24064
|
let batchIndex = 0;
|
|
24015
24065
|
let batchStopped = false;
|
|
24016
24066
|
const runBatch = async (virtualPaths, final) => {
|
|
24017
24067
|
if (options.onBatch === undefined)
|
|
24018
24068
|
return;
|
|
24019
24069
|
batchIndex += 1;
|
|
24020
|
-
const outcome = await options.onBatch({ index: batchIndex,
|
|
24070
|
+
const outcome = await options.onBatch({ index: batchIndex, virtualPaths, final });
|
|
24021
24071
|
if (outcome?.stop === true)
|
|
24022
24072
|
batchStopped = true;
|
|
24023
24073
|
};
|
|
24024
24074
|
if (!deferDeletions)
|
|
24025
24075
|
await applyDeleteDecisions();
|
|
24026
24076
|
await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
|
|
24027
|
-
|
|
24028
|
-
|
|
24029
|
-
const
|
|
24077
|
+
let batchStart = 0;
|
|
24078
|
+
for (;; ) {
|
|
24079
|
+
const batchLimit = Math.min(batchStart + batchSize, current.length);
|
|
24080
|
+
let batchEnd = batchLimit;
|
|
24030
24081
|
const changedBefore = written + deleted;
|
|
24031
|
-
|
|
24082
|
+
gate.startBatch();
|
|
24083
|
+
for (let position = batchStart;position < batchLimit; position += 1) {
|
|
24032
24084
|
const entry = current[position];
|
|
24033
24085
|
options.onProgress?.({
|
|
24034
24086
|
processed: position,
|
|
@@ -24078,15 +24130,9 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
24078
24130
|
});
|
|
24079
24131
|
continue;
|
|
24080
24132
|
}
|
|
24081
|
-
if (cloudDecision.kind === "
|
|
24082
|
-
|
|
24083
|
-
|
|
24084
|
-
code: "cloud-download-deferred",
|
|
24085
|
-
severity: "info",
|
|
24086
|
-
message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
|
|
24087
|
-
source: entry.virtualPath
|
|
24088
|
-
});
|
|
24089
|
-
continue;
|
|
24133
|
+
if (cloudDecision.kind === "next-batch") {
|
|
24134
|
+
batchEnd = position;
|
|
24135
|
+
break;
|
|
24090
24136
|
}
|
|
24091
24137
|
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
24092
24138
|
let bytes;
|
|
@@ -24159,7 +24205,7 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
24159
24205
|
continue;
|
|
24160
24206
|
}
|
|
24161
24207
|
await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
|
|
24162
|
-
path:
|
|
24208
|
+
path: basename13(outputPath),
|
|
24163
24209
|
size: Buffer.byteLength(markdown)
|
|
24164
24210
|
});
|
|
24165
24211
|
for (const diagnostic of parsed.diagnostics ?? [])
|
|
@@ -24173,7 +24219,6 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
24173
24219
|
cloudFingerprintEntries[entry.sourcePath] = {
|
|
24174
24220
|
sourceVersion: entry.sourceVersion,
|
|
24175
24221
|
exact: fingerprintResult.exact,
|
|
24176
|
-
chars: fingerprintResult.chars,
|
|
24177
24222
|
modified: fingerprintResult.modified,
|
|
24178
24223
|
revision: fingerprintResult.revision
|
|
24179
24224
|
};
|
|
@@ -24201,13 +24246,15 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
24201
24246
|
}
|
|
24202
24247
|
}
|
|
24203
24248
|
}
|
|
24249
|
+
const finalBatch = batchEnd >= current.length;
|
|
24204
24250
|
if (finalBatch && deferDeletions)
|
|
24205
24251
|
await applyDeleteDecisions();
|
|
24206
24252
|
if (finalBatch || written + deleted !== changedBefore) {
|
|
24207
24253
|
await runBatch(current.slice(batchStart, batchEnd).map((item) => item.virtualPath), finalBatch);
|
|
24208
24254
|
}
|
|
24209
|
-
if (batchStopped)
|
|
24255
|
+
if (finalBatch || batchStopped)
|
|
24210
24256
|
break;
|
|
24257
|
+
batchStart = batchEnd;
|
|
24211
24258
|
}
|
|
24212
24259
|
await session.finish();
|
|
24213
24260
|
if (fingerprintCloud) {
|
|
@@ -24297,7 +24344,7 @@ async function detectMirrorStaleness(options) {
|
|
|
24297
24344
|
}
|
|
24298
24345
|
}
|
|
24299
24346
|
function standaloneImageTooSmallMessage(virtualPath, bytes) {
|
|
24300
|
-
if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(
|
|
24347
|
+
if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname9(virtualPath))))
|
|
24301
24348
|
return;
|
|
24302
24349
|
return ocrImageTooSmallMessage(bytes);
|
|
24303
24350
|
}
|
|
@@ -24465,7 +24512,7 @@ async function collectFiles(state, directory, depth) {
|
|
|
24465
24512
|
}
|
|
24466
24513
|
if (!entry.isFile())
|
|
24467
24514
|
continue;
|
|
24468
|
-
const extension = normalizeExtension2(
|
|
24515
|
+
const extension = normalizeExtension2(extname9(entry.name));
|
|
24469
24516
|
if (!state.supportedExtensions.has(extension))
|
|
24470
24517
|
continue;
|
|
24471
24518
|
let fileStat;
|
|
@@ -30028,7 +30075,7 @@ var init_automation_tool = __esm(() => {
|
|
|
30028
30075
|
});
|
|
30029
30076
|
|
|
30030
30077
|
// src/agent/citation-links.ts
|
|
30031
|
-
import { basename as
|
|
30078
|
+
import { basename as basename14, isAbsolute as isAbsolute12 } from "node:path";
|
|
30032
30079
|
import { pathToFileURL as pathToFileURL3 } from "node:url";
|
|
30033
30080
|
function openableTarget(source) {
|
|
30034
30081
|
if (/^https?:\/\//iu.test(source)) {
|
|
@@ -30046,7 +30093,7 @@ function linkCitations(markdown, sourceOf, isFile) {
|
|
|
30046
30093
|
for (const marker of answerMarkers(markdown, isFile)) {
|
|
30047
30094
|
if (marker.kind === "number")
|
|
30048
30095
|
continue;
|
|
30049
|
-
const labels = marker.kind === "file" ? [{ label:
|
|
30096
|
+
const labels = marker.kind === "file" ? [{ label: basename14(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
|
|
30050
30097
|
let linked = false;
|
|
30051
30098
|
const parts = labels.map(({ label, source }) => {
|
|
30052
30099
|
const target = source === undefined ? undefined : openableTarget(source);
|
|
@@ -32172,10 +32219,7 @@ function renderScanReport(details) {
|
|
|
32172
32219
|
const lines = [
|
|
32173
32220
|
`dupey scanned ${details.scans.length} configured root(s).`,
|
|
32174
32221
|
`families=${details.familyCount} exactDuplicates=${details.exactDuplicateCount} extractionErrors=${errors}`,
|
|
32175
|
-
"Review exact families before cleanup; near/contains families are not safe deletion evidence."
|
|
32176
|
-
...details.skippedRoots.length === 0 ? [] : [
|
|
32177
|
-
`Not scanned (inside a cloud or network folder; scanning would download or re-read every file): ${details.skippedRoots.join(", ")}. Duplicates there are not reported, and an exact duplicate is only found against files an earlier refresh fingerprinted.`
|
|
32178
|
-
]
|
|
32222
|
+
"Review exact families before cleanup; near/contains families are not safe deletion evidence."
|
|
32179
32223
|
];
|
|
32180
32224
|
let rendered = 0;
|
|
32181
32225
|
let omitted = 0;
|
|
@@ -33153,6 +33197,56 @@ var init_packaged_skill = __esm(() => {
|
|
|
33153
33197
|
PACKAGED_SKILL_ROOTS = ["skills", "internal-skills"];
|
|
33154
33198
|
});
|
|
33155
33199
|
|
|
33200
|
+
// src/agent/pipelined-sync.ts
|
|
33201
|
+
class PipelinedSync {
|
|
33202
|
+
#sync;
|
|
33203
|
+
#covered = [];
|
|
33204
|
+
#dirty = false;
|
|
33205
|
+
#running;
|
|
33206
|
+
#failure;
|
|
33207
|
+
#stopped = false;
|
|
33208
|
+
#runs = 0;
|
|
33209
|
+
constructor(options) {
|
|
33210
|
+
this.#sync = options.sync;
|
|
33211
|
+
}
|
|
33212
|
+
async batch(batch) {
|
|
33213
|
+
this.#raise();
|
|
33214
|
+
this.#covered.push(...batch.virtualPaths);
|
|
33215
|
+
this.#dirty = true;
|
|
33216
|
+
if (!this.#stopped)
|
|
33217
|
+
this.#running ??= this.#loop();
|
|
33218
|
+
if (batch.final)
|
|
33219
|
+
await this.#running;
|
|
33220
|
+
this.#raise();
|
|
33221
|
+
return this.#stopped ? { stop: true } : undefined;
|
|
33222
|
+
}
|
|
33223
|
+
async settle() {
|
|
33224
|
+
this.#stopped = true;
|
|
33225
|
+
await this.#running;
|
|
33226
|
+
}
|
|
33227
|
+
#raise() {
|
|
33228
|
+
if (this.#failure !== undefined)
|
|
33229
|
+
throw this.#failure.error;
|
|
33230
|
+
}
|
|
33231
|
+
async#loop() {
|
|
33232
|
+
try {
|
|
33233
|
+
while (this.#dirty && !this.#stopped) {
|
|
33234
|
+
this.#dirty = false;
|
|
33235
|
+
const virtualPaths = this.#covered;
|
|
33236
|
+
this.#covered = [];
|
|
33237
|
+
this.#runs += 1;
|
|
33238
|
+
const outcome = await this.#sync({ index: this.#runs, virtualPaths });
|
|
33239
|
+
if (outcome?.stop === true)
|
|
33240
|
+
this.#stopped = true;
|
|
33241
|
+
}
|
|
33242
|
+
} catch (error) {
|
|
33243
|
+
this.#failure = { error };
|
|
33244
|
+
} finally {
|
|
33245
|
+
this.#running = undefined;
|
|
33246
|
+
}
|
|
33247
|
+
}
|
|
33248
|
+
}
|
|
33249
|
+
|
|
33156
33250
|
// src/agent/query-decomposition.ts
|
|
33157
33251
|
import { completeSimple as completeSimple2 } from "@earendil-works/pi-ai/compat";
|
|
33158
33252
|
function buildDecompositionPrompt(query) {
|
|
@@ -33392,7 +33486,7 @@ var init_query_routing = __esm(() => {
|
|
|
33392
33486
|
false: "Every part of the question is answered with a concrete, specific, consistent fact (a name, date, number, or definite statement)."
|
|
33393
33487
|
});
|
|
33394
33488
|
BASE_ROUTE_OPTIONS = {
|
|
33395
|
-
local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. When unsure whether intrinsic knowledge suffices, choose local.",
|
|
33489
|
+
local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. A request to find, locate, open, or look up a document, file, slide deck, presentation, paper, note, message, or other material is local even when it names no owner ('find the slides about X', 'where is the Y presentation', 'Z 관련 자료 찾아줘'): such material lives in the user's own files and chats, and a well-known topic is not a reason to answer from general knowledge or the web. When unsure whether intrinsic knowledge suffices, choose local.",
|
|
33396
33490
|
extra: "None of the other options fits and no search of the user's own records is needed first. Either the request is answerable without looking anything up — stable common knowledge, a definition, simple reasoning, or small talk — or it is a task for the assistant's tools that is not file organization and not AutoRAG configuration: committing code, pushing a branch, opening a pull request, running tests or a build, installing a package, running a script, writing or editing code. Programming work is extra even when the code names the user's files or folders ('write a script that lowercases every PNG', 'add this to my parser'): the assistant is asked to produce or change code, not to act on those files now. So is a request to run something (tests, a build, a script, a command) and report the outcome, even when what it runs lives in the user's own repository: running it is a task for the tool, not a search of the user's records. Never extra when the request refers to the user's own life, files, records, chats, calendar, or history (e.g. 'my', 'I', 'our', 'my girlfriend', 'my case', 'my routine'), and never extra when the answer depends on facts that may be recent, specific, or changing (current events, recent releases, prices, versions): search those locally instead. If any other option fits, do not choose extra.",
|
|
33397
33491
|
organize: "The user asks the assistant to DO something to files or folders instead of finding information: move or copy files into folders, tidy or sort a folder, bulk or pattern rename (for example by date or number), rename files according to what their content says, convert files to another format, create folder structures, group files by type or topic, archive, or move to the trash ('tidy this up', 'move these into folders', 'bulk rename these files', 'rename the files to match their content', 'convert these to PDF', 'sort my Downloads folder'). It is about acting on files and folders, never about looking up what a document says or where something is. A question that only asks to find or read a file's content is local, not organize."
|
|
33398
33492
|
};
|
|
@@ -34630,7 +34724,7 @@ var init_page_loader = __esm(() => {
|
|
|
34630
34724
|
});
|
|
34631
34725
|
|
|
34632
34726
|
// src/web/fetch/url-target.ts
|
|
34633
|
-
import { extname as
|
|
34727
|
+
import { extname as extname10 } from "node:path";
|
|
34634
34728
|
function repairCollapsedScheme(value) {
|
|
34635
34729
|
const m = value.match(/^(https?):\/(?!\/)/i);
|
|
34636
34730
|
return m ? `${m[1]}://${value.slice(m[0].length)}` : value;
|
|
@@ -34649,7 +34743,7 @@ function getFilenameExtensionHint(filename) {
|
|
|
34649
34743
|
const lower = filename.toLowerCase();
|
|
34650
34744
|
if (lower.endsWith(".tar.gz"))
|
|
34651
34745
|
return ".tar.gz";
|
|
34652
|
-
return
|
|
34746
|
+
return extname10(filename).toLowerCase();
|
|
34653
34747
|
}
|
|
34654
34748
|
function getExtensionHint(url, contentDisposition) {
|
|
34655
34749
|
if (contentDisposition) {
|
|
@@ -34826,7 +34920,7 @@ var init_convert2 = __esm(() => {
|
|
|
34826
34920
|
});
|
|
34827
34921
|
|
|
34828
34922
|
// src/web/fetch/feeds.ts
|
|
34829
|
-
import { extname as
|
|
34923
|
+
import { extname as extname11 } from "node:path";
|
|
34830
34924
|
import { XMLParser as XMLParser3 } from "fast-xml-parser";
|
|
34831
34925
|
function getHtmlAttribute(tag, attribute) {
|
|
34832
34926
|
const pattern = new RegExp(`\\b${attribute}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'=<>]+))`, "i");
|
|
@@ -34889,7 +34983,7 @@ function extractDocumentLinks(html, baseUrl) {
|
|
|
34889
34983
|
const href = getHtmlAttribute(tag, "href");
|
|
34890
34984
|
if (!href)
|
|
34891
34985
|
continue;
|
|
34892
|
-
const ext =
|
|
34986
|
+
const ext = extname11(href).toLowerCase();
|
|
34893
34987
|
if (!CONVERTIBLE_EXTENSIONS2.has(ext))
|
|
34894
34988
|
continue;
|
|
34895
34989
|
const resolved = href.startsWith("http") ? href : new URL(href, baseUrl).href;
|
|
@@ -36557,7 +36651,7 @@ var init_web_search_tool = __esm(() => {
|
|
|
36557
36651
|
// src/agent/agent.ts
|
|
36558
36652
|
import { randomUUID as randomUUID7 } from "node:crypto";
|
|
36559
36653
|
import { existsSync as existsSync34, watch as fsWatch, mkdirSync as mkdirSync24, realpathSync as realpathSync8, statSync as statSync11, writeFileSync as writeFileSync17 } from "node:fs";
|
|
36560
|
-
import { basename as
|
|
36654
|
+
import { basename as basename15, dirname as dirname27, join as join54, relative as relative8, resolve as resolve18, sep as sep10 } from "node:path";
|
|
36561
36655
|
import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
|
|
36562
36656
|
import { getAgentDir as getAgentDir6, keyText } from "@earendil-works/pi-coding-agent";
|
|
36563
36657
|
function describeDecisionModel(status) {
|
|
@@ -36992,10 +37086,20 @@ class AutoRAGAgent {
|
|
|
36992
37086
|
}
|
|
36993
37087
|
async scanDuplicateDocuments() {
|
|
36994
37088
|
if (this.dupeyOptions === false) {
|
|
36995
|
-
return { scans: [],
|
|
37089
|
+
return { scans: [], familyCount: 0, exactDuplicateCount: 0 };
|
|
36996
37090
|
}
|
|
36997
|
-
const
|
|
36998
|
-
const
|
|
37091
|
+
const cloudRoots = await this.cloudRoots();
|
|
37092
|
+
const scanRoots = [];
|
|
37093
|
+
for (const searchPath of this.searchPaths) {
|
|
37094
|
+
const scope = cloudScope(searchPath, cloudRoots);
|
|
37095
|
+
if (scope.insideCloudRoot)
|
|
37096
|
+
continue;
|
|
37097
|
+
scanRoots.push({
|
|
37098
|
+
path: searchPath,
|
|
37099
|
+
excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path))
|
|
37100
|
+
});
|
|
37101
|
+
}
|
|
37102
|
+
const scans = await Promise.all(scanRoots.map(({ path, excludeDirs }) => scanWithDupey(path, {
|
|
36999
37103
|
...this.dupeyOptions || {},
|
|
37000
37104
|
...excludeDirs.length === 0 ? {} : { excludeDirs }
|
|
37001
37105
|
})));
|
|
@@ -37009,7 +37113,7 @@ class AutoRAGAgent {
|
|
|
37009
37113
|
}
|
|
37010
37114
|
return count + [...hashes.values()].reduce((sum, size) => sum + Math.max(0, size - 1), 0);
|
|
37011
37115
|
}, 0);
|
|
37012
|
-
return { scans,
|
|
37116
|
+
return { scans, familyCount, exactDuplicateCount };
|
|
37013
37117
|
}
|
|
37014
37118
|
loadMemoryContextFor(query) {
|
|
37015
37119
|
if (this.memoryContextCache?.query === query)
|
|
@@ -38662,6 +38766,10 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38662
38766
|
inFlight: true,
|
|
38663
38767
|
lastStartedAt: startedAt
|
|
38664
38768
|
};
|
|
38769
|
+
const syncAbort = new AbortController;
|
|
38770
|
+
const forwardAbort = () => syncAbort.abort(signal?.reason);
|
|
38771
|
+
signal?.addEventListener("abort", forwardAbort, { once: true });
|
|
38772
|
+
let minSyncPipeline;
|
|
38665
38773
|
try {
|
|
38666
38774
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38667
38775
|
const batchedMinSync = needsParsed && wants("minsync");
|
|
@@ -38669,44 +38777,53 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38669
38777
|
const minSyncBatchFailures = [];
|
|
38670
38778
|
let minSyncSyncedBase = 0;
|
|
38671
38779
|
let minSyncTotalBase = 0;
|
|
38672
|
-
|
|
38673
|
-
|
|
38674
|
-
|
|
38675
|
-
|
|
38676
|
-
|
|
38677
|
-
|
|
38678
|
-
|
|
38679
|
-
|
|
38680
|
-
|
|
38681
|
-
|
|
38682
|
-
|
|
38683
|
-
|
|
38684
|
-
|
|
38685
|
-
|
|
38686
|
-
total: batchTotal
|
|
38687
|
-
});
|
|
38688
|
-
});
|
|
38689
|
-
minSyncBatches.push(result);
|
|
38690
|
-
if (result.ok)
|
|
38691
|
-
minSyncBatchFailures.length = 0;
|
|
38692
|
-
else if (result.reason !== "lock-held") {
|
|
38693
|
-
for (const virtualPath of batch.virtualPaths) {
|
|
38694
|
-
minSyncBatchFailures.push({
|
|
38695
|
-
code: "minsync-sync-failed",
|
|
38696
|
-
severity: "warning",
|
|
38697
|
-
message: `This file stayed at its previous index entry: its batch could not be indexed and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
|
|
38698
|
-
source: virtualPath
|
|
38780
|
+
minSyncPipeline = batchedMinSync ? new PipelinedSync({
|
|
38781
|
+
sync: async ({ index, virtualPaths }) => {
|
|
38782
|
+
syncAbort.signal.throwIfAborted();
|
|
38783
|
+
let syncTotal = minSyncTotalBase;
|
|
38784
|
+
const result = await this.syncMinSync(force && index === 1, syncAbort.signal, (minsyncProgress) => {
|
|
38785
|
+
syncTotal = minSyncTotalBase + minsyncProgress.total;
|
|
38786
|
+
progress = updateRefreshProgress(progress, {
|
|
38787
|
+
minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: syncTotal }
|
|
38788
|
+
});
|
|
38789
|
+
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38790
|
+
emitProgress({
|
|
38791
|
+
phase: "minsync",
|
|
38792
|
+
processed: minSyncSyncedBase + minsyncProgress.processed,
|
|
38793
|
+
total: syncTotal
|
|
38699
38794
|
});
|
|
38795
|
+
});
|
|
38796
|
+
minSyncBatches.push(result);
|
|
38797
|
+
if (result.ok)
|
|
38798
|
+
minSyncBatchFailures.length = 0;
|
|
38799
|
+
else if (result.reason !== "lock-held") {
|
|
38800
|
+
for (const virtualPath of virtualPaths) {
|
|
38801
|
+
minSyncBatchFailures.push({
|
|
38802
|
+
code: "minsync-sync-failed",
|
|
38803
|
+
severity: "warning",
|
|
38804
|
+
message: `This file stayed at its previous index entry: the sync that covered it could not index it and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
|
|
38805
|
+
source: virtualPath
|
|
38806
|
+
});
|
|
38807
|
+
}
|
|
38700
38808
|
}
|
|
38809
|
+
minSyncSyncedBase = syncTotal;
|
|
38810
|
+
minSyncTotalBase = syncTotal;
|
|
38811
|
+
return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
|
|
38701
38812
|
}
|
|
38702
|
-
|
|
38703
|
-
|
|
38704
|
-
return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
|
|
38705
|
-
};
|
|
38813
|
+
}) : undefined;
|
|
38814
|
+
const pipeline = minSyncPipeline;
|
|
38706
38815
|
const summary = needsParsed ? await this.syncParsedMirrors(force, {
|
|
38707
38816
|
...opts?.retrySkipped === true ? { retrySkipped: true } : {},
|
|
38708
38817
|
...signal === undefined ? {} : { signal },
|
|
38709
|
-
...
|
|
38818
|
+
...pipeline === undefined ? {} : {
|
|
38819
|
+
onBatch: (batch) => {
|
|
38820
|
+
if (batch.final) {
|
|
38821
|
+
progress = updateRefreshProgress(progress, { phase: "minsync" });
|
|
38822
|
+
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38823
|
+
}
|
|
38824
|
+
return pipeline.batch(batch);
|
|
38825
|
+
}
|
|
38826
|
+
},
|
|
38710
38827
|
onProgress: (parsedProgress) => {
|
|
38711
38828
|
progress = updateRefreshProgress(progress, {
|
|
38712
38829
|
phase: "parsed",
|
|
@@ -38842,6 +38959,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38842
38959
|
...fsearch !== undefined ? { fsearch } : {}
|
|
38843
38960
|
};
|
|
38844
38961
|
} catch (error) {
|
|
38962
|
+
syncAbort.abort(error);
|
|
38845
38963
|
this.refreshState = {
|
|
38846
38964
|
...this.refreshState,
|
|
38847
38965
|
lastOutcome: "failed",
|
|
@@ -38855,6 +38973,8 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38855
38973
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38856
38974
|
throw error;
|
|
38857
38975
|
} finally {
|
|
38976
|
+
signal?.removeEventListener("abort", forwardAbort);
|
|
38977
|
+
await minSyncPipeline?.settle();
|
|
38858
38978
|
this.refreshState = {
|
|
38859
38979
|
...this.refreshState,
|
|
38860
38980
|
inFlight: false,
|
|
@@ -39076,15 +39196,15 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
39076
39196
|
const cloudRoots = await this.cloudRoots();
|
|
39077
39197
|
const fingerprints = cloudFingerprintExclusions(loadCloudFingerprints(this.workspaceProjectRoot));
|
|
39078
39198
|
for (const searchPath of this.searchPaths) {
|
|
39079
|
-
const
|
|
39199
|
+
const scope = cloudScope(searchPath, cloudRoots);
|
|
39080
39200
|
let scan;
|
|
39081
|
-
if (
|
|
39201
|
+
if (scope.insideCloudRoot) {
|
|
39082
39202
|
scan = { dir: searchPath, files: [], families: [], errors: [] };
|
|
39083
39203
|
} else {
|
|
39084
39204
|
try {
|
|
39085
39205
|
scan = await scanWithDupey(searchPath, {
|
|
39086
39206
|
...this.dupeyOptions || {},
|
|
39087
|
-
...
|
|
39207
|
+
...scope.nestedCloudRoots.length === 0 ? {} : { excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path)) }
|
|
39088
39208
|
});
|
|
39089
39209
|
} catch (error) {
|
|
39090
39210
|
if (!(error instanceof DupeyCliError))
|
|
@@ -39649,7 +39769,7 @@ function toMinSyncReasonDiagnostic(reason) {
|
|
|
39649
39769
|
function pinExcludedPath(path) {
|
|
39650
39770
|
const resolvedPath = resolve18(path);
|
|
39651
39771
|
try {
|
|
39652
|
-
return join54(realpathSync8(dirname27(resolvedPath)),
|
|
39772
|
+
return join54(realpathSync8(dirname27(resolvedPath)), basename15(resolvedPath));
|
|
39653
39773
|
} catch {
|
|
39654
39774
|
return resolvedPath;
|
|
39655
39775
|
}
|