@autorag/librarian 2.13.5 → 2.13.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/agent.d.ts +5 -5
- package/dist/agent/dupey-tool.d.ts +0 -2
- package/dist/agent/pipelined-sync.d.ts +34 -0
- package/dist/agent/search-documents.d.ts +1 -1
- package/dist/cli/commands/duplicates.d.ts +2 -3
- package/dist/cli/index.js +1721 -1542
- package/dist/core.js +2505 -2320
- package/dist/datasource/skills/factory.d.ts +4 -1
- package/dist/dupey/filter.d.ts +0 -2
- package/dist/dupey/index.d.ts +0 -1
- package/dist/dupey/version.d.ts +17 -0
- package/dist/index.js +2302 -2122
- package/dist/mcp/index.js +2302 -2120
- package/dist/mcp/server.d.ts +0 -3
- package/dist/minsync/installer.d.ts +1 -1
- package/dist/minsync/version.d.ts +3 -2
- package/dist/mirror/cloud-fingerprints.d.ts +6 -5
- package/dist/mirror/cloud.d.ts +9 -6
- package/dist/mirror/sync.d.ts +53 -3
- package/dist/profiles.js +1714 -1529
- package/package.json +1 -1
- package/dist/dupey/scan-plan.d.ts +0 -23
package/dist/profiles.js
CHANGED
|
@@ -3627,7 +3627,7 @@ function outdatedMinSyncVersion(binaryPath) {
|
|
|
3627
3627
|
}
|
|
3628
3628
|
return version !== undefined && !isSupportedMinSyncVersion(version) ? version : undefined;
|
|
3629
3629
|
}
|
|
3630
|
-
var MINSYNC_MIN_VERSION = "0.
|
|
3630
|
+
var MINSYNC_MIN_VERSION = "0.7.1", VERSION_PROBE_TIMEOUT_MS = 5000, VERSION_LINE, probeCache;
|
|
3631
3631
|
var init_version = __esm(() => {
|
|
3632
3632
|
VERSION_LINE = /^minsync\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
|
|
3633
3633
|
probeCache = new Map;
|
|
@@ -3871,7 +3871,7 @@ function isGitHubAsset(value) {
|
|
|
3871
3871
|
function isRecord2(value) {
|
|
3872
3872
|
return typeof value === "object" && value !== null;
|
|
3873
3873
|
}
|
|
3874
|
-
var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.
|
|
3874
|
+
var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.7.3", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
|
|
3875
3875
|
var init_installer = __esm(() => {
|
|
3876
3876
|
init_atomic_write();
|
|
3877
3877
|
init_process();
|
|
@@ -13749,7 +13749,8 @@ class WombatMethod {
|
|
|
13749
13749
|
return matchesDatasourceScope(datasourceSourcePath("wombat", this.instanceId, hit.chunkId), options.scope);
|
|
13750
13750
|
}).map((hit) => ({
|
|
13751
13751
|
id: `wombat:${this.instanceId}:${hit.chunkId}`,
|
|
13752
|
-
content: hit
|
|
13752
|
+
content: `${sessionLabel(hit)}
|
|
13753
|
+
${hit.snippet}`,
|
|
13753
13754
|
source: datasourceSourcePath("wombat", this.instanceId, hit.chunkId),
|
|
13754
13755
|
score: hit.score,
|
|
13755
13756
|
metadata: {
|
|
@@ -13772,6 +13773,20 @@ class WombatMethod {
|
|
|
13772
13773
|
})).slice(0, topK);
|
|
13773
13774
|
}
|
|
13774
13775
|
}
|
|
13776
|
+
function oneLine(value) {
|
|
13777
|
+
return value.replace(/\s+/gu, " ").trim();
|
|
13778
|
+
}
|
|
13779
|
+
function sessionLabel(hit) {
|
|
13780
|
+
const turns = hit.startTurn === undefined ? undefined : hit.endTurn === undefined || hit.endTurn === hit.startTurn ? `turn ${hit.startTurn}` : `turns ${hit.startTurn}-${hit.endTurn}`;
|
|
13781
|
+
const parts = [
|
|
13782
|
+
oneLine(hit.catalogKey),
|
|
13783
|
+
turns,
|
|
13784
|
+
hit.startedAt === undefined ? undefined : hit.startedAt.slice(0, 10),
|
|
13785
|
+
hit.title.length === 0 ? undefined : oneLine(hit.title),
|
|
13786
|
+
hit.cwd === undefined ? undefined : oneLine(hit.cwd)
|
|
13787
|
+
].filter((part) => part !== undefined && part.length > 0);
|
|
13788
|
+
return `[${parts.join(" | ")}]`;
|
|
13789
|
+
}
|
|
13775
13790
|
var WOMBAT_MODES, METHOD_TYPE;
|
|
13776
13791
|
var init_methods7 = __esm(() => {
|
|
13777
13792
|
init_errors2();
|
|
@@ -13884,7 +13899,7 @@ class WombatSkill {
|
|
|
13884
13899
|
`);
|
|
13885
13900
|
return {
|
|
13886
13901
|
name: "datasource-wombat",
|
|
13887
|
-
description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built.",
|
|
13902
|
+
description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built. Each hit names its session. Load this skill to open that session's full conversation or its original log with the tool calls that the search does not index.",
|
|
13888
13903
|
content: [
|
|
13889
13904
|
"# wombat coding-agent session datasource",
|
|
13890
13905
|
"",
|
|
@@ -13898,10 +13913,10 @@ class WombatSkill {
|
|
|
13898
13913
|
scopes,
|
|
13899
13914
|
"",
|
|
13900
13915
|
"The selectable retrieval modes are keyword (SPLADE), semantic, and hybrid. Scope arguments can only narrow trusted access. Source identifiers remain opaque and hierarchical.",
|
|
13901
|
-
"
|
|
13916
|
+
"Every result starts with a label line, `[<sessionKey> | turns A-B | date | title | project]`, followed by the snippet. `sessionKey` is `<agent>:<session id>`.",
|
|
13902
13917
|
"",
|
|
13903
13918
|
"## Native CLI",
|
|
13904
|
-
"The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the result's
|
|
13919
|
+
"The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the `sessionKey` from the result's label:",
|
|
13905
13920
|
"- `wombat show <sessionKey> [--turn N]`: the full cleaned conversation, or one turn.",
|
|
13906
13921
|
"- `wombat trace <sessionKey> --turn N --context K`: the original session log around that turn, including the tool calls and outputs that the index leaves out.",
|
|
13907
13922
|
"- `wombat chunk context <chunk id> --json`: the turns before and after a result.",
|
|
@@ -14055,6 +14070,7 @@ function common(config, workspaceRoot) {
|
|
|
14055
14070
|
function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
|
|
14056
14071
|
const skills = [];
|
|
14057
14072
|
const unknown = [];
|
|
14073
|
+
const retired = [];
|
|
14058
14074
|
for (const [name, raw] of Object.entries(config ?? {})) {
|
|
14059
14075
|
if (raw === false)
|
|
14060
14076
|
continue;
|
|
@@ -14066,6 +14082,10 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
|
|
|
14066
14082
|
if (entry.enabled === false)
|
|
14067
14083
|
continue;
|
|
14068
14084
|
const templateName = entry.type ?? name;
|
|
14085
|
+
if (Object.hasOwn(RETIRED_TEMPLATES, templateName)) {
|
|
14086
|
+
retired.push(name);
|
|
14087
|
+
continue;
|
|
14088
|
+
}
|
|
14069
14089
|
const builder = Object.hasOwn(BUILDERS, templateName) && datasourceSkillSupported(templateName) ? BUILDERS[templateName] : undefined;
|
|
14070
14090
|
if (builder === undefined) {
|
|
14071
14091
|
unknown.push(name);
|
|
@@ -14089,9 +14109,9 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
|
|
|
14089
14109
|
}
|
|
14090
14110
|
skills.push(typeof entry.description === "string" && entry.description.trim().length > 0 ? new DescribedDatasourceSkill(aliased, entry.description.trim()) : aliased);
|
|
14091
14111
|
}
|
|
14092
|
-
return { skills, unknown };
|
|
14112
|
+
return { skills, unknown, retired };
|
|
14093
14113
|
}
|
|
14094
|
-
var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES;
|
|
14114
|
+
var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES, RETIRED_TEMPLATES;
|
|
14095
14115
|
var init_factory = __esm(() => {
|
|
14096
14116
|
init_aliased_skill();
|
|
14097
14117
|
init_config2();
|
|
@@ -14247,6 +14267,7 @@ var init_factory = __esm(() => {
|
|
|
14247
14267
|
};
|
|
14248
14268
|
PLATFORM_ONLY_SKILLS = { kakao: "darwin" };
|
|
14249
14269
|
BUILTIN_DATASOURCE_SKILL_NAMES = builtinDatasourceSkillNames();
|
|
14270
|
+
RETIRED_TEMPLATES = { spotlight: true };
|
|
14250
14271
|
});
|
|
14251
14272
|
|
|
14252
14273
|
// src/embedding/http-embedder.ts
|
|
@@ -15833,7 +15854,7 @@ function buildAgentOptions(config, agentDir) {
|
|
|
15833
15854
|
const embedder = createEmbedderFromConfig({ embedder: config.minSync?.embedder });
|
|
15834
15855
|
opts.embedder = embedder;
|
|
15835
15856
|
if (config.datasources !== undefined) {
|
|
15836
|
-
const { skills, unknown } = buildDatasourceSkills(config.datasources, config.workspacePath, {
|
|
15857
|
+
const { skills, unknown, retired } = buildDatasourceSkills(config.datasources, config.workspacePath, {
|
|
15837
15858
|
embedder,
|
|
15838
15859
|
discrawl: async () => discrawlEmbeddingRuntimeFor(await embedder.describeEndpoint()),
|
|
15839
15860
|
cliEndpoint: embeddingEndpointSource(embedder),
|
|
@@ -15846,16 +15867,27 @@ function buildAgentOptions(config, agentDir) {
|
|
|
15846
15867
|
});
|
|
15847
15868
|
if (skills.length > 0)
|
|
15848
15869
|
opts.datasourceSkills = skills;
|
|
15870
|
+
const diagnostics = [];
|
|
15871
|
+
if (retired.length > 0) {
|
|
15872
|
+
const names = retired.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
|
|
15873
|
+
diagnostics.push({
|
|
15874
|
+
code: "retired-datasource-skill",
|
|
15875
|
+
severity: "warning",
|
|
15876
|
+
message: `Datasource config ignored because the Spotlight datasource was removed: ${names.join(", ")}. Delete these entries; use fsearch/mirror search (searchPaths) instead.`,
|
|
15877
|
+
source: "datasources"
|
|
15878
|
+
});
|
|
15879
|
+
}
|
|
15849
15880
|
if (unknown.length > 0) {
|
|
15850
|
-
const
|
|
15851
|
-
|
|
15881
|
+
const names = unknown.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
|
|
15882
|
+
diagnostics.push({
|
|
15852
15883
|
code: "unknown-datasource-skill",
|
|
15853
15884
|
severity: "warning",
|
|
15854
|
-
message: `Unknown datasource skill(s) in config were skipped: ${
|
|
15885
|
+
message: `Unknown datasource skill(s) in config were skipped: ${names.join(", ")}`,
|
|
15855
15886
|
source: "datasources"
|
|
15856
|
-
};
|
|
15857
|
-
opts.startupDiagnostics = [diagnostic];
|
|
15887
|
+
});
|
|
15858
15888
|
}
|
|
15889
|
+
if (diagnostics.length > 0)
|
|
15890
|
+
opts.startupDiagnostics = diagnostics;
|
|
15859
15891
|
}
|
|
15860
15892
|
return opts;
|
|
15861
15893
|
}
|
|
@@ -17680,11 +17712,49 @@ var init_matrix = __esm(() => {
|
|
|
17680
17712
|
};
|
|
17681
17713
|
});
|
|
17682
17714
|
|
|
17715
|
+
// src/dupey/version.ts
|
|
17716
|
+
function parseDupeyVersion(output) {
|
|
17717
|
+
return VERSION_LINE2.exec(output)?.[1];
|
|
17718
|
+
}
|
|
17719
|
+
function isSupportedDupeyVersion(version) {
|
|
17720
|
+
const have = version.split(".").map((part) => Number.parseInt(part, 10));
|
|
17721
|
+
const need = DUPEY_MIN_VERSION.split(".").map((part) => Number.parseInt(part, 10));
|
|
17722
|
+
for (let index = 0;index < need.length; index += 1) {
|
|
17723
|
+
const left = have[index] ?? 0;
|
|
17724
|
+
const right = need[index] ?? 0;
|
|
17725
|
+
if (left !== right)
|
|
17726
|
+
return left > right;
|
|
17727
|
+
}
|
|
17728
|
+
return true;
|
|
17729
|
+
}
|
|
17730
|
+
var DUPEY_MIN_VERSION = "0.2.0", VERSION_LINE2;
|
|
17731
|
+
var init_version2 = __esm(() => {
|
|
17732
|
+
VERSION_LINE2 = /^dupey\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
|
|
17733
|
+
});
|
|
17734
|
+
|
|
17683
17735
|
// src/dupey/cli.ts
|
|
17684
17736
|
import { spawn as spawn13 } from "node:child_process";
|
|
17685
17737
|
import { resolve as resolve9 } from "node:path";
|
|
17686
|
-
function
|
|
17687
|
-
|
|
17738
|
+
function checkDupeyVersion(executable, options) {
|
|
17739
|
+
const cached = versionChecks.get(executable);
|
|
17740
|
+
if (cached !== undefined)
|
|
17741
|
+
return cached;
|
|
17742
|
+
const check = spawnDupey(executable, ["--version"], options).then((stdout) => {
|
|
17743
|
+
const version = parseDupeyVersion(stdout);
|
|
17744
|
+
if (version !== undefined && !isSupportedDupeyVersion(version)) {
|
|
17745
|
+
throw new DupeyCliError(`dupey ${version} at ${executable} is older than the ${DUPEY_MIN_VERSION} AutoRAG requires; run \`cargo install dupey\` to update it.`);
|
|
17746
|
+
}
|
|
17747
|
+
});
|
|
17748
|
+
check.catch(() => versionChecks.delete(executable));
|
|
17749
|
+
versionChecks.set(executable, check);
|
|
17750
|
+
return check;
|
|
17751
|
+
}
|
|
17752
|
+
async function runDupey(args, options) {
|
|
17753
|
+
if (options.run)
|
|
17754
|
+
return options.run(args);
|
|
17755
|
+
const executable = options.executable ?? "dupey";
|
|
17756
|
+
await checkDupeyVersion(executable, options);
|
|
17757
|
+
return spawnDupey(executable, args, options);
|
|
17688
17758
|
}
|
|
17689
17759
|
async function scanWithDupey(dir, options = {}) {
|
|
17690
17760
|
const absoluteDir = resolve9(dir);
|
|
@@ -17818,28 +17888,26 @@ function spawnDupey(executable, args, options) {
|
|
|
17818
17888
|
});
|
|
17819
17889
|
});
|
|
17820
17890
|
}
|
|
17821
|
-
var DupeyCliError, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
|
|
17891
|
+
var DupeyCliError, versionChecks, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
|
|
17822
17892
|
var init_cli2 = __esm(() => {
|
|
17823
17893
|
init_portable_spawn();
|
|
17894
|
+
init_version2();
|
|
17824
17895
|
DupeyCliError = class DupeyCliError extends Error {
|
|
17825
17896
|
constructor(message) {
|
|
17826
17897
|
super(message);
|
|
17827
17898
|
this.name = "DupeyCliError";
|
|
17828
17899
|
}
|
|
17829
17900
|
};
|
|
17901
|
+
versionChecks = new Map;
|
|
17830
17902
|
});
|
|
17831
17903
|
|
|
17832
17904
|
// src/dupey/filter.ts
|
|
17833
17905
|
import { stat as stat2 } from "node:fs/promises";
|
|
17834
17906
|
import { resolve as resolve10 } from "node:path";
|
|
17835
|
-
function extractedNoText(file) {
|
|
17836
|
-
const signals = file.signals;
|
|
17837
|
-
return typeof signals === "object" && signals !== null && "chars" in signals && signals.chars === 0;
|
|
17838
|
-
}
|
|
17839
17907
|
async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerprints) {
|
|
17840
17908
|
const byHash = new Map;
|
|
17841
17909
|
for (const file of scan.files) {
|
|
17842
|
-
if (typeof file.content_hash !== "string" || file.content_hash.length === 0
|
|
17910
|
+
if (typeof file.content_hash !== "string" || file.content_hash.length === 0)
|
|
17843
17911
|
continue;
|
|
17844
17912
|
const path = resolve10(root, file.path);
|
|
17845
17913
|
const group = byHash.get(file.content_hash) ?? [];
|
|
@@ -17848,7 +17916,7 @@ async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerp
|
|
|
17848
17916
|
}
|
|
17849
17917
|
const fingerprintModified = new Map;
|
|
17850
17918
|
for (const [path, fingerprint] of fingerprints ?? []) {
|
|
17851
|
-
if (fingerprint.exact.length === 0
|
|
17919
|
+
if (fingerprint.exact.length === 0)
|
|
17852
17920
|
continue;
|
|
17853
17921
|
fingerprintModified.set(path, fingerprint.modified);
|
|
17854
17922
|
const group = byHash.get(fingerprint.exact) ?? [];
|
|
@@ -17915,1397 +17983,1368 @@ async function rankExactCandidates(paths, dupeyRanking, fingerprintModified, err
|
|
|
17915
17983
|
}
|
|
17916
17984
|
var init_filter = () => {};
|
|
17917
17985
|
|
|
17918
|
-
// src/
|
|
17919
|
-
|
|
17920
|
-
|
|
17921
|
-
|
|
17922
|
-
|
|
17923
|
-
|
|
17924
|
-
|
|
17925
|
-
function
|
|
17926
|
-
const
|
|
17927
|
-
|
|
17928
|
-
}
|
|
17929
|
-
|
|
17930
|
-
|
|
17986
|
+
// src/dupey/index.ts
|
|
17987
|
+
var init_dupey = __esm(() => {
|
|
17988
|
+
init_cli2();
|
|
17989
|
+
init_filter();
|
|
17990
|
+
});
|
|
17991
|
+
|
|
17992
|
+
// src/convert/relate.ts
|
|
17993
|
+
async function convertAndRelate(request, options = {}) {
|
|
17994
|
+
const result = await convertWithFilee(request, options.filee);
|
|
17995
|
+
if (options.dupey === false)
|
|
17996
|
+
return { result, links: [] };
|
|
17997
|
+
const dupeyOptions = options.dupey ?? {};
|
|
17998
|
+
const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
|
|
17999
|
+
const links = await Promise.all(pairs.map(async ({ source, output }) => {
|
|
18000
|
+
try {
|
|
18001
|
+
return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
|
|
18002
|
+
} catch (error) {
|
|
18003
|
+
if (!(error instanceof DupeyCliError))
|
|
18004
|
+
throw error;
|
|
18005
|
+
return { source, output, dupey: { relation: "unavailable", reason: error.message } };
|
|
18006
|
+
}
|
|
18007
|
+
}));
|
|
18008
|
+
return { result, links };
|
|
17931
18009
|
}
|
|
17932
|
-
|
|
17933
|
-
|
|
18010
|
+
var init_relate = __esm(() => {
|
|
18011
|
+
init_dupey();
|
|
18012
|
+
init_cli();
|
|
18013
|
+
});
|
|
18014
|
+
|
|
18015
|
+
// src/convert/index.ts
|
|
18016
|
+
var init_convert = __esm(() => {
|
|
18017
|
+
init_cli();
|
|
18018
|
+
init_installer2();
|
|
18019
|
+
init_matrix();
|
|
18020
|
+
init_relate();
|
|
18021
|
+
});
|
|
18022
|
+
|
|
18023
|
+
// src/datasource/diagnostics.ts
|
|
18024
|
+
function mapDatasourceDiagnostic(ds) {
|
|
18025
|
+
return {
|
|
18026
|
+
code: datasourceCodeToRetrieval(ds.code),
|
|
18027
|
+
severity: ds.severity,
|
|
18028
|
+
message: `[datasource:${ds.code}] ${ds.message}`,
|
|
18029
|
+
source: ds.source ?? (ds.instanceId ? `datasource:${ds.instanceId}` : "datasource")
|
|
18030
|
+
};
|
|
17934
18031
|
}
|
|
17935
|
-
function
|
|
17936
|
-
|
|
17937
|
-
if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
|
|
17938
|
-
return "google-drive";
|
|
17939
|
-
if (lower.includes("onedrive"))
|
|
17940
|
-
return "onedrive";
|
|
17941
|
-
if (lower.includes("dropbox"))
|
|
17942
|
-
return "dropbox";
|
|
17943
|
-
if (lower.includes("mobile documents") || lower.includes("clouddocs"))
|
|
17944
|
-
return "icloud";
|
|
17945
|
-
return "file-provider";
|
|
18032
|
+
function mapDatasourceDiagnostics(diagnostics) {
|
|
18033
|
+
return diagnostics.map(mapDatasourceDiagnostic);
|
|
17946
18034
|
}
|
|
17947
|
-
function
|
|
17948
|
-
|
|
17949
|
-
|
|
17950
|
-
|
|
17951
|
-
|
|
17952
|
-
|
|
17953
|
-
|
|
17954
|
-
|
|
17955
|
-
|
|
17956
|
-
|
|
17957
|
-
|
|
18035
|
+
function datasourceCodeToRetrieval(code) {
|
|
18036
|
+
switch (code) {
|
|
18037
|
+
case "datasource-unavailable":
|
|
18038
|
+
case "datasource-cli-error":
|
|
18039
|
+
case "datasource-empty":
|
|
18040
|
+
case "datasource-rate-limited":
|
|
18041
|
+
case "datasource-auth-error":
|
|
18042
|
+
case "datasource-embedding-egress-rejected":
|
|
18043
|
+
case "datasource-index-failed":
|
|
18044
|
+
case "datasource-index-partial":
|
|
18045
|
+
case "datasource-permission-denied":
|
|
18046
|
+
case "datasource-remote-embedding-rejected":
|
|
18047
|
+
return "retrieval-method-failed";
|
|
18048
|
+
}
|
|
17958
18049
|
}
|
|
17959
|
-
|
|
17960
|
-
|
|
17961
|
-
|
|
17962
|
-
|
|
17963
|
-
|
|
18050
|
+
|
|
18051
|
+
// src/datasource/retry.ts
|
|
18052
|
+
import { setTimeout as delay2 } from "node:timers/promises";
|
|
18053
|
+
function datasourceRetryDelayMs(retry, policy) {
|
|
18054
|
+
const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
|
|
18055
|
+
return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
|
|
17964
18056
|
}
|
|
17965
|
-
function
|
|
17966
|
-
const
|
|
17967
|
-
|
|
17968
|
-
|
|
17969
|
-
|
|
17970
|
-
|
|
17971
|
-
|
|
18057
|
+
async function indexWithRetry(skill, options = {}, signal) {
|
|
18058
|
+
const policy = {
|
|
18059
|
+
maxRetries: options.maxRetries ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxRetries,
|
|
18060
|
+
baseDelayMs: options.baseDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.baseDelayMs,
|
|
18061
|
+
maxDelayMs: options.maxDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxDelayMs
|
|
18062
|
+
};
|
|
18063
|
+
const sleep = options.sleep ?? ((ms) => delay2(ms, undefined, signal === undefined ? undefined : { signal }));
|
|
18064
|
+
let lastError = "";
|
|
18065
|
+
for (let attempt = 1;; attempt += 1) {
|
|
18066
|
+
const result = await runIndex(skill, signal);
|
|
18067
|
+
signal?.throwIfAborted();
|
|
18068
|
+
if (result.ok) {
|
|
18069
|
+
if (attempt === 1)
|
|
18070
|
+
return result;
|
|
18071
|
+
const recovered = {
|
|
18072
|
+
code: "datasource-index-failed",
|
|
18073
|
+
severity: "info",
|
|
18074
|
+
message: `recovered after ${attempt} attempts; last error: ${lastError}`,
|
|
18075
|
+
instanceId: result.instanceId,
|
|
18076
|
+
source: result.skill
|
|
18077
|
+
};
|
|
18078
|
+
return { ...result, attempts: attempt, diagnostics: [...result.diagnostics, recovered] };
|
|
18079
|
+
}
|
|
18080
|
+
lastError = result.message ?? result.error ?? "datasource-index-failed";
|
|
18081
|
+
if (attempt > policy.maxRetries) {
|
|
18082
|
+
if (attempt === 1)
|
|
18083
|
+
return { ...result, attempts: attempt };
|
|
18084
|
+
const suffix = ` (gave up after ${attempt} attempts)`;
|
|
18085
|
+
const last = result.diagnostics.at(-1);
|
|
18086
|
+
return {
|
|
18087
|
+
...result,
|
|
18088
|
+
attempts: attempt,
|
|
18089
|
+
message: `${lastError}${suffix}`,
|
|
18090
|
+
diagnostics: last === undefined ? [
|
|
18091
|
+
{
|
|
18092
|
+
code: result.code ?? "datasource-index-failed",
|
|
18093
|
+
severity: "error",
|
|
18094
|
+
message: `${lastError}${suffix}`,
|
|
18095
|
+
instanceId: result.instanceId,
|
|
18096
|
+
source: result.skill
|
|
18097
|
+
}
|
|
18098
|
+
] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
|
|
18099
|
+
};
|
|
18100
|
+
}
|
|
18101
|
+
await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
|
|
18102
|
+
signal?.throwIfAborted();
|
|
18103
|
+
}
|
|
17972
18104
|
}
|
|
17973
|
-
function
|
|
18105
|
+
async function runIndex(skill, signal) {
|
|
17974
18106
|
try {
|
|
17975
|
-
return
|
|
17976
|
-
} catch {
|
|
17977
|
-
|
|
18107
|
+
return await skill.index(signal === undefined ? undefined : { signal });
|
|
18108
|
+
} catch (error) {
|
|
18109
|
+
signal?.throwIfAborted();
|
|
18110
|
+
const descriptor = skill.describe();
|
|
18111
|
+
const message = error instanceof Error ? error.message : "Datasource indexing failed.";
|
|
18112
|
+
const failed = {
|
|
18113
|
+
ok: false,
|
|
18114
|
+
instanceId: descriptor.instanceId ?? "default",
|
|
18115
|
+
skill: descriptor.name,
|
|
18116
|
+
indexedAt: Date.now(),
|
|
18117
|
+
diagnostics: [
|
|
18118
|
+
{
|
|
18119
|
+
code: "datasource-index-failed",
|
|
18120
|
+
severity: "error",
|
|
18121
|
+
message,
|
|
18122
|
+
source: descriptor.name,
|
|
18123
|
+
instanceId: descriptor.instanceId
|
|
18124
|
+
}
|
|
18125
|
+
],
|
|
18126
|
+
error: "datasource-index-failed",
|
|
18127
|
+
code: "datasource-index-failed",
|
|
18128
|
+
message
|
|
18129
|
+
};
|
|
18130
|
+
return failed;
|
|
17978
18131
|
}
|
|
17979
18132
|
}
|
|
17980
|
-
|
|
17981
|
-
|
|
17982
|
-
|
|
17983
|
-
|
|
17984
|
-
|
|
17985
|
-
|
|
17986
|
-
for (const { raw, canonical } of forms) {
|
|
17987
|
-
if (isPathInside(normalized, raw))
|
|
17988
|
-
return `${canonical}${normalized.slice(raw.length)}`;
|
|
17989
|
-
}
|
|
17990
|
-
return path;
|
|
18133
|
+
var DEFAULT_DATASOURCE_RETRY_POLICY;
|
|
18134
|
+
var init_retry = __esm(() => {
|
|
18135
|
+
DEFAULT_DATASOURCE_RETRY_POLICY = {
|
|
18136
|
+
maxRetries: 2,
|
|
18137
|
+
baseDelayMs: 1000,
|
|
18138
|
+
maxDelayMs: 30000
|
|
17991
18139
|
};
|
|
18140
|
+
});
|
|
18141
|
+
|
|
18142
|
+
// src/everything/bundle.ts
|
|
18143
|
+
import { createHash as createHash11 } from "node:crypto";
|
|
18144
|
+
import { existsSync as existsSync17, mkdirSync as mkdirSync14, readFileSync as readFileSync17, rmSync as rmSync7, writeFileSync as writeFileSync10 } from "node:fs";
|
|
18145
|
+
import { dirname as dirname14, join as join27 } from "node:path";
|
|
18146
|
+
import { fileURLToPath } from "node:url";
|
|
18147
|
+
import JSZip from "jszip";
|
|
18148
|
+
function resolveEverythingBundleDir() {
|
|
18149
|
+
let dir = dirname14(fileURLToPath(import.meta.url));
|
|
18150
|
+
for (let depth = 0;depth < 6; depth += 1) {
|
|
18151
|
+
const candidate = join27(dir, "vendor", "everything");
|
|
18152
|
+
if (existsSync17(join27(candidate, MANIFEST_FILENAME)))
|
|
18153
|
+
return candidate;
|
|
18154
|
+
const parent = dirname14(dir);
|
|
18155
|
+
if (parent === dir)
|
|
18156
|
+
break;
|
|
18157
|
+
dir = parent;
|
|
18158
|
+
}
|
|
18159
|
+
return;
|
|
17992
18160
|
}
|
|
17993
|
-
function
|
|
17994
|
-
|
|
17995
|
-
|
|
17996
|
-
return
|
|
17997
|
-
insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
|
|
17998
|
-
nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
|
|
17999
|
-
};
|
|
18161
|
+
function everythingArch(arch) {
|
|
18162
|
+
if (arch === "x64" || arch === "arm64")
|
|
18163
|
+
return arch;
|
|
18164
|
+
return;
|
|
18000
18165
|
}
|
|
18001
|
-
function
|
|
18002
|
-
|
|
18166
|
+
function everythingBinaryPaths(root, version, arch) {
|
|
18167
|
+
const dir = join27(root, ".autorag", "everything", version, arch);
|
|
18168
|
+
return { everythingPath: join27(dir, "everything.exe"), esPath: join27(dir, "es.exe") };
|
|
18169
|
+
}
|
|
18170
|
+
function loadEverythingBundleManifest(bundleDir) {
|
|
18171
|
+
return JSON.parse(readFileSync17(join27(bundleDir, MANIFEST_FILENAME), "utf8"));
|
|
18172
|
+
}
|
|
18173
|
+
async function ensureEverythingBinaries(options) {
|
|
18174
|
+
const platform = options.platform ?? process.platform;
|
|
18175
|
+
const rawArch = options.arch ?? process.arch;
|
|
18176
|
+
if (platform !== "win32") {
|
|
18177
|
+
return {
|
|
18178
|
+
ok: false,
|
|
18179
|
+
reason: "unsupported-platform",
|
|
18180
|
+
message: `Everything is Windows-only; this host is ${platform}.`
|
|
18181
|
+
};
|
|
18182
|
+
}
|
|
18183
|
+
const arch = everythingArch(rawArch);
|
|
18184
|
+
const bundleDir = options.bundleDir ?? resolveEverythingBundleDir();
|
|
18185
|
+
if (bundleDir === undefined) {
|
|
18186
|
+
return {
|
|
18187
|
+
ok: false,
|
|
18188
|
+
reason: "bundle-missing",
|
|
18189
|
+
message: `Bundled Everything assets (vendor/everything/${MANIFEST_FILENAME}) were not found above ${dirname14(fileURLToPath(import.meta.url))}.`
|
|
18190
|
+
};
|
|
18191
|
+
}
|
|
18192
|
+
let manifest;
|
|
18003
18193
|
try {
|
|
18004
|
-
|
|
18005
|
-
} catch {
|
|
18006
|
-
return;
|
|
18194
|
+
manifest = options.manifest ?? loadEverythingBundleManifest(bundleDir);
|
|
18195
|
+
} catch (error) {
|
|
18196
|
+
return { ok: false, reason: "bundle-missing", message: errorMessage2(error) };
|
|
18007
18197
|
}
|
|
18008
|
-
|
|
18009
|
-
|
|
18198
|
+
const assets = arch === undefined ? undefined : manifest.architectures[arch];
|
|
18199
|
+
if (arch === undefined || assets === undefined) {
|
|
18200
|
+
return {
|
|
18201
|
+
ok: false,
|
|
18202
|
+
reason: "unsupported-platform",
|
|
18203
|
+
message: `No bundled Everything build for Windows architecture ${rawArch}; bundled: ${Object.keys(manifest.architectures).join(", ")}.`
|
|
18204
|
+
};
|
|
18010
18205
|
}
|
|
18011
|
-
|
|
18012
|
-
|
|
18013
|
-
|
|
18014
|
-
|
|
18015
|
-
|
|
18016
|
-
}
|
|
18017
|
-
|
|
18018
|
-
|
|
18019
|
-
|
|
18020
|
-
|
|
18021
|
-
|
|
18022
|
-
|
|
18023
|
-
|
|
18024
|
-
|
|
18206
|
+
const { everythingPath, esPath } = everythingBinaryPaths(options.root, manifest.everythingVersion, arch);
|
|
18207
|
+
const everythingCached = fileHasSha256(everythingPath, assets.everythingSha256);
|
|
18208
|
+
const esCached = fileHasSha256(esPath, assets.esSha256);
|
|
18209
|
+
if (everythingCached && esCached) {
|
|
18210
|
+
return { ok: true, everythingPath, esPath, source: "cached" };
|
|
18211
|
+
}
|
|
18212
|
+
if ((options.mode ?? "install") === "cached") {
|
|
18213
|
+
const state = everythingCached || esCached ? "are incomplete or do not match their pinned SHA-256 digests" : "are not installed";
|
|
18214
|
+
return {
|
|
18215
|
+
ok: false,
|
|
18216
|
+
reason: "not-installed",
|
|
18217
|
+
message: `Everything ${manifest.everythingVersion} (${arch}) binaries ${state} under ${dirname14(everythingPath)}; run \`autorag refresh\` to (re)install and index them.`
|
|
18218
|
+
};
|
|
18219
|
+
}
|
|
18220
|
+
try {
|
|
18221
|
+
mkdirSync14(dirname14(everythingPath), { recursive: true });
|
|
18222
|
+
const everythingBytes = await extractVerified(join27(bundleDir, assets.everythingArchive), assets.everythingArchiveSha256, assets.everythingMember, assets.everythingSha256);
|
|
18223
|
+
const esBytes = await extractVerified(join27(bundleDir, assets.esArchive), assets.esArchiveSha256, assets.esMember, assets.esSha256);
|
|
18224
|
+
writeAtomic(everythingPath, everythingBytes);
|
|
18225
|
+
writeAtomic(esPath, esBytes);
|
|
18226
|
+
} catch (error) {
|
|
18227
|
+
return { ok: false, reason: "install-failed", message: errorMessage2(error) };
|
|
18228
|
+
}
|
|
18229
|
+
return { ok: true, everythingPath, esPath, source: "installed" };
|
|
18025
18230
|
}
|
|
18026
|
-
function
|
|
18027
|
-
const
|
|
18028
|
-
const
|
|
18029
|
-
if (
|
|
18030
|
-
|
|
18031
|
-
} else if (typeof failure.code === "number") {
|
|
18032
|
-
parts.push(`the helper exited with code ${failure.code}`);
|
|
18231
|
+
async function extractVerified(archivePath, archiveSha256, member, memberSha256) {
|
|
18232
|
+
const archive = readFileSync17(archivePath);
|
|
18233
|
+
const actualArchive = sha256(archive);
|
|
18234
|
+
if (actualArchive !== archiveSha256) {
|
|
18235
|
+
throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
|
|
18033
18236
|
}
|
|
18034
|
-
const
|
|
18035
|
-
|
|
18036
|
-
|
|
18037
|
-
|
|
18038
|
-
|
|
18039
|
-
|
|
18237
|
+
const zip = await JSZip.loadAsync(archive);
|
|
18238
|
+
const entry = zip.file(member);
|
|
18239
|
+
if (entry === null)
|
|
18240
|
+
throw new Error(`${archivePath} does not contain ${member}`);
|
|
18241
|
+
const bytes = await entry.async("nodebuffer");
|
|
18242
|
+
const actualMember = sha256(bytes);
|
|
18243
|
+
if (actualMember !== memberSha256) {
|
|
18244
|
+
throw new Error(`${member} from ${archivePath} SHA-256 ${actualMember} does not match pinned ${memberSha256}`);
|
|
18245
|
+
}
|
|
18246
|
+
return bytes;
|
|
18040
18247
|
}
|
|
18041
|
-
|
|
18042
|
-
|
|
18043
|
-
return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
|
|
18248
|
+
function fileHasSha256(path, expected) {
|
|
18249
|
+
return existsSync17(path) && sha256(readFileSync17(path)) === expected;
|
|
18044
18250
|
}
|
|
18045
|
-
function
|
|
18046
|
-
const
|
|
18047
|
-
|
|
18048
|
-
|
|
18049
|
-
|
|
18050
|
-
|
|
18051
|
-
|
|
18052
|
-
if (open <= 0 || !line.endsWith(")"))
|
|
18053
|
-
continue;
|
|
18054
|
-
const head = line.slice(0, open);
|
|
18055
|
-
const on = head.indexOf(" on ");
|
|
18056
|
-
if (on <= 0)
|
|
18057
|
-
continue;
|
|
18058
|
-
const mountPoint = head.slice(on + " on ".length);
|
|
18059
|
-
if (mountPoint === "")
|
|
18060
|
-
continue;
|
|
18061
|
-
const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
|
|
18062
|
-
mounts.push({
|
|
18063
|
-
device: head.slice(0, on),
|
|
18064
|
-
mountPoint,
|
|
18065
|
-
fileSystem: fields[0] ?? "",
|
|
18066
|
-
options: fields.slice(1)
|
|
18067
|
-
});
|
|
18251
|
+
function writeAtomic(path, bytes) {
|
|
18252
|
+
const temp = `${path}.${process.pid}.tmp`;
|
|
18253
|
+
try {
|
|
18254
|
+
writeFileSync10(temp, bytes);
|
|
18255
|
+
renameWithRetry(temp, path);
|
|
18256
|
+
} finally {
|
|
18257
|
+
rmSync7(temp, { force: true });
|
|
18068
18258
|
}
|
|
18069
|
-
return mounts;
|
|
18070
18259
|
}
|
|
18071
|
-
function
|
|
18072
|
-
|
|
18073
|
-
const seen = new Set;
|
|
18074
|
-
for (const mount of mounts) {
|
|
18075
|
-
if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
|
|
18076
|
-
continue;
|
|
18077
|
-
if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
|
|
18078
|
-
continue;
|
|
18079
|
-
if (seen.has(mount.mountPoint))
|
|
18080
|
-
continue;
|
|
18081
|
-
seen.add(mount.mountPoint);
|
|
18082
|
-
points.push(mount.mountPoint);
|
|
18083
|
-
}
|
|
18084
|
-
return points;
|
|
18260
|
+
function sha256(bytes) {
|
|
18261
|
+
return createHash11("sha256").update(bytes).digest("hex");
|
|
18085
18262
|
}
|
|
18086
|
-
function
|
|
18087
|
-
|
|
18088
|
-
|
|
18089
|
-
|
|
18090
|
-
|
|
18091
|
-
|
|
18092
|
-
|
|
18093
|
-
|
|
18094
|
-
|
|
18095
|
-
|
|
18096
|
-
|
|
18097
|
-
|
|
18098
|
-
|
|
18099
|
-
|
|
18100
|
-
|
|
18101
|
-
}
|
|
18102
|
-
|
|
18263
|
+
function errorMessage2(error) {
|
|
18264
|
+
return error instanceof Error ? error.message : String(error);
|
|
18265
|
+
}
|
|
18266
|
+
var MANIFEST_FILENAME = "manifest.json";
|
|
18267
|
+
var init_bundle = __esm(() => {
|
|
18268
|
+
init_atomic_write();
|
|
18269
|
+
});
|
|
18270
|
+
|
|
18271
|
+
// src/filesystem/path-exclusions.ts
|
|
18272
|
+
import { posix as posix3, win32 as win322 } from "node:path";
|
|
18273
|
+
function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
|
|
18274
|
+
const paths = platform === "win32" ? win322 : posix3;
|
|
18275
|
+
const normalize = (path) => {
|
|
18276
|
+
const absolute = paths.resolve(path);
|
|
18277
|
+
return platform === "win32" ? absolute.toLowerCase() : absolute;
|
|
18278
|
+
};
|
|
18279
|
+
const excluded = excludedPaths.map((path) => {
|
|
18280
|
+
const absolute = normalize(path);
|
|
18281
|
+
return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
|
|
18282
|
+
});
|
|
18283
|
+
return (path) => {
|
|
18284
|
+
const absolute = normalize(path);
|
|
18285
|
+
return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
|
|
18286
|
+
};
|
|
18103
18287
|
}
|
|
18288
|
+
var init_path_exclusions = () => {};
|
|
18104
18289
|
|
|
18105
|
-
|
|
18106
|
-
|
|
18107
|
-
|
|
18108
|
-
|
|
18109
|
-
|
|
18110
|
-
|
|
18111
|
-
|
|
18112
|
-
|
|
18113
|
-
|
|
18114
|
-
|
|
18115
|
-
|
|
18116
|
-
|
|
18117
|
-
|
|
18118
|
-
const canonical = canonicalPath(path);
|
|
18119
|
-
const key = normalizeCloudPath(canonical);
|
|
18120
|
-
if (key === "" || seen.has(key))
|
|
18121
|
-
return;
|
|
18122
|
-
seen.add(key);
|
|
18123
|
-
roots.push({ path: canonical, kind });
|
|
18124
|
-
};
|
|
18125
|
-
const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
|
|
18126
|
-
for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
|
|
18127
|
-
if (entry.isDirectory())
|
|
18128
|
-
add(join27(cloudStorage, entry.name), "file-provider");
|
|
18129
|
-
}
|
|
18130
|
-
if (await this.tryReadDir(mobileDocuments) !== undefined)
|
|
18131
|
-
add(mobileDocuments, "file-provider");
|
|
18132
|
-
for (const mountPoint of await this.networkMountPoints())
|
|
18133
|
-
add(mountPoint, "network");
|
|
18134
|
-
for (const directory of await this.fileProviderDirectories(searchRoots))
|
|
18135
|
-
add(directory, "file-provider");
|
|
18136
|
-
return roots;
|
|
18290
|
+
// src/filesystem/shared-exclusive-queue.ts
|
|
18291
|
+
class SharedExclusiveQueue {
|
|
18292
|
+
sharedSinceExclusive = [];
|
|
18293
|
+
lastExclusive = Promise.resolve();
|
|
18294
|
+
shared(task) {
|
|
18295
|
+
const run = this.lastExclusive.then(task);
|
|
18296
|
+
const settled = run.then(() => {
|
|
18297
|
+
return;
|
|
18298
|
+
}, () => {
|
|
18299
|
+
return;
|
|
18300
|
+
});
|
|
18301
|
+
this.sharedSinceExclusive.push(settled);
|
|
18302
|
+
return run;
|
|
18137
18303
|
}
|
|
18138
|
-
|
|
18139
|
-
|
|
18140
|
-
|
|
18141
|
-
|
|
18142
|
-
} catch {
|
|
18143
|
-
return [];
|
|
18144
|
-
}
|
|
18145
|
-
}
|
|
18146
|
-
async fileProviderDirectories(searchRoots) {
|
|
18147
|
-
if (searchRoots.length === 0)
|
|
18148
|
-
return [];
|
|
18149
|
-
const candidates = new Set;
|
|
18150
|
-
for (const searchRoot of searchRoots) {
|
|
18151
|
-
const canonical = canonicalPath(searchRoot);
|
|
18152
|
-
for (let cursor = canonical;; cursor = dirname14(cursor)) {
|
|
18153
|
-
candidates.add(cursor);
|
|
18154
|
-
const parent = dirname14(cursor);
|
|
18155
|
-
if (parent === cursor)
|
|
18156
|
-
break;
|
|
18157
|
-
}
|
|
18158
|
-
for (const entry of await this.tryReadDir(canonical) ?? []) {
|
|
18159
|
-
if (entry.isDirectory())
|
|
18160
|
-
candidates.add(join27(canonical, entry.name));
|
|
18161
|
-
}
|
|
18162
|
-
}
|
|
18163
|
-
const paths = [...candidates];
|
|
18164
|
-
const managed = [];
|
|
18165
|
-
for (let start = 0;start < paths.length; start += XATTR_BATCH) {
|
|
18166
|
-
for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
|
|
18167
|
-
managed.push(path);
|
|
18168
|
-
}
|
|
18169
|
-
return managed;
|
|
18170
|
-
}
|
|
18171
|
-
async xattrValues(paths) {
|
|
18172
|
-
try {
|
|
18173
|
-
const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
|
|
18174
|
-
timeoutMs: 15000,
|
|
18175
|
-
maxBuffer: 8 * 1024 * 1024
|
|
18176
|
-
});
|
|
18177
|
-
return parseXattrValues(stdout, paths);
|
|
18178
|
-
} catch (error) {
|
|
18179
|
-
const failure = error;
|
|
18180
|
-
return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
|
|
18181
|
-
}
|
|
18182
|
-
}
|
|
18183
|
-
async tryReadDir(path) {
|
|
18184
|
-
try {
|
|
18185
|
-
return await this.readDir(path);
|
|
18186
|
-
} catch {
|
|
18304
|
+
exclusive(task) {
|
|
18305
|
+
const earlier = Promise.all([this.lastExclusive, ...this.sharedSinceExclusive]);
|
|
18306
|
+
const run = earlier.then(task);
|
|
18307
|
+
const settled = run.then(() => {
|
|
18187
18308
|
return;
|
|
18188
|
-
}
|
|
18189
|
-
|
|
18190
|
-
|
|
18191
|
-
|
|
18192
|
-
|
|
18193
|
-
|
|
18194
|
-
const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
|
|
18195
|
-
timeoutMs: 30000,
|
|
18196
|
-
maxBuffer: 4 * 1024 * 1024
|
|
18197
|
-
});
|
|
18198
|
-
const flags = stdout.split(`
|
|
18199
|
-
`).filter(Boolean);
|
|
18200
|
-
for (const [index, path] of batch.entries()) {
|
|
18201
|
-
if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
|
|
18202
|
-
dataless.add(path);
|
|
18203
|
-
}
|
|
18204
|
-
}
|
|
18205
|
-
return dataless;
|
|
18206
|
-
}
|
|
18207
|
-
async providerOf(path) {
|
|
18208
|
-
const domain = await readFileProviderDomainUpwards(path);
|
|
18209
|
-
return domain === undefined ? undefined : cloudProviderFromDomain(domain);
|
|
18210
|
-
}
|
|
18211
|
-
canRelease(provider) {
|
|
18212
|
-
return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
|
|
18213
|
-
}
|
|
18214
|
-
async evict(path) {
|
|
18215
|
-
let stdout;
|
|
18216
|
-
try {
|
|
18217
|
-
stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
|
|
18218
|
-
timeoutMs: 30000
|
|
18219
|
-
})).stdout;
|
|
18220
|
-
} catch (error) {
|
|
18221
|
-
return { ok: false, message: describeReleaseHelperFailure(error) };
|
|
18222
|
-
}
|
|
18223
|
-
const outcome = parseReleaseOutcome(stdout);
|
|
18224
|
-
if (outcome === undefined) {
|
|
18225
|
-
const detail = stdout.trim();
|
|
18226
|
-
return {
|
|
18227
|
-
ok: false,
|
|
18228
|
-
message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
|
|
18229
|
-
};
|
|
18230
|
-
}
|
|
18231
|
-
return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
|
|
18232
|
-
}
|
|
18233
|
-
}
|
|
18234
|
-
function buildWindowsAttributeScript(paths) {
|
|
18235
|
-
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
18236
|
-
` + `$ErrorActionPreference='SilentlyContinue'
|
|
18237
|
-
` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
|
|
18238
|
-
`;
|
|
18239
|
-
}
|
|
18240
|
-
function buildWindowsSyncRootsScript() {
|
|
18241
|
-
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
18242
|
-
` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
|
|
18243
|
-
` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
|
|
18244
|
-
`;
|
|
18245
|
-
}
|
|
18246
|
-
function buildWindowsDehydrateScript(path) {
|
|
18247
|
-
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
18248
|
-
` + `$ErrorActionPreference='Stop'
|
|
18249
|
-
` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
|
|
18250
|
-
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
|
|
18251
|
-
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
|
|
18252
|
-
` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
|
|
18253
|
-
` + `'@
|
|
18254
|
-
` + `$h=[IntPtr]::Zero
|
|
18255
|
-
` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
|
|
18256
|
-
` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
|
|
18257
|
-
` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
|
|
18258
|
-
` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
|
|
18259
|
-
` + `[Autorag.CfApi]::CfCloseHandle($h)
|
|
18260
|
-
` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
|
|
18261
|
-
` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
|
|
18262
|
-
`;
|
|
18263
|
-
}
|
|
18264
|
-
function parseWindowsAttributeOutput(stdout, paths) {
|
|
18265
|
-
const lines = stdout.split(/\r?\n/);
|
|
18266
|
-
const attributes = new Map;
|
|
18267
|
-
for (const [index, path] of paths.entries()) {
|
|
18268
|
-
const value = Number.parseInt((lines[index] ?? "").trim(), 10);
|
|
18269
|
-
if (Number.isFinite(value))
|
|
18270
|
-
attributes.set(path, value);
|
|
18271
|
-
}
|
|
18272
|
-
return attributes;
|
|
18273
|
-
}
|
|
18274
|
-
function parseWindowsSyncRoots(stdout) {
|
|
18275
|
-
const roots = [];
|
|
18276
|
-
for (const line of stdout.split(/\r?\n/)) {
|
|
18277
|
-
const tab = line.indexOf("\t");
|
|
18278
|
-
if (tab <= 0)
|
|
18279
|
-
continue;
|
|
18280
|
-
const path = line.slice(0, tab).trim();
|
|
18281
|
-
if (path !== "")
|
|
18282
|
-
roots.push({ path, label: line.slice(tab + 1).trim() });
|
|
18309
|
+
}, () => {
|
|
18310
|
+
return;
|
|
18311
|
+
});
|
|
18312
|
+
this.sharedSinceExclusive = [];
|
|
18313
|
+
this.lastExclusive = settled;
|
|
18314
|
+
return run;
|
|
18283
18315
|
}
|
|
18284
|
-
return roots;
|
|
18285
18316
|
}
|
|
18286
|
-
|
|
18287
|
-
|
|
18288
|
-
|
|
18289
|
-
|
|
18290
|
-
|
|
18291
|
-
|
|
18292
|
-
|
|
18317
|
+
|
|
18318
|
+
// src/everything/client.ts
|
|
18319
|
+
import { spawn as spawn14 } from "node:child_process";
|
|
18320
|
+
import { createHash as createHash12 } from "node:crypto";
|
|
18321
|
+
import { mkdirSync as mkdirSync15, writeFileSync as writeFileSync11 } from "node:fs";
|
|
18322
|
+
import { join as join28, resolve as resolve11 } from "node:path";
|
|
18323
|
+
import { setTimeout as sleep2 } from "node:timers/promises";
|
|
18324
|
+
function everythingInstanceName(root) {
|
|
18325
|
+
return `autorag-${createHash12("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
|
|
18293
18326
|
}
|
|
18294
|
-
function
|
|
18295
|
-
|
|
18296
|
-
|
|
18297
|
-
|
|
18327
|
+
function buildEverythingIni(input) {
|
|
18328
|
+
const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
|
|
18329
|
+
const perFolder = (value) => input.folders.map(() => value).join(",");
|
|
18330
|
+
return [
|
|
18331
|
+
"[Everything]",
|
|
18332
|
+
"app_data=0",
|
|
18333
|
+
"run_as_admin=0",
|
|
18334
|
+
"run_in_background=1",
|
|
18335
|
+
"show_tray_icon=0",
|
|
18336
|
+
"show_in_taskbar=0",
|
|
18337
|
+
"check_for_updates_on_startup=0",
|
|
18338
|
+
"beta_updates=0",
|
|
18339
|
+
"language=1033",
|
|
18340
|
+
"allow_http_server=0",
|
|
18341
|
+
"allow_etp_server=0",
|
|
18342
|
+
"search_history_enabled=0",
|
|
18343
|
+
"run_history_enabled=0",
|
|
18344
|
+
"auto_include_fixed_volumes=0",
|
|
18345
|
+
"auto_include_removable_volumes=0",
|
|
18346
|
+
"auto_include_fixed_refs_volumes=0",
|
|
18347
|
+
"auto_include_removable_refs_volumes=0",
|
|
18348
|
+
"ntfs_volume_paths=",
|
|
18349
|
+
"ntfs_volume_includes=",
|
|
18350
|
+
"refs_volume_paths=",
|
|
18351
|
+
"refs_volume_includes=",
|
|
18352
|
+
"index_size=1",
|
|
18353
|
+
"index_date_modified=1",
|
|
18354
|
+
`folders=${list(input.folders)}`,
|
|
18355
|
+
`folder_monitor_changes=${perFolder("1")}`,
|
|
18356
|
+
`folder_rescan_if_full_list=${perFolder("1")}`,
|
|
18357
|
+
`folder_update_types=${perFolder("0")}`,
|
|
18358
|
+
"exclude_list_enabled=1",
|
|
18359
|
+
`exclude_folders=${list(input.excludeFolders)}`,
|
|
18360
|
+
""
|
|
18361
|
+
].join(`\r
|
|
18362
|
+
`);
|
|
18298
18363
|
}
|
|
18299
|
-
function
|
|
18300
|
-
const
|
|
18301
|
-
|
|
18302
|
-
|
|
18303
|
-
|
|
18304
|
-
|
|
18305
|
-
|
|
18306
|
-
|
|
18307
|
-
|
|
18308
|
-
|
|
18309
|
-
|
|
18310
|
-
|
|
18311
|
-
|
|
18312
|
-
|
|
18313
|
-
|
|
18314
|
-
|
|
18315
|
-
|
|
18364
|
+
function buildEverythingSearchArgs(instance, request, timeoutMs) {
|
|
18365
|
+
const args = [
|
|
18366
|
+
"-argv",
|
|
18367
|
+
"-instance",
|
|
18368
|
+
instance,
|
|
18369
|
+
"-cp",
|
|
18370
|
+
"65001",
|
|
18371
|
+
"-json",
|
|
18372
|
+
"-full-path-and-name",
|
|
18373
|
+
"-size",
|
|
18374
|
+
"-date-modified",
|
|
18375
|
+
"-date-format",
|
|
18376
|
+
"1"
|
|
18377
|
+
];
|
|
18378
|
+
if (timeoutMs !== undefined)
|
|
18379
|
+
args.push("-timeout", String(timeoutMs));
|
|
18380
|
+
if (request.matchCase)
|
|
18381
|
+
args.push("-case");
|
|
18382
|
+
if (request.matchPath)
|
|
18383
|
+
args.push("-match-path");
|
|
18384
|
+
if (request.wholeWord)
|
|
18385
|
+
args.push("-whole-word");
|
|
18386
|
+
if (request.kind === "files")
|
|
18387
|
+
args.push("/a-d");
|
|
18388
|
+
if (request.kind === "folders")
|
|
18389
|
+
args.push("/ad");
|
|
18390
|
+
if (request.path !== undefined)
|
|
18391
|
+
args.push("-path", request.path);
|
|
18392
|
+
if (request.sort !== undefined)
|
|
18393
|
+
args.push("-sort", request.sort);
|
|
18394
|
+
if (request.offset !== undefined && request.offset > 0)
|
|
18395
|
+
args.push("-offset", String(request.offset));
|
|
18396
|
+
args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
|
|
18397
|
+
if (request.regex)
|
|
18398
|
+
args.push("-regex", "--", request.query);
|
|
18399
|
+
else
|
|
18400
|
+
args.push("--", request.query);
|
|
18401
|
+
return args;
|
|
18316
18402
|
}
|
|
18317
|
-
function
|
|
18318
|
-
const
|
|
18319
|
-
|
|
18320
|
-
|
|
18321
|
-
|
|
18322
|
-
|
|
18323
|
-
|
|
18324
|
-
|
|
18325
|
-
|
|
18326
|
-
continue;
|
|
18327
|
-
}
|
|
18328
|
-
if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
|
|
18329
|
-
roots.push({ path, kind: "network" });
|
|
18330
|
-
continue;
|
|
18331
|
-
}
|
|
18332
|
-
if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
|
|
18333
|
-
roots.push({ path, kind: "cloud-files" });
|
|
18334
|
-
}
|
|
18403
|
+
function parseEverythingJson(stdout) {
|
|
18404
|
+
const text = stdout.trim();
|
|
18405
|
+
if (text.length === 0)
|
|
18406
|
+
return [];
|
|
18407
|
+
let parsed;
|
|
18408
|
+
try {
|
|
18409
|
+
parsed = JSON.parse(text);
|
|
18410
|
+
} catch {
|
|
18411
|
+
throw new Error(`ES did not return JSON: ${text.slice(0, 2000)}`);
|
|
18335
18412
|
}
|
|
18336
|
-
|
|
18337
|
-
}
|
|
18338
|
-
|
|
18339
|
-
|
|
18340
|
-
|
|
18341
|
-
|
|
18342
|
-
const rest = normalized.slice(2);
|
|
18343
|
-
const separator = rest.indexOf("\\");
|
|
18344
|
-
if (separator <= 0)
|
|
18345
|
-
return;
|
|
18346
|
-
const server = rest.slice(0, separator);
|
|
18347
|
-
const share = rest.slice(separator + 1).split("\\")[0] ?? "";
|
|
18348
|
-
if (server === "" || share === "")
|
|
18349
|
-
return;
|
|
18350
|
-
return `\\\\${server}\\${share}`;
|
|
18351
|
-
}
|
|
18352
|
-
function parseWindowsDehydrateOutput(result) {
|
|
18353
|
-
const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
18354
|
-
if (open === null)
|
|
18355
|
-
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
18356
|
-
if (Number.parseInt(open[1], 16) !== 0) {
|
|
18413
|
+
if (!Array.isArray(parsed))
|
|
18414
|
+
throw new Error(`ES did not return a JSON array: ${text.slice(0, 2000)}`);
|
|
18415
|
+
return parsed.map((item) => {
|
|
18416
|
+
const record = item;
|
|
18417
|
+
const filename = String(record.filename ?? "");
|
|
18418
|
+
const isFolder = filename.endsWith("\\") && filename.length > 3;
|
|
18357
18419
|
return {
|
|
18358
|
-
|
|
18359
|
-
|
|
18420
|
+
path: isFolder ? filename.slice(0, -1) : filename,
|
|
18421
|
+
type: filename.endsWith("\\") ? "folder" : "file",
|
|
18422
|
+
size: typeof record.size === "number" ? record.size : undefined,
|
|
18423
|
+
dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
|
|
18360
18424
|
};
|
|
18361
|
-
}
|
|
18362
|
-
const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
18363
|
-
if (dehydrate === null) {
|
|
18364
|
-
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
18365
|
-
}
|
|
18366
|
-
const hresult = dehydrate[1];
|
|
18367
|
-
if (Number.parseInt(hresult, 16) === 0)
|
|
18368
|
-
return { ok: true };
|
|
18369
|
-
const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
|
|
18370
|
-
return {
|
|
18371
|
-
ok: false,
|
|
18372
|
-
message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
|
|
18373
|
-
};
|
|
18374
|
-
}
|
|
18375
|
-
function windowsFailureMessage(result, fallback) {
|
|
18376
|
-
const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
|
|
18377
|
-
if (printed !== undefined && printed !== "")
|
|
18378
|
-
return printed;
|
|
18379
|
-
const stderr = result.stderr.trim();
|
|
18380
|
-
return stderr !== "" ? stderr : fallback;
|
|
18425
|
+
});
|
|
18381
18426
|
}
|
|
18382
18427
|
|
|
18383
|
-
class
|
|
18428
|
+
class EverythingClient {
|
|
18429
|
+
instanceName;
|
|
18430
|
+
options;
|
|
18431
|
+
platform;
|
|
18384
18432
|
run;
|
|
18385
|
-
|
|
18386
|
-
|
|
18387
|
-
|
|
18388
|
-
|
|
18433
|
+
launch;
|
|
18434
|
+
stateDir;
|
|
18435
|
+
isExcludedPath;
|
|
18436
|
+
binaries;
|
|
18437
|
+
queue = new SharedExclusiveQueue;
|
|
18438
|
+
constructor(options) {
|
|
18439
|
+
this.options = options;
|
|
18440
|
+
this.platform = options.platform ?? process.platform;
|
|
18441
|
+
this.instanceName = everythingInstanceName(options.root);
|
|
18442
|
+
this.run = options.run ?? runProcess;
|
|
18443
|
+
this.launch = options.launch ?? launchDetached;
|
|
18444
|
+
this.stateDir = join28(options.root, ".autorag", "everything");
|
|
18445
|
+
this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
|
|
18389
18446
|
}
|
|
18390
|
-
|
|
18391
|
-
|
|
18392
|
-
const seen = new Set;
|
|
18393
|
-
const add = (path, kind) => {
|
|
18394
|
-
const canonical = canonicalPath(path);
|
|
18395
|
-
const key = normalizeCloudPath(canonical).toLowerCase();
|
|
18396
|
-
if (key === "" || seen.has(key))
|
|
18397
|
-
return;
|
|
18398
|
-
seen.add(key);
|
|
18399
|
-
roots.push({ path: canonical, kind });
|
|
18400
|
-
};
|
|
18401
|
-
for (const root of await this.loadSyncRoots())
|
|
18402
|
-
add(root.path, "cloud-files");
|
|
18403
|
-
for (const root of await this.loadDiskRoots())
|
|
18404
|
-
add(root.path, root.kind);
|
|
18405
|
-
for (const searchRoot of searchRoots) {
|
|
18406
|
-
const share = uncShareRoot(searchRoot);
|
|
18407
|
-
if (share !== undefined)
|
|
18408
|
-
add(share, "network");
|
|
18409
|
-
}
|
|
18410
|
-
return roots;
|
|
18447
|
+
isSupported() {
|
|
18448
|
+
return this.platform === "win32";
|
|
18411
18449
|
}
|
|
18412
|
-
|
|
18413
|
-
|
|
18414
|
-
|
|
18415
|
-
|
|
18416
|
-
|
|
18417
|
-
|
|
18418
|
-
|
|
18450
|
+
search(request) {
|
|
18451
|
+
return this.queue.shared(async () => {
|
|
18452
|
+
const binaries = await this.resolveBinaries("cached");
|
|
18453
|
+
if (!binaries.ok)
|
|
18454
|
+
return binaries;
|
|
18455
|
+
const ping = await this.ping(binaries.esPath);
|
|
18456
|
+
if (ping.code !== 0) {
|
|
18457
|
+
return {
|
|
18458
|
+
ok: false,
|
|
18459
|
+
reason: "not-running",
|
|
18460
|
+
message: `Everything instance ${this.instanceName} is not running; run \`autorag refresh\` to start and index it.`
|
|
18461
|
+
};
|
|
18419
18462
|
}
|
|
18420
|
-
|
|
18421
|
-
|
|
18422
|
-
|
|
18463
|
+
const es = { ok: true, esPath: binaries.esPath };
|
|
18464
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
18465
|
+
if ((this.options.excludeFolders?.length ?? 0) > 0) {
|
|
18466
|
+
return this.searchWithExclusions(es.esPath, request, timeoutMs);
|
|
18423
18467
|
}
|
|
18424
|
-
|
|
18425
|
-
|
|
18426
|
-
|
|
18427
|
-
|
|
18428
|
-
|
|
18429
|
-
|
|
18468
|
+
const args = buildEverythingSearchArgs(this.instanceName, request, timeoutMs);
|
|
18469
|
+
const result = await this.run(es.esPath, args, timeoutMs + 5000);
|
|
18470
|
+
if (result.code !== 0)
|
|
18471
|
+
return this.failure("search-failed", es.esPath, args, result);
|
|
18472
|
+
try {
|
|
18473
|
+
return { ok: true, results: parseEverythingJson(result.stdout) };
|
|
18474
|
+
} catch (error) {
|
|
18475
|
+
return { ok: false, reason: "search-failed", message: error.message };
|
|
18476
|
+
}
|
|
18477
|
+
});
|
|
18430
18478
|
}
|
|
18431
|
-
|
|
18432
|
-
|
|
18479
|
+
async searchWithExclusions(esPath, request, timeoutMs) {
|
|
18480
|
+
const deadline = Date.now() + timeoutMs;
|
|
18481
|
+
const offset = request.offset ?? 0;
|
|
18482
|
+
const maxResults = request.maxResults ?? DEFAULT_MAX_RESULTS;
|
|
18483
|
+
const results = [];
|
|
18484
|
+
let skipped = 0;
|
|
18485
|
+
let nativeOffset = 0;
|
|
18486
|
+
const timedOut = () => ({
|
|
18487
|
+
ok: false,
|
|
18488
|
+
reason: "search-failed",
|
|
18489
|
+
message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
|
|
18490
|
+
});
|
|
18491
|
+
for (;; ) {
|
|
18492
|
+
const remainingMs = deadline - Date.now();
|
|
18493
|
+
if (remainingMs <= 0)
|
|
18494
|
+
return timedOut();
|
|
18495
|
+
const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
|
|
18496
|
+
const response = await this.run(esPath, args, remainingMs + 5000);
|
|
18497
|
+
if (response.code !== 0)
|
|
18498
|
+
return this.failure("search-failed", esPath, args, response);
|
|
18499
|
+
if (Date.now() >= deadline)
|
|
18500
|
+
return timedOut();
|
|
18501
|
+
let entries;
|
|
18502
|
+
try {
|
|
18503
|
+
entries = parseEverythingJson(response.stdout);
|
|
18504
|
+
} catch (error) {
|
|
18505
|
+
return { ok: false, reason: "search-failed", message: error.message };
|
|
18506
|
+
}
|
|
18507
|
+
for (const entry of entries) {
|
|
18508
|
+
if (this.isExcludedPath(entry.path))
|
|
18509
|
+
continue;
|
|
18510
|
+
if (skipped < offset) {
|
|
18511
|
+
skipped += 1;
|
|
18512
|
+
continue;
|
|
18513
|
+
}
|
|
18514
|
+
results.push(entry);
|
|
18515
|
+
if (results.length >= maxResults)
|
|
18516
|
+
return { ok: true, results };
|
|
18517
|
+
}
|
|
18518
|
+
if (entries.length < MAX_FILTER_CANDIDATES)
|
|
18519
|
+
return { ok: true, results };
|
|
18520
|
+
nativeOffset += entries.length;
|
|
18521
|
+
}
|
|
18433
18522
|
}
|
|
18434
|
-
|
|
18435
|
-
return
|
|
18523
|
+
index() {
|
|
18524
|
+
return this.queue.exclusive(async () => {
|
|
18525
|
+
const es = await this.ensureRunning(true);
|
|
18526
|
+
if (!es.ok)
|
|
18527
|
+
return es;
|
|
18528
|
+
const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
|
|
18529
|
+
const saveArgs = ["-instance", this.instanceName, "-save-db"];
|
|
18530
|
+
const saved = await this.run(es.esPath, saveArgs, timeoutMs);
|
|
18531
|
+
if (saved.code !== 0)
|
|
18532
|
+
return this.failure("index-failed", es.esPath, saveArgs, saved);
|
|
18533
|
+
const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
|
|
18534
|
+
const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
|
|
18535
|
+
if (counted.code !== 0)
|
|
18536
|
+
return this.failure("index-failed", es.esPath, countArgs, counted);
|
|
18537
|
+
return {
|
|
18538
|
+
ok: true,
|
|
18539
|
+
indexedFolders: [...this.options.folders],
|
|
18540
|
+
indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
|
|
18541
|
+
};
|
|
18542
|
+
});
|
|
18436
18543
|
}
|
|
18437
|
-
|
|
18438
|
-
|
|
18439
|
-
|
|
18544
|
+
stop() {
|
|
18545
|
+
return this.queue.exclusive(async () => {
|
|
18546
|
+
const binaries = await this.resolveBinaries("cached");
|
|
18547
|
+
if (!binaries.ok)
|
|
18548
|
+
return;
|
|
18549
|
+
await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
18550
|
+
});
|
|
18440
18551
|
}
|
|
18441
|
-
|
|
18442
|
-
|
|
18443
|
-
|
|
18552
|
+
async resolveBinaries(mode) {
|
|
18553
|
+
if (!this.isSupported()) {
|
|
18554
|
+
return {
|
|
18555
|
+
ok: false,
|
|
18556
|
+
reason: "unsupported-platform",
|
|
18557
|
+
message: `Everything is Windows-only; this host is ${this.platform}.`
|
|
18558
|
+
};
|
|
18559
|
+
}
|
|
18560
|
+
if (this.binaries?.ok)
|
|
18561
|
+
return this.binaries;
|
|
18562
|
+
const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
|
|
18563
|
+
const result = await resolve(mode);
|
|
18564
|
+
if (result.ok)
|
|
18565
|
+
this.binaries = result;
|
|
18566
|
+
return result;
|
|
18444
18567
|
}
|
|
18445
|
-
|
|
18446
|
-
|
|
18447
|
-
|
|
18448
|
-
|
|
18449
|
-
|
|
18450
|
-
|
|
18451
|
-
|
|
18452
|
-
|
|
18453
|
-
|
|
18454
|
-
|
|
18455
|
-
|
|
18456
|
-
|
|
18568
|
+
async ensureRunning(reconfigure) {
|
|
18569
|
+
const binaries = await this.resolveBinaries("install");
|
|
18570
|
+
if (!binaries.ok)
|
|
18571
|
+
return binaries;
|
|
18572
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
18573
|
+
let ping = await this.ping(binaries.esPath);
|
|
18574
|
+
if (ping.code === 0 && !reconfigure)
|
|
18575
|
+
return { ok: true, esPath: binaries.esPath };
|
|
18576
|
+
if (ping.code === 0) {
|
|
18577
|
+
const exitArgs = ["-instance", this.instanceName, "-exit"];
|
|
18578
|
+
const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
|
|
18579
|
+
if (exited.code !== 0)
|
|
18580
|
+
return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
|
|
18581
|
+
}
|
|
18582
|
+
mkdirSync15(this.stateDir, { recursive: true });
|
|
18583
|
+
const iniPath = join28(this.stateDir, "Everything.ini");
|
|
18584
|
+
writeFileSync11(iniPath, buildEverythingIni({
|
|
18585
|
+
folders: this.options.folders,
|
|
18586
|
+
excludeFolders: [
|
|
18587
|
+
...new Set([
|
|
18588
|
+
join28(this.options.root, ".autorag"),
|
|
18589
|
+
...this.options.folders.map((folder) => join28(folder, ".autorag")),
|
|
18590
|
+
...this.options.excludeFolders ?? []
|
|
18591
|
+
])
|
|
18592
|
+
]
|
|
18593
|
+
}));
|
|
18594
|
+
const launchArgs = [
|
|
18595
|
+
"-instance",
|
|
18596
|
+
this.instanceName,
|
|
18597
|
+
"-config",
|
|
18598
|
+
iniPath,
|
|
18599
|
+
"-db",
|
|
18600
|
+
join28(this.stateDir, "Everything.db"),
|
|
18601
|
+
"-startup"
|
|
18602
|
+
];
|
|
18603
|
+
try {
|
|
18604
|
+
this.launch(binaries.everythingPath, launchArgs);
|
|
18605
|
+
} catch (error) {
|
|
18606
|
+
return {
|
|
18607
|
+
ok: false,
|
|
18608
|
+
reason: "startup-failed",
|
|
18609
|
+
message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
|
|
18610
|
+
};
|
|
18611
|
+
}
|
|
18612
|
+
const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
|
|
18613
|
+
const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
18614
|
+
for (;; ) {
|
|
18615
|
+
ping = await this.ping(binaries.esPath);
|
|
18616
|
+
if (ping.code === 0)
|
|
18617
|
+
return { ok: true, esPath: binaries.esPath };
|
|
18618
|
+
if (Date.now() >= deadline)
|
|
18619
|
+
break;
|
|
18620
|
+
await sleep2(pollMs);
|
|
18457
18621
|
}
|
|
18458
|
-
}
|
|
18459
|
-
return match;
|
|
18460
|
-
}
|
|
18461
|
-
function powerShellLaunch(script) {
|
|
18462
|
-
return {
|
|
18463
|
-
args: [
|
|
18464
|
-
"-NoProfile",
|
|
18465
|
-
"-NonInteractive",
|
|
18466
|
-
"-Command",
|
|
18467
|
-
`[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; Invoke-Expression $env:${POWERSHELL_SCRIPT_VARIABLE}`
|
|
18468
|
-
],
|
|
18469
|
-
env: { [POWERSHELL_SCRIPT_VARIABLE]: script }
|
|
18470
|
-
};
|
|
18471
|
-
}
|
|
18472
|
-
async function runPowerShell(script, timeoutMs) {
|
|
18473
|
-
const launch = powerShellLaunch(script);
|
|
18474
|
-
try {
|
|
18475
|
-
const { stdout, stderr } = await execFileAsync("powershell.exe", [...launch.args], {
|
|
18476
|
-
timeout: timeoutMs,
|
|
18477
|
-
maxBuffer: 16 * 1024 * 1024,
|
|
18478
|
-
windowsHide: true,
|
|
18479
|
-
env: { ...process.env, ...launch.env }
|
|
18480
|
-
});
|
|
18481
|
-
return { ok: true, stdout, stderr };
|
|
18482
|
-
} catch (error) {
|
|
18483
|
-
const record = typeof error === "object" && error !== null ? error : {};
|
|
18484
18622
|
return {
|
|
18485
18623
|
ok: false,
|
|
18486
|
-
|
|
18487
|
-
|
|
18624
|
+
reason: "startup-failed",
|
|
18625
|
+
message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
|
|
18488
18626
|
};
|
|
18489
18627
|
}
|
|
18490
|
-
|
|
18491
|
-
|
|
18492
|
-
|
|
18493
|
-
|
|
18494
|
-
|
|
18495
|
-
return
|
|
18496
|
-
|
|
18497
|
-
|
|
18498
|
-
|
|
18499
|
-
|
|
18500
|
-
await lstat(target);
|
|
18501
|
-
const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
|
|
18502
|
-
timeout: 2000
|
|
18503
|
-
});
|
|
18504
|
-
const value = stdout.trim();
|
|
18505
|
-
return value.length > 0 ? value : undefined;
|
|
18506
|
-
} catch {
|
|
18507
|
-
return;
|
|
18628
|
+
async ping(esPath) {
|
|
18629
|
+
const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
18630
|
+
return result.code === 0 && !VERSION_PATTERN.test(result.stdout.trim()) ? { ...result, code: -1 } : result;
|
|
18631
|
+
}
|
|
18632
|
+
failure(reason, esPath, args, result) {
|
|
18633
|
+
return {
|
|
18634
|
+
ok: false,
|
|
18635
|
+
reason,
|
|
18636
|
+
message: `${esPath} ${args.join(" ")} exit ${result.code}: ${(result.stderr || result.stdout).trim()}`
|
|
18637
|
+
};
|
|
18508
18638
|
}
|
|
18509
18639
|
}
|
|
18510
|
-
|
|
18511
|
-
let
|
|
18512
|
-
|
|
18513
|
-
|
|
18514
|
-
|
|
18515
|
-
|
|
18516
|
-
|
|
18517
|
-
|
|
18640
|
+
function runProcess(command, args, timeoutMs) {
|
|
18641
|
+
let resolvePromise;
|
|
18642
|
+
const promise = new Promise((resolve) => {
|
|
18643
|
+
resolvePromise = resolve;
|
|
18644
|
+
});
|
|
18645
|
+
const child = spawn14(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
|
|
18646
|
+
const stdout = [];
|
|
18647
|
+
const stderr = [];
|
|
18648
|
+
let bytes = 0;
|
|
18649
|
+
let overflow = false;
|
|
18650
|
+
const collect = (target) => (chunk) => {
|
|
18651
|
+
bytes += chunk.length;
|
|
18652
|
+
if (bytes > MAX_OUTPUT_BYTES) {
|
|
18653
|
+
overflow = true;
|
|
18654
|
+
child.kill();
|
|
18518
18655
|
return;
|
|
18519
|
-
|
|
18520
|
-
|
|
18521
|
-
|
|
18656
|
+
}
|
|
18657
|
+
target.push(chunk);
|
|
18658
|
+
};
|
|
18659
|
+
child.stdout.on("data", collect(stdout));
|
|
18660
|
+
child.stderr.on("data", collect(stderr));
|
|
18661
|
+
const timer = setTimeout(() => child.kill(), timeoutMs);
|
|
18662
|
+
child.on("error", (error) => {
|
|
18663
|
+
clearTimeout(timer);
|
|
18664
|
+
resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
|
|
18665
|
+
});
|
|
18666
|
+
child.on("close", (code, signal) => {
|
|
18667
|
+
clearTimeout(timer);
|
|
18668
|
+
const err = Buffer.concat(stderr).toString("utf8");
|
|
18669
|
+
resolvePromise({
|
|
18670
|
+
code,
|
|
18671
|
+
stdout: Buffer.concat(stdout).toString("utf8"),
|
|
18672
|
+
stderr: overflow ? `${err}
|
|
18673
|
+
output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
|
|
18674
|
+
killed by ${signal} after ${timeoutMs}ms` : err
|
|
18675
|
+
});
|
|
18676
|
+
});
|
|
18677
|
+
return promise;
|
|
18522
18678
|
}
|
|
18523
|
-
function
|
|
18524
|
-
|
|
18679
|
+
function launchDetached(command, args) {
|
|
18680
|
+
const child = spawn14(command, args, { detached: true, stdio: "ignore", windowsHide: true });
|
|
18681
|
+
child.on("error", () => {});
|
|
18682
|
+
child.unref();
|
|
18525
18683
|
}
|
|
18526
|
-
var
|
|
18527
|
-
|
|
18528
|
-
|
|
18529
|
-
|
|
18530
|
-
|
|
18531
|
-
|
|
18532
|
-
|
|
18533
|
-
var released = $.NSFileManager.defaultManager.evictUbiquitousItemAtURLError(url, errorRef);
|
|
18534
|
-
if (released) return JSON.stringify({ ok: true });
|
|
18535
|
-
function resourceValue(key) {
|
|
18536
|
-
var ref = Ref();
|
|
18537
|
-
if (!url.getResourceValueForKeyError(ref, key, null)) return null;
|
|
18538
|
-
var value = ref[0];
|
|
18539
|
-
return value === undefined || value === null ? null : ObjC.unwrap(value);
|
|
18540
|
-
}
|
|
18541
|
-
return JSON.stringify({
|
|
18542
|
-
ok: false,
|
|
18543
|
-
isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
|
|
18544
|
-
isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
|
|
18545
|
-
downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
|
|
18546
|
-
});
|
|
18547
|
-
}`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
|
|
18548
|
-
var init_cloud_placeholder = __esm(() => {
|
|
18549
|
-
execFileAsync = promisify(execFile2);
|
|
18550
|
-
RELEASABLE_DARWIN_PROVIDERS = {
|
|
18551
|
-
icloud: true,
|
|
18552
|
-
"google-drive": true,
|
|
18553
|
-
onedrive: true,
|
|
18554
|
-
dropbox: true
|
|
18555
|
-
};
|
|
18556
|
-
DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
|
|
18557
|
-
WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
|
|
18558
|
-
WINDOWS_DEHYDRATE_REFUSALS = {
|
|
18559
|
-
"80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
|
|
18560
|
-
"80070188": "The file is pinned to stay on this device, so it is not released."
|
|
18561
|
-
};
|
|
18684
|
+
var DEFAULT_TIMEOUT_MS8 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
|
|
18685
|
+
var init_client9 = __esm(() => {
|
|
18686
|
+
init_path_exclusions();
|
|
18687
|
+
init_bundle();
|
|
18688
|
+
DEFAULT_INDEX_TIMEOUT_MS3 = 10 * 60000;
|
|
18689
|
+
MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
|
|
18690
|
+
VERSION_PATTERN = /^\d+\.\d+\.\d+\.\d+$/;
|
|
18562
18691
|
});
|
|
18563
18692
|
|
|
18564
|
-
// src/
|
|
18565
|
-
|
|
18566
|
-
|
|
18567
|
-
|
|
18568
|
-
|
|
18693
|
+
// src/everything/index.ts
|
|
18694
|
+
var init_everything = __esm(() => {
|
|
18695
|
+
init_bundle();
|
|
18696
|
+
init_client9();
|
|
18697
|
+
});
|
|
18698
|
+
|
|
18699
|
+
// src/filesystem/cloud-placeholder.ts
|
|
18700
|
+
import { execFile as execFile2 } from "node:child_process";
|
|
18701
|
+
import { realpathSync as realpathSync3 } from "node:fs";
|
|
18702
|
+
import { lstat, readdir } from "node:fs/promises";
|
|
18703
|
+
import { homedir as homedir6 } from "node:os";
|
|
18704
|
+
import { dirname as dirname15, join as join29 } from "node:path";
|
|
18705
|
+
import { promisify } from "node:util";
|
|
18706
|
+
function homeCloudRoots(home = homedir6()) {
|
|
18707
|
+
const normalizedHome = home.replaceAll("\\", "/").replace(/\/+$/, "");
|
|
18708
|
+
return [`${normalizedHome}/Library/CloudStorage`, `${normalizedHome}/Library/Mobile Documents`];
|
|
18709
|
+
}
|
|
18710
|
+
function datalessFromDarwinFlags(flags) {
|
|
18711
|
+
return Number.isFinite(flags) && (flags >>> 0 & SF_DATALESS) !== 0;
|
|
18712
|
+
}
|
|
18713
|
+
function datalessFromWindowsAttributes(attributes) {
|
|
18714
|
+
return Number.isFinite(attributes) && (attributes & WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) !== 0;
|
|
18715
|
+
}
|
|
18716
|
+
function cloudProviderFromPath(target) {
|
|
18717
|
+
const lower = target.toLowerCase();
|
|
18718
|
+
if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
|
|
18719
|
+
return "google-drive";
|
|
18720
|
+
if (lower.includes("onedrive"))
|
|
18721
|
+
return "onedrive";
|
|
18722
|
+
if (lower.includes("dropbox"))
|
|
18723
|
+
return "dropbox";
|
|
18724
|
+
if (lower.includes("mobile documents") || lower.includes("clouddocs"))
|
|
18725
|
+
return "icloud";
|
|
18726
|
+
return "file-provider";
|
|
18727
|
+
}
|
|
18728
|
+
function cloudProviderFromDomain(domain) {
|
|
18729
|
+
const lower = domain.toLowerCase();
|
|
18730
|
+
if (lower.includes("clouddocs") || lower.includes("icloud"))
|
|
18731
|
+
return "icloud";
|
|
18732
|
+
if (lower.includes("google"))
|
|
18733
|
+
return "google-drive";
|
|
18734
|
+
if (lower.includes("onedrive"))
|
|
18735
|
+
return "onedrive";
|
|
18736
|
+
if (lower.includes("dropbox"))
|
|
18737
|
+
return "dropbox";
|
|
18738
|
+
return "file-provider";
|
|
18739
|
+
}
|
|
18740
|
+
function normalizeCloudPath(value) {
|
|
18741
|
+
let normalized = value.replaceAll("\\", "/");
|
|
18742
|
+
if (normalized.length > 1)
|
|
18743
|
+
normalized = normalized.replace(/\/+$/u, "");
|
|
18744
|
+
return normalized;
|
|
18745
|
+
}
|
|
18746
|
+
function isPathInside(target, parent) {
|
|
18747
|
+
const normalizedParent = normalizeCloudPath(parent);
|
|
18748
|
+
if (normalizedParent === "")
|
|
18749
|
+
return false;
|
|
18750
|
+
const left = normalizeCloudPath(target);
|
|
18751
|
+
const [a, b] = process.platform === "win32" ? [left.toLowerCase(), normalizedParent.toLowerCase()] : [left, normalizedParent];
|
|
18752
|
+
return a === b || a.startsWith(`${b}/`);
|
|
18753
|
+
}
|
|
18754
|
+
function canonicalPath(value) {
|
|
18569
18755
|
try {
|
|
18570
|
-
return
|
|
18756
|
+
return realpathSync3(value);
|
|
18571
18757
|
} catch {
|
|
18572
|
-
return
|
|
18758
|
+
return value;
|
|
18573
18759
|
}
|
|
18574
18760
|
}
|
|
18575
|
-
function
|
|
18576
|
-
const
|
|
18577
|
-
|
|
18578
|
-
|
|
18579
|
-
|
|
18580
|
-
|
|
18581
|
-
|
|
18582
|
-
|
|
18761
|
+
function canonicalPathMapper(searchRoots) {
|
|
18762
|
+
const forms = searchRoots.map((root) => ({ raw: normalizeCloudPath(root), canonical: normalizeCloudPath(canonicalPath(root)) })).filter(({ raw, canonical }) => raw !== "" && raw !== canonical).sort((left, right) => right.raw.length - left.raw.length);
|
|
18763
|
+
if (forms.length === 0)
|
|
18764
|
+
return (path) => path;
|
|
18765
|
+
return (path) => {
|
|
18766
|
+
const normalized = normalizeCloudPath(path);
|
|
18767
|
+
for (const { raw, canonical } of forms) {
|
|
18768
|
+
if (isPathInside(normalized, raw))
|
|
18769
|
+
return `${canonical}${normalized.slice(raw.length)}`;
|
|
18583
18770
|
}
|
|
18584
|
-
|
|
18585
|
-
}
|
|
18586
|
-
return { targets, skipped };
|
|
18771
|
+
return path;
|
|
18772
|
+
};
|
|
18587
18773
|
}
|
|
18588
|
-
|
|
18589
|
-
|
|
18590
|
-
});
|
|
18591
|
-
|
|
18592
|
-
|
|
18593
|
-
|
|
18594
|
-
|
|
18595
|
-
init_filter();
|
|
18596
|
-
init_scan_plan();
|
|
18597
|
-
});
|
|
18598
|
-
|
|
18599
|
-
// src/convert/relate.ts
|
|
18600
|
-
async function convertAndRelate(request, options = {}) {
|
|
18601
|
-
const result = await convertWithFilee(request, options.filee);
|
|
18602
|
-
if (options.dupey === false)
|
|
18603
|
-
return { result, links: [] };
|
|
18604
|
-
const dupeyOptions = options.dupey ?? {};
|
|
18605
|
-
const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
|
|
18606
|
-
const links = await Promise.all(pairs.map(async ({ source, output }) => {
|
|
18607
|
-
try {
|
|
18608
|
-
return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
|
|
18609
|
-
} catch (error) {
|
|
18610
|
-
if (!(error instanceof DupeyCliError))
|
|
18611
|
-
throw error;
|
|
18612
|
-
return { source, output, dupey: { relation: "unavailable", reason: error.message } };
|
|
18613
|
-
}
|
|
18614
|
-
}));
|
|
18615
|
-
return { result, links };
|
|
18774
|
+
function cloudScope(searchRoot, cloudRoots) {
|
|
18775
|
+
const canonicalSearchRoot = canonicalPath(searchRoot);
|
|
18776
|
+
const roots = cloudRoots.map((root) => ({ path: canonicalPath(root.path), kind: root.kind }));
|
|
18777
|
+
return {
|
|
18778
|
+
insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
|
|
18779
|
+
nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
|
|
18780
|
+
};
|
|
18616
18781
|
}
|
|
18617
|
-
|
|
18618
|
-
|
|
18619
|
-
|
|
18620
|
-
|
|
18621
|
-
|
|
18622
|
-
|
|
18623
|
-
|
|
18624
|
-
|
|
18625
|
-
|
|
18626
|
-
|
|
18627
|
-
init_relate();
|
|
18628
|
-
});
|
|
18629
|
-
|
|
18630
|
-
// src/datasource/diagnostics.ts
|
|
18631
|
-
function mapDatasourceDiagnostic(ds) {
|
|
18782
|
+
function parseReleaseOutcome(stdout) {
|
|
18783
|
+
let parsed;
|
|
18784
|
+
try {
|
|
18785
|
+
parsed = JSON.parse(stdout.trim());
|
|
18786
|
+
} catch {
|
|
18787
|
+
return;
|
|
18788
|
+
}
|
|
18789
|
+
if (typeof parsed !== "object" || parsed === null || !("ok" in parsed) || typeof parsed.ok !== "boolean") {
|
|
18790
|
+
return;
|
|
18791
|
+
}
|
|
18632
18792
|
return {
|
|
18633
|
-
|
|
18634
|
-
|
|
18635
|
-
|
|
18636
|
-
|
|
18793
|
+
ok: parsed.ok,
|
|
18794
|
+
..."isUploaded" in parsed ? { isUploaded: parsed.isUploaded } : {},
|
|
18795
|
+
..."isUploading" in parsed ? { isUploading: parsed.isUploading } : {},
|
|
18796
|
+
..."downloadingStatus" in parsed ? { downloadingStatus: parsed.downloadingStatus } : {}
|
|
18637
18797
|
};
|
|
18638
18798
|
}
|
|
18639
|
-
function
|
|
18640
|
-
|
|
18799
|
+
function describeReleaseRefusal(outcome) {
|
|
18800
|
+
if (outcome.isUploading === true)
|
|
18801
|
+
return "The file is still uploading to the cloud, so it cannot be released yet.";
|
|
18802
|
+
if (outcome.isUploaded === false)
|
|
18803
|
+
return "The file has not finished uploading to the cloud, so it cannot be released yet.";
|
|
18804
|
+
const status = typeof outcome.downloadingStatus === "string" ? outcome.downloadingStatus : "unknown";
|
|
18805
|
+
return `The cloud provider declined to release the local copy (download status ${status}).`;
|
|
18641
18806
|
}
|
|
18642
|
-
function
|
|
18643
|
-
|
|
18644
|
-
|
|
18645
|
-
|
|
18646
|
-
|
|
18647
|
-
|
|
18648
|
-
|
|
18649
|
-
case "datasource-embedding-egress-rejected":
|
|
18650
|
-
case "datasource-index-failed":
|
|
18651
|
-
case "datasource-index-partial":
|
|
18652
|
-
case "datasource-permission-denied":
|
|
18653
|
-
case "datasource-remote-embedding-rejected":
|
|
18654
|
-
return "retrieval-method-failed";
|
|
18807
|
+
function describeReleaseHelperFailure(error) {
|
|
18808
|
+
const failure = error;
|
|
18809
|
+
const parts = [];
|
|
18810
|
+
if (failure.killed === true || typeof failure.signal === "string") {
|
|
18811
|
+
parts.push(`the helper was killed${typeof failure.signal === "string" ? ` by ${failure.signal}` : ""}`);
|
|
18812
|
+
} else if (typeof failure.code === "number") {
|
|
18813
|
+
parts.push(`the helper exited with code ${failure.code}`);
|
|
18655
18814
|
}
|
|
18815
|
+
const stderr = typeof failure.stderr === "string" ? failure.stderr.trim() : "";
|
|
18816
|
+
if (stderr !== "")
|
|
18817
|
+
parts.push(stderr);
|
|
18818
|
+
if (parts.length === 0)
|
|
18819
|
+
parts.push(error instanceof Error ? error.message : String(error));
|
|
18820
|
+
return `osascript could not release the local copy: ${parts.join("; ")}`;
|
|
18656
18821
|
}
|
|
18657
|
-
|
|
18658
|
-
|
|
18659
|
-
|
|
18660
|
-
function datasourceRetryDelayMs(retry, policy) {
|
|
18661
|
-
const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
|
|
18662
|
-
return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
|
|
18822
|
+
async function defaultDarwinReadDir(path) {
|
|
18823
|
+
const entries = await readdir(path, { withFileTypes: true });
|
|
18824
|
+
return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
|
|
18663
18825
|
}
|
|
18664
|
-
|
|
18665
|
-
const
|
|
18666
|
-
|
|
18667
|
-
|
|
18668
|
-
|
|
18669
|
-
|
|
18670
|
-
|
|
18671
|
-
|
|
18672
|
-
|
|
18673
|
-
const
|
|
18674
|
-
|
|
18675
|
-
if (
|
|
18676
|
-
|
|
18677
|
-
|
|
18678
|
-
|
|
18679
|
-
|
|
18680
|
-
|
|
18681
|
-
|
|
18682
|
-
|
|
18683
|
-
|
|
18684
|
-
|
|
18685
|
-
|
|
18686
|
-
}
|
|
18687
|
-
lastError = result.message ?? result.error ?? "datasource-index-failed";
|
|
18688
|
-
if (attempt > policy.maxRetries) {
|
|
18689
|
-
if (attempt === 1)
|
|
18690
|
-
return { ...result, attempts: attempt };
|
|
18691
|
-
const suffix = ` (gave up after ${attempt} attempts)`;
|
|
18692
|
-
const last = result.diagnostics.at(-1);
|
|
18693
|
-
return {
|
|
18694
|
-
...result,
|
|
18695
|
-
attempts: attempt,
|
|
18696
|
-
message: `${lastError}${suffix}`,
|
|
18697
|
-
diagnostics: last === undefined ? [
|
|
18698
|
-
{
|
|
18699
|
-
code: result.code ?? "datasource-index-failed",
|
|
18700
|
-
severity: "error",
|
|
18701
|
-
message: `${lastError}${suffix}`,
|
|
18702
|
-
instanceId: result.instanceId,
|
|
18703
|
-
source: result.skill
|
|
18704
|
-
}
|
|
18705
|
-
] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
|
|
18706
|
-
};
|
|
18707
|
-
}
|
|
18708
|
-
await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
|
|
18709
|
-
signal?.throwIfAborted();
|
|
18826
|
+
function parseDarwinMounts(stdout) {
|
|
18827
|
+
const mounts = [];
|
|
18828
|
+
for (const line of stdout.split(`
|
|
18829
|
+
`)) {
|
|
18830
|
+
if (line.trim() === "")
|
|
18831
|
+
continue;
|
|
18832
|
+
const open = line.lastIndexOf(" (");
|
|
18833
|
+
if (open <= 0 || !line.endsWith(")"))
|
|
18834
|
+
continue;
|
|
18835
|
+
const head = line.slice(0, open);
|
|
18836
|
+
const on = head.indexOf(" on ");
|
|
18837
|
+
if (on <= 0)
|
|
18838
|
+
continue;
|
|
18839
|
+
const mountPoint = head.slice(on + " on ".length);
|
|
18840
|
+
if (mountPoint === "")
|
|
18841
|
+
continue;
|
|
18842
|
+
const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
|
|
18843
|
+
mounts.push({
|
|
18844
|
+
device: head.slice(0, on),
|
|
18845
|
+
mountPoint,
|
|
18846
|
+
fileSystem: fields[0] ?? "",
|
|
18847
|
+
options: fields.slice(1)
|
|
18848
|
+
});
|
|
18710
18849
|
}
|
|
18850
|
+
return mounts;
|
|
18711
18851
|
}
|
|
18712
|
-
|
|
18713
|
-
|
|
18714
|
-
|
|
18715
|
-
|
|
18716
|
-
|
|
18717
|
-
|
|
18718
|
-
|
|
18719
|
-
|
|
18720
|
-
|
|
18721
|
-
|
|
18722
|
-
|
|
18723
|
-
|
|
18724
|
-
diagnostics: [
|
|
18725
|
-
{
|
|
18726
|
-
code: "datasource-index-failed",
|
|
18727
|
-
severity: "error",
|
|
18728
|
-
message,
|
|
18729
|
-
source: descriptor.name,
|
|
18730
|
-
instanceId: descriptor.instanceId
|
|
18731
|
-
}
|
|
18732
|
-
],
|
|
18733
|
-
error: "datasource-index-failed",
|
|
18734
|
-
code: "datasource-index-failed",
|
|
18735
|
-
message
|
|
18736
|
-
};
|
|
18737
|
-
return failed;
|
|
18852
|
+
function networkMountPoints(mounts) {
|
|
18853
|
+
const points = [];
|
|
18854
|
+
const seen = new Set;
|
|
18855
|
+
for (const mount of mounts) {
|
|
18856
|
+
if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
|
|
18857
|
+
continue;
|
|
18858
|
+
if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
|
|
18859
|
+
continue;
|
|
18860
|
+
if (seen.has(mount.mountPoint))
|
|
18861
|
+
continue;
|
|
18862
|
+
seen.add(mount.mountPoint);
|
|
18863
|
+
points.push(mount.mountPoint);
|
|
18738
18864
|
}
|
|
18865
|
+
return points;
|
|
18739
18866
|
}
|
|
18740
|
-
|
|
18741
|
-
|
|
18742
|
-
|
|
18743
|
-
|
|
18744
|
-
|
|
18745
|
-
|
|
18746
|
-
|
|
18747
|
-
|
|
18748
|
-
|
|
18749
|
-
|
|
18750
|
-
|
|
18751
|
-
|
|
18752
|
-
|
|
18753
|
-
|
|
18754
|
-
|
|
18755
|
-
function resolveEverythingBundleDir() {
|
|
18756
|
-
let dir = dirname15(fileURLToPath(import.meta.url));
|
|
18757
|
-
for (let depth = 0;depth < 6; depth += 1) {
|
|
18758
|
-
const candidate = join28(dir, "vendor", "everything");
|
|
18759
|
-
if (existsSync17(join28(candidate, MANIFEST_FILENAME)))
|
|
18760
|
-
return candidate;
|
|
18761
|
-
const parent = dirname15(dir);
|
|
18762
|
-
if (parent === dir)
|
|
18763
|
-
break;
|
|
18764
|
-
dir = parent;
|
|
18867
|
+
function parseXattrValues(stdout, candidates) {
|
|
18868
|
+
const values = new Map;
|
|
18869
|
+
if (candidates.length === 0)
|
|
18870
|
+
return values;
|
|
18871
|
+
const ordered = [...candidates].sort((left, right) => right.length - left.length);
|
|
18872
|
+
for (const line of stdout.split(`
|
|
18873
|
+
`)) {
|
|
18874
|
+
if (line === "")
|
|
18875
|
+
continue;
|
|
18876
|
+
for (const candidate of ordered) {
|
|
18877
|
+
if (line.startsWith(`${candidate}: `)) {
|
|
18878
|
+
values.set(candidate, line.slice(candidate.length + 2));
|
|
18879
|
+
break;
|
|
18880
|
+
}
|
|
18881
|
+
}
|
|
18765
18882
|
}
|
|
18766
|
-
return;
|
|
18767
|
-
}
|
|
18768
|
-
function everythingArch(arch) {
|
|
18769
|
-
if (arch === "x64" || arch === "arm64")
|
|
18770
|
-
return arch;
|
|
18771
|
-
return;
|
|
18772
|
-
}
|
|
18773
|
-
function everythingBinaryPaths(root, version, arch) {
|
|
18774
|
-
const dir = join28(root, ".autorag", "everything", version, arch);
|
|
18775
|
-
return { everythingPath: join28(dir, "everything.exe"), esPath: join28(dir, "es.exe") };
|
|
18776
|
-
}
|
|
18777
|
-
function loadEverythingBundleManifest(bundleDir) {
|
|
18778
|
-
return JSON.parse(readFileSync17(join28(bundleDir, MANIFEST_FILENAME), "utf8"));
|
|
18883
|
+
return values;
|
|
18779
18884
|
}
|
|
18780
|
-
|
|
18781
|
-
|
|
18782
|
-
|
|
18783
|
-
|
|
18784
|
-
|
|
18785
|
-
|
|
18786
|
-
|
|
18787
|
-
|
|
18788
|
-
|
|
18885
|
+
|
|
18886
|
+
class DarwinCloudProbe {
|
|
18887
|
+
exec;
|
|
18888
|
+
readDir;
|
|
18889
|
+
home;
|
|
18890
|
+
constructor(options = {}) {
|
|
18891
|
+
this.exec = options.exec ?? defaultCloudExec;
|
|
18892
|
+
this.readDir = options.readDir ?? defaultDarwinReadDir;
|
|
18893
|
+
this.home = options.home ?? homedir6();
|
|
18789
18894
|
}
|
|
18790
|
-
|
|
18791
|
-
|
|
18792
|
-
|
|
18793
|
-
|
|
18794
|
-
|
|
18795
|
-
|
|
18796
|
-
|
|
18895
|
+
async cloudRoots(searchRoots = []) {
|
|
18896
|
+
const roots = [];
|
|
18897
|
+
const seen = new Set;
|
|
18898
|
+
const add = (path, kind) => {
|
|
18899
|
+
const canonical = canonicalPath(path);
|
|
18900
|
+
const key = normalizeCloudPath(canonical);
|
|
18901
|
+
if (key === "" || seen.has(key))
|
|
18902
|
+
return;
|
|
18903
|
+
seen.add(key);
|
|
18904
|
+
roots.push({ path: canonical, kind });
|
|
18797
18905
|
};
|
|
18906
|
+
const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
|
|
18907
|
+
for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
|
|
18908
|
+
if (entry.isDirectory())
|
|
18909
|
+
add(join29(cloudStorage, entry.name), "file-provider");
|
|
18910
|
+
}
|
|
18911
|
+
if (await this.tryReadDir(mobileDocuments) !== undefined)
|
|
18912
|
+
add(mobileDocuments, "file-provider");
|
|
18913
|
+
for (const mountPoint of await this.networkMountPoints())
|
|
18914
|
+
add(mountPoint, "network");
|
|
18915
|
+
for (const directory of await this.fileProviderDirectories(searchRoots))
|
|
18916
|
+
add(directory, "file-provider");
|
|
18917
|
+
return roots;
|
|
18798
18918
|
}
|
|
18799
|
-
|
|
18800
|
-
|
|
18801
|
-
|
|
18802
|
-
|
|
18803
|
-
|
|
18919
|
+
async networkMountPoints() {
|
|
18920
|
+
try {
|
|
18921
|
+
const { stdout } = await this.exec("/sbin/mount", [], { timeoutMs: 1e4, maxBuffer: 4 * 1024 * 1024 });
|
|
18922
|
+
return networkMountPoints(parseDarwinMounts(stdout));
|
|
18923
|
+
} catch {
|
|
18924
|
+
return [];
|
|
18925
|
+
}
|
|
18804
18926
|
}
|
|
18805
|
-
|
|
18806
|
-
|
|
18807
|
-
|
|
18808
|
-
|
|
18809
|
-
|
|
18810
|
-
|
|
18811
|
-
|
|
18927
|
+
async fileProviderDirectories(searchRoots) {
|
|
18928
|
+
if (searchRoots.length === 0)
|
|
18929
|
+
return [];
|
|
18930
|
+
const candidates = new Set;
|
|
18931
|
+
for (const searchRoot of searchRoots) {
|
|
18932
|
+
const canonical = canonicalPath(searchRoot);
|
|
18933
|
+
for (let cursor = canonical;; cursor = dirname15(cursor)) {
|
|
18934
|
+
candidates.add(cursor);
|
|
18935
|
+
const parent = dirname15(cursor);
|
|
18936
|
+
if (parent === cursor)
|
|
18937
|
+
break;
|
|
18938
|
+
}
|
|
18939
|
+
for (const entry of await this.tryReadDir(canonical) ?? []) {
|
|
18940
|
+
if (entry.isDirectory())
|
|
18941
|
+
candidates.add(join29(canonical, entry.name));
|
|
18942
|
+
}
|
|
18943
|
+
}
|
|
18944
|
+
const paths = [...candidates];
|
|
18945
|
+
const managed = [];
|
|
18946
|
+
for (let start = 0;start < paths.length; start += XATTR_BATCH) {
|
|
18947
|
+
for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
|
|
18948
|
+
managed.push(path);
|
|
18949
|
+
}
|
|
18950
|
+
return managed;
|
|
18812
18951
|
}
|
|
18813
|
-
|
|
18814
|
-
|
|
18815
|
-
|
|
18816
|
-
|
|
18817
|
-
|
|
18952
|
+
async xattrValues(paths) {
|
|
18953
|
+
try {
|
|
18954
|
+
const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
|
|
18955
|
+
timeoutMs: 15000,
|
|
18956
|
+
maxBuffer: 8 * 1024 * 1024
|
|
18957
|
+
});
|
|
18958
|
+
return parseXattrValues(stdout, paths);
|
|
18959
|
+
} catch (error) {
|
|
18960
|
+
const failure = error;
|
|
18961
|
+
return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
|
|
18962
|
+
}
|
|
18818
18963
|
}
|
|
18819
|
-
|
|
18820
|
-
|
|
18821
|
-
|
|
18822
|
-
|
|
18823
|
-
|
|
18824
|
-
|
|
18825
|
-
};
|
|
18964
|
+
async tryReadDir(path) {
|
|
18965
|
+
try {
|
|
18966
|
+
return await this.readDir(path);
|
|
18967
|
+
} catch {
|
|
18968
|
+
return;
|
|
18969
|
+
}
|
|
18826
18970
|
}
|
|
18827
|
-
|
|
18828
|
-
|
|
18829
|
-
|
|
18830
|
-
|
|
18831
|
-
|
|
18832
|
-
|
|
18833
|
-
|
|
18834
|
-
|
|
18971
|
+
async datalessAmong(paths) {
|
|
18972
|
+
const dataless = new Set;
|
|
18973
|
+
for (let start = 0;start < paths.length; start += STAT_BATCH) {
|
|
18974
|
+
const batch = paths.slice(start, start + STAT_BATCH);
|
|
18975
|
+
const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
|
|
18976
|
+
timeoutMs: 30000,
|
|
18977
|
+
maxBuffer: 4 * 1024 * 1024
|
|
18978
|
+
});
|
|
18979
|
+
const flags = stdout.split(`
|
|
18980
|
+
`).filter(Boolean);
|
|
18981
|
+
for (const [index, path] of batch.entries()) {
|
|
18982
|
+
if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
|
|
18983
|
+
dataless.add(path);
|
|
18984
|
+
}
|
|
18985
|
+
}
|
|
18986
|
+
return dataless;
|
|
18835
18987
|
}
|
|
18836
|
-
|
|
18837
|
-
|
|
18838
|
-
|
|
18839
|
-
const archive = readFileSync17(archivePath);
|
|
18840
|
-
const actualArchive = sha256(archive);
|
|
18841
|
-
if (actualArchive !== archiveSha256) {
|
|
18842
|
-
throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
|
|
18988
|
+
async providerOf(path) {
|
|
18989
|
+
const domain = await readFileProviderDomainUpwards(path);
|
|
18990
|
+
return domain === undefined ? undefined : cloudProviderFromDomain(domain);
|
|
18843
18991
|
}
|
|
18844
|
-
|
|
18845
|
-
|
|
18846
|
-
if (entry === null)
|
|
18847
|
-
throw new Error(`${archivePath} does not contain ${member}`);
|
|
18848
|
-
const bytes = await entry.async("nodebuffer");
|
|
18849
|
-
const actualMember = sha256(bytes);
|
|
18850
|
-
if (actualMember !== memberSha256) {
|
|
18851
|
-
throw new Error(`${member} from ${archivePath} SHA-256 ${actualMember} does not match pinned ${memberSha256}`);
|
|
18992
|
+
canRelease(provider) {
|
|
18993
|
+
return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
|
|
18852
18994
|
}
|
|
18853
|
-
|
|
18854
|
-
|
|
18855
|
-
|
|
18856
|
-
|
|
18857
|
-
|
|
18858
|
-
|
|
18859
|
-
|
|
18860
|
-
|
|
18861
|
-
|
|
18862
|
-
|
|
18863
|
-
|
|
18864
|
-
|
|
18995
|
+
async evict(path) {
|
|
18996
|
+
let stdout;
|
|
18997
|
+
try {
|
|
18998
|
+
stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
|
|
18999
|
+
timeoutMs: 30000
|
|
19000
|
+
})).stdout;
|
|
19001
|
+
} catch (error) {
|
|
19002
|
+
return { ok: false, message: describeReleaseHelperFailure(error) };
|
|
19003
|
+
}
|
|
19004
|
+
const outcome = parseReleaseOutcome(stdout);
|
|
19005
|
+
if (outcome === undefined) {
|
|
19006
|
+
const detail = stdout.trim();
|
|
19007
|
+
return {
|
|
19008
|
+
ok: false,
|
|
19009
|
+
message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
|
|
19010
|
+
};
|
|
19011
|
+
}
|
|
19012
|
+
return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
|
|
18865
19013
|
}
|
|
18866
19014
|
}
|
|
18867
|
-
function
|
|
18868
|
-
return
|
|
19015
|
+
function buildWindowsAttributeScript(paths) {
|
|
19016
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19017
|
+
` + `$ErrorActionPreference='SilentlyContinue'
|
|
19018
|
+
` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
|
|
19019
|
+
`;
|
|
18869
19020
|
}
|
|
18870
|
-
function
|
|
18871
|
-
return
|
|
19021
|
+
function buildWindowsSyncRootsScript() {
|
|
19022
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19023
|
+
` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
|
|
19024
|
+
` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
|
|
19025
|
+
`;
|
|
18872
19026
|
}
|
|
18873
|
-
|
|
18874
|
-
|
|
18875
|
-
|
|
18876
|
-
|
|
18877
|
-
|
|
18878
|
-
|
|
18879
|
-
|
|
18880
|
-
|
|
18881
|
-
|
|
18882
|
-
|
|
18883
|
-
|
|
18884
|
-
|
|
18885
|
-
|
|
18886
|
-
|
|
18887
|
-
|
|
18888
|
-
|
|
18889
|
-
|
|
18890
|
-
return (path) => {
|
|
18891
|
-
const absolute = normalize(path);
|
|
18892
|
-
return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
|
|
18893
|
-
};
|
|
19027
|
+
function buildWindowsDehydrateScript(path) {
|
|
19028
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19029
|
+
` + `$ErrorActionPreference='Stop'
|
|
19030
|
+
` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
|
|
19031
|
+
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
|
|
19032
|
+
` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
|
|
19033
|
+
` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
|
|
19034
|
+
` + `'@
|
|
19035
|
+
` + `$h=[IntPtr]::Zero
|
|
19036
|
+
` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
|
|
19037
|
+
` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
|
|
19038
|
+
` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
|
|
19039
|
+
` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
|
|
19040
|
+
` + `[Autorag.CfApi]::CfCloseHandle($h)
|
|
19041
|
+
` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
|
|
19042
|
+
` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
|
|
19043
|
+
`;
|
|
18894
19044
|
}
|
|
18895
|
-
|
|
18896
|
-
|
|
18897
|
-
|
|
18898
|
-
|
|
18899
|
-
|
|
18900
|
-
|
|
18901
|
-
|
|
18902
|
-
const run = this.lastExclusive.then(task);
|
|
18903
|
-
const settled = run.then(() => {
|
|
18904
|
-
return;
|
|
18905
|
-
}, () => {
|
|
18906
|
-
return;
|
|
18907
|
-
});
|
|
18908
|
-
this.sharedSinceExclusive.push(settled);
|
|
18909
|
-
return run;
|
|
19045
|
+
function parseWindowsAttributeOutput(stdout, paths) {
|
|
19046
|
+
const lines = stdout.split(/\r?\n/);
|
|
19047
|
+
const attributes = new Map;
|
|
19048
|
+
for (const [index, path] of paths.entries()) {
|
|
19049
|
+
const value = Number.parseInt((lines[index] ?? "").trim(), 10);
|
|
19050
|
+
if (Number.isFinite(value))
|
|
19051
|
+
attributes.set(path, value);
|
|
18910
19052
|
}
|
|
18911
|
-
|
|
18912
|
-
|
|
18913
|
-
|
|
18914
|
-
|
|
18915
|
-
|
|
18916
|
-
|
|
18917
|
-
|
|
18918
|
-
|
|
18919
|
-
|
|
18920
|
-
|
|
18921
|
-
|
|
19053
|
+
return attributes;
|
|
19054
|
+
}
|
|
19055
|
+
function parseWindowsSyncRoots(stdout) {
|
|
19056
|
+
const roots = [];
|
|
19057
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
19058
|
+
const tab = line.indexOf("\t");
|
|
19059
|
+
if (tab <= 0)
|
|
19060
|
+
continue;
|
|
19061
|
+
const path = line.slice(0, tab).trim();
|
|
19062
|
+
if (path !== "")
|
|
19063
|
+
roots.push({ path, label: line.slice(tab + 1).trim() });
|
|
18922
19064
|
}
|
|
19065
|
+
return roots;
|
|
18923
19066
|
}
|
|
18924
|
-
|
|
18925
|
-
|
|
18926
|
-
|
|
18927
|
-
|
|
18928
|
-
|
|
18929
|
-
|
|
18930
|
-
|
|
18931
|
-
function everythingInstanceName(root) {
|
|
18932
|
-
return `autorag-${createHash12("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
|
|
19067
|
+
function providerFromWindowsSyncRoot(label) {
|
|
19068
|
+
const lower = label.toLowerCase();
|
|
19069
|
+
if (lower.includes("onedrive") || lower.includes("one drive"))
|
|
19070
|
+
return "onedrive";
|
|
19071
|
+
if (lower.includes("google"))
|
|
19072
|
+
return "google-drive";
|
|
19073
|
+
return "file-provider";
|
|
18933
19074
|
}
|
|
18934
|
-
function
|
|
18935
|
-
|
|
18936
|
-
|
|
18937
|
-
|
|
18938
|
-
"[Everything]",
|
|
18939
|
-
"app_data=0",
|
|
18940
|
-
"run_as_admin=0",
|
|
18941
|
-
"run_in_background=1",
|
|
18942
|
-
"show_tray_icon=0",
|
|
18943
|
-
"show_in_taskbar=0",
|
|
18944
|
-
"check_for_updates_on_startup=0",
|
|
18945
|
-
"beta_updates=0",
|
|
18946
|
-
"language=1033",
|
|
18947
|
-
"allow_http_server=0",
|
|
18948
|
-
"allow_etp_server=0",
|
|
18949
|
-
"search_history_enabled=0",
|
|
18950
|
-
"run_history_enabled=0",
|
|
18951
|
-
"auto_include_fixed_volumes=0",
|
|
18952
|
-
"auto_include_removable_volumes=0",
|
|
18953
|
-
"auto_include_fixed_refs_volumes=0",
|
|
18954
|
-
"auto_include_removable_refs_volumes=0",
|
|
18955
|
-
"ntfs_volume_paths=",
|
|
18956
|
-
"ntfs_volume_includes=",
|
|
18957
|
-
"refs_volume_paths=",
|
|
18958
|
-
"refs_volume_includes=",
|
|
18959
|
-
"index_size=1",
|
|
18960
|
-
"index_date_modified=1",
|
|
18961
|
-
`folders=${list(input.folders)}`,
|
|
18962
|
-
`folder_monitor_changes=${perFolder("1")}`,
|
|
18963
|
-
`folder_rescan_if_full_list=${perFolder("1")}`,
|
|
18964
|
-
`folder_update_types=${perFolder("0")}`,
|
|
18965
|
-
"exclude_list_enabled=1",
|
|
18966
|
-
`exclude_folders=${list(input.excludeFolders)}`,
|
|
18967
|
-
""
|
|
18968
|
-
].join(`\r
|
|
18969
|
-
`);
|
|
19075
|
+
function buildWindowsDiskScript() {
|
|
19076
|
+
return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
|
|
19077
|
+
` + "Get-CimInstance -ClassName Win32_LogicalDisk | ForEach-Object {" + `$_.DeviceID + [char]9 + $_.DriveType + [char]9 + $_.FileSystem + [char]9 + $_.VolumeName}
|
|
19078
|
+
`;
|
|
18970
19079
|
}
|
|
18971
|
-
function
|
|
18972
|
-
const
|
|
18973
|
-
|
|
18974
|
-
"
|
|
18975
|
-
|
|
18976
|
-
"
|
|
18977
|
-
"
|
|
18978
|
-
|
|
18979
|
-
|
|
18980
|
-
|
|
18981
|
-
|
|
18982
|
-
|
|
18983
|
-
|
|
18984
|
-
|
|
18985
|
-
|
|
18986
|
-
|
|
18987
|
-
|
|
18988
|
-
args.push("-case");
|
|
18989
|
-
if (request.matchPath)
|
|
18990
|
-
args.push("-match-path");
|
|
18991
|
-
if (request.wholeWord)
|
|
18992
|
-
args.push("-whole-word");
|
|
18993
|
-
if (request.kind === "files")
|
|
18994
|
-
args.push("/a-d");
|
|
18995
|
-
if (request.kind === "folders")
|
|
18996
|
-
args.push("/ad");
|
|
18997
|
-
if (request.path !== undefined)
|
|
18998
|
-
args.push("-path", request.path);
|
|
18999
|
-
if (request.sort !== undefined)
|
|
19000
|
-
args.push("-sort", request.sort);
|
|
19001
|
-
if (request.offset !== undefined && request.offset > 0)
|
|
19002
|
-
args.push("-offset", String(request.offset));
|
|
19003
|
-
args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
|
|
19004
|
-
if (request.regex)
|
|
19005
|
-
args.push("-regex", "--", request.query);
|
|
19006
|
-
else
|
|
19007
|
-
args.push("--", request.query);
|
|
19008
|
-
return args;
|
|
19080
|
+
function parseWindowsLogicalDisks(stdout) {
|
|
19081
|
+
const disks = [];
|
|
19082
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
19083
|
+
if (line.trim() === "")
|
|
19084
|
+
continue;
|
|
19085
|
+
const [deviceId, driveType, fileSystem, ...rest] = line.split("\t");
|
|
19086
|
+
const type = Number.parseInt((driveType ?? "").trim(), 10);
|
|
19087
|
+
if (!Number.isFinite(type))
|
|
19088
|
+
continue;
|
|
19089
|
+
disks.push({
|
|
19090
|
+
deviceId: (deviceId ?? "").trim(),
|
|
19091
|
+
driveType: type,
|
|
19092
|
+
fileSystem: (fileSystem ?? "").trim(),
|
|
19093
|
+
volumeName: rest.join("\t").trim()
|
|
19094
|
+
});
|
|
19095
|
+
}
|
|
19096
|
+
return disks;
|
|
19009
19097
|
}
|
|
19010
|
-
function
|
|
19011
|
-
const
|
|
19012
|
-
|
|
19013
|
-
|
|
19014
|
-
|
|
19015
|
-
|
|
19016
|
-
|
|
19017
|
-
|
|
19018
|
-
|
|
19098
|
+
function windowsRootsFromDisks(disks) {
|
|
19099
|
+
const roots = [];
|
|
19100
|
+
for (const disk of disks) {
|
|
19101
|
+
if (disk.deviceId === "")
|
|
19102
|
+
continue;
|
|
19103
|
+
const path = `${disk.deviceId}\\`;
|
|
19104
|
+
const fileSystem = disk.fileSystem.toLowerCase();
|
|
19105
|
+
if (disk.driveType === WINDOWS_DRIVE_TYPE_NETWORK) {
|
|
19106
|
+
roots.push({ path, kind: "network" });
|
|
19107
|
+
continue;
|
|
19108
|
+
}
|
|
19109
|
+
if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
|
|
19110
|
+
roots.push({ path, kind: "network" });
|
|
19111
|
+
continue;
|
|
19112
|
+
}
|
|
19113
|
+
if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
|
|
19114
|
+
roots.push({ path, kind: "cloud-files" });
|
|
19115
|
+
}
|
|
19019
19116
|
}
|
|
19020
|
-
|
|
19021
|
-
|
|
19022
|
-
|
|
19023
|
-
|
|
19024
|
-
|
|
19025
|
-
|
|
19117
|
+
return roots;
|
|
19118
|
+
}
|
|
19119
|
+
function uncShareRoot(target) {
|
|
19120
|
+
const normalized = target.replaceAll("/", "\\");
|
|
19121
|
+
if (!normalized.startsWith("\\\\") || normalized.startsWith("\\\\?\\") || normalized.startsWith("\\\\.\\"))
|
|
19122
|
+
return;
|
|
19123
|
+
const rest = normalized.slice(2);
|
|
19124
|
+
const separator = rest.indexOf("\\");
|
|
19125
|
+
if (separator <= 0)
|
|
19126
|
+
return;
|
|
19127
|
+
const server = rest.slice(0, separator);
|
|
19128
|
+
const share = rest.slice(separator + 1).split("\\")[0] ?? "";
|
|
19129
|
+
if (server === "" || share === "")
|
|
19130
|
+
return;
|
|
19131
|
+
return `\\\\${server}\\${share}`;
|
|
19132
|
+
}
|
|
19133
|
+
function parseWindowsDehydrateOutput(result) {
|
|
19134
|
+
const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
19135
|
+
if (open === null)
|
|
19136
|
+
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
19137
|
+
if (Number.parseInt(open[1], 16) !== 0) {
|
|
19026
19138
|
return {
|
|
19027
|
-
|
|
19028
|
-
|
|
19029
|
-
size: typeof record.size === "number" ? record.size : undefined,
|
|
19030
|
-
dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
|
|
19139
|
+
ok: false,
|
|
19140
|
+
message: windowsFailureMessage(result, `could not open the file for release (0x${open[1]})`)
|
|
19031
19141
|
};
|
|
19032
|
-
}
|
|
19142
|
+
}
|
|
19143
|
+
const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
|
|
19144
|
+
if (dehydrate === null) {
|
|
19145
|
+
return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
|
|
19146
|
+
}
|
|
19147
|
+
const hresult = dehydrate[1];
|
|
19148
|
+
if (Number.parseInt(hresult, 16) === 0)
|
|
19149
|
+
return { ok: true };
|
|
19150
|
+
const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
|
|
19151
|
+
return {
|
|
19152
|
+
ok: false,
|
|
19153
|
+
message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
|
|
19154
|
+
};
|
|
19155
|
+
}
|
|
19156
|
+
function windowsFailureMessage(result, fallback) {
|
|
19157
|
+
const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
|
|
19158
|
+
if (printed !== undefined && printed !== "")
|
|
19159
|
+
return printed;
|
|
19160
|
+
const stderr = result.stderr.trim();
|
|
19161
|
+
return stderr !== "" ? stderr : fallback;
|
|
19033
19162
|
}
|
|
19034
19163
|
|
|
19035
|
-
class
|
|
19036
|
-
instanceName;
|
|
19037
|
-
options;
|
|
19038
|
-
platform;
|
|
19164
|
+
class WindowsCloudProbe {
|
|
19039
19165
|
run;
|
|
19040
|
-
|
|
19041
|
-
|
|
19042
|
-
|
|
19043
|
-
|
|
19044
|
-
queue = new SharedExclusiveQueue;
|
|
19045
|
-
constructor(options) {
|
|
19046
|
-
this.options = options;
|
|
19047
|
-
this.platform = options.platform ?? process.platform;
|
|
19048
|
-
this.instanceName = everythingInstanceName(options.root);
|
|
19049
|
-
this.run = options.run ?? runProcess;
|
|
19050
|
-
this.launch = options.launch ?? launchDetached;
|
|
19051
|
-
this.stateDir = join29(options.root, ".autorag", "everything");
|
|
19052
|
-
this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
|
|
19053
|
-
}
|
|
19054
|
-
isSupported() {
|
|
19055
|
-
return this.platform === "win32";
|
|
19166
|
+
syncRoots;
|
|
19167
|
+
disks;
|
|
19168
|
+
constructor(options = {}) {
|
|
19169
|
+
this.run = options.run ?? runPowerShell;
|
|
19056
19170
|
}
|
|
19057
|
-
|
|
19058
|
-
|
|
19059
|
-
|
|
19060
|
-
|
|
19061
|
-
|
|
19062
|
-
const
|
|
19063
|
-
if (
|
|
19064
|
-
return
|
|
19065
|
-
|
|
19066
|
-
|
|
19067
|
-
|
|
19068
|
-
|
|
19069
|
-
|
|
19070
|
-
|
|
19071
|
-
|
|
19072
|
-
|
|
19073
|
-
|
|
19074
|
-
|
|
19075
|
-
|
|
19076
|
-
|
|
19077
|
-
|
|
19078
|
-
return this.failure("search-failed", es.esPath, args, result);
|
|
19079
|
-
try {
|
|
19080
|
-
return { ok: true, results: parseEverythingJson(result.stdout) };
|
|
19081
|
-
} catch (error) {
|
|
19082
|
-
return { ok: false, reason: "search-failed", message: error.message };
|
|
19083
|
-
}
|
|
19084
|
-
});
|
|
19171
|
+
async cloudRoots(searchRoots = []) {
|
|
19172
|
+
const roots = [];
|
|
19173
|
+
const seen = new Set;
|
|
19174
|
+
const add = (path, kind) => {
|
|
19175
|
+
const canonical = canonicalPath(path);
|
|
19176
|
+
const key = normalizeCloudPath(canonical).toLowerCase();
|
|
19177
|
+
if (key === "" || seen.has(key))
|
|
19178
|
+
return;
|
|
19179
|
+
seen.add(key);
|
|
19180
|
+
roots.push({ path: canonical, kind });
|
|
19181
|
+
};
|
|
19182
|
+
for (const root of await this.loadSyncRoots())
|
|
19183
|
+
add(root.path, "cloud-files");
|
|
19184
|
+
for (const root of await this.loadDiskRoots())
|
|
19185
|
+
add(root.path, root.kind);
|
|
19186
|
+
for (const searchRoot of searchRoots) {
|
|
19187
|
+
const share = uncShareRoot(searchRoot);
|
|
19188
|
+
if (share !== undefined)
|
|
19189
|
+
add(share, "network");
|
|
19190
|
+
}
|
|
19191
|
+
return roots;
|
|
19085
19192
|
}
|
|
19086
|
-
async
|
|
19087
|
-
const
|
|
19088
|
-
|
|
19089
|
-
|
|
19090
|
-
|
|
19091
|
-
|
|
19092
|
-
|
|
19093
|
-
const timedOut = () => ({
|
|
19094
|
-
ok: false,
|
|
19095
|
-
reason: "search-failed",
|
|
19096
|
-
message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
|
|
19097
|
-
});
|
|
19098
|
-
for (;; ) {
|
|
19099
|
-
const remainingMs = deadline - Date.now();
|
|
19100
|
-
if (remainingMs <= 0)
|
|
19101
|
-
return timedOut();
|
|
19102
|
-
const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
|
|
19103
|
-
const response = await this.run(esPath, args, remainingMs + 5000);
|
|
19104
|
-
if (response.code !== 0)
|
|
19105
|
-
return this.failure("search-failed", esPath, args, response);
|
|
19106
|
-
if (Date.now() >= deadline)
|
|
19107
|
-
return timedOut();
|
|
19108
|
-
let entries;
|
|
19109
|
-
try {
|
|
19110
|
-
entries = parseEverythingJson(response.stdout);
|
|
19111
|
-
} catch (error) {
|
|
19112
|
-
return { ok: false, reason: "search-failed", message: error.message };
|
|
19193
|
+
async datalessAmong(paths) {
|
|
19194
|
+
const dataless = new Set;
|
|
19195
|
+
for (let start = 0;start < paths.length; start += WINDOWS_ATTRIBUTE_BATCH) {
|
|
19196
|
+
const batch = paths.slice(start, start + WINDOWS_ATTRIBUTE_BATCH);
|
|
19197
|
+
const result = await this.run(buildWindowsAttributeScript(batch), 30000);
|
|
19198
|
+
if (!result.ok) {
|
|
19199
|
+
throw new Error(`Could not read Windows cloud file attributes: ${result.stderr.trim() || "powershell.exe failed"}`);
|
|
19113
19200
|
}
|
|
19114
|
-
for (const
|
|
19115
|
-
if (
|
|
19116
|
-
|
|
19117
|
-
if (skipped < offset) {
|
|
19118
|
-
skipped += 1;
|
|
19119
|
-
continue;
|
|
19120
|
-
}
|
|
19121
|
-
results.push(entry);
|
|
19122
|
-
if (results.length >= maxResults)
|
|
19123
|
-
return { ok: true, results };
|
|
19201
|
+
for (const [path, attributes] of parseWindowsAttributeOutput(result.stdout, batch)) {
|
|
19202
|
+
if (datalessFromWindowsAttributes(attributes))
|
|
19203
|
+
dataless.add(path);
|
|
19124
19204
|
}
|
|
19125
|
-
if (entries.length < MAX_FILTER_CANDIDATES)
|
|
19126
|
-
return { ok: true, results };
|
|
19127
|
-
nativeOffset += entries.length;
|
|
19128
19205
|
}
|
|
19206
|
+
return dataless;
|
|
19129
19207
|
}
|
|
19130
|
-
|
|
19131
|
-
|
|
19132
|
-
|
|
19133
|
-
if (!es.ok)
|
|
19134
|
-
return es;
|
|
19135
|
-
const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
|
|
19136
|
-
const saveArgs = ["-instance", this.instanceName, "-save-db"];
|
|
19137
|
-
const saved = await this.run(es.esPath, saveArgs, timeoutMs);
|
|
19138
|
-
if (saved.code !== 0)
|
|
19139
|
-
return this.failure("index-failed", es.esPath, saveArgs, saved);
|
|
19140
|
-
const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
|
|
19141
|
-
const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
|
|
19142
|
-
if (counted.code !== 0)
|
|
19143
|
-
return this.failure("index-failed", es.esPath, countArgs, counted);
|
|
19144
|
-
return {
|
|
19145
|
-
ok: true,
|
|
19146
|
-
indexedFolders: [...this.options.folders],
|
|
19147
|
-
indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
|
|
19148
|
-
};
|
|
19149
|
-
});
|
|
19208
|
+
async providerOf(path) {
|
|
19209
|
+
const root = longestSyncRoot(await this.loadSyncRoots(), path);
|
|
19210
|
+
return root === undefined ? cloudProviderFromPath(path) : providerFromWindowsSyncRoot(root.label);
|
|
19150
19211
|
}
|
|
19151
|
-
|
|
19152
|
-
return
|
|
19153
|
-
const binaries = await this.resolveBinaries("cached");
|
|
19154
|
-
if (!binaries.ok)
|
|
19155
|
-
return;
|
|
19156
|
-
await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
19157
|
-
});
|
|
19212
|
+
canRelease(provider) {
|
|
19213
|
+
return provider !== undefined;
|
|
19158
19214
|
}
|
|
19159
|
-
async
|
|
19160
|
-
|
|
19161
|
-
return {
|
|
19162
|
-
ok: false,
|
|
19163
|
-
reason: "unsupported-platform",
|
|
19164
|
-
message: `Everything is Windows-only; this host is ${this.platform}.`
|
|
19165
|
-
};
|
|
19166
|
-
}
|
|
19167
|
-
if (this.binaries?.ok)
|
|
19168
|
-
return this.binaries;
|
|
19169
|
-
const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
|
|
19170
|
-
const result = await resolve(mode);
|
|
19171
|
-
if (result.ok)
|
|
19172
|
-
this.binaries = result;
|
|
19173
|
-
return result;
|
|
19215
|
+
async evict(path) {
|
|
19216
|
+
return parseWindowsDehydrateOutput(await this.run(buildWindowsDehydrateScript(path), 60000));
|
|
19174
19217
|
}
|
|
19175
|
-
|
|
19176
|
-
|
|
19177
|
-
|
|
19178
|
-
return binaries;
|
|
19179
|
-
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
19180
|
-
let ping = await this.ping(binaries.esPath);
|
|
19181
|
-
if (ping.code === 0 && !reconfigure)
|
|
19182
|
-
return { ok: true, esPath: binaries.esPath };
|
|
19183
|
-
if (ping.code === 0) {
|
|
19184
|
-
const exitArgs = ["-instance", this.instanceName, "-exit"];
|
|
19185
|
-
const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
|
|
19186
|
-
if (exited.code !== 0)
|
|
19187
|
-
return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
|
|
19188
|
-
}
|
|
19189
|
-
mkdirSync15(this.stateDir, { recursive: true });
|
|
19190
|
-
const iniPath = join29(this.stateDir, "Everything.ini");
|
|
19191
|
-
writeFileSync11(iniPath, buildEverythingIni({
|
|
19192
|
-
folders: this.options.folders,
|
|
19193
|
-
excludeFolders: [
|
|
19194
|
-
...new Set([
|
|
19195
|
-
join29(this.options.root, ".autorag"),
|
|
19196
|
-
...this.options.folders.map((folder) => join29(folder, ".autorag")),
|
|
19197
|
-
...this.options.excludeFolders ?? []
|
|
19198
|
-
])
|
|
19199
|
-
]
|
|
19200
|
-
}));
|
|
19201
|
-
const launchArgs = [
|
|
19202
|
-
"-instance",
|
|
19203
|
-
this.instanceName,
|
|
19204
|
-
"-config",
|
|
19205
|
-
iniPath,
|
|
19206
|
-
"-db",
|
|
19207
|
-
join29(this.stateDir, "Everything.db"),
|
|
19208
|
-
"-startup"
|
|
19209
|
-
];
|
|
19210
|
-
try {
|
|
19211
|
-
this.launch(binaries.everythingPath, launchArgs);
|
|
19212
|
-
} catch (error) {
|
|
19213
|
-
return {
|
|
19214
|
-
ok: false,
|
|
19215
|
-
reason: "startup-failed",
|
|
19216
|
-
message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
|
|
19217
|
-
};
|
|
19218
|
-
}
|
|
19219
|
-
const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
|
|
19220
|
-
const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
19221
|
-
for (;; ) {
|
|
19222
|
-
ping = await this.ping(binaries.esPath);
|
|
19223
|
-
if (ping.code === 0)
|
|
19224
|
-
return { ok: true, esPath: binaries.esPath };
|
|
19225
|
-
if (Date.now() >= deadline)
|
|
19226
|
-
break;
|
|
19227
|
-
await sleep2(pollMs);
|
|
19228
|
-
}
|
|
19229
|
-
return {
|
|
19230
|
-
ok: false,
|
|
19231
|
-
reason: "startup-failed",
|
|
19232
|
-
message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
|
|
19233
|
-
};
|
|
19218
|
+
loadSyncRoots() {
|
|
19219
|
+
this.syncRoots ??= this.run(buildWindowsSyncRootsScript(), 15000).then((result) => result.ok ? parseWindowsSyncRoots(result.stdout) : [], () => []);
|
|
19220
|
+
return this.syncRoots;
|
|
19234
19221
|
}
|
|
19235
|
-
|
|
19236
|
-
|
|
19237
|
-
return
|
|
19222
|
+
loadDiskRoots() {
|
|
19223
|
+
this.disks ??= this.run(buildWindowsDiskScript(), 30000).then((result) => result.ok ? windowsRootsFromDisks(parseWindowsLogicalDisks(result.stdout)) : [], () => []);
|
|
19224
|
+
return this.disks;
|
|
19238
19225
|
}
|
|
19239
|
-
|
|
19226
|
+
}
|
|
19227
|
+
function longestSyncRoot(roots, target) {
|
|
19228
|
+
const normalized = target.replaceAll("/", "\\").toLowerCase();
|
|
19229
|
+
let match;
|
|
19230
|
+
let matchLength = -1;
|
|
19231
|
+
for (const root of roots) {
|
|
19232
|
+
const prefix = root.path.replaceAll("/", "\\").replace(/\\+$/u, "").toLowerCase();
|
|
19233
|
+
if (prefix === "" || prefix.length <= matchLength)
|
|
19234
|
+
continue;
|
|
19235
|
+
if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
|
|
19236
|
+
match = root;
|
|
19237
|
+
matchLength = prefix.length;
|
|
19238
|
+
}
|
|
19239
|
+
}
|
|
19240
|
+
return match;
|
|
19241
|
+
}
|
|
19242
|
+
function powerShellLaunch(script) {
|
|
19243
|
+
return {
|
|
19244
|
+
args: [
|
|
19245
|
+
"-NoProfile",
|
|
19246
|
+
"-NonInteractive",
|
|
19247
|
+
"-Command",
|
|
19248
|
+
`[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; Invoke-Expression $env:${POWERSHELL_SCRIPT_VARIABLE}`
|
|
19249
|
+
],
|
|
19250
|
+
env: { [POWERSHELL_SCRIPT_VARIABLE]: script }
|
|
19251
|
+
};
|
|
19252
|
+
}
|
|
19253
|
+
async function runPowerShell(script, timeoutMs) {
|
|
19254
|
+
const launch = powerShellLaunch(script);
|
|
19255
|
+
try {
|
|
19256
|
+
const { stdout, stderr } = await execFileAsync("powershell.exe", [...launch.args], {
|
|
19257
|
+
timeout: timeoutMs,
|
|
19258
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
19259
|
+
windowsHide: true,
|
|
19260
|
+
env: { ...process.env, ...launch.env }
|
|
19261
|
+
});
|
|
19262
|
+
return { ok: true, stdout, stderr };
|
|
19263
|
+
} catch (error) {
|
|
19264
|
+
const record = typeof error === "object" && error !== null ? error : {};
|
|
19240
19265
|
return {
|
|
19241
19266
|
ok: false,
|
|
19242
|
-
|
|
19243
|
-
|
|
19267
|
+
stdout: typeof record.stdout === "string" ? record.stdout : "",
|
|
19268
|
+
stderr: typeof record.stderr === "string" ? record.stderr : String(error)
|
|
19244
19269
|
};
|
|
19245
19270
|
}
|
|
19246
19271
|
}
|
|
19247
|
-
function
|
|
19248
|
-
|
|
19249
|
-
|
|
19250
|
-
|
|
19251
|
-
|
|
19252
|
-
|
|
19253
|
-
|
|
19254
|
-
|
|
19255
|
-
|
|
19256
|
-
|
|
19257
|
-
|
|
19258
|
-
|
|
19259
|
-
if (bytes > MAX_OUTPUT_BYTES) {
|
|
19260
|
-
overflow = true;
|
|
19261
|
-
child.kill();
|
|
19262
|
-
return;
|
|
19263
|
-
}
|
|
19264
|
-
target.push(chunk);
|
|
19265
|
-
};
|
|
19266
|
-
child.stdout.on("data", collect(stdout));
|
|
19267
|
-
child.stderr.on("data", collect(stderr));
|
|
19268
|
-
const timer = setTimeout(() => child.kill(), timeoutMs);
|
|
19269
|
-
child.on("error", (error) => {
|
|
19270
|
-
clearTimeout(timer);
|
|
19271
|
-
resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
|
|
19272
|
-
});
|
|
19273
|
-
child.on("close", (code, signal) => {
|
|
19274
|
-
clearTimeout(timer);
|
|
19275
|
-
const err = Buffer.concat(stderr).toString("utf8");
|
|
19276
|
-
resolvePromise({
|
|
19277
|
-
code,
|
|
19278
|
-
stdout: Buffer.concat(stdout).toString("utf8"),
|
|
19279
|
-
stderr: overflow ? `${err}
|
|
19280
|
-
output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
|
|
19281
|
-
killed by ${signal} after ${timeoutMs}ms` : err
|
|
19272
|
+
function defaultCloudProbe() {
|
|
19273
|
+
if (process.platform === "darwin")
|
|
19274
|
+
return new DarwinCloudProbe;
|
|
19275
|
+
if (process.platform === "win32")
|
|
19276
|
+
return new WindowsCloudProbe;
|
|
19277
|
+
return;
|
|
19278
|
+
}
|
|
19279
|
+
async function readFileProviderDomain(target) {
|
|
19280
|
+
try {
|
|
19281
|
+
await lstat(target);
|
|
19282
|
+
const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
|
|
19283
|
+
timeout: 2000
|
|
19282
19284
|
});
|
|
19283
|
-
|
|
19284
|
-
|
|
19285
|
+
const value = stdout.trim();
|
|
19286
|
+
return value.length > 0 ? value : undefined;
|
|
19287
|
+
} catch {
|
|
19288
|
+
return;
|
|
19289
|
+
}
|
|
19285
19290
|
}
|
|
19286
|
-
function
|
|
19287
|
-
|
|
19288
|
-
|
|
19289
|
-
|
|
19291
|
+
async function readFileProviderDomainUpwards(target) {
|
|
19292
|
+
let cursor = target;
|
|
19293
|
+
for (let depth = 0;depth < 12; depth += 1) {
|
|
19294
|
+
const domain = await readFileProviderDomain(cursor);
|
|
19295
|
+
if (domain !== undefined)
|
|
19296
|
+
return domain;
|
|
19297
|
+
const parent = dirname15(cursor);
|
|
19298
|
+
if (parent === cursor)
|
|
19299
|
+
return;
|
|
19300
|
+
cursor = parent;
|
|
19301
|
+
}
|
|
19302
|
+
return;
|
|
19290
19303
|
}
|
|
19291
|
-
|
|
19292
|
-
|
|
19293
|
-
|
|
19294
|
-
|
|
19295
|
-
|
|
19296
|
-
|
|
19297
|
-
|
|
19298
|
-
|
|
19299
|
-
|
|
19300
|
-
|
|
19301
|
-
var
|
|
19302
|
-
|
|
19303
|
-
|
|
19304
|
+
function quotePowerShell(value) {
|
|
19305
|
+
return `'${value.replace(/['\u2018\u2019\u201A\u201B]/gu, (quote) => quote + quote)}'`;
|
|
19306
|
+
}
|
|
19307
|
+
var execFileAsync, SF_DATALESS = 1073741824, FILE_PROVIDER_XATTR = "com.apple.file-provider-domain-id", WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 4194304, STAT_BATCH = 200, defaultCloudExec = (file, args, options) => execFileAsync(file, [...args], {
|
|
19308
|
+
timeout: options?.timeoutMs ?? 30000,
|
|
19309
|
+
...options?.maxBuffer === undefined ? {} : { maxBuffer: options.maxBuffer }
|
|
19310
|
+
}), JXA_RELEASE_PROGRAM = `ObjC.import("Foundation");
|
|
19311
|
+
function run(argv) {
|
|
19312
|
+
var url = $.NSURL.fileURLWithPath(argv[0]);
|
|
19313
|
+
var errorRef = Ref();
|
|
19314
|
+
var released = $.NSFileManager.defaultManager.evictUbiquitousItemAtURLError(url, errorRef);
|
|
19315
|
+
if (released) return JSON.stringify({ ok: true });
|
|
19316
|
+
function resourceValue(key) {
|
|
19317
|
+
var ref = Ref();
|
|
19318
|
+
if (!url.getResourceValueForKeyError(ref, key, null)) return null;
|
|
19319
|
+
var value = ref[0];
|
|
19320
|
+
return value === undefined || value === null ? null : ObjC.unwrap(value);
|
|
19321
|
+
}
|
|
19322
|
+
return JSON.stringify({
|
|
19323
|
+
ok: false,
|
|
19324
|
+
isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
|
|
19325
|
+
isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
|
|
19326
|
+
downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
|
|
19327
|
+
});
|
|
19328
|
+
}`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
|
|
19329
|
+
var init_cloud_placeholder = __esm(() => {
|
|
19330
|
+
execFileAsync = promisify(execFile2);
|
|
19331
|
+
RELEASABLE_DARWIN_PROVIDERS = {
|
|
19332
|
+
icloud: true,
|
|
19333
|
+
"google-drive": true,
|
|
19334
|
+
onedrive: true,
|
|
19335
|
+
dropbox: true
|
|
19336
|
+
};
|
|
19337
|
+
DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
|
|
19338
|
+
WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
|
|
19339
|
+
WINDOWS_DEHYDRATE_REFUSALS = {
|
|
19340
|
+
"80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
|
|
19341
|
+
"80070188": "The file is pinned to stay on this device, so it is not released."
|
|
19342
|
+
};
|
|
19304
19343
|
});
|
|
19305
19344
|
|
|
19306
19345
|
// src/fsearch/walk.ts
|
|
19307
19346
|
import { readdir as readdir2, stat as stat3 } from "node:fs/promises";
|
|
19308
|
-
import { basename as
|
|
19347
|
+
import { basename as basename7, join as join30, sep as sep5 } from "node:path";
|
|
19309
19348
|
function compare(a, b) {
|
|
19310
19349
|
return a < b ? -1 : a > b ? 1 : 0;
|
|
19311
19350
|
}
|
|
@@ -19316,11 +19355,11 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
19316
19355
|
const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_DEADLINE_MS);
|
|
19317
19356
|
const matches = request.regex ? (() => {
|
|
19318
19357
|
const pattern = new RegExp(request.query, request.matchCase ? "" : "i");
|
|
19319
|
-
return (path) => pattern.test(request.matchPath ? path :
|
|
19358
|
+
return (path) => pattern.test(request.matchPath ? path : basename7(path));
|
|
19320
19359
|
})() : (() => {
|
|
19321
19360
|
const needle = request.matchCase ? request.query : request.query.toLowerCase();
|
|
19322
19361
|
return (path) => {
|
|
19323
|
-
const haystack = request.matchPath ? path :
|
|
19362
|
+
const haystack = request.matchPath ? path : basename7(path);
|
|
19324
19363
|
return (request.matchCase ? haystack : haystack.toLowerCase()).includes(needle);
|
|
19325
19364
|
};
|
|
19326
19365
|
})();
|
|
@@ -19375,7 +19414,7 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
19375
19414
|
continue;
|
|
19376
19415
|
collected.push({
|
|
19377
19416
|
path,
|
|
19378
|
-
name:
|
|
19417
|
+
name: basename7(path),
|
|
19379
19418
|
type: info.isDirectory() ? "folder" : "file",
|
|
19380
19419
|
size: info.isDirectory() ? undefined : info.size,
|
|
19381
19420
|
dateModified: info.mtime.toISOString()
|
|
@@ -19400,7 +19439,7 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
19400
19439
|
continue;
|
|
19401
19440
|
collected.push({
|
|
19402
19441
|
path: folder,
|
|
19403
|
-
name:
|
|
19442
|
+
name: basename7(folder),
|
|
19404
19443
|
type: "folder",
|
|
19405
19444
|
size: undefined,
|
|
19406
19445
|
dateModified: info.mtime.toISOString()
|
|
@@ -19433,7 +19472,7 @@ var init_walk = __esm(() => {
|
|
|
19433
19472
|
import { spawn as spawn15 } from "node:child_process";
|
|
19434
19473
|
import { createHash as createHash13 } from "node:crypto";
|
|
19435
19474
|
import { existsSync as existsSync18, mkdirSync as mkdirSync16, readFileSync as readFileSync18, realpathSync as realpathSync4, rmSync as rmSync8, writeFileSync as writeFileSync12 } from "node:fs";
|
|
19436
|
-
import { basename as
|
|
19475
|
+
import { basename as basename8, join as join31, resolve as resolve12, sep as sep6 } from "node:path";
|
|
19437
19476
|
import { setTimeout as sleep3 } from "node:timers/promises";
|
|
19438
19477
|
function fsearchSocketPath(dbPath) {
|
|
19439
19478
|
let canonical;
|
|
@@ -19497,7 +19536,7 @@ function parseFSearchJsonLines(stdout) {
|
|
|
19497
19536
|
const path = String(record.path ?? "");
|
|
19498
19537
|
entries.push({
|
|
19499
19538
|
path,
|
|
19500
|
-
name:
|
|
19539
|
+
name: basename8(path),
|
|
19501
19540
|
type: record.type === "folder" ? "folder" : "file",
|
|
19502
19541
|
size: typeof record.size === "number" ? record.size : undefined,
|
|
19503
19542
|
dateModified: typeof record.mtime === "number" ? new Date(record.mtime * 1000).toISOString() : undefined
|
|
@@ -19915,7 +19954,7 @@ var init_diagnostics = __esm(() => {
|
|
|
19915
19954
|
});
|
|
19916
19955
|
|
|
19917
19956
|
// src/jikji/retry-command.ts
|
|
19918
|
-
import { basename as
|
|
19957
|
+
import { basename as basename9 } from "node:path";
|
|
19919
19958
|
function shellWords(command) {
|
|
19920
19959
|
const words = [];
|
|
19921
19960
|
let current = "";
|
|
@@ -19982,7 +20021,7 @@ function parseJikjiRetryCommand(command, envelopeProof) {
|
|
|
19982
20021
|
const words = shellWords(command);
|
|
19983
20022
|
if (words === undefined || words.length < 4)
|
|
19984
20023
|
return;
|
|
19985
|
-
const program =
|
|
20024
|
+
const program = basename9(words[0]).replace(/\.exe$/iu, "");
|
|
19986
20025
|
if (program !== "jikji" || words[1] !== "discover")
|
|
19987
20026
|
return;
|
|
19988
20027
|
const root = words[2];
|
|
@@ -20390,7 +20429,7 @@ class SerialQueue {
|
|
|
20390
20429
|
|
|
20391
20430
|
// src/jikji/version.ts
|
|
20392
20431
|
function parseJikjiVersion(output) {
|
|
20393
|
-
return
|
|
20432
|
+
return VERSION_LINE3.exec(output)?.[1];
|
|
20394
20433
|
}
|
|
20395
20434
|
function parts(version) {
|
|
20396
20435
|
return version.split(".").map((part) => Number.parseInt(part, 10));
|
|
@@ -20406,9 +20445,9 @@ function isSupportedJikjiVersion(version) {
|
|
|
20406
20445
|
}
|
|
20407
20446
|
return true;
|
|
20408
20447
|
}
|
|
20409
|
-
var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1",
|
|
20410
|
-
var
|
|
20411
|
-
|
|
20448
|
+
var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1", VERSION_LINE3;
|
|
20449
|
+
var init_version3 = __esm(() => {
|
|
20450
|
+
VERSION_LINE3 = /^jikji(?:-cli)?\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
|
|
20412
20451
|
});
|
|
20413
20452
|
|
|
20414
20453
|
// src/jikji/client.ts
|
|
@@ -20776,7 +20815,7 @@ var init_client11 = __esm(() => {
|
|
|
20776
20815
|
init_answer_pack();
|
|
20777
20816
|
init_installer3();
|
|
20778
20817
|
init_prepare_budget();
|
|
20779
|
-
|
|
20818
|
+
init_version3();
|
|
20780
20819
|
ABORTED_BEFORE_START = { ok: false, reason: "aborted", stdout: "", stderr: "", code: null };
|
|
20781
20820
|
});
|
|
20782
20821
|
|
|
@@ -21861,7 +21900,7 @@ import {
|
|
|
21861
21900
|
rmSync as rmSync10,
|
|
21862
21901
|
writeFileSync as writeFileSync14
|
|
21863
21902
|
} from "node:fs";
|
|
21864
|
-
import { basename as
|
|
21903
|
+
import { basename as basename10, dirname as dirname18, isAbsolute as isAbsolute9, join as join36, relative as relative7, sep as sep8 } from "node:path";
|
|
21865
21904
|
function syncMinSyncWorkspace(root, options = {}) {
|
|
21866
21905
|
const workspacePath = options.workspacePath ?? minSyncWorkspaceRoot(root);
|
|
21867
21906
|
const filesRoot = join36(workspacePath, "files");
|
|
@@ -21902,7 +21941,7 @@ function syncMinSyncWorkspace(root, options = {}) {
|
|
|
21902
21941
|
return { workspacePath, entries, excluded };
|
|
21903
21942
|
}
|
|
21904
21943
|
function stagingStatePath(workspacePath) {
|
|
21905
|
-
return join36(dirname18(workspacePath), `.${
|
|
21944
|
+
return join36(dirname18(workspacePath), `.${basename10(workspacePath)}-autorag-staging.json`);
|
|
21906
21945
|
}
|
|
21907
21946
|
function loadStagingState2(workspacePath) {
|
|
21908
21947
|
const path = stagingStatePath(workspacePath);
|
|
@@ -21953,7 +21992,7 @@ function ensureManagedDirectory(filesRoot, directory) {
|
|
|
21953
21992
|
}
|
|
21954
21993
|
}
|
|
21955
21994
|
function copyFileAtomically(source, destination) {
|
|
21956
|
-
const temporaryPath = join36(dirname18(destination), `.${
|
|
21995
|
+
const temporaryPath = join36(dirname18(destination), `.${basename10(destination)}.${randomUUID6()}.tmp`);
|
|
21957
21996
|
try {
|
|
21958
21997
|
copyFileSync4(source, temporaryPath);
|
|
21959
21998
|
renameWithRetry(temporaryPath, destination);
|
|
@@ -22007,7 +22046,7 @@ var init_workspace = __esm(() => {
|
|
|
22007
22046
|
|
|
22008
22047
|
// src/minsync/method.ts
|
|
22009
22048
|
import { existsSync as existsSync23, readFileSync as readFileSync21, realpathSync as realpathSync7, statSync as statSync7 } from "node:fs";
|
|
22010
|
-
import { basename as
|
|
22049
|
+
import { basename as basename11, join as join37, normalize } from "node:path";
|
|
22011
22050
|
|
|
22012
22051
|
class MinSyncVectorMethod {
|
|
22013
22052
|
root;
|
|
@@ -22108,7 +22147,7 @@ class MinSyncVectorMethod {
|
|
|
22108
22147
|
const entry = byPath.get(hit.path);
|
|
22109
22148
|
if (!entry || !matchesVirtualPathScope(entry.virtualPath, options.scope))
|
|
22110
22149
|
continue;
|
|
22111
|
-
const chunkDiscriminator = hit.docId ??
|
|
22150
|
+
const chunkDiscriminator = hit.docId ?? basename11(hit.path);
|
|
22112
22151
|
const chunkId = `minsync:${entry.virtualPath}:${chunkDiscriminator}`;
|
|
22113
22152
|
const location = locateChunkInMirror(mirrorTexts, entry, hit.text);
|
|
22114
22153
|
results.push({
|
|
@@ -22208,18 +22247,24 @@ var init_minsync = __esm(() => {
|
|
|
22208
22247
|
|
|
22209
22248
|
// src/mirror/cloud-fingerprints.ts
|
|
22210
22249
|
import { existsSync as existsSync24, readFileSync as readFileSync22 } from "node:fs";
|
|
22211
|
-
import { join as join38 } from "node:path";
|
|
22250
|
+
import { extname as extname5, join as join38 } from "node:path";
|
|
22212
22251
|
function cloudFingerprintStorePath(root) {
|
|
22213
22252
|
return join38(root, ".autorag", CLOUD_FINGERPRINTS_FILE);
|
|
22214
22253
|
}
|
|
22215
22254
|
function emptyCloudFingerprints() {
|
|
22216
|
-
return { version:
|
|
22255
|
+
return { version: 2, entries: {} };
|
|
22217
22256
|
}
|
|
22218
|
-
function
|
|
22257
|
+
function readCloudFingerprint(sourcePath, value, storeVersion) {
|
|
22219
22258
|
if (typeof value !== "object" || value === null)
|
|
22220
|
-
return
|
|
22259
|
+
return;
|
|
22221
22260
|
const record = value;
|
|
22222
|
-
|
|
22261
|
+
const { sourceVersion, exact, modified, revision } = record;
|
|
22262
|
+
if (typeof sourceVersion !== "string" || typeof exact !== "string" || modified !== null && typeof modified !== "string" || revision !== null && typeof revision !== "string") {
|
|
22263
|
+
return;
|
|
22264
|
+
}
|
|
22265
|
+
if (storeVersion !== 2 && (record.chars === 0 || extname5(sourcePath).toLowerCase() === ".hwp"))
|
|
22266
|
+
return;
|
|
22267
|
+
return { sourceVersion, exact, modified, revision };
|
|
22223
22268
|
}
|
|
22224
22269
|
function loadCloudFingerprints(root) {
|
|
22225
22270
|
const path = cloudFingerprintStorePath(root);
|
|
@@ -22232,24 +22277,26 @@ function loadCloudFingerprints(root) {
|
|
|
22232
22277
|
const { entries } = parsed;
|
|
22233
22278
|
if (typeof entries !== "object" || entries === null)
|
|
22234
22279
|
return emptyCloudFingerprints();
|
|
22280
|
+
const storeVersion = "version" in parsed ? parsed.version : undefined;
|
|
22235
22281
|
const valid = {};
|
|
22236
|
-
for (const [sourcePath,
|
|
22237
|
-
|
|
22282
|
+
for (const [sourcePath, value] of Object.entries(entries)) {
|
|
22283
|
+
const fingerprint = readCloudFingerprint(sourcePath, value, storeVersion);
|
|
22284
|
+
if (fingerprint !== undefined)
|
|
22238
22285
|
valid[sourcePath] = fingerprint;
|
|
22239
22286
|
}
|
|
22240
|
-
return { version:
|
|
22287
|
+
return { version: 2, entries: valid };
|
|
22241
22288
|
} catch {
|
|
22242
22289
|
return emptyCloudFingerprints();
|
|
22243
22290
|
}
|
|
22244
22291
|
}
|
|
22245
22292
|
function saveCloudFingerprints(root, entries) {
|
|
22246
|
-
writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version:
|
|
22293
|
+
writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version: 2, entries }, null, 2)}
|
|
22247
22294
|
`);
|
|
22248
22295
|
}
|
|
22249
22296
|
function cloudFingerprintExclusions(store) {
|
|
22250
22297
|
const map = new Map;
|
|
22251
22298
|
for (const [sourcePath, fingerprint] of Object.entries(store.entries)) {
|
|
22252
|
-
map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified
|
|
22299
|
+
map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified });
|
|
22253
22300
|
}
|
|
22254
22301
|
return map;
|
|
22255
22302
|
}
|
|
@@ -22641,7 +22688,7 @@ class Parser {
|
|
|
22641
22688
|
}
|
|
22642
22689
|
|
|
22643
22690
|
// src/parser/csv.ts
|
|
22644
|
-
import { basename as
|
|
22691
|
+
import { basename as basename12, extname as extname6 } from "node:path";
|
|
22645
22692
|
function toCell(text) {
|
|
22646
22693
|
return text.trim() === "" ? undefined : { text, kind: classifyText(text) };
|
|
22647
22694
|
}
|
|
@@ -22772,7 +22819,7 @@ var init_csv = __esm(() => {
|
|
|
22772
22819
|
this.#maxRows = options.maxRows ?? DEFAULT_MAX_ROWS;
|
|
22773
22820
|
}
|
|
22774
22821
|
async parse(input) {
|
|
22775
|
-
const extension =
|
|
22822
|
+
const extension = extname6(input.virtualPath);
|
|
22776
22823
|
const format = extension.slice(1).toLowerCase();
|
|
22777
22824
|
let text = decodeText(input.bytes).replace(/^\uFEFF/, "");
|
|
22778
22825
|
const directive = SEPARATOR_DIRECTIVE.exec(text);
|
|
@@ -22784,10 +22831,10 @@ var init_csv = __esm(() => {
|
|
|
22784
22831
|
{
|
|
22785
22832
|
code: "parser-warning",
|
|
22786
22833
|
severity: "warning",
|
|
22787
|
-
message: `"${
|
|
22834
|
+
message: `"${basename12(input.virtualPath)}": indexed the first ${this.#maxRows} of ${total} rows`
|
|
22788
22835
|
}
|
|
22789
22836
|
] : [];
|
|
22790
|
-
const grid = { name:
|
|
22837
|
+
const grid = { name: basename12(input.virtualPath, extension), rows: rows.map((row) => row.map(toCell)) };
|
|
22791
22838
|
return {
|
|
22792
22839
|
markdown: normalizeMarkdown(renderWorkbook([grid])),
|
|
22793
22840
|
metadata: { parser: this.name, format },
|
|
@@ -23326,7 +23373,7 @@ var init_plain_text = __esm(() => {
|
|
|
23326
23373
|
});
|
|
23327
23374
|
|
|
23328
23375
|
// src/parser/registry.ts
|
|
23329
|
-
import { extname as
|
|
23376
|
+
import { extname as extname7 } from "node:path";
|
|
23330
23377
|
|
|
23331
23378
|
class ParserRegistry {
|
|
23332
23379
|
parsersByExtension = new Map;
|
|
@@ -23345,7 +23392,7 @@ class ParserRegistry {
|
|
|
23345
23392
|
}
|
|
23346
23393
|
}
|
|
23347
23394
|
getForVirtualPath(virtualPath) {
|
|
23348
|
-
return this.parsersByExtension.get(normalizeExtension(
|
|
23395
|
+
return this.parsersByExtension.get(normalizeExtension(extname7(virtualPath)));
|
|
23349
23396
|
}
|
|
23350
23397
|
list() {
|
|
23351
23398
|
return Array.from(new Set(this.parsersByExtension.values()));
|
|
@@ -23358,7 +23405,7 @@ function normalizeExtension(extension) {
|
|
|
23358
23405
|
var init_registry = () => {};
|
|
23359
23406
|
|
|
23360
23407
|
// src/parser/spreadsheet.ts
|
|
23361
|
-
import { extname as
|
|
23408
|
+
import { extname as extname8 } from "node:path";
|
|
23362
23409
|
import * as XLSX from "xlsx";
|
|
23363
23410
|
function assertWorkbookSignature(format, bytes) {
|
|
23364
23411
|
if (startsWith(bytes, ZIP_SIGNATURE) || startsWith(bytes, OLE_SIGNATURE))
|
|
@@ -23457,7 +23504,7 @@ var init_spreadsheet = __esm(() => {
|
|
|
23457
23504
|
this.#maxRowsPerSheet = options.maxRowsPerSheet ?? DEFAULT_MAX_ROWS_PER_SHEET;
|
|
23458
23505
|
}
|
|
23459
23506
|
async parse(input) {
|
|
23460
|
-
const format =
|
|
23507
|
+
const format = extname8(input.virtualPath).slice(1).toLowerCase();
|
|
23461
23508
|
let workbook;
|
|
23462
23509
|
try {
|
|
23463
23510
|
assertWorkbookSignature(format, input.bytes);
|
|
@@ -23731,6 +23778,10 @@ class CloudGate {
|
|
|
23731
23778
|
isPlaceholder(path) {
|
|
23732
23779
|
return this.placeholders.has(path);
|
|
23733
23780
|
}
|
|
23781
|
+
startBatch() {
|
|
23782
|
+
this.downloadedFiles = 0;
|
|
23783
|
+
this.downloadedBytes = 0;
|
|
23784
|
+
}
|
|
23734
23785
|
async decide(path, sizeBytes, directory) {
|
|
23735
23786
|
if (this.probe === undefined || !this.placeholders.has(path))
|
|
23736
23787
|
return LOCAL;
|
|
@@ -23740,7 +23791,7 @@ class CloudGate {
|
|
|
23740
23791
|
const overFiles = this.downloadedFiles >= this.maxFiles;
|
|
23741
23792
|
const overBytes = this.downloadedFiles > 0 && this.downloadedBytes + sizeBytes > this.maxBytes;
|
|
23742
23793
|
if (overFiles || overBytes)
|
|
23743
|
-
return { kind: "
|
|
23794
|
+
return { kind: "next-batch" };
|
|
23744
23795
|
this.downloadedFiles += 1;
|
|
23745
23796
|
this.downloadedBytes += sizeBytes;
|
|
23746
23797
|
return { kind: "download" };
|
|
@@ -23785,7 +23836,7 @@ var init_cloud = __esm(() => {
|
|
|
23785
23836
|
// src/mirror/sync.ts
|
|
23786
23837
|
import { createHash as createHash17 } from "node:crypto";
|
|
23787
23838
|
import { opendir as opendir2, readFile, stat as stat4 } from "node:fs/promises";
|
|
23788
|
-
import { basename as
|
|
23839
|
+
import { basename as basename13, dirname as dirname21, extname as extname9, join as join42, resolve as resolve15, sep as sep9 } from "node:path";
|
|
23789
23840
|
import { setTimeout as sleep4 } from "node:timers/promises";
|
|
23790
23841
|
function errorCode(error) {
|
|
23791
23842
|
if (!(error instanceof Error))
|
|
@@ -23989,195 +24040,221 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
23989
24040
|
});
|
|
23990
24041
|
const pending = new Set(plan.decisions.filter((decision) => decision.action === "process" || decision.action === "retry").map((decision) => decision.key));
|
|
23991
24042
|
const deferred = new Map(plan.decisions.filter((decision) => decision.deferred === true).map((decision) => [decision.key, decision]));
|
|
23992
|
-
|
|
23993
|
-
|
|
23994
|
-
|
|
23995
|
-
const path = pathsByKey.get(decision.key);
|
|
23996
|
-
await session.delete(decision.key);
|
|
23997
|
-
if (path === undefined)
|
|
23998
|
-
continue;
|
|
23999
|
-
if (previous.entries[path])
|
|
24000
|
-
deleted += 1;
|
|
24001
|
-
if (!currentPaths.has(path))
|
|
24002
|
-
diagnostics.push({
|
|
24003
|
-
code: "deleted-mirror",
|
|
24004
|
-
severity: "info",
|
|
24005
|
-
message: "A previously indexed document is gone; its parsed mirror was removed.",
|
|
24006
|
-
source: path
|
|
24007
|
-
});
|
|
24008
|
-
}
|
|
24009
|
-
await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
|
|
24010
|
-
for (const [position, entry] of current.entries()) {
|
|
24011
|
-
options.onProgress?.({
|
|
24012
|
-
processed: position,
|
|
24013
|
-
total: current.length,
|
|
24014
|
-
written,
|
|
24015
|
-
deleted,
|
|
24016
|
-
skipped,
|
|
24017
|
-
currentVirtualPath: entry.virtualPath
|
|
24018
|
-
});
|
|
24019
|
-
if (options.signal?.aborted)
|
|
24020
|
-
throw abortReason(options.signal);
|
|
24021
|
-
const omission = omitted.get(entry.virtualPath);
|
|
24022
|
-
if (omission !== undefined) {
|
|
24023
|
-
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
24024
|
-
if (previous.entries[entry.virtualPath])
|
|
24025
|
-
deleted += 1;
|
|
24026
|
-
await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
|
|
24027
|
-
}
|
|
24028
|
-
skipped += 1;
|
|
24029
|
-
if (omission.diagnostic)
|
|
24030
|
-
diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
|
|
24031
|
-
continue;
|
|
24032
|
-
}
|
|
24033
|
-
const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
|
|
24034
|
-
if (deferredFailure !== undefined) {
|
|
24035
|
-
skipped += 1;
|
|
24036
|
-
const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
|
|
24037
|
-
diagnostics.push({
|
|
24038
|
-
code,
|
|
24039
|
-
severity: "warning",
|
|
24040
|
-
source: entry.virtualPath,
|
|
24041
|
-
message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
|
|
24042
|
-
});
|
|
24043
|
-
continue;
|
|
24044
|
-
}
|
|
24045
|
-
const parser = registry.getForVirtualPath(entry.virtualPath);
|
|
24046
|
-
const outputPath = parsedOutputPath(options.root, entry.virtualPath);
|
|
24047
|
-
const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname21(entry.sourcePath)) : { kind: "local" };
|
|
24048
|
-
if (cloudDecision.kind === "skip") {
|
|
24049
|
-
skipped += 1;
|
|
24050
|
-
diagnostics.push({
|
|
24051
|
-
code: "cloud-placeholder-skipped",
|
|
24052
|
-
severity: "info",
|
|
24053
|
-
message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
|
|
24054
|
-
source: entry.virtualPath,
|
|
24055
|
-
...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
|
|
24056
|
-
});
|
|
24057
|
-
continue;
|
|
24058
|
-
}
|
|
24059
|
-
if (cloudDecision.kind === "defer") {
|
|
24060
|
-
skipped += 1;
|
|
24061
|
-
diagnostics.push({
|
|
24062
|
-
code: "cloud-download-deferred",
|
|
24063
|
-
severity: "info",
|
|
24064
|
-
message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
|
|
24065
|
-
source: entry.virtualPath
|
|
24066
|
-
});
|
|
24067
|
-
continue;
|
|
24068
|
-
}
|
|
24069
|
-
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
24070
|
-
let bytes;
|
|
24071
|
-
try {
|
|
24072
|
-
bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
|
|
24073
|
-
} catch (error) {
|
|
24074
|
-
const failure = sourceReadFailure(error);
|
|
24075
|
-
await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
|
|
24076
|
-
continue;
|
|
24077
|
-
}
|
|
24078
|
-
const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
|
|
24079
|
-
if (tooSmallImage !== undefined) {
|
|
24080
|
-
await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
|
|
24043
|
+
const applyDeleteDecisions = async () => {
|
|
24044
|
+
for (const decision of plan.decisions) {
|
|
24045
|
+
if (decision.action !== "delete")
|
|
24081
24046
|
continue;
|
|
24082
|
-
|
|
24083
|
-
|
|
24084
|
-
|
|
24085
|
-
const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
|
|
24086
|
-
parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
|
|
24087
|
-
} catch (error) {
|
|
24088
|
-
if (options.signal?.aborted)
|
|
24089
|
-
throw abortReason(options.signal);
|
|
24090
|
-
if (error instanceof ParseError)
|
|
24091
|
-
await skipFile(entry, {
|
|
24092
|
-
code: "parser-failed",
|
|
24093
|
-
severity: "warning",
|
|
24094
|
-
message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
|
|
24095
|
-
}, "parser-failed");
|
|
24096
|
-
else
|
|
24097
|
-
await skipFile(entry, {
|
|
24098
|
-
code: "parser-crashed",
|
|
24099
|
-
severity: "warning",
|
|
24100
|
-
message: PARSER_CRASH_MESSAGE,
|
|
24101
|
-
...error instanceof Error ? { reason: error.name } : {}
|
|
24102
|
-
});
|
|
24047
|
+
const path = pathsByKey.get(decision.key);
|
|
24048
|
+
await session.delete(decision.key);
|
|
24049
|
+
if (path === undefined)
|
|
24103
24050
|
continue;
|
|
24104
|
-
|
|
24105
|
-
|
|
24106
|
-
|
|
24107
|
-
|
|
24108
|
-
|
|
24109
|
-
|
|
24051
|
+
if (previous.entries[path])
|
|
24052
|
+
deleted += 1;
|
|
24053
|
+
if (!currentPaths.has(path))
|
|
24054
|
+
diagnostics.push({
|
|
24055
|
+
code: "deleted-mirror",
|
|
24056
|
+
severity: "info",
|
|
24057
|
+
message: "A previously indexed document is gone; its parsed mirror was removed.",
|
|
24058
|
+
source: path
|
|
24059
|
+
});
|
|
24060
|
+
}
|
|
24061
|
+
};
|
|
24062
|
+
const deferDeletions = options.onBatch !== undefined;
|
|
24063
|
+
const batchSize = Math.max(1, Math.floor(options.batchSize ?? DEFAULT_MIRROR_BATCH_SIZE));
|
|
24064
|
+
let batchIndex = 0;
|
|
24065
|
+
let batchStopped = false;
|
|
24066
|
+
const runBatch = async (virtualPaths, final) => {
|
|
24067
|
+
if (options.onBatch === undefined)
|
|
24068
|
+
return;
|
|
24069
|
+
batchIndex += 1;
|
|
24070
|
+
const outcome = await options.onBatch({ index: batchIndex, virtualPaths, final });
|
|
24071
|
+
if (outcome?.stop === true)
|
|
24072
|
+
batchStopped = true;
|
|
24073
|
+
};
|
|
24074
|
+
if (!deferDeletions)
|
|
24075
|
+
await applyDeleteDecisions();
|
|
24076
|
+
await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
|
|
24077
|
+
let batchStart = 0;
|
|
24078
|
+
for (;; ) {
|
|
24079
|
+
const batchLimit = Math.min(batchStart + batchSize, current.length);
|
|
24080
|
+
let batchEnd = batchLimit;
|
|
24081
|
+
const changedBefore = written + deleted;
|
|
24082
|
+
gate.startBatch();
|
|
24083
|
+
for (let position = batchStart;position < batchLimit; position += 1) {
|
|
24084
|
+
const entry = current[position];
|
|
24085
|
+
options.onProgress?.({
|
|
24086
|
+
processed: position,
|
|
24087
|
+
total: current.length,
|
|
24088
|
+
written,
|
|
24089
|
+
deleted,
|
|
24090
|
+
skipped,
|
|
24091
|
+
currentVirtualPath: entry.virtualPath
|
|
24092
|
+
});
|
|
24093
|
+
if (options.signal?.aborted)
|
|
24094
|
+
throw abortReason(options.signal);
|
|
24095
|
+
const omission = omitted.get(entry.virtualPath);
|
|
24096
|
+
if (omission !== undefined) {
|
|
24097
|
+
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
24098
|
+
if (previous.entries[entry.virtualPath])
|
|
24099
|
+
deleted += 1;
|
|
24100
|
+
await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
|
|
24101
|
+
}
|
|
24102
|
+
skipped += 1;
|
|
24103
|
+
if (omission.diagnostic)
|
|
24104
|
+
diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
|
|
24110
24105
|
continue;
|
|
24111
24106
|
}
|
|
24112
|
-
const
|
|
24113
|
-
if (
|
|
24107
|
+
const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
|
|
24108
|
+
if (deferredFailure !== undefined) {
|
|
24114
24109
|
skipped += 1;
|
|
24110
|
+
const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
|
|
24115
24111
|
diagnostics.push({
|
|
24116
|
-
code
|
|
24112
|
+
code,
|
|
24117
24113
|
severity: "warning",
|
|
24118
|
-
|
|
24119
|
-
|
|
24114
|
+
source: entry.virtualPath,
|
|
24115
|
+
message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
|
|
24120
24116
|
});
|
|
24121
|
-
await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
|
|
24122
24117
|
continue;
|
|
24123
24118
|
}
|
|
24124
|
-
const
|
|
24125
|
-
|
|
24126
|
-
|
|
24127
|
-
|
|
24128
|
-
|
|
24129
|
-
|
|
24130
|
-
|
|
24131
|
-
|
|
24132
|
-
|
|
24133
|
-
|
|
24134
|
-
|
|
24135
|
-
...code !== undefined ? { reason: code } : {}
|
|
24119
|
+
const parser = registry.getForVirtualPath(entry.virtualPath);
|
|
24120
|
+
const outputPath = parsedOutputPath(options.root, entry.virtualPath);
|
|
24121
|
+
const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname21(entry.sourcePath)) : { kind: "local" };
|
|
24122
|
+
if (cloudDecision.kind === "skip") {
|
|
24123
|
+
skipped += 1;
|
|
24124
|
+
diagnostics.push({
|
|
24125
|
+
code: "cloud-placeholder-skipped",
|
|
24126
|
+
severity: "info",
|
|
24127
|
+
message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
|
|
24128
|
+
source: entry.virtualPath,
|
|
24129
|
+
...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
|
|
24136
24130
|
});
|
|
24137
24131
|
continue;
|
|
24138
24132
|
}
|
|
24139
|
-
|
|
24140
|
-
|
|
24141
|
-
|
|
24142
|
-
}
|
|
24143
|
-
|
|
24144
|
-
|
|
24145
|
-
|
|
24146
|
-
|
|
24147
|
-
|
|
24148
|
-
|
|
24149
|
-
|
|
24150
|
-
|
|
24151
|
-
|
|
24152
|
-
|
|
24153
|
-
|
|
24154
|
-
|
|
24155
|
-
|
|
24156
|
-
|
|
24157
|
-
|
|
24158
|
-
|
|
24159
|
-
|
|
24160
|
-
|
|
24161
|
-
|
|
24133
|
+
if (cloudDecision.kind === "next-batch") {
|
|
24134
|
+
batchEnd = position;
|
|
24135
|
+
break;
|
|
24136
|
+
}
|
|
24137
|
+
if (pending.has(mirrorStageKey(entry.virtualPath))) {
|
|
24138
|
+
let bytes;
|
|
24139
|
+
try {
|
|
24140
|
+
bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
|
|
24141
|
+
} catch (error) {
|
|
24142
|
+
const failure = sourceReadFailure(error);
|
|
24143
|
+
await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
|
|
24144
|
+
continue;
|
|
24145
|
+
}
|
|
24146
|
+
const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
|
|
24147
|
+
if (tooSmallImage !== undefined) {
|
|
24148
|
+
await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
|
|
24149
|
+
continue;
|
|
24150
|
+
}
|
|
24151
|
+
let parsed;
|
|
24152
|
+
try {
|
|
24153
|
+
const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
|
|
24154
|
+
parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
|
|
24155
|
+
} catch (error) {
|
|
24156
|
+
if (options.signal?.aborted)
|
|
24157
|
+
throw abortReason(options.signal);
|
|
24158
|
+
if (error instanceof ParseError)
|
|
24159
|
+
await skipFile(entry, {
|
|
24160
|
+
code: "parser-failed",
|
|
24162
24161
|
severity: "warning",
|
|
24163
|
-
message:
|
|
24164
|
-
|
|
24162
|
+
message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
|
|
24163
|
+
}, "parser-failed");
|
|
24164
|
+
else
|
|
24165
|
+
await skipFile(entry, {
|
|
24166
|
+
code: "parser-crashed",
|
|
24167
|
+
severity: "warning",
|
|
24168
|
+
message: PARSER_CRASH_MESSAGE,
|
|
24169
|
+
...error instanceof Error ? { reason: error.name } : {}
|
|
24165
24170
|
});
|
|
24166
|
-
|
|
24171
|
+
continue;
|
|
24167
24172
|
}
|
|
24168
|
-
|
|
24169
|
-
|
|
24170
|
-
|
|
24171
|
-
|
|
24173
|
+
let info;
|
|
24174
|
+
try {
|
|
24175
|
+
info = await stat4(entry.sourcePath, { bigint: true });
|
|
24176
|
+
} catch (error) {
|
|
24177
|
+
await skipFile(entry, sourceReadFailure(error));
|
|
24178
|
+
continue;
|
|
24179
|
+
}
|
|
24180
|
+
const unchanged = cloudDecision.kind === "download" ? contentIdentity(info) === entry.contentIdentity : statIdentity(info) === entry.statIdentity;
|
|
24181
|
+
if (!unchanged) {
|
|
24182
|
+
skipped += 1;
|
|
24172
24183
|
diagnostics.push({
|
|
24173
|
-
code: "
|
|
24184
|
+
code: "source-changed",
|
|
24174
24185
|
severity: "warning",
|
|
24175
|
-
message:
|
|
24186
|
+
message: "The source changed while parsing; its last good mirror was kept and it will be retried on the next refresh.",
|
|
24176
24187
|
source: entry.virtualPath
|
|
24177
24188
|
});
|
|
24189
|
+
await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
|
|
24190
|
+
continue;
|
|
24191
|
+
}
|
|
24192
|
+
const markdown = normalizeMarkdown(parsed.markdown);
|
|
24193
|
+
try {
|
|
24194
|
+
await withTransientRetry(() => writeFileAtomic(outputPath, markdown), transientRetries, transientBackoffMs);
|
|
24195
|
+
} catch (error) {
|
|
24196
|
+
const code = errorCode(error);
|
|
24197
|
+
if (code !== undefined && Object.hasOwn(WORKSPACE_ERROR_CODES, code))
|
|
24198
|
+
throw new MirrorWorkspaceError(options.root, error);
|
|
24199
|
+
await skipFile(entry, {
|
|
24200
|
+
code: "mirror-write-failed",
|
|
24201
|
+
severity: "warning",
|
|
24202
|
+
message: error instanceof Error ? error.message : String(error),
|
|
24203
|
+
...code !== undefined ? { reason: code } : {}
|
|
24204
|
+
});
|
|
24205
|
+
continue;
|
|
24206
|
+
}
|
|
24207
|
+
await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
|
|
24208
|
+
path: basename13(outputPath),
|
|
24209
|
+
size: Buffer.byteLength(markdown)
|
|
24210
|
+
});
|
|
24211
|
+
for (const diagnostic of parsed.diagnostics ?? [])
|
|
24212
|
+
diagnostics.push({ ...diagnostic, source: entry.virtualPath });
|
|
24213
|
+
written += 1;
|
|
24214
|
+
if (fingerprintCloud && cloudFilePaths.has(entry.sourcePath)) {
|
|
24215
|
+
const existing = cloudFingerprintEntries[entry.sourcePath];
|
|
24216
|
+
if (existing === undefined || existing.sourceVersion !== entry.sourceVersion) {
|
|
24217
|
+
try {
|
|
24218
|
+
const fingerprintResult = await fingerprintWithDupey(entry.sourcePath, options.cloudFingerprints?.dupey ?? {});
|
|
24219
|
+
cloudFingerprintEntries[entry.sourcePath] = {
|
|
24220
|
+
sourceVersion: entry.sourceVersion,
|
|
24221
|
+
exact: fingerprintResult.exact,
|
|
24222
|
+
modified: fingerprintResult.modified,
|
|
24223
|
+
revision: fingerprintResult.revision
|
|
24224
|
+
};
|
|
24225
|
+
cloudFingerprintsDirty = true;
|
|
24226
|
+
} catch (error) {
|
|
24227
|
+
diagnostics.push({
|
|
24228
|
+
code: "cloud-fingerprint-failed",
|
|
24229
|
+
severity: "warning",
|
|
24230
|
+
message: `The cloud or network file was indexed but its duplicate fingerprint could not be read, so it will not take part in exact-duplicate exclusion: ${error instanceof Error ? error.message : String(error)}`,
|
|
24231
|
+
source: entry.virtualPath
|
|
24232
|
+
});
|
|
24233
|
+
}
|
|
24234
|
+
}
|
|
24235
|
+
}
|
|
24236
|
+
if (cloudDecision.kind === "download" && gate.evictAfterParse) {
|
|
24237
|
+
const released = await gate.release(entry.sourcePath);
|
|
24238
|
+
if (!released.ok) {
|
|
24239
|
+
diagnostics.push({
|
|
24240
|
+
code: "cloud-evict-failed",
|
|
24241
|
+
severity: "warning",
|
|
24242
|
+
message: `The file was indexed but its local copy could not be released: ${released.message}`,
|
|
24243
|
+
source: entry.virtualPath
|
|
24244
|
+
});
|
|
24245
|
+
}
|
|
24178
24246
|
}
|
|
24179
24247
|
}
|
|
24180
24248
|
}
|
|
24249
|
+
const finalBatch = batchEnd >= current.length;
|
|
24250
|
+
if (finalBatch && deferDeletions)
|
|
24251
|
+
await applyDeleteDecisions();
|
|
24252
|
+
if (finalBatch || written + deleted !== changedBefore) {
|
|
24253
|
+
await runBatch(current.slice(batchStart, batchEnd).map((item) => item.virtualPath), finalBatch);
|
|
24254
|
+
}
|
|
24255
|
+
if (finalBatch || batchStopped)
|
|
24256
|
+
break;
|
|
24257
|
+
batchStart = batchEnd;
|
|
24181
24258
|
}
|
|
24182
24259
|
await session.finish();
|
|
24183
24260
|
if (fingerprintCloud) {
|
|
@@ -24267,7 +24344,7 @@ async function detectMirrorStaleness(options) {
|
|
|
24267
24344
|
}
|
|
24268
24345
|
}
|
|
24269
24346
|
function standaloneImageTooSmallMessage(virtualPath, bytes) {
|
|
24270
|
-
if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(
|
|
24347
|
+
if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname9(virtualPath))))
|
|
24271
24348
|
return;
|
|
24272
24349
|
return ocrImageTooSmallMessage(bytes);
|
|
24273
24350
|
}
|
|
@@ -24435,7 +24512,7 @@ async function collectFiles(state, directory, depth) {
|
|
|
24435
24512
|
}
|
|
24436
24513
|
if (!entry.isFile())
|
|
24437
24514
|
continue;
|
|
24438
|
-
const extension = normalizeExtension2(
|
|
24515
|
+
const extension = normalizeExtension2(extname9(entry.name));
|
|
24439
24516
|
if (!state.supportedExtensions.has(extension))
|
|
24440
24517
|
continue;
|
|
24441
24518
|
let fileStat;
|
|
@@ -24469,7 +24546,7 @@ async function collectFiles(state, directory, depth) {
|
|
|
24469
24546
|
warnDirectoryUnreadable(state, directory, code);
|
|
24470
24547
|
}
|
|
24471
24548
|
}
|
|
24472
|
-
var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
|
|
24549
|
+
var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
|
|
24473
24550
|
var init_sync = __esm(() => {
|
|
24474
24551
|
init_cli2();
|
|
24475
24552
|
init_cloud_placeholder();
|
|
@@ -29998,7 +30075,7 @@ var init_automation_tool = __esm(() => {
|
|
|
29998
30075
|
});
|
|
29999
30076
|
|
|
30000
30077
|
// src/agent/citation-links.ts
|
|
30001
|
-
import { basename as
|
|
30078
|
+
import { basename as basename14, isAbsolute as isAbsolute12 } from "node:path";
|
|
30002
30079
|
import { pathToFileURL as pathToFileURL3 } from "node:url";
|
|
30003
30080
|
function openableTarget(source) {
|
|
30004
30081
|
if (/^https?:\/\//iu.test(source)) {
|
|
@@ -30016,7 +30093,7 @@ function linkCitations(markdown, sourceOf, isFile) {
|
|
|
30016
30093
|
for (const marker of answerMarkers(markdown, isFile)) {
|
|
30017
30094
|
if (marker.kind === "number")
|
|
30018
30095
|
continue;
|
|
30019
|
-
const labels = marker.kind === "file" ? [{ label:
|
|
30096
|
+
const labels = marker.kind === "file" ? [{ label: basename14(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
|
|
30020
30097
|
let linked = false;
|
|
30021
30098
|
const parts = labels.map(({ label, source }) => {
|
|
30022
30099
|
const target = source === undefined ? undefined : openableTarget(source);
|
|
@@ -32142,10 +32219,7 @@ function renderScanReport(details) {
|
|
|
32142
32219
|
const lines = [
|
|
32143
32220
|
`dupey scanned ${details.scans.length} configured root(s).`,
|
|
32144
32221
|
`families=${details.familyCount} exactDuplicates=${details.exactDuplicateCount} extractionErrors=${errors}`,
|
|
32145
|
-
"Review exact families before cleanup; near/contains families are not safe deletion evidence."
|
|
32146
|
-
...details.skippedRoots.length === 0 ? [] : [
|
|
32147
|
-
`Not scanned (inside a cloud or network folder; scanning would download or re-read every file): ${details.skippedRoots.join(", ")}. Duplicates there are not reported, and an exact duplicate is only found against files an earlier refresh fingerprinted.`
|
|
32148
|
-
]
|
|
32222
|
+
"Review exact families before cleanup; near/contains families are not safe deletion evidence."
|
|
32149
32223
|
];
|
|
32150
32224
|
let rendered = 0;
|
|
32151
32225
|
let omitted = 0;
|
|
@@ -33123,6 +33197,56 @@ var init_packaged_skill = __esm(() => {
|
|
|
33123
33197
|
PACKAGED_SKILL_ROOTS = ["skills", "internal-skills"];
|
|
33124
33198
|
});
|
|
33125
33199
|
|
|
33200
|
+
// src/agent/pipelined-sync.ts
|
|
33201
|
+
class PipelinedSync {
|
|
33202
|
+
#sync;
|
|
33203
|
+
#covered = [];
|
|
33204
|
+
#dirty = false;
|
|
33205
|
+
#running;
|
|
33206
|
+
#failure;
|
|
33207
|
+
#stopped = false;
|
|
33208
|
+
#runs = 0;
|
|
33209
|
+
constructor(options) {
|
|
33210
|
+
this.#sync = options.sync;
|
|
33211
|
+
}
|
|
33212
|
+
async batch(batch) {
|
|
33213
|
+
this.#raise();
|
|
33214
|
+
this.#covered.push(...batch.virtualPaths);
|
|
33215
|
+
this.#dirty = true;
|
|
33216
|
+
if (!this.#stopped)
|
|
33217
|
+
this.#running ??= this.#loop();
|
|
33218
|
+
if (batch.final)
|
|
33219
|
+
await this.#running;
|
|
33220
|
+
this.#raise();
|
|
33221
|
+
return this.#stopped ? { stop: true } : undefined;
|
|
33222
|
+
}
|
|
33223
|
+
async settle() {
|
|
33224
|
+
this.#stopped = true;
|
|
33225
|
+
await this.#running;
|
|
33226
|
+
}
|
|
33227
|
+
#raise() {
|
|
33228
|
+
if (this.#failure !== undefined)
|
|
33229
|
+
throw this.#failure.error;
|
|
33230
|
+
}
|
|
33231
|
+
async#loop() {
|
|
33232
|
+
try {
|
|
33233
|
+
while (this.#dirty && !this.#stopped) {
|
|
33234
|
+
this.#dirty = false;
|
|
33235
|
+
const virtualPaths = this.#covered;
|
|
33236
|
+
this.#covered = [];
|
|
33237
|
+
this.#runs += 1;
|
|
33238
|
+
const outcome = await this.#sync({ index: this.#runs, virtualPaths });
|
|
33239
|
+
if (outcome?.stop === true)
|
|
33240
|
+
this.#stopped = true;
|
|
33241
|
+
}
|
|
33242
|
+
} catch (error) {
|
|
33243
|
+
this.#failure = { error };
|
|
33244
|
+
} finally {
|
|
33245
|
+
this.#running = undefined;
|
|
33246
|
+
}
|
|
33247
|
+
}
|
|
33248
|
+
}
|
|
33249
|
+
|
|
33126
33250
|
// src/agent/query-decomposition.ts
|
|
33127
33251
|
import { completeSimple as completeSimple2 } from "@earendil-works/pi-ai/compat";
|
|
33128
33252
|
function buildDecompositionPrompt(query) {
|
|
@@ -33362,7 +33486,7 @@ var init_query_routing = __esm(() => {
|
|
|
33362
33486
|
false: "Every part of the question is answered with a concrete, specific, consistent fact (a name, date, number, or definite statement)."
|
|
33363
33487
|
});
|
|
33364
33488
|
BASE_ROUTE_OPTIONS = {
|
|
33365
|
-
local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. When unsure whether intrinsic knowledge suffices, choose local.",
|
|
33489
|
+
local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. A request to find, locate, open, or look up a document, file, slide deck, presentation, paper, note, message, or other material is local even when it names no owner ('find the slides about X', 'where is the Y presentation', 'Z 관련 자료 찾아줘'): such material lives in the user's own files and chats, and a well-known topic is not a reason to answer from general knowledge or the web. When unsure whether intrinsic knowledge suffices, choose local.",
|
|
33366
33490
|
extra: "None of the other options fits and no search of the user's own records is needed first. Either the request is answerable without looking anything up — stable common knowledge, a definition, simple reasoning, or small talk — or it is a task for the assistant's tools that is not file organization and not AutoRAG configuration: committing code, pushing a branch, opening a pull request, running tests or a build, installing a package, running a script, writing or editing code. Programming work is extra even when the code names the user's files or folders ('write a script that lowercases every PNG', 'add this to my parser'): the assistant is asked to produce or change code, not to act on those files now. So is a request to run something (tests, a build, a script, a command) and report the outcome, even when what it runs lives in the user's own repository: running it is a task for the tool, not a search of the user's records. Never extra when the request refers to the user's own life, files, records, chats, calendar, or history (e.g. 'my', 'I', 'our', 'my girlfriend', 'my case', 'my routine'), and never extra when the answer depends on facts that may be recent, specific, or changing (current events, recent releases, prices, versions): search those locally instead. If any other option fits, do not choose extra.",
|
|
33367
33491
|
organize: "The user asks the assistant to DO something to files or folders instead of finding information: move or copy files into folders, tidy or sort a folder, bulk or pattern rename (for example by date or number), rename files according to what their content says, convert files to another format, create folder structures, group files by type or topic, archive, or move to the trash ('tidy this up', 'move these into folders', 'bulk rename these files', 'rename the files to match their content', 'convert these to PDF', 'sort my Downloads folder'). It is about acting on files and folders, never about looking up what a document says or where something is. A question that only asks to find or read a file's content is local, not organize."
|
|
33368
33492
|
};
|
|
@@ -34600,7 +34724,7 @@ var init_page_loader = __esm(() => {
|
|
|
34600
34724
|
});
|
|
34601
34725
|
|
|
34602
34726
|
// src/web/fetch/url-target.ts
|
|
34603
|
-
import { extname as
|
|
34727
|
+
import { extname as extname10 } from "node:path";
|
|
34604
34728
|
function repairCollapsedScheme(value) {
|
|
34605
34729
|
const m = value.match(/^(https?):\/(?!\/)/i);
|
|
34606
34730
|
return m ? `${m[1]}://${value.slice(m[0].length)}` : value;
|
|
@@ -34619,7 +34743,7 @@ function getFilenameExtensionHint(filename) {
|
|
|
34619
34743
|
const lower = filename.toLowerCase();
|
|
34620
34744
|
if (lower.endsWith(".tar.gz"))
|
|
34621
34745
|
return ".tar.gz";
|
|
34622
|
-
return
|
|
34746
|
+
return extname10(filename).toLowerCase();
|
|
34623
34747
|
}
|
|
34624
34748
|
function getExtensionHint(url, contentDisposition) {
|
|
34625
34749
|
if (contentDisposition) {
|
|
@@ -34796,7 +34920,7 @@ var init_convert2 = __esm(() => {
|
|
|
34796
34920
|
});
|
|
34797
34921
|
|
|
34798
34922
|
// src/web/fetch/feeds.ts
|
|
34799
|
-
import { extname as
|
|
34923
|
+
import { extname as extname11 } from "node:path";
|
|
34800
34924
|
import { XMLParser as XMLParser3 } from "fast-xml-parser";
|
|
34801
34925
|
function getHtmlAttribute(tag, attribute) {
|
|
34802
34926
|
const pattern = new RegExp(`\\b${attribute}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'=<>]+))`, "i");
|
|
@@ -34859,7 +34983,7 @@ function extractDocumentLinks(html, baseUrl) {
|
|
|
34859
34983
|
const href = getHtmlAttribute(tag, "href");
|
|
34860
34984
|
if (!href)
|
|
34861
34985
|
continue;
|
|
34862
|
-
const ext =
|
|
34986
|
+
const ext = extname11(href).toLowerCase();
|
|
34863
34987
|
if (!CONVERTIBLE_EXTENSIONS2.has(ext))
|
|
34864
34988
|
continue;
|
|
34865
34989
|
const resolved = href.startsWith("http") ? href : new URL(href, baseUrl).href;
|
|
@@ -36527,7 +36651,7 @@ var init_web_search_tool = __esm(() => {
|
|
|
36527
36651
|
// src/agent/agent.ts
|
|
36528
36652
|
import { randomUUID as randomUUID7 } from "node:crypto";
|
|
36529
36653
|
import { existsSync as existsSync34, watch as fsWatch, mkdirSync as mkdirSync24, realpathSync as realpathSync8, statSync as statSync11, writeFileSync as writeFileSync17 } from "node:fs";
|
|
36530
|
-
import { basename as
|
|
36654
|
+
import { basename as basename15, dirname as dirname27, join as join54, relative as relative8, resolve as resolve18, sep as sep10 } from "node:path";
|
|
36531
36655
|
import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
|
|
36532
36656
|
import { getAgentDir as getAgentDir6, keyText } from "@earendil-works/pi-coding-agent";
|
|
36533
36657
|
function describeDecisionModel(status) {
|
|
@@ -36962,10 +37086,20 @@ class AutoRAGAgent {
|
|
|
36962
37086
|
}
|
|
36963
37087
|
async scanDuplicateDocuments() {
|
|
36964
37088
|
if (this.dupeyOptions === false) {
|
|
36965
|
-
return { scans: [],
|
|
37089
|
+
return { scans: [], familyCount: 0, exactDuplicateCount: 0 };
|
|
37090
|
+
}
|
|
37091
|
+
const cloudRoots = await this.cloudRoots();
|
|
37092
|
+
const scanRoots = [];
|
|
37093
|
+
for (const searchPath of this.searchPaths) {
|
|
37094
|
+
const scope = cloudScope(searchPath, cloudRoots);
|
|
37095
|
+
if (scope.insideCloudRoot)
|
|
37096
|
+
continue;
|
|
37097
|
+
scanRoots.push({
|
|
37098
|
+
path: searchPath,
|
|
37099
|
+
excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path))
|
|
37100
|
+
});
|
|
36966
37101
|
}
|
|
36967
|
-
const {
|
|
36968
|
-
const scans = await Promise.all(targets.map(({ path, excludeDirs }) => scanWithDupey(path, {
|
|
37102
|
+
const scans = await Promise.all(scanRoots.map(({ path, excludeDirs }) => scanWithDupey(path, {
|
|
36969
37103
|
...this.dupeyOptions || {},
|
|
36970
37104
|
...excludeDirs.length === 0 ? {} : { excludeDirs }
|
|
36971
37105
|
})));
|
|
@@ -36979,7 +37113,7 @@ class AutoRAGAgent {
|
|
|
36979
37113
|
}
|
|
36980
37114
|
return count + [...hashes.values()].reduce((sum, size) => sum + Math.max(0, size - 1), 0);
|
|
36981
37115
|
}, 0);
|
|
36982
|
-
return { scans,
|
|
37116
|
+
return { scans, familyCount, exactDuplicateCount };
|
|
36983
37117
|
}
|
|
36984
37118
|
loadMemoryContextFor(query) {
|
|
36985
37119
|
if (this.memoryContextCache?.query === query)
|
|
@@ -38632,13 +38766,67 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38632
38766
|
inFlight: true,
|
|
38633
38767
|
lastStartedAt: startedAt
|
|
38634
38768
|
};
|
|
38769
|
+
const syncAbort = new AbortController;
|
|
38770
|
+
const forwardAbort = () => syncAbort.abort(signal?.reason);
|
|
38771
|
+
signal?.addEventListener("abort", forwardAbort, { once: true });
|
|
38772
|
+
let minSyncPipeline;
|
|
38635
38773
|
try {
|
|
38636
38774
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38775
|
+
const batchedMinSync = needsParsed && wants("minsync");
|
|
38776
|
+
const minSyncBatches = [];
|
|
38777
|
+
const minSyncBatchFailures = [];
|
|
38778
|
+
let minSyncSyncedBase = 0;
|
|
38779
|
+
let minSyncTotalBase = 0;
|
|
38780
|
+
minSyncPipeline = batchedMinSync ? new PipelinedSync({
|
|
38781
|
+
sync: async ({ index, virtualPaths }) => {
|
|
38782
|
+
syncAbort.signal.throwIfAborted();
|
|
38783
|
+
let syncTotal = minSyncTotalBase;
|
|
38784
|
+
const result = await this.syncMinSync(force && index === 1, syncAbort.signal, (minsyncProgress) => {
|
|
38785
|
+
syncTotal = minSyncTotalBase + minsyncProgress.total;
|
|
38786
|
+
progress = updateRefreshProgress(progress, {
|
|
38787
|
+
minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: syncTotal }
|
|
38788
|
+
});
|
|
38789
|
+
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38790
|
+
emitProgress({
|
|
38791
|
+
phase: "minsync",
|
|
38792
|
+
processed: minSyncSyncedBase + minsyncProgress.processed,
|
|
38793
|
+
total: syncTotal
|
|
38794
|
+
});
|
|
38795
|
+
});
|
|
38796
|
+
minSyncBatches.push(result);
|
|
38797
|
+
if (result.ok)
|
|
38798
|
+
minSyncBatchFailures.length = 0;
|
|
38799
|
+
else if (result.reason !== "lock-held") {
|
|
38800
|
+
for (const virtualPath of virtualPaths) {
|
|
38801
|
+
minSyncBatchFailures.push({
|
|
38802
|
+
code: "minsync-sync-failed",
|
|
38803
|
+
severity: "warning",
|
|
38804
|
+
message: `This file stayed at its previous index entry: the sync that covered it could not index it and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
|
|
38805
|
+
source: virtualPath
|
|
38806
|
+
});
|
|
38807
|
+
}
|
|
38808
|
+
}
|
|
38809
|
+
minSyncSyncedBase = syncTotal;
|
|
38810
|
+
minSyncTotalBase = syncTotal;
|
|
38811
|
+
return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
|
|
38812
|
+
}
|
|
38813
|
+
}) : undefined;
|
|
38814
|
+
const pipeline = minSyncPipeline;
|
|
38637
38815
|
const summary = needsParsed ? await this.syncParsedMirrors(force, {
|
|
38638
38816
|
...opts?.retrySkipped === true ? { retrySkipped: true } : {},
|
|
38639
38817
|
...signal === undefined ? {} : { signal },
|
|
38818
|
+
...pipeline === undefined ? {} : {
|
|
38819
|
+
onBatch: (batch) => {
|
|
38820
|
+
if (batch.final) {
|
|
38821
|
+
progress = updateRefreshProgress(progress, { phase: "minsync" });
|
|
38822
|
+
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38823
|
+
}
|
|
38824
|
+
return pipeline.batch(batch);
|
|
38825
|
+
}
|
|
38826
|
+
},
|
|
38640
38827
|
onProgress: (parsedProgress) => {
|
|
38641
38828
|
progress = updateRefreshProgress(progress, {
|
|
38829
|
+
phase: "parsed",
|
|
38642
38830
|
sourceFiles: { total: parsedProgress.total },
|
|
38643
38831
|
parsedCounts: {
|
|
38644
38832
|
scanned: parsedProgress.processed,
|
|
@@ -38660,7 +38848,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38660
38848
|
}) : await this.scanMirrorStaleness();
|
|
38661
38849
|
signal?.throwIfAborted();
|
|
38662
38850
|
progress = updateRefreshProgress(progress, {
|
|
38663
|
-
phase: wants("
|
|
38851
|
+
phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
|
|
38664
38852
|
sourceFiles: { total: summary.scanned },
|
|
38665
38853
|
parsedCounts: {
|
|
38666
38854
|
scanned: summary.scanned,
|
|
@@ -38670,24 +38858,12 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38670
38858
|
}
|
|
38671
38859
|
});
|
|
38672
38860
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38673
|
-
const minsync =
|
|
38674
|
-
progress = updateRefreshProgress(progress, {
|
|
38675
|
-
minsync: { synced: minsyncProgress.processed, total: minsyncProgress.total }
|
|
38676
|
-
});
|
|
38677
|
-
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38678
|
-
emitProgress({
|
|
38679
|
-
phase: "minsync",
|
|
38680
|
-
processed: minsyncProgress.processed,
|
|
38681
|
-
total: minsyncProgress.total
|
|
38682
|
-
});
|
|
38683
|
-
}) : undefined;
|
|
38684
|
-
signal?.throwIfAborted();
|
|
38861
|
+
const minsync = batchedMinSync ? aggregateMinSyncBatches(minSyncBatches) : undefined;
|
|
38685
38862
|
if (minsync !== undefined) {
|
|
38686
38863
|
progress = updateRefreshProgress(progress, {
|
|
38687
|
-
phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
|
|
38688
38864
|
minsync: {
|
|
38689
|
-
...progress.minsync,
|
|
38690
38865
|
synced: minsync.synced,
|
|
38866
|
+
total: Math.max(minSyncTotalBase, minsync.synced),
|
|
38691
38867
|
...minsync.filesFailed !== undefined && minsync.filesFailed.length > 0 ? { filesSkipped: minsync.filesFailed.length } : {}
|
|
38692
38868
|
}
|
|
38693
38869
|
});
|
|
@@ -38749,7 +38925,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38749
38925
|
finishedAt: new Date().toISOString()
|
|
38750
38926
|
});
|
|
38751
38927
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38752
|
-
const minsyncDiagnostics = minSyncRefreshDiagnostics(minsync);
|
|
38928
|
+
const minsyncDiagnostics = [...minSyncRefreshDiagnostics(minsync), ...minSyncBatchFailures];
|
|
38753
38929
|
const publicMinsync = minsync ? {
|
|
38754
38930
|
ok: minsync.ok,
|
|
38755
38931
|
synced: minsync.synced,
|
|
@@ -38783,6 +38959,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38783
38959
|
...fsearch !== undefined ? { fsearch } : {}
|
|
38784
38960
|
};
|
|
38785
38961
|
} catch (error) {
|
|
38962
|
+
syncAbort.abort(error);
|
|
38786
38963
|
this.refreshState = {
|
|
38787
38964
|
...this.refreshState,
|
|
38788
38965
|
lastOutcome: "failed",
|
|
@@ -38796,6 +38973,8 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
38796
38973
|
writeRefreshProgress(this.workspaceProjectRoot, progress);
|
|
38797
38974
|
throw error;
|
|
38798
38975
|
} finally {
|
|
38976
|
+
signal?.removeEventListener("abort", forwardAbort);
|
|
38977
|
+
await minSyncPipeline?.settle();
|
|
38799
38978
|
this.refreshState = {
|
|
38800
38979
|
...this.refreshState,
|
|
38801
38980
|
inFlight: false,
|
|
@@ -39017,15 +39196,15 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
|
|
|
39017
39196
|
const cloudRoots = await this.cloudRoots();
|
|
39018
39197
|
const fingerprints = cloudFingerprintExclusions(loadCloudFingerprints(this.workspaceProjectRoot));
|
|
39019
39198
|
for (const searchPath of this.searchPaths) {
|
|
39020
|
-
const
|
|
39199
|
+
const scope = cloudScope(searchPath, cloudRoots);
|
|
39021
39200
|
let scan;
|
|
39022
|
-
if (
|
|
39201
|
+
if (scope.insideCloudRoot) {
|
|
39023
39202
|
scan = { dir: searchPath, files: [], families: [], errors: [] };
|
|
39024
39203
|
} else {
|
|
39025
39204
|
try {
|
|
39026
39205
|
scan = await scanWithDupey(searchPath, {
|
|
39027
39206
|
...this.dupeyOptions || {},
|
|
39028
|
-
...
|
|
39207
|
+
...scope.nestedCloudRoots.length === 0 ? {} : { excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path)) }
|
|
39029
39208
|
});
|
|
39030
39209
|
} catch (error) {
|
|
39031
39210
|
if (!(error instanceof DupeyCliError))
|
|
@@ -39554,6 +39733,12 @@ function fsearchRefreshDiagnostics(fsearch) {
|
|
|
39554
39733
|
}
|
|
39555
39734
|
];
|
|
39556
39735
|
}
|
|
39736
|
+
function aggregateMinSyncBatches(batches) {
|
|
39737
|
+
const last = batches.at(-1);
|
|
39738
|
+
if (last === undefined)
|
|
39739
|
+
return;
|
|
39740
|
+
return { ...last, synced: batches.reduce((total, batch) => total + batch.synced, 0) };
|
|
39741
|
+
}
|
|
39557
39742
|
function minSyncRefreshDiagnostics(minsync) {
|
|
39558
39743
|
if (!minsync)
|
|
39559
39744
|
return [];
|
|
@@ -39584,7 +39769,7 @@ function toMinSyncReasonDiagnostic(reason) {
|
|
|
39584
39769
|
function pinExcludedPath(path) {
|
|
39585
39770
|
const resolvedPath = resolve18(path);
|
|
39586
39771
|
try {
|
|
39587
|
-
return join54(realpathSync8(dirname27(resolvedPath)),
|
|
39772
|
+
return join54(realpathSync8(dirname27(resolvedPath)), basename15(resolvedPath));
|
|
39588
39773
|
} catch {
|
|
39589
39774
|
return resolvedPath;
|
|
39590
39775
|
}
|