@autorag/librarian 2.13.6 → 2.13.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/profiles.js CHANGED
@@ -3627,7 +3627,7 @@ function outdatedMinSyncVersion(binaryPath) {
3627
3627
  }
3628
3628
  return version !== undefined && !isSupportedMinSyncVersion(version) ? version : undefined;
3629
3629
  }
3630
- var MINSYNC_MIN_VERSION = "0.6.0", VERSION_PROBE_TIMEOUT_MS = 5000, VERSION_LINE, probeCache;
3630
+ var MINSYNC_MIN_VERSION = "0.7.1", VERSION_PROBE_TIMEOUT_MS = 5000, VERSION_LINE, probeCache;
3631
3631
  var init_version = __esm(() => {
3632
3632
  VERSION_LINE = /^minsync\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
3633
3633
  probeCache = new Map;
@@ -3871,7 +3871,7 @@ function isGitHubAsset(value) {
3871
3871
  function isRecord2(value) {
3872
3872
  return typeof value === "object" && value !== null;
3873
3873
  }
3874
- var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.6.0", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
3874
+ var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.7.3", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
3875
3875
  var init_installer = __esm(() => {
3876
3876
  init_atomic_write();
3877
3877
  init_process();
@@ -13749,7 +13749,8 @@ class WombatMethod {
13749
13749
  return matchesDatasourceScope(datasourceSourcePath("wombat", this.instanceId, hit.chunkId), options.scope);
13750
13750
  }).map((hit) => ({
13751
13751
  id: `wombat:${this.instanceId}:${hit.chunkId}`,
13752
- content: hit.snippet,
13752
+ content: `${sessionLabel(hit)}
13753
+ ${hit.snippet}`,
13753
13754
  source: datasourceSourcePath("wombat", this.instanceId, hit.chunkId),
13754
13755
  score: hit.score,
13755
13756
  metadata: {
@@ -13772,6 +13773,20 @@ class WombatMethod {
13772
13773
  })).slice(0, topK);
13773
13774
  }
13774
13775
  }
13776
+ function oneLine(value) {
13777
+ return value.replace(/\s+/gu, " ").trim();
13778
+ }
13779
+ function sessionLabel(hit) {
13780
+ const turns = hit.startTurn === undefined ? undefined : hit.endTurn === undefined || hit.endTurn === hit.startTurn ? `turn ${hit.startTurn}` : `turns ${hit.startTurn}-${hit.endTurn}`;
13781
+ const parts = [
13782
+ oneLine(hit.catalogKey),
13783
+ turns,
13784
+ hit.startedAt === undefined ? undefined : hit.startedAt.slice(0, 10),
13785
+ hit.title.length === 0 ? undefined : oneLine(hit.title),
13786
+ hit.cwd === undefined ? undefined : oneLine(hit.cwd)
13787
+ ].filter((part) => part !== undefined && part.length > 0);
13788
+ return `[${parts.join(" | ")}]`;
13789
+ }
13775
13790
  var WOMBAT_MODES, METHOD_TYPE;
13776
13791
  var init_methods7 = __esm(() => {
13777
13792
  init_errors2();
@@ -13884,7 +13899,7 @@ class WombatSkill {
13884
13899
  `);
13885
13900
  return {
13886
13901
  name: "datasource-wombat",
13887
- description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built.",
13902
+ description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built. Each hit names its session. Load this skill to open that session's full conversation or its original log with the tool calls that the search does not index.",
13888
13903
  content: [
13889
13904
  "# wombat coding-agent session datasource",
13890
13905
  "",
@@ -13898,10 +13913,10 @@ class WombatSkill {
13898
13913
  scopes,
13899
13914
  "",
13900
13915
  "The selectable retrieval modes are keyword (SPLADE), semantic, and hybrid. Scope arguments can only narrow trusted access. Source identifiers remain opaque and hierarchical.",
13901
- "Each result carries `agent`, `title`, `startedAt`, `startTurn`/`endTurn` and `sessionKey` (`<agent>:<session id>`).",
13916
+ "Every result starts with a label line, `[<sessionKey> | turns A-B | date | title | project]`, followed by the snippet. `sessionKey` is `<agent>:<session id>`.",
13902
13917
  "",
13903
13918
  "## Native CLI",
13904
- "The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the result's `sessionKey`:",
13919
+ "The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the `sessionKey` from the result's label:",
13905
13920
  "- `wombat show <sessionKey> [--turn N]`: the full cleaned conversation, or one turn.",
13906
13921
  "- `wombat trace <sessionKey> --turn N --context K`: the original session log around that turn, including the tool calls and outputs that the index leaves out.",
13907
13922
  "- `wombat chunk context <chunk id> --json`: the turns before and after a result.",
@@ -14055,6 +14070,7 @@ function common(config, workspaceRoot) {
14055
14070
  function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
14056
14071
  const skills = [];
14057
14072
  const unknown = [];
14073
+ const retired = [];
14058
14074
  for (const [name, raw] of Object.entries(config ?? {})) {
14059
14075
  if (raw === false)
14060
14076
  continue;
@@ -14066,6 +14082,10 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
14066
14082
  if (entry.enabled === false)
14067
14083
  continue;
14068
14084
  const templateName = entry.type ?? name;
14085
+ if (Object.hasOwn(RETIRED_TEMPLATES, templateName)) {
14086
+ retired.push(name);
14087
+ continue;
14088
+ }
14069
14089
  const builder = Object.hasOwn(BUILDERS, templateName) && datasourceSkillSupported(templateName) ? BUILDERS[templateName] : undefined;
14070
14090
  if (builder === undefined) {
14071
14091
  unknown.push(name);
@@ -14089,9 +14109,9 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
14089
14109
  }
14090
14110
  skills.push(typeof entry.description === "string" && entry.description.trim().length > 0 ? new DescribedDatasourceSkill(aliased, entry.description.trim()) : aliased);
14091
14111
  }
14092
- return { skills, unknown };
14112
+ return { skills, unknown, retired };
14093
14113
  }
14094
- var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES;
14114
+ var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES, RETIRED_TEMPLATES;
14095
14115
  var init_factory = __esm(() => {
14096
14116
  init_aliased_skill();
14097
14117
  init_config2();
@@ -14247,6 +14267,7 @@ var init_factory = __esm(() => {
14247
14267
  };
14248
14268
  PLATFORM_ONLY_SKILLS = { kakao: "darwin" };
14249
14269
  BUILTIN_DATASOURCE_SKILL_NAMES = builtinDatasourceSkillNames();
14270
+ RETIRED_TEMPLATES = { spotlight: true };
14250
14271
  });
14251
14272
 
14252
14273
  // src/embedding/http-embedder.ts
@@ -15833,7 +15854,7 @@ function buildAgentOptions(config, agentDir) {
15833
15854
  const embedder = createEmbedderFromConfig({ embedder: config.minSync?.embedder });
15834
15855
  opts.embedder = embedder;
15835
15856
  if (config.datasources !== undefined) {
15836
- const { skills, unknown } = buildDatasourceSkills(config.datasources, config.workspacePath, {
15857
+ const { skills, unknown, retired } = buildDatasourceSkills(config.datasources, config.workspacePath, {
15837
15858
  embedder,
15838
15859
  discrawl: async () => discrawlEmbeddingRuntimeFor(await embedder.describeEndpoint()),
15839
15860
  cliEndpoint: embeddingEndpointSource(embedder),
@@ -15846,16 +15867,27 @@ function buildAgentOptions(config, agentDir) {
15846
15867
  });
15847
15868
  if (skills.length > 0)
15848
15869
  opts.datasourceSkills = skills;
15870
+ const diagnostics = [];
15871
+ if (retired.length > 0) {
15872
+ const names = retired.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
15873
+ diagnostics.push({
15874
+ code: "retired-datasource-skill",
15875
+ severity: "warning",
15876
+ message: `Datasource config ignored because the Spotlight datasource was removed: ${names.join(", ")}. Delete these entries; use fsearch/mirror search (searchPaths) instead.`,
15877
+ source: "datasources"
15878
+ });
15879
+ }
15849
15880
  if (unknown.length > 0) {
15850
- const safeNames = unknown.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
15851
- const diagnostic = {
15881
+ const names = unknown.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
15882
+ diagnostics.push({
15852
15883
  code: "unknown-datasource-skill",
15853
15884
  severity: "warning",
15854
- message: `Unknown datasource skill(s) in config were skipped: ${safeNames.join(", ")}`,
15885
+ message: `Unknown datasource skill(s) in config were skipped: ${names.join(", ")}`,
15855
15886
  source: "datasources"
15856
- };
15857
- opts.startupDiagnostics = [diagnostic];
15887
+ });
15858
15888
  }
15889
+ if (diagnostics.length > 0)
15890
+ opts.startupDiagnostics = diagnostics;
15859
15891
  }
15860
15892
  return opts;
15861
15893
  }
@@ -17680,11 +17712,49 @@ var init_matrix = __esm(() => {
17680
17712
  };
17681
17713
  });
17682
17714
 
17715
+ // src/dupey/version.ts
17716
+ function parseDupeyVersion(output) {
17717
+ return VERSION_LINE2.exec(output)?.[1];
17718
+ }
17719
+ function isSupportedDupeyVersion(version) {
17720
+ const have = version.split(".").map((part) => Number.parseInt(part, 10));
17721
+ const need = DUPEY_MIN_VERSION.split(".").map((part) => Number.parseInt(part, 10));
17722
+ for (let index = 0;index < need.length; index += 1) {
17723
+ const left = have[index] ?? 0;
17724
+ const right = need[index] ?? 0;
17725
+ if (left !== right)
17726
+ return left > right;
17727
+ }
17728
+ return true;
17729
+ }
17730
+ var DUPEY_MIN_VERSION = "0.2.0", VERSION_LINE2;
17731
+ var init_version2 = __esm(() => {
17732
+ VERSION_LINE2 = /^dupey\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
17733
+ });
17734
+
17683
17735
  // src/dupey/cli.ts
17684
17736
  import { spawn as spawn13 } from "node:child_process";
17685
17737
  import { resolve as resolve9 } from "node:path";
17686
- function runDupey(args, options) {
17687
- return options.run ? options.run(args) : spawnDupey(options.executable ?? "dupey", args, options);
17738
+ function checkDupeyVersion(executable, options) {
17739
+ const cached = versionChecks.get(executable);
17740
+ if (cached !== undefined)
17741
+ return cached;
17742
+ const check = spawnDupey(executable, ["--version"], options).then((stdout) => {
17743
+ const version = parseDupeyVersion(stdout);
17744
+ if (version !== undefined && !isSupportedDupeyVersion(version)) {
17745
+ throw new DupeyCliError(`dupey ${version} at ${executable} is older than the ${DUPEY_MIN_VERSION} AutoRAG requires; run \`cargo install dupey\` to update it.`);
17746
+ }
17747
+ });
17748
+ check.catch(() => versionChecks.delete(executable));
17749
+ versionChecks.set(executable, check);
17750
+ return check;
17751
+ }
17752
+ async function runDupey(args, options) {
17753
+ if (options.run)
17754
+ return options.run(args);
17755
+ const executable = options.executable ?? "dupey";
17756
+ await checkDupeyVersion(executable, options);
17757
+ return spawnDupey(executable, args, options);
17688
17758
  }
17689
17759
  async function scanWithDupey(dir, options = {}) {
17690
17760
  const absoluteDir = resolve9(dir);
@@ -17818,28 +17888,26 @@ function spawnDupey(executable, args, options) {
17818
17888
  });
17819
17889
  });
17820
17890
  }
17821
- var DupeyCliError, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
17891
+ var DupeyCliError, versionChecks, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
17822
17892
  var init_cli2 = __esm(() => {
17823
17893
  init_portable_spawn();
17894
+ init_version2();
17824
17895
  DupeyCliError = class DupeyCliError extends Error {
17825
17896
  constructor(message) {
17826
17897
  super(message);
17827
17898
  this.name = "DupeyCliError";
17828
17899
  }
17829
17900
  };
17901
+ versionChecks = new Map;
17830
17902
  });
17831
17903
 
17832
17904
  // src/dupey/filter.ts
17833
17905
  import { stat as stat2 } from "node:fs/promises";
17834
17906
  import { resolve as resolve10 } from "node:path";
17835
- function extractedNoText(file) {
17836
- const signals = file.signals;
17837
- return typeof signals === "object" && signals !== null && "chars" in signals && signals.chars === 0;
17838
- }
17839
17907
  async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerprints) {
17840
17908
  const byHash = new Map;
17841
17909
  for (const file of scan.files) {
17842
- if (typeof file.content_hash !== "string" || file.content_hash.length === 0 || extractedNoText(file))
17910
+ if (typeof file.content_hash !== "string" || file.content_hash.length === 0)
17843
17911
  continue;
17844
17912
  const path = resolve10(root, file.path);
17845
17913
  const group = byHash.get(file.content_hash) ?? [];
@@ -17848,7 +17916,7 @@ async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerp
17848
17916
  }
17849
17917
  const fingerprintModified = new Map;
17850
17918
  for (const [path, fingerprint] of fingerprints ?? []) {
17851
- if (fingerprint.exact.length === 0 || fingerprint.chars === 0)
17919
+ if (fingerprint.exact.length === 0)
17852
17920
  continue;
17853
17921
  fingerprintModified.set(path, fingerprint.modified);
17854
17922
  const group = byHash.get(fingerprint.exact) ?? [];
@@ -17915,1397 +17983,1368 @@ async function rankExactCandidates(paths, dupeyRanking, fingerprintModified, err
17915
17983
  }
17916
17984
  var init_filter = () => {};
17917
17985
 
17918
- // src/filesystem/cloud-placeholder.ts
17919
- import { execFile as execFile2 } from "node:child_process";
17920
- import { realpathSync as realpathSync3 } from "node:fs";
17921
- import { lstat, readdir } from "node:fs/promises";
17922
- import { homedir as homedir6 } from "node:os";
17923
- import { dirname as dirname14, join as join27 } from "node:path";
17924
- import { promisify } from "node:util";
17925
- function homeCloudRoots(home = homedir6()) {
17926
- const normalizedHome = home.replaceAll("\\", "/").replace(/\/+$/, "");
17927
- return [`${normalizedHome}/Library/CloudStorage`, `${normalizedHome}/Library/Mobile Documents`];
17928
- }
17929
- function datalessFromDarwinFlags(flags) {
17930
- return Number.isFinite(flags) && (flags >>> 0 & SF_DATALESS) !== 0;
17986
+ // src/dupey/index.ts
17987
+ var init_dupey = __esm(() => {
17988
+ init_cli2();
17989
+ init_filter();
17990
+ });
17991
+
17992
+ // src/convert/relate.ts
17993
+ async function convertAndRelate(request, options = {}) {
17994
+ const result = await convertWithFilee(request, options.filee);
17995
+ if (options.dupey === false)
17996
+ return { result, links: [] };
17997
+ const dupeyOptions = options.dupey ?? {};
17998
+ const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
17999
+ const links = await Promise.all(pairs.map(async ({ source, output }) => {
18000
+ try {
18001
+ return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
18002
+ } catch (error) {
18003
+ if (!(error instanceof DupeyCliError))
18004
+ throw error;
18005
+ return { source, output, dupey: { relation: "unavailable", reason: error.message } };
18006
+ }
18007
+ }));
18008
+ return { result, links };
17931
18009
  }
17932
- function datalessFromWindowsAttributes(attributes) {
17933
- return Number.isFinite(attributes) && (attributes & WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) !== 0;
18010
+ var init_relate = __esm(() => {
18011
+ init_dupey();
18012
+ init_cli();
18013
+ });
18014
+
18015
+ // src/convert/index.ts
18016
+ var init_convert = __esm(() => {
18017
+ init_cli();
18018
+ init_installer2();
18019
+ init_matrix();
18020
+ init_relate();
18021
+ });
18022
+
18023
+ // src/datasource/diagnostics.ts
18024
+ function mapDatasourceDiagnostic(ds) {
18025
+ return {
18026
+ code: datasourceCodeToRetrieval(ds.code),
18027
+ severity: ds.severity,
18028
+ message: `[datasource:${ds.code}] ${ds.message}`,
18029
+ source: ds.source ?? (ds.instanceId ? `datasource:${ds.instanceId}` : "datasource")
18030
+ };
17934
18031
  }
17935
- function cloudProviderFromPath(target) {
17936
- const lower = target.toLowerCase();
17937
- if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
17938
- return "google-drive";
17939
- if (lower.includes("onedrive"))
17940
- return "onedrive";
17941
- if (lower.includes("dropbox"))
17942
- return "dropbox";
17943
- if (lower.includes("mobile documents") || lower.includes("clouddocs"))
17944
- return "icloud";
17945
- return "file-provider";
18032
+ function mapDatasourceDiagnostics(diagnostics) {
18033
+ return diagnostics.map(mapDatasourceDiagnostic);
17946
18034
  }
17947
- function cloudProviderFromDomain(domain) {
17948
- const lower = domain.toLowerCase();
17949
- if (lower.includes("clouddocs") || lower.includes("icloud"))
17950
- return "icloud";
17951
- if (lower.includes("google"))
17952
- return "google-drive";
17953
- if (lower.includes("onedrive"))
17954
- return "onedrive";
17955
- if (lower.includes("dropbox"))
17956
- return "dropbox";
17957
- return "file-provider";
18035
+ function datasourceCodeToRetrieval(code) {
18036
+ switch (code) {
18037
+ case "datasource-unavailable":
18038
+ case "datasource-cli-error":
18039
+ case "datasource-empty":
18040
+ case "datasource-rate-limited":
18041
+ case "datasource-auth-error":
18042
+ case "datasource-embedding-egress-rejected":
18043
+ case "datasource-index-failed":
18044
+ case "datasource-index-partial":
18045
+ case "datasource-permission-denied":
18046
+ case "datasource-remote-embedding-rejected":
18047
+ return "retrieval-method-failed";
18048
+ }
17958
18049
  }
17959
- function normalizeCloudPath(value) {
17960
- let normalized = value.replaceAll("\\", "/");
17961
- if (normalized.length > 1)
17962
- normalized = normalized.replace(/\/+$/u, "");
17963
- return normalized;
18050
+
18051
+ // src/datasource/retry.ts
18052
+ import { setTimeout as delay2 } from "node:timers/promises";
18053
+ function datasourceRetryDelayMs(retry, policy) {
18054
+ const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
18055
+ return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
17964
18056
  }
17965
- function isPathInside(target, parent) {
17966
- const normalizedParent = normalizeCloudPath(parent);
17967
- if (normalizedParent === "")
17968
- return false;
17969
- const left = normalizeCloudPath(target);
17970
- const [a, b] = process.platform === "win32" ? [left.toLowerCase(), normalizedParent.toLowerCase()] : [left, normalizedParent];
17971
- return a === b || a.startsWith(`${b}/`);
18057
+ async function indexWithRetry(skill, options = {}, signal) {
18058
+ const policy = {
18059
+ maxRetries: options.maxRetries ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxRetries,
18060
+ baseDelayMs: options.baseDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.baseDelayMs,
18061
+ maxDelayMs: options.maxDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxDelayMs
18062
+ };
18063
+ const sleep = options.sleep ?? ((ms) => delay2(ms, undefined, signal === undefined ? undefined : { signal }));
18064
+ let lastError = "";
18065
+ for (let attempt = 1;; attempt += 1) {
18066
+ const result = await runIndex(skill, signal);
18067
+ signal?.throwIfAborted();
18068
+ if (result.ok) {
18069
+ if (attempt === 1)
18070
+ return result;
18071
+ const recovered = {
18072
+ code: "datasource-index-failed",
18073
+ severity: "info",
18074
+ message: `recovered after ${attempt} attempts; last error: ${lastError}`,
18075
+ instanceId: result.instanceId,
18076
+ source: result.skill
18077
+ };
18078
+ return { ...result, attempts: attempt, diagnostics: [...result.diagnostics, recovered] };
18079
+ }
18080
+ lastError = result.message ?? result.error ?? "datasource-index-failed";
18081
+ if (attempt > policy.maxRetries) {
18082
+ if (attempt === 1)
18083
+ return { ...result, attempts: attempt };
18084
+ const suffix = ` (gave up after ${attempt} attempts)`;
18085
+ const last = result.diagnostics.at(-1);
18086
+ return {
18087
+ ...result,
18088
+ attempts: attempt,
18089
+ message: `${lastError}${suffix}`,
18090
+ diagnostics: last === undefined ? [
18091
+ {
18092
+ code: result.code ?? "datasource-index-failed",
18093
+ severity: "error",
18094
+ message: `${lastError}${suffix}`,
18095
+ instanceId: result.instanceId,
18096
+ source: result.skill
18097
+ }
18098
+ ] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
18099
+ };
18100
+ }
18101
+ await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
18102
+ signal?.throwIfAborted();
18103
+ }
17972
18104
  }
17973
- function canonicalPath(value) {
18105
+ async function runIndex(skill, signal) {
17974
18106
  try {
17975
- return realpathSync3(value);
17976
- } catch {
17977
- return value;
18107
+ return await skill.index(signal === undefined ? undefined : { signal });
18108
+ } catch (error) {
18109
+ signal?.throwIfAborted();
18110
+ const descriptor = skill.describe();
18111
+ const message = error instanceof Error ? error.message : "Datasource indexing failed.";
18112
+ const failed = {
18113
+ ok: false,
18114
+ instanceId: descriptor.instanceId ?? "default",
18115
+ skill: descriptor.name,
18116
+ indexedAt: Date.now(),
18117
+ diagnostics: [
18118
+ {
18119
+ code: "datasource-index-failed",
18120
+ severity: "error",
18121
+ message,
18122
+ source: descriptor.name,
18123
+ instanceId: descriptor.instanceId
18124
+ }
18125
+ ],
18126
+ error: "datasource-index-failed",
18127
+ code: "datasource-index-failed",
18128
+ message
18129
+ };
18130
+ return failed;
17978
18131
  }
17979
18132
  }
17980
- function canonicalPathMapper(searchRoots) {
17981
- const forms = searchRoots.map((root) => ({ raw: normalizeCloudPath(root), canonical: normalizeCloudPath(canonicalPath(root)) })).filter(({ raw, canonical }) => raw !== "" && raw !== canonical).sort((left, right) => right.raw.length - left.raw.length);
17982
- if (forms.length === 0)
17983
- return (path) => path;
17984
- return (path) => {
17985
- const normalized = normalizeCloudPath(path);
17986
- for (const { raw, canonical } of forms) {
17987
- if (isPathInside(normalized, raw))
17988
- return `${canonical}${normalized.slice(raw.length)}`;
17989
- }
17990
- return path;
18133
+ var DEFAULT_DATASOURCE_RETRY_POLICY;
18134
+ var init_retry = __esm(() => {
18135
+ DEFAULT_DATASOURCE_RETRY_POLICY = {
18136
+ maxRetries: 2,
18137
+ baseDelayMs: 1000,
18138
+ maxDelayMs: 30000
17991
18139
  };
18140
+ });
18141
+
18142
+ // src/everything/bundle.ts
18143
+ import { createHash as createHash11 } from "node:crypto";
18144
+ import { existsSync as existsSync17, mkdirSync as mkdirSync14, readFileSync as readFileSync17, rmSync as rmSync7, writeFileSync as writeFileSync10 } from "node:fs";
18145
+ import { dirname as dirname14, join as join27 } from "node:path";
18146
+ import { fileURLToPath } from "node:url";
18147
+ import JSZip from "jszip";
18148
+ function resolveEverythingBundleDir() {
18149
+ let dir = dirname14(fileURLToPath(import.meta.url));
18150
+ for (let depth = 0;depth < 6; depth += 1) {
18151
+ const candidate = join27(dir, "vendor", "everything");
18152
+ if (existsSync17(join27(candidate, MANIFEST_FILENAME)))
18153
+ return candidate;
18154
+ const parent = dirname14(dir);
18155
+ if (parent === dir)
18156
+ break;
18157
+ dir = parent;
18158
+ }
18159
+ return;
17992
18160
  }
17993
- function cloudScope(searchRoot, cloudRoots) {
17994
- const canonicalSearchRoot = canonicalPath(searchRoot);
17995
- const roots = cloudRoots.map((root) => ({ path: canonicalPath(root.path), kind: root.kind }));
17996
- return {
17997
- insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
17998
- nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
17999
- };
18161
+ function everythingArch(arch) {
18162
+ if (arch === "x64" || arch === "arm64")
18163
+ return arch;
18164
+ return;
18000
18165
  }
18001
- function parseReleaseOutcome(stdout) {
18002
- let parsed;
18166
+ function everythingBinaryPaths(root, version, arch) {
18167
+ const dir = join27(root, ".autorag", "everything", version, arch);
18168
+ return { everythingPath: join27(dir, "everything.exe"), esPath: join27(dir, "es.exe") };
18169
+ }
18170
+ function loadEverythingBundleManifest(bundleDir) {
18171
+ return JSON.parse(readFileSync17(join27(bundleDir, MANIFEST_FILENAME), "utf8"));
18172
+ }
18173
+ async function ensureEverythingBinaries(options) {
18174
+ const platform = options.platform ?? process.platform;
18175
+ const rawArch = options.arch ?? process.arch;
18176
+ if (platform !== "win32") {
18177
+ return {
18178
+ ok: false,
18179
+ reason: "unsupported-platform",
18180
+ message: `Everything is Windows-only; this host is ${platform}.`
18181
+ };
18182
+ }
18183
+ const arch = everythingArch(rawArch);
18184
+ const bundleDir = options.bundleDir ?? resolveEverythingBundleDir();
18185
+ if (bundleDir === undefined) {
18186
+ return {
18187
+ ok: false,
18188
+ reason: "bundle-missing",
18189
+ message: `Bundled Everything assets (vendor/everything/${MANIFEST_FILENAME}) were not found above ${dirname14(fileURLToPath(import.meta.url))}.`
18190
+ };
18191
+ }
18192
+ let manifest;
18003
18193
  try {
18004
- parsed = JSON.parse(stdout.trim());
18005
- } catch {
18006
- return;
18194
+ manifest = options.manifest ?? loadEverythingBundleManifest(bundleDir);
18195
+ } catch (error) {
18196
+ return { ok: false, reason: "bundle-missing", message: errorMessage2(error) };
18007
18197
  }
18008
- if (typeof parsed !== "object" || parsed === null || !("ok" in parsed) || typeof parsed.ok !== "boolean") {
18009
- return;
18198
+ const assets = arch === undefined ? undefined : manifest.architectures[arch];
18199
+ if (arch === undefined || assets === undefined) {
18200
+ return {
18201
+ ok: false,
18202
+ reason: "unsupported-platform",
18203
+ message: `No bundled Everything build for Windows architecture ${rawArch}; bundled: ${Object.keys(manifest.architectures).join(", ")}.`
18204
+ };
18010
18205
  }
18011
- return {
18012
- ok: parsed.ok,
18013
- ..."isUploaded" in parsed ? { isUploaded: parsed.isUploaded } : {},
18014
- ..."isUploading" in parsed ? { isUploading: parsed.isUploading } : {},
18015
- ..."downloadingStatus" in parsed ? { downloadingStatus: parsed.downloadingStatus } : {}
18016
- };
18017
- }
18018
- function describeReleaseRefusal(outcome) {
18019
- if (outcome.isUploading === true)
18020
- return "The file is still uploading to the cloud, so it cannot be released yet.";
18021
- if (outcome.isUploaded === false)
18022
- return "The file has not finished uploading to the cloud, so it cannot be released yet.";
18023
- const status = typeof outcome.downloadingStatus === "string" ? outcome.downloadingStatus : "unknown";
18024
- return `The cloud provider declined to release the local copy (download status ${status}).`;
18206
+ const { everythingPath, esPath } = everythingBinaryPaths(options.root, manifest.everythingVersion, arch);
18207
+ const everythingCached = fileHasSha256(everythingPath, assets.everythingSha256);
18208
+ const esCached = fileHasSha256(esPath, assets.esSha256);
18209
+ if (everythingCached && esCached) {
18210
+ return { ok: true, everythingPath, esPath, source: "cached" };
18211
+ }
18212
+ if ((options.mode ?? "install") === "cached") {
18213
+ const state = everythingCached || esCached ? "are incomplete or do not match their pinned SHA-256 digests" : "are not installed";
18214
+ return {
18215
+ ok: false,
18216
+ reason: "not-installed",
18217
+ message: `Everything ${manifest.everythingVersion} (${arch}) binaries ${state} under ${dirname14(everythingPath)}; run \`autorag refresh\` to (re)install and index them.`
18218
+ };
18219
+ }
18220
+ try {
18221
+ mkdirSync14(dirname14(everythingPath), { recursive: true });
18222
+ const everythingBytes = await extractVerified(join27(bundleDir, assets.everythingArchive), assets.everythingArchiveSha256, assets.everythingMember, assets.everythingSha256);
18223
+ const esBytes = await extractVerified(join27(bundleDir, assets.esArchive), assets.esArchiveSha256, assets.esMember, assets.esSha256);
18224
+ writeAtomic(everythingPath, everythingBytes);
18225
+ writeAtomic(esPath, esBytes);
18226
+ } catch (error) {
18227
+ return { ok: false, reason: "install-failed", message: errorMessage2(error) };
18228
+ }
18229
+ return { ok: true, everythingPath, esPath, source: "installed" };
18025
18230
  }
18026
- function describeReleaseHelperFailure(error) {
18027
- const failure = error;
18028
- const parts = [];
18029
- if (failure.killed === true || typeof failure.signal === "string") {
18030
- parts.push(`the helper was killed${typeof failure.signal === "string" ? ` by ${failure.signal}` : ""}`);
18031
- } else if (typeof failure.code === "number") {
18032
- parts.push(`the helper exited with code ${failure.code}`);
18231
+ async function extractVerified(archivePath, archiveSha256, member, memberSha256) {
18232
+ const archive = readFileSync17(archivePath);
18233
+ const actualArchive = sha256(archive);
18234
+ if (actualArchive !== archiveSha256) {
18235
+ throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
18033
18236
  }
18034
- const stderr = typeof failure.stderr === "string" ? failure.stderr.trim() : "";
18035
- if (stderr !== "")
18036
- parts.push(stderr);
18037
- if (parts.length === 0)
18038
- parts.push(error instanceof Error ? error.message : String(error));
18039
- return `osascript could not release the local copy: ${parts.join("; ")}`;
18237
+ const zip = await JSZip.loadAsync(archive);
18238
+ const entry = zip.file(member);
18239
+ if (entry === null)
18240
+ throw new Error(`${archivePath} does not contain ${member}`);
18241
+ const bytes = await entry.async("nodebuffer");
18242
+ const actualMember = sha256(bytes);
18243
+ if (actualMember !== memberSha256) {
18244
+ throw new Error(`${member} from ${archivePath} SHA-256 ${actualMember} does not match pinned ${memberSha256}`);
18245
+ }
18246
+ return bytes;
18040
18247
  }
18041
- async function defaultDarwinReadDir(path) {
18042
- const entries = await readdir(path, { withFileTypes: true });
18043
- return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
18248
+ function fileHasSha256(path, expected) {
18249
+ return existsSync17(path) && sha256(readFileSync17(path)) === expected;
18044
18250
  }
18045
- function parseDarwinMounts(stdout) {
18046
- const mounts = [];
18047
- for (const line of stdout.split(`
18048
- `)) {
18049
- if (line.trim() === "")
18050
- continue;
18051
- const open = line.lastIndexOf(" (");
18052
- if (open <= 0 || !line.endsWith(")"))
18053
- continue;
18054
- const head = line.slice(0, open);
18055
- const on = head.indexOf(" on ");
18056
- if (on <= 0)
18057
- continue;
18058
- const mountPoint = head.slice(on + " on ".length);
18059
- if (mountPoint === "")
18060
- continue;
18061
- const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
18062
- mounts.push({
18063
- device: head.slice(0, on),
18064
- mountPoint,
18065
- fileSystem: fields[0] ?? "",
18066
- options: fields.slice(1)
18067
- });
18251
+ function writeAtomic(path, bytes) {
18252
+ const temp = `${path}.${process.pid}.tmp`;
18253
+ try {
18254
+ writeFileSync10(temp, bytes);
18255
+ renameWithRetry(temp, path);
18256
+ } finally {
18257
+ rmSync7(temp, { force: true });
18068
18258
  }
18069
- return mounts;
18070
18259
  }
18071
- function networkMountPoints(mounts) {
18072
- const points = [];
18073
- const seen = new Set;
18074
- for (const mount of mounts) {
18075
- if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
18076
- continue;
18077
- if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
18078
- continue;
18079
- if (seen.has(mount.mountPoint))
18080
- continue;
18081
- seen.add(mount.mountPoint);
18082
- points.push(mount.mountPoint);
18083
- }
18084
- return points;
18260
+ function sha256(bytes) {
18261
+ return createHash11("sha256").update(bytes).digest("hex");
18085
18262
  }
18086
- function parseXattrValues(stdout, candidates) {
18087
- const values = new Map;
18088
- if (candidates.length === 0)
18089
- return values;
18090
- const ordered = [...candidates].sort((left, right) => right.length - left.length);
18091
- for (const line of stdout.split(`
18092
- `)) {
18093
- if (line === "")
18094
- continue;
18095
- for (const candidate of ordered) {
18096
- if (line.startsWith(`${candidate}: `)) {
18097
- values.set(candidate, line.slice(candidate.length + 2));
18098
- break;
18099
- }
18100
- }
18101
- }
18102
- return values;
18263
+ function errorMessage2(error) {
18264
+ return error instanceof Error ? error.message : String(error);
18265
+ }
18266
+ var MANIFEST_FILENAME = "manifest.json";
18267
+ var init_bundle = __esm(() => {
18268
+ init_atomic_write();
18269
+ });
18270
+
18271
+ // src/filesystem/path-exclusions.ts
18272
+ import { posix as posix3, win32 as win322 } from "node:path";
18273
+ function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
18274
+ const paths = platform === "win32" ? win322 : posix3;
18275
+ const normalize = (path) => {
18276
+ const absolute = paths.resolve(path);
18277
+ return platform === "win32" ? absolute.toLowerCase() : absolute;
18278
+ };
18279
+ const excluded = excludedPaths.map((path) => {
18280
+ const absolute = normalize(path);
18281
+ return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
18282
+ });
18283
+ return (path) => {
18284
+ const absolute = normalize(path);
18285
+ return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
18286
+ };
18103
18287
  }
18288
+ var init_path_exclusions = () => {};
18104
18289
 
18105
- class DarwinCloudProbe {
18106
- exec;
18107
- readDir;
18108
- home;
18109
- constructor(options = {}) {
18110
- this.exec = options.exec ?? defaultCloudExec;
18111
- this.readDir = options.readDir ?? defaultDarwinReadDir;
18112
- this.home = options.home ?? homedir6();
18113
- }
18114
- async cloudRoots(searchRoots = []) {
18115
- const roots = [];
18116
- const seen = new Set;
18117
- const add = (path, kind) => {
18118
- const canonical = canonicalPath(path);
18119
- const key = normalizeCloudPath(canonical);
18120
- if (key === "" || seen.has(key))
18121
- return;
18122
- seen.add(key);
18123
- roots.push({ path: canonical, kind });
18124
- };
18125
- const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
18126
- for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
18127
- if (entry.isDirectory())
18128
- add(join27(cloudStorage, entry.name), "file-provider");
18129
- }
18130
- if (await this.tryReadDir(mobileDocuments) !== undefined)
18131
- add(mobileDocuments, "file-provider");
18132
- for (const mountPoint of await this.networkMountPoints())
18133
- add(mountPoint, "network");
18134
- for (const directory of await this.fileProviderDirectories(searchRoots))
18135
- add(directory, "file-provider");
18136
- return roots;
18290
+ // src/filesystem/shared-exclusive-queue.ts
18291
+ class SharedExclusiveQueue {
18292
+ sharedSinceExclusive = [];
18293
+ lastExclusive = Promise.resolve();
18294
+ shared(task) {
18295
+ const run = this.lastExclusive.then(task);
18296
+ const settled = run.then(() => {
18297
+ return;
18298
+ }, () => {
18299
+ return;
18300
+ });
18301
+ this.sharedSinceExclusive.push(settled);
18302
+ return run;
18137
18303
  }
18138
- async networkMountPoints() {
18139
- try {
18140
- const { stdout } = await this.exec("/sbin/mount", [], { timeoutMs: 1e4, maxBuffer: 4 * 1024 * 1024 });
18141
- return networkMountPoints(parseDarwinMounts(stdout));
18142
- } catch {
18143
- return [];
18144
- }
18145
- }
18146
- async fileProviderDirectories(searchRoots) {
18147
- if (searchRoots.length === 0)
18148
- return [];
18149
- const candidates = new Set;
18150
- for (const searchRoot of searchRoots) {
18151
- const canonical = canonicalPath(searchRoot);
18152
- for (let cursor = canonical;; cursor = dirname14(cursor)) {
18153
- candidates.add(cursor);
18154
- const parent = dirname14(cursor);
18155
- if (parent === cursor)
18156
- break;
18157
- }
18158
- for (const entry of await this.tryReadDir(canonical) ?? []) {
18159
- if (entry.isDirectory())
18160
- candidates.add(join27(canonical, entry.name));
18161
- }
18162
- }
18163
- const paths = [...candidates];
18164
- const managed = [];
18165
- for (let start = 0;start < paths.length; start += XATTR_BATCH) {
18166
- for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
18167
- managed.push(path);
18168
- }
18169
- return managed;
18170
- }
18171
- async xattrValues(paths) {
18172
- try {
18173
- const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
18174
- timeoutMs: 15000,
18175
- maxBuffer: 8 * 1024 * 1024
18176
- });
18177
- return parseXattrValues(stdout, paths);
18178
- } catch (error) {
18179
- const failure = error;
18180
- return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
18181
- }
18182
- }
18183
- async tryReadDir(path) {
18184
- try {
18185
- return await this.readDir(path);
18186
- } catch {
18304
+ exclusive(task) {
18305
+ const earlier = Promise.all([this.lastExclusive, ...this.sharedSinceExclusive]);
18306
+ const run = earlier.then(task);
18307
+ const settled = run.then(() => {
18187
18308
  return;
18188
- }
18189
- }
18190
- async datalessAmong(paths) {
18191
- const dataless = new Set;
18192
- for (let start = 0;start < paths.length; start += STAT_BATCH) {
18193
- const batch = paths.slice(start, start + STAT_BATCH);
18194
- const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
18195
- timeoutMs: 30000,
18196
- maxBuffer: 4 * 1024 * 1024
18197
- });
18198
- const flags = stdout.split(`
18199
- `).filter(Boolean);
18200
- for (const [index, path] of batch.entries()) {
18201
- if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
18202
- dataless.add(path);
18203
- }
18204
- }
18205
- return dataless;
18206
- }
18207
- async providerOf(path) {
18208
- const domain = await readFileProviderDomainUpwards(path);
18209
- return domain === undefined ? undefined : cloudProviderFromDomain(domain);
18210
- }
18211
- canRelease(provider) {
18212
- return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
18213
- }
18214
- async evict(path) {
18215
- let stdout;
18216
- try {
18217
- stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
18218
- timeoutMs: 30000
18219
- })).stdout;
18220
- } catch (error) {
18221
- return { ok: false, message: describeReleaseHelperFailure(error) };
18222
- }
18223
- const outcome = parseReleaseOutcome(stdout);
18224
- if (outcome === undefined) {
18225
- const detail = stdout.trim();
18226
- return {
18227
- ok: false,
18228
- message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
18229
- };
18230
- }
18231
- return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
18232
- }
18233
- }
18234
- function buildWindowsAttributeScript(paths) {
18235
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18236
- ` + `$ErrorActionPreference='SilentlyContinue'
18237
- ` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
18238
- `;
18239
- }
18240
- function buildWindowsSyncRootsScript() {
18241
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18242
- ` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
18243
- ` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
18244
- `;
18245
- }
18246
- function buildWindowsDehydrateScript(path) {
18247
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18248
- ` + `$ErrorActionPreference='Stop'
18249
- ` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
18250
- ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
18251
- ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
18252
- ` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
18253
- ` + `'@
18254
- ` + `$h=[IntPtr]::Zero
18255
- ` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
18256
- ` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
18257
- ` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
18258
- ` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
18259
- ` + `[Autorag.CfApi]::CfCloseHandle($h)
18260
- ` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
18261
- ` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
18262
- `;
18263
- }
18264
- function parseWindowsAttributeOutput(stdout, paths) {
18265
- const lines = stdout.split(/\r?\n/);
18266
- const attributes = new Map;
18267
- for (const [index, path] of paths.entries()) {
18268
- const value = Number.parseInt((lines[index] ?? "").trim(), 10);
18269
- if (Number.isFinite(value))
18270
- attributes.set(path, value);
18271
- }
18272
- return attributes;
18273
- }
18274
- function parseWindowsSyncRoots(stdout) {
18275
- const roots = [];
18276
- for (const line of stdout.split(/\r?\n/)) {
18277
- const tab = line.indexOf("\t");
18278
- if (tab <= 0)
18279
- continue;
18280
- const path = line.slice(0, tab).trim();
18281
- if (path !== "")
18282
- roots.push({ path, label: line.slice(tab + 1).trim() });
18309
+ }, () => {
18310
+ return;
18311
+ });
18312
+ this.sharedSinceExclusive = [];
18313
+ this.lastExclusive = settled;
18314
+ return run;
18283
18315
  }
18284
- return roots;
18285
18316
  }
18286
- function providerFromWindowsSyncRoot(label) {
18287
- const lower = label.toLowerCase();
18288
- if (lower.includes("onedrive") || lower.includes("one drive"))
18289
- return "onedrive";
18290
- if (lower.includes("google"))
18291
- return "google-drive";
18292
- return "file-provider";
18317
+
18318
+ // src/everything/client.ts
18319
+ import { spawn as spawn14 } from "node:child_process";
18320
+ import { createHash as createHash12 } from "node:crypto";
18321
+ import { mkdirSync as mkdirSync15, writeFileSync as writeFileSync11 } from "node:fs";
18322
+ import { join as join28, resolve as resolve11 } from "node:path";
18323
+ import { setTimeout as sleep2 } from "node:timers/promises";
18324
+ function everythingInstanceName(root) {
18325
+ return `autorag-${createHash12("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
18293
18326
  }
18294
- function buildWindowsDiskScript() {
18295
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18296
- ` + "Get-CimInstance -ClassName Win32_LogicalDisk | ForEach-Object {" + `$_.DeviceID + [char]9 + $_.DriveType + [char]9 + $_.FileSystem + [char]9 + $_.VolumeName}
18297
- `;
18327
+ function buildEverythingIni(input) {
18328
+ const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
18329
+ const perFolder = (value) => input.folders.map(() => value).join(",");
18330
+ return [
18331
+ "[Everything]",
18332
+ "app_data=0",
18333
+ "run_as_admin=0",
18334
+ "run_in_background=1",
18335
+ "show_tray_icon=0",
18336
+ "show_in_taskbar=0",
18337
+ "check_for_updates_on_startup=0",
18338
+ "beta_updates=0",
18339
+ "language=1033",
18340
+ "allow_http_server=0",
18341
+ "allow_etp_server=0",
18342
+ "search_history_enabled=0",
18343
+ "run_history_enabled=0",
18344
+ "auto_include_fixed_volumes=0",
18345
+ "auto_include_removable_volumes=0",
18346
+ "auto_include_fixed_refs_volumes=0",
18347
+ "auto_include_removable_refs_volumes=0",
18348
+ "ntfs_volume_paths=",
18349
+ "ntfs_volume_includes=",
18350
+ "refs_volume_paths=",
18351
+ "refs_volume_includes=",
18352
+ "index_size=1",
18353
+ "index_date_modified=1",
18354
+ `folders=${list(input.folders)}`,
18355
+ `folder_monitor_changes=${perFolder("1")}`,
18356
+ `folder_rescan_if_full_list=${perFolder("1")}`,
18357
+ `folder_update_types=${perFolder("0")}`,
18358
+ "exclude_list_enabled=1",
18359
+ `exclude_folders=${list(input.excludeFolders)}`,
18360
+ ""
18361
+ ].join(`\r
18362
+ `);
18298
18363
  }
18299
- function parseWindowsLogicalDisks(stdout) {
18300
- const disks = [];
18301
- for (const line of stdout.split(/\r?\n/)) {
18302
- if (line.trim() === "")
18303
- continue;
18304
- const [deviceId, driveType, fileSystem, ...rest] = line.split("\t");
18305
- const type = Number.parseInt((driveType ?? "").trim(), 10);
18306
- if (!Number.isFinite(type))
18307
- continue;
18308
- disks.push({
18309
- deviceId: (deviceId ?? "").trim(),
18310
- driveType: type,
18311
- fileSystem: (fileSystem ?? "").trim(),
18312
- volumeName: rest.join("\t").trim()
18313
- });
18314
- }
18315
- return disks;
18364
+ function buildEverythingSearchArgs(instance, request, timeoutMs) {
18365
+ const args = [
18366
+ "-argv",
18367
+ "-instance",
18368
+ instance,
18369
+ "-cp",
18370
+ "65001",
18371
+ "-json",
18372
+ "-full-path-and-name",
18373
+ "-size",
18374
+ "-date-modified",
18375
+ "-date-format",
18376
+ "1"
18377
+ ];
18378
+ if (timeoutMs !== undefined)
18379
+ args.push("-timeout", String(timeoutMs));
18380
+ if (request.matchCase)
18381
+ args.push("-case");
18382
+ if (request.matchPath)
18383
+ args.push("-match-path");
18384
+ if (request.wholeWord)
18385
+ args.push("-whole-word");
18386
+ if (request.kind === "files")
18387
+ args.push("/a-d");
18388
+ if (request.kind === "folders")
18389
+ args.push("/ad");
18390
+ if (request.path !== undefined)
18391
+ args.push("-path", request.path);
18392
+ if (request.sort !== undefined)
18393
+ args.push("-sort", request.sort);
18394
+ if (request.offset !== undefined && request.offset > 0)
18395
+ args.push("-offset", String(request.offset));
18396
+ args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
18397
+ if (request.regex)
18398
+ args.push("-regex", "--", request.query);
18399
+ else
18400
+ args.push("--", request.query);
18401
+ return args;
18316
18402
  }
18317
- function windowsRootsFromDisks(disks) {
18318
- const roots = [];
18319
- for (const disk of disks) {
18320
- if (disk.deviceId === "")
18321
- continue;
18322
- const path = `${disk.deviceId}\\`;
18323
- const fileSystem = disk.fileSystem.toLowerCase();
18324
- if (disk.driveType === WINDOWS_DRIVE_TYPE_NETWORK) {
18325
- roots.push({ path, kind: "network" });
18326
- continue;
18327
- }
18328
- if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
18329
- roots.push({ path, kind: "network" });
18330
- continue;
18331
- }
18332
- if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
18333
- roots.push({ path, kind: "cloud-files" });
18334
- }
18403
+ function parseEverythingJson(stdout) {
18404
+ const text = stdout.trim();
18405
+ if (text.length === 0)
18406
+ return [];
18407
+ let parsed;
18408
+ try {
18409
+ parsed = JSON.parse(text);
18410
+ } catch {
18411
+ throw new Error(`ES did not return JSON: ${text.slice(0, 2000)}`);
18335
18412
  }
18336
- return roots;
18337
- }
18338
- function uncShareRoot(target) {
18339
- const normalized = target.replaceAll("/", "\\");
18340
- if (!normalized.startsWith("\\\\") || normalized.startsWith("\\\\?\\") || normalized.startsWith("\\\\.\\"))
18341
- return;
18342
- const rest = normalized.slice(2);
18343
- const separator = rest.indexOf("\\");
18344
- if (separator <= 0)
18345
- return;
18346
- const server = rest.slice(0, separator);
18347
- const share = rest.slice(separator + 1).split("\\")[0] ?? "";
18348
- if (server === "" || share === "")
18349
- return;
18350
- return `\\\\${server}\\${share}`;
18351
- }
18352
- function parseWindowsDehydrateOutput(result) {
18353
- const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
18354
- if (open === null)
18355
- return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
18356
- if (Number.parseInt(open[1], 16) !== 0) {
18413
+ if (!Array.isArray(parsed))
18414
+ throw new Error(`ES did not return a JSON array: ${text.slice(0, 2000)}`);
18415
+ return parsed.map((item) => {
18416
+ const record = item;
18417
+ const filename = String(record.filename ?? "");
18418
+ const isFolder = filename.endsWith("\\") && filename.length > 3;
18357
18419
  return {
18358
- ok: false,
18359
- message: windowsFailureMessage(result, `could not open the file for release (0x${open[1]})`)
18420
+ path: isFolder ? filename.slice(0, -1) : filename,
18421
+ type: filename.endsWith("\\") ? "folder" : "file",
18422
+ size: typeof record.size === "number" ? record.size : undefined,
18423
+ dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
18360
18424
  };
18361
- }
18362
- const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
18363
- if (dehydrate === null) {
18364
- return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
18365
- }
18366
- const hresult = dehydrate[1];
18367
- if (Number.parseInt(hresult, 16) === 0)
18368
- return { ok: true };
18369
- const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
18370
- return {
18371
- ok: false,
18372
- message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
18373
- };
18374
- }
18375
- function windowsFailureMessage(result, fallback) {
18376
- const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
18377
- if (printed !== undefined && printed !== "")
18378
- return printed;
18379
- const stderr = result.stderr.trim();
18380
- return stderr !== "" ? stderr : fallback;
18425
+ });
18381
18426
  }
18382
18427
 
18383
- class WindowsCloudProbe {
18428
+ class EverythingClient {
18429
+ instanceName;
18430
+ options;
18431
+ platform;
18384
18432
  run;
18385
- syncRoots;
18386
- disks;
18387
- constructor(options = {}) {
18388
- this.run = options.run ?? runPowerShell;
18433
+ launch;
18434
+ stateDir;
18435
+ isExcludedPath;
18436
+ binaries;
18437
+ queue = new SharedExclusiveQueue;
18438
+ constructor(options) {
18439
+ this.options = options;
18440
+ this.platform = options.platform ?? process.platform;
18441
+ this.instanceName = everythingInstanceName(options.root);
18442
+ this.run = options.run ?? runProcess;
18443
+ this.launch = options.launch ?? launchDetached;
18444
+ this.stateDir = join28(options.root, ".autorag", "everything");
18445
+ this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
18389
18446
  }
18390
- async cloudRoots(searchRoots = []) {
18391
- const roots = [];
18392
- const seen = new Set;
18393
- const add = (path, kind) => {
18394
- const canonical = canonicalPath(path);
18395
- const key = normalizeCloudPath(canonical).toLowerCase();
18396
- if (key === "" || seen.has(key))
18397
- return;
18398
- seen.add(key);
18399
- roots.push({ path: canonical, kind });
18400
- };
18401
- for (const root of await this.loadSyncRoots())
18402
- add(root.path, "cloud-files");
18403
- for (const root of await this.loadDiskRoots())
18404
- add(root.path, root.kind);
18405
- for (const searchRoot of searchRoots) {
18406
- const share = uncShareRoot(searchRoot);
18407
- if (share !== undefined)
18408
- add(share, "network");
18409
- }
18410
- return roots;
18447
+ isSupported() {
18448
+ return this.platform === "win32";
18411
18449
  }
18412
- async datalessAmong(paths) {
18413
- const dataless = new Set;
18414
- for (let start = 0;start < paths.length; start += WINDOWS_ATTRIBUTE_BATCH) {
18415
- const batch = paths.slice(start, start + WINDOWS_ATTRIBUTE_BATCH);
18416
- const result = await this.run(buildWindowsAttributeScript(batch), 30000);
18417
- if (!result.ok) {
18418
- throw new Error(`Could not read Windows cloud file attributes: ${result.stderr.trim() || "powershell.exe failed"}`);
18450
+ search(request) {
18451
+ return this.queue.shared(async () => {
18452
+ const binaries = await this.resolveBinaries("cached");
18453
+ if (!binaries.ok)
18454
+ return binaries;
18455
+ const ping = await this.ping(binaries.esPath);
18456
+ if (ping.code !== 0) {
18457
+ return {
18458
+ ok: false,
18459
+ reason: "not-running",
18460
+ message: `Everything instance ${this.instanceName} is not running; run \`autorag refresh\` to start and index it.`
18461
+ };
18419
18462
  }
18420
- for (const [path, attributes] of parseWindowsAttributeOutput(result.stdout, batch)) {
18421
- if (datalessFromWindowsAttributes(attributes))
18422
- dataless.add(path);
18463
+ const es = { ok: true, esPath: binaries.esPath };
18464
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
18465
+ if ((this.options.excludeFolders?.length ?? 0) > 0) {
18466
+ return this.searchWithExclusions(es.esPath, request, timeoutMs);
18423
18467
  }
18424
- }
18425
- return dataless;
18426
- }
18427
- async providerOf(path) {
18428
- const root = longestSyncRoot(await this.loadSyncRoots(), path);
18429
- return root === undefined ? cloudProviderFromPath(path) : providerFromWindowsSyncRoot(root.label);
18430
- }
18431
- canRelease(provider) {
18432
- return provider !== undefined;
18468
+ const args = buildEverythingSearchArgs(this.instanceName, request, timeoutMs);
18469
+ const result = await this.run(es.esPath, args, timeoutMs + 5000);
18470
+ if (result.code !== 0)
18471
+ return this.failure("search-failed", es.esPath, args, result);
18472
+ try {
18473
+ return { ok: true, results: parseEverythingJson(result.stdout) };
18474
+ } catch (error) {
18475
+ return { ok: false, reason: "search-failed", message: error.message };
18476
+ }
18477
+ });
18433
18478
  }
18434
- async evict(path) {
18435
- return parseWindowsDehydrateOutput(await this.run(buildWindowsDehydrateScript(path), 60000));
18479
+ async searchWithExclusions(esPath, request, timeoutMs) {
18480
+ const deadline = Date.now() + timeoutMs;
18481
+ const offset = request.offset ?? 0;
18482
+ const maxResults = request.maxResults ?? DEFAULT_MAX_RESULTS;
18483
+ const results = [];
18484
+ let skipped = 0;
18485
+ let nativeOffset = 0;
18486
+ const timedOut = () => ({
18487
+ ok: false,
18488
+ reason: "search-failed",
18489
+ message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
18490
+ });
18491
+ for (;; ) {
18492
+ const remainingMs = deadline - Date.now();
18493
+ if (remainingMs <= 0)
18494
+ return timedOut();
18495
+ const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
18496
+ const response = await this.run(esPath, args, remainingMs + 5000);
18497
+ if (response.code !== 0)
18498
+ return this.failure("search-failed", esPath, args, response);
18499
+ if (Date.now() >= deadline)
18500
+ return timedOut();
18501
+ let entries;
18502
+ try {
18503
+ entries = parseEverythingJson(response.stdout);
18504
+ } catch (error) {
18505
+ return { ok: false, reason: "search-failed", message: error.message };
18506
+ }
18507
+ for (const entry of entries) {
18508
+ if (this.isExcludedPath(entry.path))
18509
+ continue;
18510
+ if (skipped < offset) {
18511
+ skipped += 1;
18512
+ continue;
18513
+ }
18514
+ results.push(entry);
18515
+ if (results.length >= maxResults)
18516
+ return { ok: true, results };
18517
+ }
18518
+ if (entries.length < MAX_FILTER_CANDIDATES)
18519
+ return { ok: true, results };
18520
+ nativeOffset += entries.length;
18521
+ }
18436
18522
  }
18437
- loadSyncRoots() {
18438
- this.syncRoots ??= this.run(buildWindowsSyncRootsScript(), 15000).then((result) => result.ok ? parseWindowsSyncRoots(result.stdout) : [], () => []);
18439
- return this.syncRoots;
18523
+ index() {
18524
+ return this.queue.exclusive(async () => {
18525
+ const es = await this.ensureRunning(true);
18526
+ if (!es.ok)
18527
+ return es;
18528
+ const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
18529
+ const saveArgs = ["-instance", this.instanceName, "-save-db"];
18530
+ const saved = await this.run(es.esPath, saveArgs, timeoutMs);
18531
+ if (saved.code !== 0)
18532
+ return this.failure("index-failed", es.esPath, saveArgs, saved);
18533
+ const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
18534
+ const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
18535
+ if (counted.code !== 0)
18536
+ return this.failure("index-failed", es.esPath, countArgs, counted);
18537
+ return {
18538
+ ok: true,
18539
+ indexedFolders: [...this.options.folders],
18540
+ indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
18541
+ };
18542
+ });
18440
18543
  }
18441
- loadDiskRoots() {
18442
- this.disks ??= this.run(buildWindowsDiskScript(), 30000).then((result) => result.ok ? windowsRootsFromDisks(parseWindowsLogicalDisks(result.stdout)) : [], () => []);
18443
- return this.disks;
18544
+ stop() {
18545
+ return this.queue.exclusive(async () => {
18546
+ const binaries = await this.resolveBinaries("cached");
18547
+ if (!binaries.ok)
18548
+ return;
18549
+ await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
18550
+ });
18444
18551
  }
18445
- }
18446
- function longestSyncRoot(roots, target) {
18447
- const normalized = target.replaceAll("/", "\\").toLowerCase();
18448
- let match;
18449
- let matchLength = -1;
18450
- for (const root of roots) {
18451
- const prefix = root.path.replaceAll("/", "\\").replace(/\\+$/u, "").toLowerCase();
18452
- if (prefix === "" || prefix.length <= matchLength)
18453
- continue;
18454
- if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
18455
- match = root;
18456
- matchLength = prefix.length;
18552
+ async resolveBinaries(mode) {
18553
+ if (!this.isSupported()) {
18554
+ return {
18555
+ ok: false,
18556
+ reason: "unsupported-platform",
18557
+ message: `Everything is Windows-only; this host is ${this.platform}.`
18558
+ };
18457
18559
  }
18560
+ if (this.binaries?.ok)
18561
+ return this.binaries;
18562
+ const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
18563
+ const result = await resolve(mode);
18564
+ if (result.ok)
18565
+ this.binaries = result;
18566
+ return result;
18458
18567
  }
18459
- return match;
18460
- }
18461
- function powerShellLaunch(script) {
18462
- return {
18463
- args: [
18464
- "-NoProfile",
18465
- "-NonInteractive",
18466
- "-Command",
18467
- `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; Invoke-Expression $env:${POWERSHELL_SCRIPT_VARIABLE}`
18468
- ],
18469
- env: { [POWERSHELL_SCRIPT_VARIABLE]: script }
18470
- };
18471
- }
18472
- async function runPowerShell(script, timeoutMs) {
18473
- const launch = powerShellLaunch(script);
18474
- try {
18475
- const { stdout, stderr } = await execFileAsync("powershell.exe", [...launch.args], {
18476
- timeout: timeoutMs,
18477
- maxBuffer: 16 * 1024 * 1024,
18478
- windowsHide: true,
18479
- env: { ...process.env, ...launch.env }
18480
- });
18481
- return { ok: true, stdout, stderr };
18482
- } catch (error) {
18483
- const record = typeof error === "object" && error !== null ? error : {};
18568
+ async ensureRunning(reconfigure) {
18569
+ const binaries = await this.resolveBinaries("install");
18570
+ if (!binaries.ok)
18571
+ return binaries;
18572
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
18573
+ let ping = await this.ping(binaries.esPath);
18574
+ if (ping.code === 0 && !reconfigure)
18575
+ return { ok: true, esPath: binaries.esPath };
18576
+ if (ping.code === 0) {
18577
+ const exitArgs = ["-instance", this.instanceName, "-exit"];
18578
+ const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
18579
+ if (exited.code !== 0)
18580
+ return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
18581
+ }
18582
+ mkdirSync15(this.stateDir, { recursive: true });
18583
+ const iniPath = join28(this.stateDir, "Everything.ini");
18584
+ writeFileSync11(iniPath, buildEverythingIni({
18585
+ folders: this.options.folders,
18586
+ excludeFolders: [
18587
+ ...new Set([
18588
+ join28(this.options.root, ".autorag"),
18589
+ ...this.options.folders.map((folder) => join28(folder, ".autorag")),
18590
+ ...this.options.excludeFolders ?? []
18591
+ ])
18592
+ ]
18593
+ }));
18594
+ const launchArgs = [
18595
+ "-instance",
18596
+ this.instanceName,
18597
+ "-config",
18598
+ iniPath,
18599
+ "-db",
18600
+ join28(this.stateDir, "Everything.db"),
18601
+ "-startup"
18602
+ ];
18603
+ try {
18604
+ this.launch(binaries.everythingPath, launchArgs);
18605
+ } catch (error) {
18606
+ return {
18607
+ ok: false,
18608
+ reason: "startup-failed",
18609
+ message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
18610
+ };
18611
+ }
18612
+ const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
18613
+ const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
18614
+ for (;; ) {
18615
+ ping = await this.ping(binaries.esPath);
18616
+ if (ping.code === 0)
18617
+ return { ok: true, esPath: binaries.esPath };
18618
+ if (Date.now() >= deadline)
18619
+ break;
18620
+ await sleep2(pollMs);
18621
+ }
18484
18622
  return {
18485
18623
  ok: false,
18486
- stdout: typeof record.stdout === "string" ? record.stdout : "",
18487
- stderr: typeof record.stderr === "string" ? record.stderr : String(error)
18624
+ reason: "startup-failed",
18625
+ message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
18488
18626
  };
18489
18627
  }
18490
- }
18491
- function defaultCloudProbe() {
18492
- if (process.platform === "darwin")
18493
- return new DarwinCloudProbe;
18494
- if (process.platform === "win32")
18495
- return new WindowsCloudProbe;
18496
- return;
18497
- }
18498
- async function readFileProviderDomain(target) {
18499
- try {
18500
- await lstat(target);
18501
- const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
18502
- timeout: 2000
18503
- });
18504
- const value = stdout.trim();
18505
- return value.length > 0 ? value : undefined;
18506
- } catch {
18507
- return;
18628
+ async ping(esPath) {
18629
+ const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
18630
+ return result.code === 0 && !VERSION_PATTERN.test(result.stdout.trim()) ? { ...result, code: -1 } : result;
18508
18631
  }
18509
- }
18510
- async function readFileProviderDomainUpwards(target) {
18511
- let cursor = target;
18512
- for (let depth = 0;depth < 12; depth += 1) {
18513
- const domain = await readFileProviderDomain(cursor);
18514
- if (domain !== undefined)
18515
- return domain;
18516
- const parent = dirname14(cursor);
18517
- if (parent === cursor)
18518
- return;
18519
- cursor = parent;
18632
+ failure(reason, esPath, args, result) {
18633
+ return {
18634
+ ok: false,
18635
+ reason,
18636
+ message: `${esPath} ${args.join(" ")} exit ${result.code}: ${(result.stderr || result.stdout).trim()}`
18637
+ };
18520
18638
  }
18521
- return;
18522
- }
18523
- function quotePowerShell(value) {
18524
- return `'${value.replace(/['\u2018\u2019\u201A\u201B]/gu, (quote) => quote + quote)}'`;
18525
18639
  }
18526
- var execFileAsync, SF_DATALESS = 1073741824, FILE_PROVIDER_XATTR = "com.apple.file-provider-domain-id", WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 4194304, STAT_BATCH = 200, defaultCloudExec = (file, args, options) => execFileAsync(file, [...args], {
18527
- timeout: options?.timeoutMs ?? 30000,
18528
- ...options?.maxBuffer === undefined ? {} : { maxBuffer: options.maxBuffer }
18529
- }), JXA_RELEASE_PROGRAM = `ObjC.import("Foundation");
18530
- function run(argv) {
18531
- var url = $.NSURL.fileURLWithPath(argv[0]);
18532
- var errorRef = Ref();
18533
- var released = $.NSFileManager.defaultManager.evictUbiquitousItemAtURLError(url, errorRef);
18534
- if (released) return JSON.stringify({ ok: true });
18535
- function resourceValue(key) {
18536
- var ref = Ref();
18537
- if (!url.getResourceValueForKeyError(ref, key, null)) return null;
18538
- var value = ref[0];
18539
- return value === undefined || value === null ? null : ObjC.unwrap(value);
18540
- }
18541
- return JSON.stringify({
18542
- ok: false,
18543
- isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
18544
- isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
18545
- downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
18546
- });
18547
- }`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
18548
- var init_cloud_placeholder = __esm(() => {
18549
- execFileAsync = promisify(execFile2);
18550
- RELEASABLE_DARWIN_PROVIDERS = {
18551
- icloud: true,
18552
- "google-drive": true,
18553
- onedrive: true,
18554
- dropbox: true
18555
- };
18556
- DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
18557
- WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
18558
- WINDOWS_DEHYDRATE_REFUSALS = {
18559
- "80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
18560
- "80070188": "The file is pinned to stay on this device, so it is not released."
18640
+ function runProcess(command, args, timeoutMs) {
18641
+ let resolvePromise;
18642
+ const promise = new Promise((resolve) => {
18643
+ resolvePromise = resolve;
18644
+ });
18645
+ const child = spawn14(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
18646
+ const stdout = [];
18647
+ const stderr = [];
18648
+ let bytes = 0;
18649
+ let overflow = false;
18650
+ const collect = (target) => (chunk) => {
18651
+ bytes += chunk.length;
18652
+ if (bytes > MAX_OUTPUT_BYTES) {
18653
+ overflow = true;
18654
+ child.kill();
18655
+ return;
18656
+ }
18657
+ target.push(chunk);
18561
18658
  };
18562
- });
18563
-
18564
- // src/dupey/scan-plan.ts
18565
- import { basename as basename7 } from "node:path";
18566
- async function listCloudRoots(probe, roots) {
18567
- if (probe === undefined)
18568
- return [];
18569
- try {
18570
- return await probe.cloudRoots(roots);
18571
- } catch {
18572
- return [];
18573
- }
18659
+ child.stdout.on("data", collect(stdout));
18660
+ child.stderr.on("data", collect(stderr));
18661
+ const timer = setTimeout(() => child.kill(), timeoutMs);
18662
+ child.on("error", (error) => {
18663
+ clearTimeout(timer);
18664
+ resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
18665
+ });
18666
+ child.on("close", (code, signal) => {
18667
+ clearTimeout(timer);
18668
+ const err = Buffer.concat(stderr).toString("utf8");
18669
+ resolvePromise({
18670
+ code,
18671
+ stdout: Buffer.concat(stdout).toString("utf8"),
18672
+ stderr: overflow ? `${err}
18673
+ output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
18674
+ killed by ${signal} after ${timeoutMs}ms` : err
18675
+ });
18676
+ });
18677
+ return promise;
18574
18678
  }
18575
- function planDupeyScans(roots, cloudRoots) {
18576
- const targets = [];
18577
- const skipped = [];
18578
- for (const path of roots) {
18579
- const scope = cloudScope(path, cloudRoots);
18580
- if (scope.insideCloudRoot) {
18581
- skipped.push(path);
18582
- continue;
18583
- }
18584
- targets.push({ path, excludeDirs: scope.nestedCloudRoots.map((root) => basename7(root.path)) });
18585
- }
18586
- return { targets, skipped };
18679
+ function launchDetached(command, args) {
18680
+ const child = spawn14(command, args, { detached: true, stdio: "ignore", windowsHide: true });
18681
+ child.on("error", () => {});
18682
+ child.unref();
18587
18683
  }
18588
- var init_scan_plan = __esm(() => {
18589
- init_cloud_placeholder();
18684
+ var DEFAULT_TIMEOUT_MS8 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
18685
+ var init_client9 = __esm(() => {
18686
+ init_path_exclusions();
18687
+ init_bundle();
18688
+ DEFAULT_INDEX_TIMEOUT_MS3 = 10 * 60000;
18689
+ MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
18690
+ VERSION_PATTERN = /^\d+\.\d+\.\d+\.\d+$/;
18590
18691
  });
18591
18692
 
18592
- // src/dupey/index.ts
18593
- var init_dupey = __esm(() => {
18594
- init_cli2();
18595
- init_filter();
18596
- init_scan_plan();
18693
+ // src/everything/index.ts
18694
+ var init_everything = __esm(() => {
18695
+ init_bundle();
18696
+ init_client9();
18597
18697
  });
18598
18698
 
18599
- // src/convert/relate.ts
18600
- async function convertAndRelate(request, options = {}) {
18601
- const result = await convertWithFilee(request, options.filee);
18602
- if (options.dupey === false)
18603
- return { result, links: [] };
18604
- const dupeyOptions = options.dupey ?? {};
18605
- const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
18606
- const links = await Promise.all(pairs.map(async ({ source, output }) => {
18607
- try {
18608
- return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
18609
- } catch (error) {
18610
- if (!(error instanceof DupeyCliError))
18611
- throw error;
18612
- return { source, output, dupey: { relation: "unavailable", reason: error.message } };
18613
- }
18614
- }));
18615
- return { result, links };
18699
+ // src/filesystem/cloud-placeholder.ts
18700
+ import { execFile as execFile2 } from "node:child_process";
18701
+ import { realpathSync as realpathSync3 } from "node:fs";
18702
+ import { lstat, readdir } from "node:fs/promises";
18703
+ import { homedir as homedir6 } from "node:os";
18704
+ import { dirname as dirname15, join as join29 } from "node:path";
18705
+ import { promisify } from "node:util";
18706
+ function homeCloudRoots(home = homedir6()) {
18707
+ const normalizedHome = home.replaceAll("\\", "/").replace(/\/+$/, "");
18708
+ return [`${normalizedHome}/Library/CloudStorage`, `${normalizedHome}/Library/Mobile Documents`];
18616
18709
  }
18617
- var init_relate = __esm(() => {
18618
- init_dupey();
18619
- init_cli();
18620
- });
18621
-
18622
- // src/convert/index.ts
18623
- var init_convert = __esm(() => {
18624
- init_cli();
18625
- init_installer2();
18626
- init_matrix();
18627
- init_relate();
18628
- });
18629
-
18630
- // src/datasource/diagnostics.ts
18631
- function mapDatasourceDiagnostic(ds) {
18632
- return {
18633
- code: datasourceCodeToRetrieval(ds.code),
18634
- severity: ds.severity,
18635
- message: `[datasource:${ds.code}] ${ds.message}`,
18636
- source: ds.source ?? (ds.instanceId ? `datasource:${ds.instanceId}` : "datasource")
18637
- };
18710
+ function datalessFromDarwinFlags(flags) {
18711
+ return Number.isFinite(flags) && (flags >>> 0 & SF_DATALESS) !== 0;
18638
18712
  }
18639
- function mapDatasourceDiagnostics(diagnostics) {
18640
- return diagnostics.map(mapDatasourceDiagnostic);
18713
+ function datalessFromWindowsAttributes(attributes) {
18714
+ return Number.isFinite(attributes) && (attributes & WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) !== 0;
18641
18715
  }
18642
- function datasourceCodeToRetrieval(code) {
18643
- switch (code) {
18644
- case "datasource-unavailable":
18645
- case "datasource-cli-error":
18646
- case "datasource-empty":
18647
- case "datasource-rate-limited":
18648
- case "datasource-auth-error":
18649
- case "datasource-embedding-egress-rejected":
18650
- case "datasource-index-failed":
18651
- case "datasource-index-partial":
18652
- case "datasource-permission-denied":
18653
- case "datasource-remote-embedding-rejected":
18654
- return "retrieval-method-failed";
18716
+ function cloudProviderFromPath(target) {
18717
+ const lower = target.toLowerCase();
18718
+ if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
18719
+ return "google-drive";
18720
+ if (lower.includes("onedrive"))
18721
+ return "onedrive";
18722
+ if (lower.includes("dropbox"))
18723
+ return "dropbox";
18724
+ if (lower.includes("mobile documents") || lower.includes("clouddocs"))
18725
+ return "icloud";
18726
+ return "file-provider";
18727
+ }
18728
+ function cloudProviderFromDomain(domain) {
18729
+ const lower = domain.toLowerCase();
18730
+ if (lower.includes("clouddocs") || lower.includes("icloud"))
18731
+ return "icloud";
18732
+ if (lower.includes("google"))
18733
+ return "google-drive";
18734
+ if (lower.includes("onedrive"))
18735
+ return "onedrive";
18736
+ if (lower.includes("dropbox"))
18737
+ return "dropbox";
18738
+ return "file-provider";
18739
+ }
18740
+ function normalizeCloudPath(value) {
18741
+ let normalized = value.replaceAll("\\", "/");
18742
+ if (normalized.length > 1)
18743
+ normalized = normalized.replace(/\/+$/u, "");
18744
+ return normalized;
18745
+ }
18746
+ function isPathInside(target, parent) {
18747
+ const normalizedParent = normalizeCloudPath(parent);
18748
+ if (normalizedParent === "")
18749
+ return false;
18750
+ const left = normalizeCloudPath(target);
18751
+ const [a, b] = process.platform === "win32" ? [left.toLowerCase(), normalizedParent.toLowerCase()] : [left, normalizedParent];
18752
+ return a === b || a.startsWith(`${b}/`);
18753
+ }
18754
+ function canonicalPath(value) {
18755
+ try {
18756
+ return realpathSync3(value);
18757
+ } catch {
18758
+ return value;
18655
18759
  }
18656
18760
  }
18657
-
18658
- // src/datasource/retry.ts
18659
- import { setTimeout as delay2 } from "node:timers/promises";
18660
- function datasourceRetryDelayMs(retry, policy) {
18661
- const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
18662
- return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
18761
+ function canonicalPathMapper(searchRoots) {
18762
+ const forms = searchRoots.map((root) => ({ raw: normalizeCloudPath(root), canonical: normalizeCloudPath(canonicalPath(root)) })).filter(({ raw, canonical }) => raw !== "" && raw !== canonical).sort((left, right) => right.raw.length - left.raw.length);
18763
+ if (forms.length === 0)
18764
+ return (path) => path;
18765
+ return (path) => {
18766
+ const normalized = normalizeCloudPath(path);
18767
+ for (const { raw, canonical } of forms) {
18768
+ if (isPathInside(normalized, raw))
18769
+ return `${canonical}${normalized.slice(raw.length)}`;
18770
+ }
18771
+ return path;
18772
+ };
18663
18773
  }
18664
- async function indexWithRetry(skill, options = {}, signal) {
18665
- const policy = {
18666
- maxRetries: options.maxRetries ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxRetries,
18667
- baseDelayMs: options.baseDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.baseDelayMs,
18668
- maxDelayMs: options.maxDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxDelayMs
18774
+ function cloudScope(searchRoot, cloudRoots) {
18775
+ const canonicalSearchRoot = canonicalPath(searchRoot);
18776
+ const roots = cloudRoots.map((root) => ({ path: canonicalPath(root.path), kind: root.kind }));
18777
+ return {
18778
+ insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
18779
+ nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
18669
18780
  };
18670
- const sleep = options.sleep ?? ((ms) => delay2(ms, undefined, signal === undefined ? undefined : { signal }));
18671
- let lastError = "";
18672
- for (let attempt = 1;; attempt += 1) {
18673
- const result = await runIndex(skill, signal);
18674
- signal?.throwIfAborted();
18675
- if (result.ok) {
18676
- if (attempt === 1)
18677
- return result;
18678
- const recovered = {
18679
- code: "datasource-index-failed",
18680
- severity: "info",
18681
- message: `recovered after ${attempt} attempts; last error: ${lastError}`,
18682
- instanceId: result.instanceId,
18683
- source: result.skill
18684
- };
18685
- return { ...result, attempts: attempt, diagnostics: [...result.diagnostics, recovered] };
18686
- }
18687
- lastError = result.message ?? result.error ?? "datasource-index-failed";
18688
- if (attempt > policy.maxRetries) {
18689
- if (attempt === 1)
18690
- return { ...result, attempts: attempt };
18691
- const suffix = ` (gave up after ${attempt} attempts)`;
18692
- const last = result.diagnostics.at(-1);
18693
- return {
18694
- ...result,
18695
- attempts: attempt,
18696
- message: `${lastError}${suffix}`,
18697
- diagnostics: last === undefined ? [
18698
- {
18699
- code: result.code ?? "datasource-index-failed",
18700
- severity: "error",
18701
- message: `${lastError}${suffix}`,
18702
- instanceId: result.instanceId,
18703
- source: result.skill
18704
- }
18705
- ] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
18706
- };
18707
- }
18708
- await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
18709
- signal?.throwIfAborted();
18710
- }
18711
18781
  }
18712
- async function runIndex(skill, signal) {
18782
+ function parseReleaseOutcome(stdout) {
18783
+ let parsed;
18713
18784
  try {
18714
- return await skill.index(signal === undefined ? undefined : { signal });
18715
- } catch (error) {
18716
- signal?.throwIfAborted();
18717
- const descriptor = skill.describe();
18718
- const message = error instanceof Error ? error.message : "Datasource indexing failed.";
18719
- const failed = {
18720
- ok: false,
18721
- instanceId: descriptor.instanceId ?? "default",
18722
- skill: descriptor.name,
18723
- indexedAt: Date.now(),
18724
- diagnostics: [
18725
- {
18726
- code: "datasource-index-failed",
18727
- severity: "error",
18728
- message,
18729
- source: descriptor.name,
18730
- instanceId: descriptor.instanceId
18731
- }
18732
- ],
18733
- error: "datasource-index-failed",
18734
- code: "datasource-index-failed",
18735
- message
18736
- };
18737
- return failed;
18785
+ parsed = JSON.parse(stdout.trim());
18786
+ } catch {
18787
+ return;
18788
+ }
18789
+ if (typeof parsed !== "object" || parsed === null || !("ok" in parsed) || typeof parsed.ok !== "boolean") {
18790
+ return;
18791
+ }
18792
+ return {
18793
+ ok: parsed.ok,
18794
+ ..."isUploaded" in parsed ? { isUploaded: parsed.isUploaded } : {},
18795
+ ..."isUploading" in parsed ? { isUploading: parsed.isUploading } : {},
18796
+ ..."downloadingStatus" in parsed ? { downloadingStatus: parsed.downloadingStatus } : {}
18797
+ };
18798
+ }
18799
+ function describeReleaseRefusal(outcome) {
18800
+ if (outcome.isUploading === true)
18801
+ return "The file is still uploading to the cloud, so it cannot be released yet.";
18802
+ if (outcome.isUploaded === false)
18803
+ return "The file has not finished uploading to the cloud, so it cannot be released yet.";
18804
+ const status = typeof outcome.downloadingStatus === "string" ? outcome.downloadingStatus : "unknown";
18805
+ return `The cloud provider declined to release the local copy (download status ${status}).`;
18806
+ }
18807
+ function describeReleaseHelperFailure(error) {
18808
+ const failure = error;
18809
+ const parts = [];
18810
+ if (failure.killed === true || typeof failure.signal === "string") {
18811
+ parts.push(`the helper was killed${typeof failure.signal === "string" ? ` by ${failure.signal}` : ""}`);
18812
+ } else if (typeof failure.code === "number") {
18813
+ parts.push(`the helper exited with code ${failure.code}`);
18738
18814
  }
18815
+ const stderr = typeof failure.stderr === "string" ? failure.stderr.trim() : "";
18816
+ if (stderr !== "")
18817
+ parts.push(stderr);
18818
+ if (parts.length === 0)
18819
+ parts.push(error instanceof Error ? error.message : String(error));
18820
+ return `osascript could not release the local copy: ${parts.join("; ")}`;
18739
18821
  }
18740
- var DEFAULT_DATASOURCE_RETRY_POLICY;
18741
- var init_retry = __esm(() => {
18742
- DEFAULT_DATASOURCE_RETRY_POLICY = {
18743
- maxRetries: 2,
18744
- baseDelayMs: 1000,
18745
- maxDelayMs: 30000
18746
- };
18747
- });
18748
-
18749
- // src/everything/bundle.ts
18750
- import { createHash as createHash11 } from "node:crypto";
18751
- import { existsSync as existsSync17, mkdirSync as mkdirSync14, readFileSync as readFileSync17, rmSync as rmSync7, writeFileSync as writeFileSync10 } from "node:fs";
18752
- import { dirname as dirname15, join as join28 } from "node:path";
18753
- import { fileURLToPath } from "node:url";
18754
- import JSZip from "jszip";
18755
- function resolveEverythingBundleDir() {
18756
- let dir = dirname15(fileURLToPath(import.meta.url));
18757
- for (let depth = 0;depth < 6; depth += 1) {
18758
- const candidate = join28(dir, "vendor", "everything");
18759
- if (existsSync17(join28(candidate, MANIFEST_FILENAME)))
18760
- return candidate;
18761
- const parent = dirname15(dir);
18762
- if (parent === dir)
18763
- break;
18764
- dir = parent;
18765
- }
18766
- return;
18822
+ async function defaultDarwinReadDir(path) {
18823
+ const entries = await readdir(path, { withFileTypes: true });
18824
+ return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
18767
18825
  }
18768
- function everythingArch(arch) {
18769
- if (arch === "x64" || arch === "arm64")
18770
- return arch;
18771
- return;
18826
+ function parseDarwinMounts(stdout) {
18827
+ const mounts = [];
18828
+ for (const line of stdout.split(`
18829
+ `)) {
18830
+ if (line.trim() === "")
18831
+ continue;
18832
+ const open = line.lastIndexOf(" (");
18833
+ if (open <= 0 || !line.endsWith(")"))
18834
+ continue;
18835
+ const head = line.slice(0, open);
18836
+ const on = head.indexOf(" on ");
18837
+ if (on <= 0)
18838
+ continue;
18839
+ const mountPoint = head.slice(on + " on ".length);
18840
+ if (mountPoint === "")
18841
+ continue;
18842
+ const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
18843
+ mounts.push({
18844
+ device: head.slice(0, on),
18845
+ mountPoint,
18846
+ fileSystem: fields[0] ?? "",
18847
+ options: fields.slice(1)
18848
+ });
18849
+ }
18850
+ return mounts;
18772
18851
  }
18773
- function everythingBinaryPaths(root, version, arch) {
18774
- const dir = join28(root, ".autorag", "everything", version, arch);
18775
- return { everythingPath: join28(dir, "everything.exe"), esPath: join28(dir, "es.exe") };
18852
+ function networkMountPoints(mounts) {
18853
+ const points = [];
18854
+ const seen = new Set;
18855
+ for (const mount of mounts) {
18856
+ if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
18857
+ continue;
18858
+ if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
18859
+ continue;
18860
+ if (seen.has(mount.mountPoint))
18861
+ continue;
18862
+ seen.add(mount.mountPoint);
18863
+ points.push(mount.mountPoint);
18864
+ }
18865
+ return points;
18776
18866
  }
18777
- function loadEverythingBundleManifest(bundleDir) {
18778
- return JSON.parse(readFileSync17(join28(bundleDir, MANIFEST_FILENAME), "utf8"));
18867
+ function parseXattrValues(stdout, candidates) {
18868
+ const values = new Map;
18869
+ if (candidates.length === 0)
18870
+ return values;
18871
+ const ordered = [...candidates].sort((left, right) => right.length - left.length);
18872
+ for (const line of stdout.split(`
18873
+ `)) {
18874
+ if (line === "")
18875
+ continue;
18876
+ for (const candidate of ordered) {
18877
+ if (line.startsWith(`${candidate}: `)) {
18878
+ values.set(candidate, line.slice(candidate.length + 2));
18879
+ break;
18880
+ }
18881
+ }
18882
+ }
18883
+ return values;
18779
18884
  }
18780
- async function ensureEverythingBinaries(options) {
18781
- const platform = options.platform ?? process.platform;
18782
- const rawArch = options.arch ?? process.arch;
18783
- if (platform !== "win32") {
18784
- return {
18785
- ok: false,
18786
- reason: "unsupported-platform",
18787
- message: `Everything is Windows-only; this host is ${platform}.`
18788
- };
18885
+
18886
+ class DarwinCloudProbe {
18887
+ exec;
18888
+ readDir;
18889
+ home;
18890
+ constructor(options = {}) {
18891
+ this.exec = options.exec ?? defaultCloudExec;
18892
+ this.readDir = options.readDir ?? defaultDarwinReadDir;
18893
+ this.home = options.home ?? homedir6();
18789
18894
  }
18790
- const arch = everythingArch(rawArch);
18791
- const bundleDir = options.bundleDir ?? resolveEverythingBundleDir();
18792
- if (bundleDir === undefined) {
18793
- return {
18794
- ok: false,
18795
- reason: "bundle-missing",
18796
- message: `Bundled Everything assets (vendor/everything/${MANIFEST_FILENAME}) were not found above ${dirname15(fileURLToPath(import.meta.url))}.`
18895
+ async cloudRoots(searchRoots = []) {
18896
+ const roots = [];
18897
+ const seen = new Set;
18898
+ const add = (path, kind) => {
18899
+ const canonical = canonicalPath(path);
18900
+ const key = normalizeCloudPath(canonical);
18901
+ if (key === "" || seen.has(key))
18902
+ return;
18903
+ seen.add(key);
18904
+ roots.push({ path: canonical, kind });
18797
18905
  };
18906
+ const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
18907
+ for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
18908
+ if (entry.isDirectory())
18909
+ add(join29(cloudStorage, entry.name), "file-provider");
18910
+ }
18911
+ if (await this.tryReadDir(mobileDocuments) !== undefined)
18912
+ add(mobileDocuments, "file-provider");
18913
+ for (const mountPoint of await this.networkMountPoints())
18914
+ add(mountPoint, "network");
18915
+ for (const directory of await this.fileProviderDirectories(searchRoots))
18916
+ add(directory, "file-provider");
18917
+ return roots;
18798
18918
  }
18799
- let manifest;
18800
- try {
18801
- manifest = options.manifest ?? loadEverythingBundleManifest(bundleDir);
18802
- } catch (error) {
18803
- return { ok: false, reason: "bundle-missing", message: errorMessage2(error) };
18919
+ async networkMountPoints() {
18920
+ try {
18921
+ const { stdout } = await this.exec("/sbin/mount", [], { timeoutMs: 1e4, maxBuffer: 4 * 1024 * 1024 });
18922
+ return networkMountPoints(parseDarwinMounts(stdout));
18923
+ } catch {
18924
+ return [];
18925
+ }
18804
18926
  }
18805
- const assets = arch === undefined ? undefined : manifest.architectures[arch];
18806
- if (arch === undefined || assets === undefined) {
18807
- return {
18808
- ok: false,
18809
- reason: "unsupported-platform",
18810
- message: `No bundled Everything build for Windows architecture ${rawArch}; bundled: ${Object.keys(manifest.architectures).join(", ")}.`
18811
- };
18927
+ async fileProviderDirectories(searchRoots) {
18928
+ if (searchRoots.length === 0)
18929
+ return [];
18930
+ const candidates = new Set;
18931
+ for (const searchRoot of searchRoots) {
18932
+ const canonical = canonicalPath(searchRoot);
18933
+ for (let cursor = canonical;; cursor = dirname15(cursor)) {
18934
+ candidates.add(cursor);
18935
+ const parent = dirname15(cursor);
18936
+ if (parent === cursor)
18937
+ break;
18938
+ }
18939
+ for (const entry of await this.tryReadDir(canonical) ?? []) {
18940
+ if (entry.isDirectory())
18941
+ candidates.add(join29(canonical, entry.name));
18942
+ }
18943
+ }
18944
+ const paths = [...candidates];
18945
+ const managed = [];
18946
+ for (let start = 0;start < paths.length; start += XATTR_BATCH) {
18947
+ for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
18948
+ managed.push(path);
18949
+ }
18950
+ return managed;
18812
18951
  }
18813
- const { everythingPath, esPath } = everythingBinaryPaths(options.root, manifest.everythingVersion, arch);
18814
- const everythingCached = fileHasSha256(everythingPath, assets.everythingSha256);
18815
- const esCached = fileHasSha256(esPath, assets.esSha256);
18816
- if (everythingCached && esCached) {
18817
- return { ok: true, everythingPath, esPath, source: "cached" };
18952
+ async xattrValues(paths) {
18953
+ try {
18954
+ const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
18955
+ timeoutMs: 15000,
18956
+ maxBuffer: 8 * 1024 * 1024
18957
+ });
18958
+ return parseXattrValues(stdout, paths);
18959
+ } catch (error) {
18960
+ const failure = error;
18961
+ return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
18962
+ }
18818
18963
  }
18819
- if ((options.mode ?? "install") === "cached") {
18820
- const state = everythingCached || esCached ? "are incomplete or do not match their pinned SHA-256 digests" : "are not installed";
18821
- return {
18822
- ok: false,
18823
- reason: "not-installed",
18824
- message: `Everything ${manifest.everythingVersion} (${arch}) binaries ${state} under ${dirname15(everythingPath)}; run \`autorag refresh\` to (re)install and index them.`
18825
- };
18964
+ async tryReadDir(path) {
18965
+ try {
18966
+ return await this.readDir(path);
18967
+ } catch {
18968
+ return;
18969
+ }
18826
18970
  }
18827
- try {
18828
- mkdirSync14(dirname15(everythingPath), { recursive: true });
18829
- const everythingBytes = await extractVerified(join28(bundleDir, assets.everythingArchive), assets.everythingArchiveSha256, assets.everythingMember, assets.everythingSha256);
18830
- const esBytes = await extractVerified(join28(bundleDir, assets.esArchive), assets.esArchiveSha256, assets.esMember, assets.esSha256);
18831
- writeAtomic(everythingPath, everythingBytes);
18832
- writeAtomic(esPath, esBytes);
18833
- } catch (error) {
18834
- return { ok: false, reason: "install-failed", message: errorMessage2(error) };
18971
+ async datalessAmong(paths) {
18972
+ const dataless = new Set;
18973
+ for (let start = 0;start < paths.length; start += STAT_BATCH) {
18974
+ const batch = paths.slice(start, start + STAT_BATCH);
18975
+ const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
18976
+ timeoutMs: 30000,
18977
+ maxBuffer: 4 * 1024 * 1024
18978
+ });
18979
+ const flags = stdout.split(`
18980
+ `).filter(Boolean);
18981
+ for (const [index, path] of batch.entries()) {
18982
+ if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
18983
+ dataless.add(path);
18984
+ }
18985
+ }
18986
+ return dataless;
18835
18987
  }
18836
- return { ok: true, everythingPath, esPath, source: "installed" };
18837
- }
18838
- async function extractVerified(archivePath, archiveSha256, member, memberSha256) {
18839
- const archive = readFileSync17(archivePath);
18840
- const actualArchive = sha256(archive);
18841
- if (actualArchive !== archiveSha256) {
18842
- throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
18988
+ async providerOf(path) {
18989
+ const domain = await readFileProviderDomainUpwards(path);
18990
+ return domain === undefined ? undefined : cloudProviderFromDomain(domain);
18843
18991
  }
18844
- const zip = await JSZip.loadAsync(archive);
18845
- const entry = zip.file(member);
18846
- if (entry === null)
18847
- throw new Error(`${archivePath} does not contain ${member}`);
18848
- const bytes = await entry.async("nodebuffer");
18849
- const actualMember = sha256(bytes);
18850
- if (actualMember !== memberSha256) {
18851
- throw new Error(`${member} from ${archivePath} SHA-256 ${actualMember} does not match pinned ${memberSha256}`);
18992
+ canRelease(provider) {
18993
+ return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
18994
+ }
18995
+ async evict(path) {
18996
+ let stdout;
18997
+ try {
18998
+ stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
18999
+ timeoutMs: 30000
19000
+ })).stdout;
19001
+ } catch (error) {
19002
+ return { ok: false, message: describeReleaseHelperFailure(error) };
19003
+ }
19004
+ const outcome = parseReleaseOutcome(stdout);
19005
+ if (outcome === undefined) {
19006
+ const detail = stdout.trim();
19007
+ return {
19008
+ ok: false,
19009
+ message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
19010
+ };
19011
+ }
19012
+ return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
18852
19013
  }
18853
- return bytes;
18854
19014
  }
18855
- function fileHasSha256(path, expected) {
18856
- return existsSync17(path) && sha256(readFileSync17(path)) === expected;
19015
+ function buildWindowsAttributeScript(paths) {
19016
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19017
+ ` + `$ErrorActionPreference='SilentlyContinue'
19018
+ ` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
19019
+ `;
18857
19020
  }
18858
- function writeAtomic(path, bytes) {
18859
- const temp = `${path}.${process.pid}.tmp`;
18860
- try {
18861
- writeFileSync10(temp, bytes);
18862
- renameWithRetry(temp, path);
18863
- } finally {
18864
- rmSync7(temp, { force: true });
19021
+ function buildWindowsSyncRootsScript() {
19022
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19023
+ ` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
19024
+ ` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
19025
+ `;
19026
+ }
19027
+ function buildWindowsDehydrateScript(path) {
19028
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19029
+ ` + `$ErrorActionPreference='Stop'
19030
+ ` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
19031
+ ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
19032
+ ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
19033
+ ` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
19034
+ ` + `'@
19035
+ ` + `$h=[IntPtr]::Zero
19036
+ ` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
19037
+ ` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
19038
+ ` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
19039
+ ` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
19040
+ ` + `[Autorag.CfApi]::CfCloseHandle($h)
19041
+ ` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
19042
+ ` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
19043
+ `;
19044
+ }
19045
+ function parseWindowsAttributeOutput(stdout, paths) {
19046
+ const lines = stdout.split(/\r?\n/);
19047
+ const attributes = new Map;
19048
+ for (const [index, path] of paths.entries()) {
19049
+ const value = Number.parseInt((lines[index] ?? "").trim(), 10);
19050
+ if (Number.isFinite(value))
19051
+ attributes.set(path, value);
18865
19052
  }
19053
+ return attributes;
18866
19054
  }
18867
- function sha256(bytes) {
18868
- return createHash11("sha256").update(bytes).digest("hex");
19055
+ function parseWindowsSyncRoots(stdout) {
19056
+ const roots = [];
19057
+ for (const line of stdout.split(/\r?\n/)) {
19058
+ const tab = line.indexOf("\t");
19059
+ if (tab <= 0)
19060
+ continue;
19061
+ const path = line.slice(0, tab).trim();
19062
+ if (path !== "")
19063
+ roots.push({ path, label: line.slice(tab + 1).trim() });
19064
+ }
19065
+ return roots;
18869
19066
  }
18870
- function errorMessage2(error) {
18871
- return error instanceof Error ? error.message : String(error);
19067
+ function providerFromWindowsSyncRoot(label) {
19068
+ const lower = label.toLowerCase();
19069
+ if (lower.includes("onedrive") || lower.includes("one drive"))
19070
+ return "onedrive";
19071
+ if (lower.includes("google"))
19072
+ return "google-drive";
19073
+ return "file-provider";
18872
19074
  }
18873
- var MANIFEST_FILENAME = "manifest.json";
18874
- var init_bundle = __esm(() => {
18875
- init_atomic_write();
18876
- });
18877
-
18878
- // src/filesystem/path-exclusions.ts
18879
- import { posix as posix3, win32 as win322 } from "node:path";
18880
- function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
18881
- const paths = platform === "win32" ? win322 : posix3;
18882
- const normalize = (path) => {
18883
- const absolute = paths.resolve(path);
18884
- return platform === "win32" ? absolute.toLowerCase() : absolute;
18885
- };
18886
- const excluded = excludedPaths.map((path) => {
18887
- const absolute = normalize(path);
18888
- return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
18889
- });
18890
- return (path) => {
18891
- const absolute = normalize(path);
18892
- return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
18893
- };
19075
+ function buildWindowsDiskScript() {
19076
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19077
+ ` + "Get-CimInstance -ClassName Win32_LogicalDisk | ForEach-Object {" + `$_.DeviceID + [char]9 + $_.DriveType + [char]9 + $_.FileSystem + [char]9 + $_.VolumeName}
19078
+ `;
18894
19079
  }
18895
- var init_path_exclusions = () => {};
18896
-
18897
- // src/filesystem/shared-exclusive-queue.ts
18898
- class SharedExclusiveQueue {
18899
- sharedSinceExclusive = [];
18900
- lastExclusive = Promise.resolve();
18901
- shared(task) {
18902
- const run = this.lastExclusive.then(task);
18903
- const settled = run.then(() => {
18904
- return;
18905
- }, () => {
18906
- return;
18907
- });
18908
- this.sharedSinceExclusive.push(settled);
18909
- return run;
18910
- }
18911
- exclusive(task) {
18912
- const earlier = Promise.all([this.lastExclusive, ...this.sharedSinceExclusive]);
18913
- const run = earlier.then(task);
18914
- const settled = run.then(() => {
18915
- return;
18916
- }, () => {
18917
- return;
19080
+ function parseWindowsLogicalDisks(stdout) {
19081
+ const disks = [];
19082
+ for (const line of stdout.split(/\r?\n/)) {
19083
+ if (line.trim() === "")
19084
+ continue;
19085
+ const [deviceId, driveType, fileSystem, ...rest] = line.split("\t");
19086
+ const type = Number.parseInt((driveType ?? "").trim(), 10);
19087
+ if (!Number.isFinite(type))
19088
+ continue;
19089
+ disks.push({
19090
+ deviceId: (deviceId ?? "").trim(),
19091
+ driveType: type,
19092
+ fileSystem: (fileSystem ?? "").trim(),
19093
+ volumeName: rest.join("\t").trim()
18918
19094
  });
18919
- this.sharedSinceExclusive = [];
18920
- this.lastExclusive = settled;
18921
- return run;
18922
19095
  }
19096
+ return disks;
18923
19097
  }
18924
-
18925
- // src/everything/client.ts
18926
- import { spawn as spawn14 } from "node:child_process";
18927
- import { createHash as createHash12 } from "node:crypto";
18928
- import { mkdirSync as mkdirSync15, writeFileSync as writeFileSync11 } from "node:fs";
18929
- import { join as join29, resolve as resolve11 } from "node:path";
18930
- import { setTimeout as sleep2 } from "node:timers/promises";
18931
- function everythingInstanceName(root) {
18932
- return `autorag-${createHash12("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
18933
- }
18934
- function buildEverythingIni(input) {
18935
- const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
18936
- const perFolder = (value) => input.folders.map(() => value).join(",");
18937
- return [
18938
- "[Everything]",
18939
- "app_data=0",
18940
- "run_as_admin=0",
18941
- "run_in_background=1",
18942
- "show_tray_icon=0",
18943
- "show_in_taskbar=0",
18944
- "check_for_updates_on_startup=0",
18945
- "beta_updates=0",
18946
- "language=1033",
18947
- "allow_http_server=0",
18948
- "allow_etp_server=0",
18949
- "search_history_enabled=0",
18950
- "run_history_enabled=0",
18951
- "auto_include_fixed_volumes=0",
18952
- "auto_include_removable_volumes=0",
18953
- "auto_include_fixed_refs_volumes=0",
18954
- "auto_include_removable_refs_volumes=0",
18955
- "ntfs_volume_paths=",
18956
- "ntfs_volume_includes=",
18957
- "refs_volume_paths=",
18958
- "refs_volume_includes=",
18959
- "index_size=1",
18960
- "index_date_modified=1",
18961
- `folders=${list(input.folders)}`,
18962
- `folder_monitor_changes=${perFolder("1")}`,
18963
- `folder_rescan_if_full_list=${perFolder("1")}`,
18964
- `folder_update_types=${perFolder("0")}`,
18965
- "exclude_list_enabled=1",
18966
- `exclude_folders=${list(input.excludeFolders)}`,
18967
- ""
18968
- ].join(`\r
18969
- `);
19098
+ function windowsRootsFromDisks(disks) {
19099
+ const roots = [];
19100
+ for (const disk of disks) {
19101
+ if (disk.deviceId === "")
19102
+ continue;
19103
+ const path = `${disk.deviceId}\\`;
19104
+ const fileSystem = disk.fileSystem.toLowerCase();
19105
+ if (disk.driveType === WINDOWS_DRIVE_TYPE_NETWORK) {
19106
+ roots.push({ path, kind: "network" });
19107
+ continue;
19108
+ }
19109
+ if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
19110
+ roots.push({ path, kind: "network" });
19111
+ continue;
19112
+ }
19113
+ if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
19114
+ roots.push({ path, kind: "cloud-files" });
19115
+ }
19116
+ }
19117
+ return roots;
18970
19118
  }
18971
- function buildEverythingSearchArgs(instance, request, timeoutMs) {
18972
- const args = [
18973
- "-argv",
18974
- "-instance",
18975
- instance,
18976
- "-cp",
18977
- "65001",
18978
- "-json",
18979
- "-full-path-and-name",
18980
- "-size",
18981
- "-date-modified",
18982
- "-date-format",
18983
- "1"
18984
- ];
18985
- if (timeoutMs !== undefined)
18986
- args.push("-timeout", String(timeoutMs));
18987
- if (request.matchCase)
18988
- args.push("-case");
18989
- if (request.matchPath)
18990
- args.push("-match-path");
18991
- if (request.wholeWord)
18992
- args.push("-whole-word");
18993
- if (request.kind === "files")
18994
- args.push("/a-d");
18995
- if (request.kind === "folders")
18996
- args.push("/ad");
18997
- if (request.path !== undefined)
18998
- args.push("-path", request.path);
18999
- if (request.sort !== undefined)
19000
- args.push("-sort", request.sort);
19001
- if (request.offset !== undefined && request.offset > 0)
19002
- args.push("-offset", String(request.offset));
19003
- args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
19004
- if (request.regex)
19005
- args.push("-regex", "--", request.query);
19006
- else
19007
- args.push("--", request.query);
19008
- return args;
19119
+ function uncShareRoot(target) {
19120
+ const normalized = target.replaceAll("/", "\\");
19121
+ if (!normalized.startsWith("\\\\") || normalized.startsWith("\\\\?\\") || normalized.startsWith("\\\\.\\"))
19122
+ return;
19123
+ const rest = normalized.slice(2);
19124
+ const separator = rest.indexOf("\\");
19125
+ if (separator <= 0)
19126
+ return;
19127
+ const server = rest.slice(0, separator);
19128
+ const share = rest.slice(separator + 1).split("\\")[0] ?? "";
19129
+ if (server === "" || share === "")
19130
+ return;
19131
+ return `\\\\${server}\\${share}`;
19009
19132
  }
19010
- function parseEverythingJson(stdout) {
19011
- const text = stdout.trim();
19012
- if (text.length === 0)
19013
- return [];
19014
- let parsed;
19015
- try {
19016
- parsed = JSON.parse(text);
19017
- } catch {
19018
- throw new Error(`ES did not return JSON: ${text.slice(0, 2000)}`);
19019
- }
19020
- if (!Array.isArray(parsed))
19021
- throw new Error(`ES did not return a JSON array: ${text.slice(0, 2000)}`);
19022
- return parsed.map((item) => {
19023
- const record = item;
19024
- const filename = String(record.filename ?? "");
19025
- const isFolder = filename.endsWith("\\") && filename.length > 3;
19133
+ function parseWindowsDehydrateOutput(result) {
19134
+ const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
19135
+ if (open === null)
19136
+ return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
19137
+ if (Number.parseInt(open[1], 16) !== 0) {
19026
19138
  return {
19027
- path: isFolder ? filename.slice(0, -1) : filename,
19028
- type: filename.endsWith("\\") ? "folder" : "file",
19029
- size: typeof record.size === "number" ? record.size : undefined,
19030
- dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
19139
+ ok: false,
19140
+ message: windowsFailureMessage(result, `could not open the file for release (0x${open[1]})`)
19031
19141
  };
19032
- });
19142
+ }
19143
+ const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
19144
+ if (dehydrate === null) {
19145
+ return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
19146
+ }
19147
+ const hresult = dehydrate[1];
19148
+ if (Number.parseInt(hresult, 16) === 0)
19149
+ return { ok: true };
19150
+ const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
19151
+ return {
19152
+ ok: false,
19153
+ message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
19154
+ };
19155
+ }
19156
+ function windowsFailureMessage(result, fallback) {
19157
+ const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
19158
+ if (printed !== undefined && printed !== "")
19159
+ return printed;
19160
+ const stderr = result.stderr.trim();
19161
+ return stderr !== "" ? stderr : fallback;
19033
19162
  }
19034
19163
 
19035
- class EverythingClient {
19036
- instanceName;
19037
- options;
19038
- platform;
19164
+ class WindowsCloudProbe {
19039
19165
  run;
19040
- launch;
19041
- stateDir;
19042
- isExcludedPath;
19043
- binaries;
19044
- queue = new SharedExclusiveQueue;
19045
- constructor(options) {
19046
- this.options = options;
19047
- this.platform = options.platform ?? process.platform;
19048
- this.instanceName = everythingInstanceName(options.root);
19049
- this.run = options.run ?? runProcess;
19050
- this.launch = options.launch ?? launchDetached;
19051
- this.stateDir = join29(options.root, ".autorag", "everything");
19052
- this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
19053
- }
19054
- isSupported() {
19055
- return this.platform === "win32";
19166
+ syncRoots;
19167
+ disks;
19168
+ constructor(options = {}) {
19169
+ this.run = options.run ?? runPowerShell;
19056
19170
  }
19057
- search(request) {
19058
- return this.queue.shared(async () => {
19059
- const binaries = await this.resolveBinaries("cached");
19060
- if (!binaries.ok)
19061
- return binaries;
19062
- const ping = await this.ping(binaries.esPath);
19063
- if (ping.code !== 0) {
19064
- return {
19065
- ok: false,
19066
- reason: "not-running",
19067
- message: `Everything instance ${this.instanceName} is not running; run \`autorag refresh\` to start and index it.`
19068
- };
19069
- }
19070
- const es = { ok: true, esPath: binaries.esPath };
19071
- const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
19072
- if ((this.options.excludeFolders?.length ?? 0) > 0) {
19073
- return this.searchWithExclusions(es.esPath, request, timeoutMs);
19074
- }
19075
- const args = buildEverythingSearchArgs(this.instanceName, request, timeoutMs);
19076
- const result = await this.run(es.esPath, args, timeoutMs + 5000);
19077
- if (result.code !== 0)
19078
- return this.failure("search-failed", es.esPath, args, result);
19079
- try {
19080
- return { ok: true, results: parseEverythingJson(result.stdout) };
19081
- } catch (error) {
19082
- return { ok: false, reason: "search-failed", message: error.message };
19083
- }
19084
- });
19171
+ async cloudRoots(searchRoots = []) {
19172
+ const roots = [];
19173
+ const seen = new Set;
19174
+ const add = (path, kind) => {
19175
+ const canonical = canonicalPath(path);
19176
+ const key = normalizeCloudPath(canonical).toLowerCase();
19177
+ if (key === "" || seen.has(key))
19178
+ return;
19179
+ seen.add(key);
19180
+ roots.push({ path: canonical, kind });
19181
+ };
19182
+ for (const root of await this.loadSyncRoots())
19183
+ add(root.path, "cloud-files");
19184
+ for (const root of await this.loadDiskRoots())
19185
+ add(root.path, root.kind);
19186
+ for (const searchRoot of searchRoots) {
19187
+ const share = uncShareRoot(searchRoot);
19188
+ if (share !== undefined)
19189
+ add(share, "network");
19190
+ }
19191
+ return roots;
19085
19192
  }
19086
- async searchWithExclusions(esPath, request, timeoutMs) {
19087
- const deadline = Date.now() + timeoutMs;
19088
- const offset = request.offset ?? 0;
19089
- const maxResults = request.maxResults ?? DEFAULT_MAX_RESULTS;
19090
- const results = [];
19091
- let skipped = 0;
19092
- let nativeOffset = 0;
19093
- const timedOut = () => ({
19094
- ok: false,
19095
- reason: "search-failed",
19096
- message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
19097
- });
19098
- for (;; ) {
19099
- const remainingMs = deadline - Date.now();
19100
- if (remainingMs <= 0)
19101
- return timedOut();
19102
- const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
19103
- const response = await this.run(esPath, args, remainingMs + 5000);
19104
- if (response.code !== 0)
19105
- return this.failure("search-failed", esPath, args, response);
19106
- if (Date.now() >= deadline)
19107
- return timedOut();
19108
- let entries;
19109
- try {
19110
- entries = parseEverythingJson(response.stdout);
19111
- } catch (error) {
19112
- return { ok: false, reason: "search-failed", message: error.message };
19193
+ async datalessAmong(paths) {
19194
+ const dataless = new Set;
19195
+ for (let start = 0;start < paths.length; start += WINDOWS_ATTRIBUTE_BATCH) {
19196
+ const batch = paths.slice(start, start + WINDOWS_ATTRIBUTE_BATCH);
19197
+ const result = await this.run(buildWindowsAttributeScript(batch), 30000);
19198
+ if (!result.ok) {
19199
+ throw new Error(`Could not read Windows cloud file attributes: ${result.stderr.trim() || "powershell.exe failed"}`);
19113
19200
  }
19114
- for (const entry of entries) {
19115
- if (this.isExcludedPath(entry.path))
19116
- continue;
19117
- if (skipped < offset) {
19118
- skipped += 1;
19119
- continue;
19120
- }
19121
- results.push(entry);
19122
- if (results.length >= maxResults)
19123
- return { ok: true, results };
19201
+ for (const [path, attributes] of parseWindowsAttributeOutput(result.stdout, batch)) {
19202
+ if (datalessFromWindowsAttributes(attributes))
19203
+ dataless.add(path);
19124
19204
  }
19125
- if (entries.length < MAX_FILTER_CANDIDATES)
19126
- return { ok: true, results };
19127
- nativeOffset += entries.length;
19128
19205
  }
19206
+ return dataless;
19129
19207
  }
19130
- index() {
19131
- return this.queue.exclusive(async () => {
19132
- const es = await this.ensureRunning(true);
19133
- if (!es.ok)
19134
- return es;
19135
- const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
19136
- const saveArgs = ["-instance", this.instanceName, "-save-db"];
19137
- const saved = await this.run(es.esPath, saveArgs, timeoutMs);
19138
- if (saved.code !== 0)
19139
- return this.failure("index-failed", es.esPath, saveArgs, saved);
19140
- const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
19141
- const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
19142
- if (counted.code !== 0)
19143
- return this.failure("index-failed", es.esPath, countArgs, counted);
19144
- return {
19145
- ok: true,
19146
- indexedFolders: [...this.options.folders],
19147
- indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
19148
- };
19149
- });
19208
+ async providerOf(path) {
19209
+ const root = longestSyncRoot(await this.loadSyncRoots(), path);
19210
+ return root === undefined ? cloudProviderFromPath(path) : providerFromWindowsSyncRoot(root.label);
19150
19211
  }
19151
- stop() {
19152
- return this.queue.exclusive(async () => {
19153
- const binaries = await this.resolveBinaries("cached");
19154
- if (!binaries.ok)
19155
- return;
19156
- await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
19157
- });
19212
+ canRelease(provider) {
19213
+ return provider !== undefined;
19158
19214
  }
19159
- async resolveBinaries(mode) {
19160
- if (!this.isSupported()) {
19161
- return {
19162
- ok: false,
19163
- reason: "unsupported-platform",
19164
- message: `Everything is Windows-only; this host is ${this.platform}.`
19165
- };
19166
- }
19167
- if (this.binaries?.ok)
19168
- return this.binaries;
19169
- const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
19170
- const result = await resolve(mode);
19171
- if (result.ok)
19172
- this.binaries = result;
19173
- return result;
19215
+ async evict(path) {
19216
+ return parseWindowsDehydrateOutput(await this.run(buildWindowsDehydrateScript(path), 60000));
19174
19217
  }
19175
- async ensureRunning(reconfigure) {
19176
- const binaries = await this.resolveBinaries("install");
19177
- if (!binaries.ok)
19178
- return binaries;
19179
- const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
19180
- let ping = await this.ping(binaries.esPath);
19181
- if (ping.code === 0 && !reconfigure)
19182
- return { ok: true, esPath: binaries.esPath };
19183
- if (ping.code === 0) {
19184
- const exitArgs = ["-instance", this.instanceName, "-exit"];
19185
- const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
19186
- if (exited.code !== 0)
19187
- return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
19188
- }
19189
- mkdirSync15(this.stateDir, { recursive: true });
19190
- const iniPath = join29(this.stateDir, "Everything.ini");
19191
- writeFileSync11(iniPath, buildEverythingIni({
19192
- folders: this.options.folders,
19193
- excludeFolders: [
19194
- ...new Set([
19195
- join29(this.options.root, ".autorag"),
19196
- ...this.options.folders.map((folder) => join29(folder, ".autorag")),
19197
- ...this.options.excludeFolders ?? []
19198
- ])
19199
- ]
19200
- }));
19201
- const launchArgs = [
19202
- "-instance",
19203
- this.instanceName,
19204
- "-config",
19205
- iniPath,
19206
- "-db",
19207
- join29(this.stateDir, "Everything.db"),
19208
- "-startup"
19209
- ];
19210
- try {
19211
- this.launch(binaries.everythingPath, launchArgs);
19212
- } catch (error) {
19213
- return {
19214
- ok: false,
19215
- reason: "startup-failed",
19216
- message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
19217
- };
19218
- }
19219
- const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
19220
- const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
19221
- for (;; ) {
19222
- ping = await this.ping(binaries.esPath);
19223
- if (ping.code === 0)
19224
- return { ok: true, esPath: binaries.esPath };
19225
- if (Date.now() >= deadline)
19226
- break;
19227
- await sleep2(pollMs);
19228
- }
19229
- return {
19230
- ok: false,
19231
- reason: "startup-failed",
19232
- message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
19233
- };
19218
+ loadSyncRoots() {
19219
+ this.syncRoots ??= this.run(buildWindowsSyncRootsScript(), 15000).then((result) => result.ok ? parseWindowsSyncRoots(result.stdout) : [], () => []);
19220
+ return this.syncRoots;
19234
19221
  }
19235
- async ping(esPath) {
19236
- const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
19237
- return result.code === 0 && !VERSION_PATTERN.test(result.stdout.trim()) ? { ...result, code: -1 } : result;
19222
+ loadDiskRoots() {
19223
+ this.disks ??= this.run(buildWindowsDiskScript(), 30000).then((result) => result.ok ? windowsRootsFromDisks(parseWindowsLogicalDisks(result.stdout)) : [], () => []);
19224
+ return this.disks;
19238
19225
  }
19239
- failure(reason, esPath, args, result) {
19226
+ }
19227
+ function longestSyncRoot(roots, target) {
19228
+ const normalized = target.replaceAll("/", "\\").toLowerCase();
19229
+ let match;
19230
+ let matchLength = -1;
19231
+ for (const root of roots) {
19232
+ const prefix = root.path.replaceAll("/", "\\").replace(/\\+$/u, "").toLowerCase();
19233
+ if (prefix === "" || prefix.length <= matchLength)
19234
+ continue;
19235
+ if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
19236
+ match = root;
19237
+ matchLength = prefix.length;
19238
+ }
19239
+ }
19240
+ return match;
19241
+ }
19242
+ function powerShellLaunch(script) {
19243
+ return {
19244
+ args: [
19245
+ "-NoProfile",
19246
+ "-NonInteractive",
19247
+ "-Command",
19248
+ `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; Invoke-Expression $env:${POWERSHELL_SCRIPT_VARIABLE}`
19249
+ ],
19250
+ env: { [POWERSHELL_SCRIPT_VARIABLE]: script }
19251
+ };
19252
+ }
19253
+ async function runPowerShell(script, timeoutMs) {
19254
+ const launch = powerShellLaunch(script);
19255
+ try {
19256
+ const { stdout, stderr } = await execFileAsync("powershell.exe", [...launch.args], {
19257
+ timeout: timeoutMs,
19258
+ maxBuffer: 16 * 1024 * 1024,
19259
+ windowsHide: true,
19260
+ env: { ...process.env, ...launch.env }
19261
+ });
19262
+ return { ok: true, stdout, stderr };
19263
+ } catch (error) {
19264
+ const record = typeof error === "object" && error !== null ? error : {};
19240
19265
  return {
19241
19266
  ok: false,
19242
- reason,
19243
- message: `${esPath} ${args.join(" ")} exit ${result.code}: ${(result.stderr || result.stdout).trim()}`
19267
+ stdout: typeof record.stdout === "string" ? record.stdout : "",
19268
+ stderr: typeof record.stderr === "string" ? record.stderr : String(error)
19244
19269
  };
19245
19270
  }
19246
19271
  }
19247
- function runProcess(command, args, timeoutMs) {
19248
- let resolvePromise;
19249
- const promise = new Promise((resolve) => {
19250
- resolvePromise = resolve;
19251
- });
19252
- const child = spawn14(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
19253
- const stdout = [];
19254
- const stderr = [];
19255
- let bytes = 0;
19256
- let overflow = false;
19257
- const collect = (target) => (chunk) => {
19258
- bytes += chunk.length;
19259
- if (bytes > MAX_OUTPUT_BYTES) {
19260
- overflow = true;
19261
- child.kill();
19262
- return;
19263
- }
19264
- target.push(chunk);
19265
- };
19266
- child.stdout.on("data", collect(stdout));
19267
- child.stderr.on("data", collect(stderr));
19268
- const timer = setTimeout(() => child.kill(), timeoutMs);
19269
- child.on("error", (error) => {
19270
- clearTimeout(timer);
19271
- resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
19272
- });
19273
- child.on("close", (code, signal) => {
19274
- clearTimeout(timer);
19275
- const err = Buffer.concat(stderr).toString("utf8");
19276
- resolvePromise({
19277
- code,
19278
- stdout: Buffer.concat(stdout).toString("utf8"),
19279
- stderr: overflow ? `${err}
19280
- output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
19281
- killed by ${signal} after ${timeoutMs}ms` : err
19272
+ function defaultCloudProbe() {
19273
+ if (process.platform === "darwin")
19274
+ return new DarwinCloudProbe;
19275
+ if (process.platform === "win32")
19276
+ return new WindowsCloudProbe;
19277
+ return;
19278
+ }
19279
+ async function readFileProviderDomain(target) {
19280
+ try {
19281
+ await lstat(target);
19282
+ const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
19283
+ timeout: 2000
19282
19284
  });
19283
- });
19284
- return promise;
19285
+ const value = stdout.trim();
19286
+ return value.length > 0 ? value : undefined;
19287
+ } catch {
19288
+ return;
19289
+ }
19285
19290
  }
19286
- function launchDetached(command, args) {
19287
- const child = spawn14(command, args, { detached: true, stdio: "ignore", windowsHide: true });
19288
- child.on("error", () => {});
19289
- child.unref();
19291
+ async function readFileProviderDomainUpwards(target) {
19292
+ let cursor = target;
19293
+ for (let depth = 0;depth < 12; depth += 1) {
19294
+ const domain = await readFileProviderDomain(cursor);
19295
+ if (domain !== undefined)
19296
+ return domain;
19297
+ const parent = dirname15(cursor);
19298
+ if (parent === cursor)
19299
+ return;
19300
+ cursor = parent;
19301
+ }
19302
+ return;
19290
19303
  }
19291
- var DEFAULT_TIMEOUT_MS8 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
19292
- var init_client9 = __esm(() => {
19293
- init_path_exclusions();
19294
- init_bundle();
19295
- DEFAULT_INDEX_TIMEOUT_MS3 = 10 * 60000;
19296
- MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
19297
- VERSION_PATTERN = /^\d+\.\d+\.\d+\.\d+$/;
19298
- });
19299
-
19300
- // src/everything/index.ts
19301
- var init_everything = __esm(() => {
19302
- init_bundle();
19303
- init_client9();
19304
+ function quotePowerShell(value) {
19305
+ return `'${value.replace(/['\u2018\u2019\u201A\u201B]/gu, (quote) => quote + quote)}'`;
19306
+ }
19307
+ var execFileAsync, SF_DATALESS = 1073741824, FILE_PROVIDER_XATTR = "com.apple.file-provider-domain-id", WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 4194304, STAT_BATCH = 200, defaultCloudExec = (file, args, options) => execFileAsync(file, [...args], {
19308
+ timeout: options?.timeoutMs ?? 30000,
19309
+ ...options?.maxBuffer === undefined ? {} : { maxBuffer: options.maxBuffer }
19310
+ }), JXA_RELEASE_PROGRAM = `ObjC.import("Foundation");
19311
+ function run(argv) {
19312
+ var url = $.NSURL.fileURLWithPath(argv[0]);
19313
+ var errorRef = Ref();
19314
+ var released = $.NSFileManager.defaultManager.evictUbiquitousItemAtURLError(url, errorRef);
19315
+ if (released) return JSON.stringify({ ok: true });
19316
+ function resourceValue(key) {
19317
+ var ref = Ref();
19318
+ if (!url.getResourceValueForKeyError(ref, key, null)) return null;
19319
+ var value = ref[0];
19320
+ return value === undefined || value === null ? null : ObjC.unwrap(value);
19321
+ }
19322
+ return JSON.stringify({
19323
+ ok: false,
19324
+ isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
19325
+ isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
19326
+ downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
19327
+ });
19328
+ }`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
19329
+ var init_cloud_placeholder = __esm(() => {
19330
+ execFileAsync = promisify(execFile2);
19331
+ RELEASABLE_DARWIN_PROVIDERS = {
19332
+ icloud: true,
19333
+ "google-drive": true,
19334
+ onedrive: true,
19335
+ dropbox: true
19336
+ };
19337
+ DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
19338
+ WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
19339
+ WINDOWS_DEHYDRATE_REFUSALS = {
19340
+ "80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
19341
+ "80070188": "The file is pinned to stay on this device, so it is not released."
19342
+ };
19304
19343
  });
19305
19344
 
19306
19345
  // src/fsearch/walk.ts
19307
19346
  import { readdir as readdir2, stat as stat3 } from "node:fs/promises";
19308
- import { basename as basename8, join as join30, sep as sep5 } from "node:path";
19347
+ import { basename as basename7, join as join30, sep as sep5 } from "node:path";
19309
19348
  function compare(a, b) {
19310
19349
  return a < b ? -1 : a > b ? 1 : 0;
19311
19350
  }
@@ -19316,11 +19355,11 @@ async function walkFileSearch(folders, request, options = {}) {
19316
19355
  const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_DEADLINE_MS);
19317
19356
  const matches = request.regex ? (() => {
19318
19357
  const pattern = new RegExp(request.query, request.matchCase ? "" : "i");
19319
- return (path) => pattern.test(request.matchPath ? path : basename8(path));
19358
+ return (path) => pattern.test(request.matchPath ? path : basename7(path));
19320
19359
  })() : (() => {
19321
19360
  const needle = request.matchCase ? request.query : request.query.toLowerCase();
19322
19361
  return (path) => {
19323
- const haystack = request.matchPath ? path : basename8(path);
19362
+ const haystack = request.matchPath ? path : basename7(path);
19324
19363
  return (request.matchCase ? haystack : haystack.toLowerCase()).includes(needle);
19325
19364
  };
19326
19365
  })();
@@ -19375,7 +19414,7 @@ async function walkFileSearch(folders, request, options = {}) {
19375
19414
  continue;
19376
19415
  collected.push({
19377
19416
  path,
19378
- name: basename8(path),
19417
+ name: basename7(path),
19379
19418
  type: info.isDirectory() ? "folder" : "file",
19380
19419
  size: info.isDirectory() ? undefined : info.size,
19381
19420
  dateModified: info.mtime.toISOString()
@@ -19400,7 +19439,7 @@ async function walkFileSearch(folders, request, options = {}) {
19400
19439
  continue;
19401
19440
  collected.push({
19402
19441
  path: folder,
19403
- name: basename8(folder),
19442
+ name: basename7(folder),
19404
19443
  type: "folder",
19405
19444
  size: undefined,
19406
19445
  dateModified: info.mtime.toISOString()
@@ -19433,7 +19472,7 @@ var init_walk = __esm(() => {
19433
19472
  import { spawn as spawn15 } from "node:child_process";
19434
19473
  import { createHash as createHash13 } from "node:crypto";
19435
19474
  import { existsSync as existsSync18, mkdirSync as mkdirSync16, readFileSync as readFileSync18, realpathSync as realpathSync4, rmSync as rmSync8, writeFileSync as writeFileSync12 } from "node:fs";
19436
- import { basename as basename9, join as join31, resolve as resolve12, sep as sep6 } from "node:path";
19475
+ import { basename as basename8, join as join31, resolve as resolve12, sep as sep6 } from "node:path";
19437
19476
  import { setTimeout as sleep3 } from "node:timers/promises";
19438
19477
  function fsearchSocketPath(dbPath) {
19439
19478
  let canonical;
@@ -19497,7 +19536,7 @@ function parseFSearchJsonLines(stdout) {
19497
19536
  const path = String(record.path ?? "");
19498
19537
  entries.push({
19499
19538
  path,
19500
- name: basename9(path),
19539
+ name: basename8(path),
19501
19540
  type: record.type === "folder" ? "folder" : "file",
19502
19541
  size: typeof record.size === "number" ? record.size : undefined,
19503
19542
  dateModified: typeof record.mtime === "number" ? new Date(record.mtime * 1000).toISOString() : undefined
@@ -19915,7 +19954,7 @@ var init_diagnostics = __esm(() => {
19915
19954
  });
19916
19955
 
19917
19956
  // src/jikji/retry-command.ts
19918
- import { basename as basename10 } from "node:path";
19957
+ import { basename as basename9 } from "node:path";
19919
19958
  function shellWords(command) {
19920
19959
  const words = [];
19921
19960
  let current = "";
@@ -19982,7 +20021,7 @@ function parseJikjiRetryCommand(command, envelopeProof) {
19982
20021
  const words = shellWords(command);
19983
20022
  if (words === undefined || words.length < 4)
19984
20023
  return;
19985
- const program = basename10(words[0]).replace(/\.exe$/iu, "");
20024
+ const program = basename9(words[0]).replace(/\.exe$/iu, "");
19986
20025
  if (program !== "jikji" || words[1] !== "discover")
19987
20026
  return;
19988
20027
  const root = words[2];
@@ -20390,7 +20429,7 @@ class SerialQueue {
20390
20429
 
20391
20430
  // src/jikji/version.ts
20392
20431
  function parseJikjiVersion(output) {
20393
- return VERSION_LINE2.exec(output)?.[1];
20432
+ return VERSION_LINE3.exec(output)?.[1];
20394
20433
  }
20395
20434
  function parts(version) {
20396
20435
  return version.split(".").map((part) => Number.parseInt(part, 10));
@@ -20406,9 +20445,9 @@ function isSupportedJikjiVersion(version) {
20406
20445
  }
20407
20446
  return true;
20408
20447
  }
20409
- var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1", VERSION_LINE2;
20410
- var init_version2 = __esm(() => {
20411
- VERSION_LINE2 = /^jikji(?:-cli)?\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
20448
+ var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1", VERSION_LINE3;
20449
+ var init_version3 = __esm(() => {
20450
+ VERSION_LINE3 = /^jikji(?:-cli)?\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
20412
20451
  });
20413
20452
 
20414
20453
  // src/jikji/client.ts
@@ -20776,7 +20815,7 @@ var init_client11 = __esm(() => {
20776
20815
  init_answer_pack();
20777
20816
  init_installer3();
20778
20817
  init_prepare_budget();
20779
- init_version2();
20818
+ init_version3();
20780
20819
  ABORTED_BEFORE_START = { ok: false, reason: "aborted", stdout: "", stderr: "", code: null };
20781
20820
  });
20782
20821
 
@@ -21861,7 +21900,7 @@ import {
21861
21900
  rmSync as rmSync10,
21862
21901
  writeFileSync as writeFileSync14
21863
21902
  } from "node:fs";
21864
- import { basename as basename11, dirname as dirname18, isAbsolute as isAbsolute9, join as join36, relative as relative7, sep as sep8 } from "node:path";
21903
+ import { basename as basename10, dirname as dirname18, isAbsolute as isAbsolute9, join as join36, relative as relative7, sep as sep8 } from "node:path";
21865
21904
  function syncMinSyncWorkspace(root, options = {}) {
21866
21905
  const workspacePath = options.workspacePath ?? minSyncWorkspaceRoot(root);
21867
21906
  const filesRoot = join36(workspacePath, "files");
@@ -21902,7 +21941,7 @@ function syncMinSyncWorkspace(root, options = {}) {
21902
21941
  return { workspacePath, entries, excluded };
21903
21942
  }
21904
21943
  function stagingStatePath(workspacePath) {
21905
- return join36(dirname18(workspacePath), `.${basename11(workspacePath)}-autorag-staging.json`);
21944
+ return join36(dirname18(workspacePath), `.${basename10(workspacePath)}-autorag-staging.json`);
21906
21945
  }
21907
21946
  function loadStagingState2(workspacePath) {
21908
21947
  const path = stagingStatePath(workspacePath);
@@ -21953,7 +21992,7 @@ function ensureManagedDirectory(filesRoot, directory) {
21953
21992
  }
21954
21993
  }
21955
21994
  function copyFileAtomically(source, destination) {
21956
- const temporaryPath = join36(dirname18(destination), `.${basename11(destination)}.${randomUUID6()}.tmp`);
21995
+ const temporaryPath = join36(dirname18(destination), `.${basename10(destination)}.${randomUUID6()}.tmp`);
21957
21996
  try {
21958
21997
  copyFileSync4(source, temporaryPath);
21959
21998
  renameWithRetry(temporaryPath, destination);
@@ -22007,7 +22046,7 @@ var init_workspace = __esm(() => {
22007
22046
 
22008
22047
  // src/minsync/method.ts
22009
22048
  import { existsSync as existsSync23, readFileSync as readFileSync21, realpathSync as realpathSync7, statSync as statSync7 } from "node:fs";
22010
- import { basename as basename12, join as join37, normalize } from "node:path";
22049
+ import { basename as basename11, join as join37, normalize } from "node:path";
22011
22050
 
22012
22051
  class MinSyncVectorMethod {
22013
22052
  root;
@@ -22108,7 +22147,7 @@ class MinSyncVectorMethod {
22108
22147
  const entry = byPath.get(hit.path);
22109
22148
  if (!entry || !matchesVirtualPathScope(entry.virtualPath, options.scope))
22110
22149
  continue;
22111
- const chunkDiscriminator = hit.docId ?? basename12(hit.path);
22150
+ const chunkDiscriminator = hit.docId ?? basename11(hit.path);
22112
22151
  const chunkId = `minsync:${entry.virtualPath}:${chunkDiscriminator}`;
22113
22152
  const location = locateChunkInMirror(mirrorTexts, entry, hit.text);
22114
22153
  results.push({
@@ -22208,18 +22247,24 @@ var init_minsync = __esm(() => {
22208
22247
 
22209
22248
  // src/mirror/cloud-fingerprints.ts
22210
22249
  import { existsSync as existsSync24, readFileSync as readFileSync22 } from "node:fs";
22211
- import { join as join38 } from "node:path";
22250
+ import { extname as extname5, join as join38 } from "node:path";
22212
22251
  function cloudFingerprintStorePath(root) {
22213
22252
  return join38(root, ".autorag", CLOUD_FINGERPRINTS_FILE);
22214
22253
  }
22215
22254
  function emptyCloudFingerprints() {
22216
- return { version: 1, entries: {} };
22255
+ return { version: 2, entries: {} };
22217
22256
  }
22218
- function isCloudFingerprint(value) {
22257
+ function readCloudFingerprint(sourcePath, value, storeVersion) {
22219
22258
  if (typeof value !== "object" || value === null)
22220
- return false;
22259
+ return;
22221
22260
  const record = value;
22222
- return typeof record.sourceVersion === "string" && typeof record.exact === "string" && typeof record.chars === "number" && (record.modified === null || typeof record.modified === "string") && (record.revision === null || typeof record.revision === "string");
22261
+ const { sourceVersion, exact, modified, revision } = record;
22262
+ if (typeof sourceVersion !== "string" || typeof exact !== "string" || modified !== null && typeof modified !== "string" || revision !== null && typeof revision !== "string") {
22263
+ return;
22264
+ }
22265
+ if (storeVersion !== 2 && (record.chars === 0 || extname5(sourcePath).toLowerCase() === ".hwp"))
22266
+ return;
22267
+ return { sourceVersion, exact, modified, revision };
22223
22268
  }
22224
22269
  function loadCloudFingerprints(root) {
22225
22270
  const path = cloudFingerprintStorePath(root);
@@ -22232,24 +22277,26 @@ function loadCloudFingerprints(root) {
22232
22277
  const { entries } = parsed;
22233
22278
  if (typeof entries !== "object" || entries === null)
22234
22279
  return emptyCloudFingerprints();
22280
+ const storeVersion = "version" in parsed ? parsed.version : undefined;
22235
22281
  const valid = {};
22236
- for (const [sourcePath, fingerprint] of Object.entries(entries)) {
22237
- if (isCloudFingerprint(fingerprint))
22282
+ for (const [sourcePath, value] of Object.entries(entries)) {
22283
+ const fingerprint = readCloudFingerprint(sourcePath, value, storeVersion);
22284
+ if (fingerprint !== undefined)
22238
22285
  valid[sourcePath] = fingerprint;
22239
22286
  }
22240
- return { version: 1, entries: valid };
22287
+ return { version: 2, entries: valid };
22241
22288
  } catch {
22242
22289
  return emptyCloudFingerprints();
22243
22290
  }
22244
22291
  }
22245
22292
  function saveCloudFingerprints(root, entries) {
22246
- writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version: 1, entries }, null, 2)}
22293
+ writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version: 2, entries }, null, 2)}
22247
22294
  `);
22248
22295
  }
22249
22296
  function cloudFingerprintExclusions(store) {
22250
22297
  const map = new Map;
22251
22298
  for (const [sourcePath, fingerprint] of Object.entries(store.entries)) {
22252
- map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified, chars: fingerprint.chars });
22299
+ map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified });
22253
22300
  }
22254
22301
  return map;
22255
22302
  }
@@ -22641,7 +22688,7 @@ class Parser {
22641
22688
  }
22642
22689
 
22643
22690
  // src/parser/csv.ts
22644
- import { basename as basename13, extname as extname5 } from "node:path";
22691
+ import { basename as basename12, extname as extname6 } from "node:path";
22645
22692
  function toCell(text) {
22646
22693
  return text.trim() === "" ? undefined : { text, kind: classifyText(text) };
22647
22694
  }
@@ -22772,7 +22819,7 @@ var init_csv = __esm(() => {
22772
22819
  this.#maxRows = options.maxRows ?? DEFAULT_MAX_ROWS;
22773
22820
  }
22774
22821
  async parse(input) {
22775
- const extension = extname5(input.virtualPath);
22822
+ const extension = extname6(input.virtualPath);
22776
22823
  const format = extension.slice(1).toLowerCase();
22777
22824
  let text = decodeText(input.bytes).replace(/^\uFEFF/, "");
22778
22825
  const directive = SEPARATOR_DIRECTIVE.exec(text);
@@ -22784,10 +22831,10 @@ var init_csv = __esm(() => {
22784
22831
  {
22785
22832
  code: "parser-warning",
22786
22833
  severity: "warning",
22787
- message: `"${basename13(input.virtualPath)}": indexed the first ${this.#maxRows} of ${total} rows`
22834
+ message: `"${basename12(input.virtualPath)}": indexed the first ${this.#maxRows} of ${total} rows`
22788
22835
  }
22789
22836
  ] : [];
22790
- const grid = { name: basename13(input.virtualPath, extension), rows: rows.map((row) => row.map(toCell)) };
22837
+ const grid = { name: basename12(input.virtualPath, extension), rows: rows.map((row) => row.map(toCell)) };
22791
22838
  return {
22792
22839
  markdown: normalizeMarkdown(renderWorkbook([grid])),
22793
22840
  metadata: { parser: this.name, format },
@@ -23326,7 +23373,7 @@ var init_plain_text = __esm(() => {
23326
23373
  });
23327
23374
 
23328
23375
  // src/parser/registry.ts
23329
- import { extname as extname6 } from "node:path";
23376
+ import { extname as extname7 } from "node:path";
23330
23377
 
23331
23378
  class ParserRegistry {
23332
23379
  parsersByExtension = new Map;
@@ -23345,7 +23392,7 @@ class ParserRegistry {
23345
23392
  }
23346
23393
  }
23347
23394
  getForVirtualPath(virtualPath) {
23348
- return this.parsersByExtension.get(normalizeExtension(extname6(virtualPath)));
23395
+ return this.parsersByExtension.get(normalizeExtension(extname7(virtualPath)));
23349
23396
  }
23350
23397
  list() {
23351
23398
  return Array.from(new Set(this.parsersByExtension.values()));
@@ -23358,7 +23405,7 @@ function normalizeExtension(extension) {
23358
23405
  var init_registry = () => {};
23359
23406
 
23360
23407
  // src/parser/spreadsheet.ts
23361
- import { extname as extname7 } from "node:path";
23408
+ import { extname as extname8 } from "node:path";
23362
23409
  import * as XLSX from "xlsx";
23363
23410
  function assertWorkbookSignature(format, bytes) {
23364
23411
  if (startsWith(bytes, ZIP_SIGNATURE) || startsWith(bytes, OLE_SIGNATURE))
@@ -23457,7 +23504,7 @@ var init_spreadsheet = __esm(() => {
23457
23504
  this.#maxRowsPerSheet = options.maxRowsPerSheet ?? DEFAULT_MAX_ROWS_PER_SHEET;
23458
23505
  }
23459
23506
  async parse(input) {
23460
- const format = extname7(input.virtualPath).slice(1).toLowerCase();
23507
+ const format = extname8(input.virtualPath).slice(1).toLowerCase();
23461
23508
  let workbook;
23462
23509
  try {
23463
23510
  assertWorkbookSignature(format, input.bytes);
@@ -23731,6 +23778,10 @@ class CloudGate {
23731
23778
  isPlaceholder(path) {
23732
23779
  return this.placeholders.has(path);
23733
23780
  }
23781
+ startBatch() {
23782
+ this.downloadedFiles = 0;
23783
+ this.downloadedBytes = 0;
23784
+ }
23734
23785
  async decide(path, sizeBytes, directory) {
23735
23786
  if (this.probe === undefined || !this.placeholders.has(path))
23736
23787
  return LOCAL;
@@ -23740,7 +23791,7 @@ class CloudGate {
23740
23791
  const overFiles = this.downloadedFiles >= this.maxFiles;
23741
23792
  const overBytes = this.downloadedFiles > 0 && this.downloadedBytes + sizeBytes > this.maxBytes;
23742
23793
  if (overFiles || overBytes)
23743
- return { kind: "defer" };
23794
+ return { kind: "next-batch" };
23744
23795
  this.downloadedFiles += 1;
23745
23796
  this.downloadedBytes += sizeBytes;
23746
23797
  return { kind: "download" };
@@ -23785,7 +23836,7 @@ var init_cloud = __esm(() => {
23785
23836
  // src/mirror/sync.ts
23786
23837
  import { createHash as createHash17 } from "node:crypto";
23787
23838
  import { opendir as opendir2, readFile, stat as stat4 } from "node:fs/promises";
23788
- import { basename as basename14, dirname as dirname21, extname as extname8, join as join42, resolve as resolve15, sep as sep9 } from "node:path";
23839
+ import { basename as basename13, dirname as dirname21, extname as extname9, join as join42, resolve as resolve15, sep as sep9 } from "node:path";
23789
23840
  import { setTimeout as sleep4 } from "node:timers/promises";
23790
23841
  function errorCode(error) {
23791
23842
  if (!(error instanceof Error))
@@ -24010,25 +24061,26 @@ async function syncWithRegistry(options, registry, isolated) {
24010
24061
  };
24011
24062
  const deferDeletions = options.onBatch !== undefined;
24012
24063
  const batchSize = Math.max(1, Math.floor(options.batchSize ?? DEFAULT_MIRROR_BATCH_SIZE));
24013
- const batchCount = Math.max(1, Math.ceil(current.length / batchSize));
24014
24064
  let batchIndex = 0;
24015
24065
  let batchStopped = false;
24016
24066
  const runBatch = async (virtualPaths, final) => {
24017
24067
  if (options.onBatch === undefined)
24018
24068
  return;
24019
24069
  batchIndex += 1;
24020
- const outcome = await options.onBatch({ index: batchIndex, total: batchCount, virtualPaths, final });
24070
+ const outcome = await options.onBatch({ index: batchIndex, virtualPaths, final });
24021
24071
  if (outcome?.stop === true)
24022
24072
  batchStopped = true;
24023
24073
  };
24024
24074
  if (!deferDeletions)
24025
24075
  await applyDeleteDecisions();
24026
24076
  await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
24027
- for (let batchStart = 0;batchStart < Math.max(current.length, 1); batchStart += batchSize) {
24028
- const batchEnd = Math.min(batchStart + batchSize, current.length);
24029
- const finalBatch = batchEnd >= current.length;
24077
+ let batchStart = 0;
24078
+ for (;; ) {
24079
+ const batchLimit = Math.min(batchStart + batchSize, current.length);
24080
+ let batchEnd = batchLimit;
24030
24081
  const changedBefore = written + deleted;
24031
- for (let position = batchStart;position < batchEnd; position += 1) {
24082
+ gate.startBatch();
24083
+ for (let position = batchStart;position < batchLimit; position += 1) {
24032
24084
  const entry = current[position];
24033
24085
  options.onProgress?.({
24034
24086
  processed: position,
@@ -24078,15 +24130,9 @@ async function syncWithRegistry(options, registry, isolated) {
24078
24130
  });
24079
24131
  continue;
24080
24132
  }
24081
- if (cloudDecision.kind === "defer") {
24082
- skipped += 1;
24083
- diagnostics.push({
24084
- code: "cloud-download-deferred",
24085
- severity: "info",
24086
- message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
24087
- source: entry.virtualPath
24088
- });
24089
- continue;
24133
+ if (cloudDecision.kind === "next-batch") {
24134
+ batchEnd = position;
24135
+ break;
24090
24136
  }
24091
24137
  if (pending.has(mirrorStageKey(entry.virtualPath))) {
24092
24138
  let bytes;
@@ -24159,7 +24205,7 @@ async function syncWithRegistry(options, registry, isolated) {
24159
24205
  continue;
24160
24206
  }
24161
24207
  await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
24162
- path: basename14(outputPath),
24208
+ path: basename13(outputPath),
24163
24209
  size: Buffer.byteLength(markdown)
24164
24210
  });
24165
24211
  for (const diagnostic of parsed.diagnostics ?? [])
@@ -24173,7 +24219,6 @@ async function syncWithRegistry(options, registry, isolated) {
24173
24219
  cloudFingerprintEntries[entry.sourcePath] = {
24174
24220
  sourceVersion: entry.sourceVersion,
24175
24221
  exact: fingerprintResult.exact,
24176
- chars: fingerprintResult.chars,
24177
24222
  modified: fingerprintResult.modified,
24178
24223
  revision: fingerprintResult.revision
24179
24224
  };
@@ -24201,13 +24246,15 @@ async function syncWithRegistry(options, registry, isolated) {
24201
24246
  }
24202
24247
  }
24203
24248
  }
24249
+ const finalBatch = batchEnd >= current.length;
24204
24250
  if (finalBatch && deferDeletions)
24205
24251
  await applyDeleteDecisions();
24206
24252
  if (finalBatch || written + deleted !== changedBefore) {
24207
24253
  await runBatch(current.slice(batchStart, batchEnd).map((item) => item.virtualPath), finalBatch);
24208
24254
  }
24209
- if (batchStopped)
24255
+ if (finalBatch || batchStopped)
24210
24256
  break;
24257
+ batchStart = batchEnd;
24211
24258
  }
24212
24259
  await session.finish();
24213
24260
  if (fingerprintCloud) {
@@ -24297,7 +24344,7 @@ async function detectMirrorStaleness(options) {
24297
24344
  }
24298
24345
  }
24299
24346
  function standaloneImageTooSmallMessage(virtualPath, bytes) {
24300
- if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname8(virtualPath))))
24347
+ if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname9(virtualPath))))
24301
24348
  return;
24302
24349
  return ocrImageTooSmallMessage(bytes);
24303
24350
  }
@@ -24465,7 +24512,7 @@ async function collectFiles(state, directory, depth) {
24465
24512
  }
24466
24513
  if (!entry.isFile())
24467
24514
  continue;
24468
- const extension = normalizeExtension2(extname8(entry.name));
24515
+ const extension = normalizeExtension2(extname9(entry.name));
24469
24516
  if (!state.supportedExtensions.has(extension))
24470
24517
  continue;
24471
24518
  let fileStat;
@@ -30028,7 +30075,7 @@ var init_automation_tool = __esm(() => {
30028
30075
  });
30029
30076
 
30030
30077
  // src/agent/citation-links.ts
30031
- import { basename as basename15, isAbsolute as isAbsolute12 } from "node:path";
30078
+ import { basename as basename14, isAbsolute as isAbsolute12 } from "node:path";
30032
30079
  import { pathToFileURL as pathToFileURL3 } from "node:url";
30033
30080
  function openableTarget(source) {
30034
30081
  if (/^https?:\/\//iu.test(source)) {
@@ -30046,7 +30093,7 @@ function linkCitations(markdown, sourceOf, isFile) {
30046
30093
  for (const marker of answerMarkers(markdown, isFile)) {
30047
30094
  if (marker.kind === "number")
30048
30095
  continue;
30049
- const labels = marker.kind === "file" ? [{ label: basename15(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
30096
+ const labels = marker.kind === "file" ? [{ label: basename14(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
30050
30097
  let linked = false;
30051
30098
  const parts = labels.map(({ label, source }) => {
30052
30099
  const target = source === undefined ? undefined : openableTarget(source);
@@ -32172,10 +32219,7 @@ function renderScanReport(details) {
32172
32219
  const lines = [
32173
32220
  `dupey scanned ${details.scans.length} configured root(s).`,
32174
32221
  `families=${details.familyCount} exactDuplicates=${details.exactDuplicateCount} extractionErrors=${errors}`,
32175
- "Review exact families before cleanup; near/contains families are not safe deletion evidence.",
32176
- ...details.skippedRoots.length === 0 ? [] : [
32177
- `Not scanned (inside a cloud or network folder; scanning would download or re-read every file): ${details.skippedRoots.join(", ")}. Duplicates there are not reported, and an exact duplicate is only found against files an earlier refresh fingerprinted.`
32178
- ]
32222
+ "Review exact families before cleanup; near/contains families are not safe deletion evidence."
32179
32223
  ];
32180
32224
  let rendered = 0;
32181
32225
  let omitted = 0;
@@ -33153,6 +33197,56 @@ var init_packaged_skill = __esm(() => {
33153
33197
  PACKAGED_SKILL_ROOTS = ["skills", "internal-skills"];
33154
33198
  });
33155
33199
 
33200
+ // src/agent/pipelined-sync.ts
33201
+ class PipelinedSync {
33202
+ #sync;
33203
+ #covered = [];
33204
+ #dirty = false;
33205
+ #running;
33206
+ #failure;
33207
+ #stopped = false;
33208
+ #runs = 0;
33209
+ constructor(options) {
33210
+ this.#sync = options.sync;
33211
+ }
33212
+ async batch(batch) {
33213
+ this.#raise();
33214
+ this.#covered.push(...batch.virtualPaths);
33215
+ this.#dirty = true;
33216
+ if (!this.#stopped)
33217
+ this.#running ??= this.#loop();
33218
+ if (batch.final)
33219
+ await this.#running;
33220
+ this.#raise();
33221
+ return this.#stopped ? { stop: true } : undefined;
33222
+ }
33223
+ async settle() {
33224
+ this.#stopped = true;
33225
+ await this.#running;
33226
+ }
33227
+ #raise() {
33228
+ if (this.#failure !== undefined)
33229
+ throw this.#failure.error;
33230
+ }
33231
+ async#loop() {
33232
+ try {
33233
+ while (this.#dirty && !this.#stopped) {
33234
+ this.#dirty = false;
33235
+ const virtualPaths = this.#covered;
33236
+ this.#covered = [];
33237
+ this.#runs += 1;
33238
+ const outcome = await this.#sync({ index: this.#runs, virtualPaths });
33239
+ if (outcome?.stop === true)
33240
+ this.#stopped = true;
33241
+ }
33242
+ } catch (error) {
33243
+ this.#failure = { error };
33244
+ } finally {
33245
+ this.#running = undefined;
33246
+ }
33247
+ }
33248
+ }
33249
+
33156
33250
  // src/agent/query-decomposition.ts
33157
33251
  import { completeSimple as completeSimple2 } from "@earendil-works/pi-ai/compat";
33158
33252
  function buildDecompositionPrompt(query) {
@@ -33392,7 +33486,7 @@ var init_query_routing = __esm(() => {
33392
33486
  false: "Every part of the question is answered with a concrete, specific, consistent fact (a name, date, number, or definite statement)."
33393
33487
  });
33394
33488
  BASE_ROUTE_OPTIONS = {
33395
- local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. When unsure whether intrinsic knowledge suffices, choose local.",
33489
+ local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. A request to find, locate, open, or look up a document, file, slide deck, presentation, paper, note, message, or other material is local even when it names no owner ('find the slides about X', 'where is the Y presentation', 'Z 관련 자료 찾아줘'): such material lives in the user's own files and chats, and a well-known topic is not a reason to answer from general knowledge or the web. When unsure whether intrinsic knowledge suffices, choose local.",
33396
33490
  extra: "None of the other options fits and no search of the user's own records is needed first. Either the request is answerable without looking anything up — stable common knowledge, a definition, simple reasoning, or small talk — or it is a task for the assistant's tools that is not file organization and not AutoRAG configuration: committing code, pushing a branch, opening a pull request, running tests or a build, installing a package, running a script, writing or editing code. Programming work is extra even when the code names the user's files or folders ('write a script that lowercases every PNG', 'add this to my parser'): the assistant is asked to produce or change code, not to act on those files now. So is a request to run something (tests, a build, a script, a command) and report the outcome, even when what it runs lives in the user's own repository: running it is a task for the tool, not a search of the user's records. Never extra when the request refers to the user's own life, files, records, chats, calendar, or history (e.g. 'my', 'I', 'our', 'my girlfriend', 'my case', 'my routine'), and never extra when the answer depends on facts that may be recent, specific, or changing (current events, recent releases, prices, versions): search those locally instead. If any other option fits, do not choose extra.",
33397
33491
  organize: "The user asks the assistant to DO something to files or folders instead of finding information: move or copy files into folders, tidy or sort a folder, bulk or pattern rename (for example by date or number), rename files according to what their content says, convert files to another format, create folder structures, group files by type or topic, archive, or move to the trash ('tidy this up', 'move these into folders', 'bulk rename these files', 'rename the files to match their content', 'convert these to PDF', 'sort my Downloads folder'). It is about acting on files and folders, never about looking up what a document says or where something is. A question that only asks to find or read a file's content is local, not organize."
33398
33492
  };
@@ -34630,7 +34724,7 @@ var init_page_loader = __esm(() => {
34630
34724
  });
34631
34725
 
34632
34726
  // src/web/fetch/url-target.ts
34633
- import { extname as extname9 } from "node:path";
34727
+ import { extname as extname10 } from "node:path";
34634
34728
  function repairCollapsedScheme(value) {
34635
34729
  const m = value.match(/^(https?):\/(?!\/)/i);
34636
34730
  return m ? `${m[1]}://${value.slice(m[0].length)}` : value;
@@ -34649,7 +34743,7 @@ function getFilenameExtensionHint(filename) {
34649
34743
  const lower = filename.toLowerCase();
34650
34744
  if (lower.endsWith(".tar.gz"))
34651
34745
  return ".tar.gz";
34652
- return extname9(filename).toLowerCase();
34746
+ return extname10(filename).toLowerCase();
34653
34747
  }
34654
34748
  function getExtensionHint(url, contentDisposition) {
34655
34749
  if (contentDisposition) {
@@ -34826,7 +34920,7 @@ var init_convert2 = __esm(() => {
34826
34920
  });
34827
34921
 
34828
34922
  // src/web/fetch/feeds.ts
34829
- import { extname as extname10 } from "node:path";
34923
+ import { extname as extname11 } from "node:path";
34830
34924
  import { XMLParser as XMLParser3 } from "fast-xml-parser";
34831
34925
  function getHtmlAttribute(tag, attribute) {
34832
34926
  const pattern = new RegExp(`\\b${attribute}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'=<>]+))`, "i");
@@ -34889,7 +34983,7 @@ function extractDocumentLinks(html, baseUrl) {
34889
34983
  const href = getHtmlAttribute(tag, "href");
34890
34984
  if (!href)
34891
34985
  continue;
34892
- const ext = extname10(href).toLowerCase();
34986
+ const ext = extname11(href).toLowerCase();
34893
34987
  if (!CONVERTIBLE_EXTENSIONS2.has(ext))
34894
34988
  continue;
34895
34989
  const resolved = href.startsWith("http") ? href : new URL(href, baseUrl).href;
@@ -36557,7 +36651,7 @@ var init_web_search_tool = __esm(() => {
36557
36651
  // src/agent/agent.ts
36558
36652
  import { randomUUID as randomUUID7 } from "node:crypto";
36559
36653
  import { existsSync as existsSync34, watch as fsWatch, mkdirSync as mkdirSync24, realpathSync as realpathSync8, statSync as statSync11, writeFileSync as writeFileSync17 } from "node:fs";
36560
- import { basename as basename16, dirname as dirname27, join as join54, relative as relative8, resolve as resolve18, sep as sep10 } from "node:path";
36654
+ import { basename as basename15, dirname as dirname27, join as join54, relative as relative8, resolve as resolve18, sep as sep10 } from "node:path";
36561
36655
  import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
36562
36656
  import { getAgentDir as getAgentDir6, keyText } from "@earendil-works/pi-coding-agent";
36563
36657
  function describeDecisionModel(status) {
@@ -36992,10 +37086,20 @@ class AutoRAGAgent {
36992
37086
  }
36993
37087
  async scanDuplicateDocuments() {
36994
37088
  if (this.dupeyOptions === false) {
36995
- return { scans: [], skippedRoots: [], familyCount: 0, exactDuplicateCount: 0 };
37089
+ return { scans: [], familyCount: 0, exactDuplicateCount: 0 };
36996
37090
  }
36997
- const { targets, skipped } = planDupeyScans(this.searchPaths, await this.cloudRoots());
36998
- const scans = await Promise.all(targets.map(({ path, excludeDirs }) => scanWithDupey(path, {
37091
+ const cloudRoots = await this.cloudRoots();
37092
+ const scanRoots = [];
37093
+ for (const searchPath of this.searchPaths) {
37094
+ const scope = cloudScope(searchPath, cloudRoots);
37095
+ if (scope.insideCloudRoot)
37096
+ continue;
37097
+ scanRoots.push({
37098
+ path: searchPath,
37099
+ excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path))
37100
+ });
37101
+ }
37102
+ const scans = await Promise.all(scanRoots.map(({ path, excludeDirs }) => scanWithDupey(path, {
36999
37103
  ...this.dupeyOptions || {},
37000
37104
  ...excludeDirs.length === 0 ? {} : { excludeDirs }
37001
37105
  })));
@@ -37009,7 +37113,7 @@ class AutoRAGAgent {
37009
37113
  }
37010
37114
  return count + [...hashes.values()].reduce((sum, size) => sum + Math.max(0, size - 1), 0);
37011
37115
  }, 0);
37012
- return { scans, skippedRoots: skipped, familyCount, exactDuplicateCount };
37116
+ return { scans, familyCount, exactDuplicateCount };
37013
37117
  }
37014
37118
  loadMemoryContextFor(query) {
37015
37119
  if (this.memoryContextCache?.query === query)
@@ -38662,6 +38766,10 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38662
38766
  inFlight: true,
38663
38767
  lastStartedAt: startedAt
38664
38768
  };
38769
+ const syncAbort = new AbortController;
38770
+ const forwardAbort = () => syncAbort.abort(signal?.reason);
38771
+ signal?.addEventListener("abort", forwardAbort, { once: true });
38772
+ let minSyncPipeline;
38665
38773
  try {
38666
38774
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38667
38775
  const batchedMinSync = needsParsed && wants("minsync");
@@ -38669,44 +38777,53 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38669
38777
  const minSyncBatchFailures = [];
38670
38778
  let minSyncSyncedBase = 0;
38671
38779
  let minSyncTotalBase = 0;
38672
- const minSyncBatch = async (batch) => {
38673
- signal?.throwIfAborted();
38674
- progress = updateRefreshProgress(progress, { phase: "minsync" });
38675
- writeRefreshProgress(this.workspaceProjectRoot, progress);
38676
- let batchTotal = minSyncTotalBase;
38677
- const result = await this.syncMinSync(force && batch.index === 1, signal, (minsyncProgress) => {
38678
- batchTotal = minSyncTotalBase + minsyncProgress.total;
38679
- progress = updateRefreshProgress(progress, {
38680
- minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: batchTotal }
38681
- });
38682
- writeRefreshProgress(this.workspaceProjectRoot, progress);
38683
- emitProgress({
38684
- phase: "minsync",
38685
- processed: minSyncSyncedBase + minsyncProgress.processed,
38686
- total: batchTotal
38687
- });
38688
- });
38689
- minSyncBatches.push(result);
38690
- if (result.ok)
38691
- minSyncBatchFailures.length = 0;
38692
- else if (result.reason !== "lock-held") {
38693
- for (const virtualPath of batch.virtualPaths) {
38694
- minSyncBatchFailures.push({
38695
- code: "minsync-sync-failed",
38696
- severity: "warning",
38697
- message: `This file stayed at its previous index entry: its batch could not be indexed and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
38698
- source: virtualPath
38780
+ minSyncPipeline = batchedMinSync ? new PipelinedSync({
38781
+ sync: async ({ index, virtualPaths }) => {
38782
+ syncAbort.signal.throwIfAborted();
38783
+ let syncTotal = minSyncTotalBase;
38784
+ const result = await this.syncMinSync(force && index === 1, syncAbort.signal, (minsyncProgress) => {
38785
+ syncTotal = minSyncTotalBase + minsyncProgress.total;
38786
+ progress = updateRefreshProgress(progress, {
38787
+ minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: syncTotal }
38788
+ });
38789
+ writeRefreshProgress(this.workspaceProjectRoot, progress);
38790
+ emitProgress({
38791
+ phase: "minsync",
38792
+ processed: minSyncSyncedBase + minsyncProgress.processed,
38793
+ total: syncTotal
38699
38794
  });
38795
+ });
38796
+ minSyncBatches.push(result);
38797
+ if (result.ok)
38798
+ minSyncBatchFailures.length = 0;
38799
+ else if (result.reason !== "lock-held") {
38800
+ for (const virtualPath of virtualPaths) {
38801
+ minSyncBatchFailures.push({
38802
+ code: "minsync-sync-failed",
38803
+ severity: "warning",
38804
+ message: `This file stayed at its previous index entry: the sync that covered it could not index it and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
38805
+ source: virtualPath
38806
+ });
38807
+ }
38700
38808
  }
38809
+ minSyncSyncedBase = syncTotal;
38810
+ minSyncTotalBase = syncTotal;
38811
+ return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
38701
38812
  }
38702
- minSyncSyncedBase = batchTotal;
38703
- minSyncTotalBase = batchTotal;
38704
- return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
38705
- };
38813
+ }) : undefined;
38814
+ const pipeline = minSyncPipeline;
38706
38815
  const summary = needsParsed ? await this.syncParsedMirrors(force, {
38707
38816
  ...opts?.retrySkipped === true ? { retrySkipped: true } : {},
38708
38817
  ...signal === undefined ? {} : { signal },
38709
- ...batchedMinSync ? { onBatch: minSyncBatch } : {},
38818
+ ...pipeline === undefined ? {} : {
38819
+ onBatch: (batch) => {
38820
+ if (batch.final) {
38821
+ progress = updateRefreshProgress(progress, { phase: "minsync" });
38822
+ writeRefreshProgress(this.workspaceProjectRoot, progress);
38823
+ }
38824
+ return pipeline.batch(batch);
38825
+ }
38826
+ },
38710
38827
  onProgress: (parsedProgress) => {
38711
38828
  progress = updateRefreshProgress(progress, {
38712
38829
  phase: "parsed",
@@ -38842,6 +38959,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38842
38959
  ...fsearch !== undefined ? { fsearch } : {}
38843
38960
  };
38844
38961
  } catch (error) {
38962
+ syncAbort.abort(error);
38845
38963
  this.refreshState = {
38846
38964
  ...this.refreshState,
38847
38965
  lastOutcome: "failed",
@@ -38855,6 +38973,8 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38855
38973
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38856
38974
  throw error;
38857
38975
  } finally {
38976
+ signal?.removeEventListener("abort", forwardAbort);
38977
+ await minSyncPipeline?.settle();
38858
38978
  this.refreshState = {
38859
38979
  ...this.refreshState,
38860
38980
  inFlight: false,
@@ -39076,15 +39196,15 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
39076
39196
  const cloudRoots = await this.cloudRoots();
39077
39197
  const fingerprints = cloudFingerprintExclusions(loadCloudFingerprints(this.workspaceProjectRoot));
39078
39198
  for (const searchPath of this.searchPaths) {
39079
- const target = planDupeyScans([searchPath], cloudRoots).targets[0];
39199
+ const scope = cloudScope(searchPath, cloudRoots);
39080
39200
  let scan;
39081
- if (target === undefined) {
39201
+ if (scope.insideCloudRoot) {
39082
39202
  scan = { dir: searchPath, files: [], families: [], errors: [] };
39083
39203
  } else {
39084
39204
  try {
39085
39205
  scan = await scanWithDupey(searchPath, {
39086
39206
  ...this.dupeyOptions || {},
39087
- ...target.excludeDirs.length === 0 ? {} : { excludeDirs: target.excludeDirs }
39207
+ ...scope.nestedCloudRoots.length === 0 ? {} : { excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path)) }
39088
39208
  });
39089
39209
  } catch (error) {
39090
39210
  if (!(error instanceof DupeyCliError))
@@ -39649,7 +39769,7 @@ function toMinSyncReasonDiagnostic(reason) {
39649
39769
  function pinExcludedPath(path) {
39650
39770
  const resolvedPath = resolve18(path);
39651
39771
  try {
39652
- return join54(realpathSync8(dirname27(resolvedPath)), basename16(resolvedPath));
39772
+ return join54(realpathSync8(dirname27(resolvedPath)), basename15(resolvedPath));
39653
39773
  } catch {
39654
39774
  return resolvedPath;
39655
39775
  }