@autorag/librarian 2.13.5 → 2.13.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/profiles.js CHANGED
@@ -3627,7 +3627,7 @@ function outdatedMinSyncVersion(binaryPath) {
3627
3627
  }
3628
3628
  return version !== undefined && !isSupportedMinSyncVersion(version) ? version : undefined;
3629
3629
  }
3630
- var MINSYNC_MIN_VERSION = "0.6.0", VERSION_PROBE_TIMEOUT_MS = 5000, VERSION_LINE, probeCache;
3630
+ var MINSYNC_MIN_VERSION = "0.7.1", VERSION_PROBE_TIMEOUT_MS = 5000, VERSION_LINE, probeCache;
3631
3631
  var init_version = __esm(() => {
3632
3632
  VERSION_LINE = /^minsync\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
3633
3633
  probeCache = new Map;
@@ -3871,7 +3871,7 @@ function isGitHubAsset(value) {
3871
3871
  function isRecord2(value) {
3872
3872
  return typeof value === "object" && value !== null;
3873
3873
  }
3874
- var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.6.0", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
3874
+ var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.7.3", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
3875
3875
  var init_installer = __esm(() => {
3876
3876
  init_atomic_write();
3877
3877
  init_process();
@@ -13749,7 +13749,8 @@ class WombatMethod {
13749
13749
  return matchesDatasourceScope(datasourceSourcePath("wombat", this.instanceId, hit.chunkId), options.scope);
13750
13750
  }).map((hit) => ({
13751
13751
  id: `wombat:${this.instanceId}:${hit.chunkId}`,
13752
- content: hit.snippet,
13752
+ content: `${sessionLabel(hit)}
13753
+ ${hit.snippet}`,
13753
13754
  source: datasourceSourcePath("wombat", this.instanceId, hit.chunkId),
13754
13755
  score: hit.score,
13755
13756
  metadata: {
@@ -13772,6 +13773,20 @@ class WombatMethod {
13772
13773
  })).slice(0, topK);
13773
13774
  }
13774
13775
  }
13776
+ function oneLine(value) {
13777
+ return value.replace(/\s+/gu, " ").trim();
13778
+ }
13779
+ function sessionLabel(hit) {
13780
+ const turns = hit.startTurn === undefined ? undefined : hit.endTurn === undefined || hit.endTurn === hit.startTurn ? `turn ${hit.startTurn}` : `turns ${hit.startTurn}-${hit.endTurn}`;
13781
+ const parts = [
13782
+ oneLine(hit.catalogKey),
13783
+ turns,
13784
+ hit.startedAt === undefined ? undefined : hit.startedAt.slice(0, 10),
13785
+ hit.title.length === 0 ? undefined : oneLine(hit.title),
13786
+ hit.cwd === undefined ? undefined : oneLine(hit.cwd)
13787
+ ].filter((part) => part !== undefined && part.length > 0);
13788
+ return `[${parts.join(" | ")}]`;
13789
+ }
13775
13790
  var WOMBAT_MODES, METHOD_TYPE;
13776
13791
  var init_methods7 = __esm(() => {
13777
13792
  init_errors2();
@@ -13884,7 +13899,7 @@ class WombatSkill {
13884
13899
  `);
13885
13900
  return {
13886
13901
  name: "datasource-wombat",
13887
- description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built.",
13902
+ description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built. Each hit names its session. Load this skill to open that session's full conversation or its original log with the tool calls that the search does not index.",
13888
13903
  content: [
13889
13904
  "# wombat coding-agent session datasource",
13890
13905
  "",
@@ -13898,10 +13913,10 @@ class WombatSkill {
13898
13913
  scopes,
13899
13914
  "",
13900
13915
  "The selectable retrieval modes are keyword (SPLADE), semantic, and hybrid. Scope arguments can only narrow trusted access. Source identifiers remain opaque and hierarchical.",
13901
- "Each result carries `agent`, `title`, `startedAt`, `startTurn`/`endTurn` and `sessionKey` (`<agent>:<session id>`).",
13916
+ "Every result starts with a label line, `[<sessionKey> | turns A-B | date | title | project]`, followed by the snippet. `sessionKey` is `<agent>:<session id>`.",
13902
13917
  "",
13903
13918
  "## Native CLI",
13904
- "The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the result's `sessionKey`:",
13919
+ "The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the `sessionKey` from the result's label:",
13905
13920
  "- `wombat show <sessionKey> [--turn N]`: the full cleaned conversation, or one turn.",
13906
13921
  "- `wombat trace <sessionKey> --turn N --context K`: the original session log around that turn, including the tool calls and outputs that the index leaves out.",
13907
13922
  "- `wombat chunk context <chunk id> --json`: the turns before and after a result.",
@@ -14055,6 +14070,7 @@ function common(config, workspaceRoot) {
14055
14070
  function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
14056
14071
  const skills = [];
14057
14072
  const unknown = [];
14073
+ const retired = [];
14058
14074
  for (const [name, raw] of Object.entries(config ?? {})) {
14059
14075
  if (raw === false)
14060
14076
  continue;
@@ -14066,6 +14082,10 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
14066
14082
  if (entry.enabled === false)
14067
14083
  continue;
14068
14084
  const templateName = entry.type ?? name;
14085
+ if (Object.hasOwn(RETIRED_TEMPLATES, templateName)) {
14086
+ retired.push(name);
14087
+ continue;
14088
+ }
14069
14089
  const builder = Object.hasOwn(BUILDERS, templateName) && datasourceSkillSupported(templateName) ? BUILDERS[templateName] : undefined;
14070
14090
  if (builder === undefined) {
14071
14091
  unknown.push(name);
@@ -14089,9 +14109,9 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
14089
14109
  }
14090
14110
  skills.push(typeof entry.description === "string" && entry.description.trim().length > 0 ? new DescribedDatasourceSkill(aliased, entry.description.trim()) : aliased);
14091
14111
  }
14092
- return { skills, unknown };
14112
+ return { skills, unknown, retired };
14093
14113
  }
14094
- var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES;
14114
+ var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES, RETIRED_TEMPLATES;
14095
14115
  var init_factory = __esm(() => {
14096
14116
  init_aliased_skill();
14097
14117
  init_config2();
@@ -14247,6 +14267,7 @@ var init_factory = __esm(() => {
14247
14267
  };
14248
14268
  PLATFORM_ONLY_SKILLS = { kakao: "darwin" };
14249
14269
  BUILTIN_DATASOURCE_SKILL_NAMES = builtinDatasourceSkillNames();
14270
+ RETIRED_TEMPLATES = { spotlight: true };
14250
14271
  });
14251
14272
 
14252
14273
  // src/embedding/http-embedder.ts
@@ -15833,7 +15854,7 @@ function buildAgentOptions(config, agentDir) {
15833
15854
  const embedder = createEmbedderFromConfig({ embedder: config.minSync?.embedder });
15834
15855
  opts.embedder = embedder;
15835
15856
  if (config.datasources !== undefined) {
15836
- const { skills, unknown } = buildDatasourceSkills(config.datasources, config.workspacePath, {
15857
+ const { skills, unknown, retired } = buildDatasourceSkills(config.datasources, config.workspacePath, {
15837
15858
  embedder,
15838
15859
  discrawl: async () => discrawlEmbeddingRuntimeFor(await embedder.describeEndpoint()),
15839
15860
  cliEndpoint: embeddingEndpointSource(embedder),
@@ -15846,16 +15867,27 @@ function buildAgentOptions(config, agentDir) {
15846
15867
  });
15847
15868
  if (skills.length > 0)
15848
15869
  opts.datasourceSkills = skills;
15870
+ const diagnostics = [];
15871
+ if (retired.length > 0) {
15872
+ const names = retired.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
15873
+ diagnostics.push({
15874
+ code: "retired-datasource-skill",
15875
+ severity: "warning",
15876
+ message: `Datasource config ignored because the Spotlight datasource was removed: ${names.join(", ")}. Delete these entries; use fsearch/mirror search (searchPaths) instead.`,
15877
+ source: "datasources"
15878
+ });
15879
+ }
15849
15880
  if (unknown.length > 0) {
15850
- const safeNames = unknown.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
15851
- const diagnostic = {
15881
+ const names = unknown.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
15882
+ diagnostics.push({
15852
15883
  code: "unknown-datasource-skill",
15853
15884
  severity: "warning",
15854
- message: `Unknown datasource skill(s) in config were skipped: ${safeNames.join(", ")}`,
15885
+ message: `Unknown datasource skill(s) in config were skipped: ${names.join(", ")}`,
15855
15886
  source: "datasources"
15856
- };
15857
- opts.startupDiagnostics = [diagnostic];
15887
+ });
15858
15888
  }
15889
+ if (diagnostics.length > 0)
15890
+ opts.startupDiagnostics = diagnostics;
15859
15891
  }
15860
15892
  return opts;
15861
15893
  }
@@ -17680,11 +17712,49 @@ var init_matrix = __esm(() => {
17680
17712
  };
17681
17713
  });
17682
17714
 
17715
+ // src/dupey/version.ts
17716
+ function parseDupeyVersion(output) {
17717
+ return VERSION_LINE2.exec(output)?.[1];
17718
+ }
17719
+ function isSupportedDupeyVersion(version) {
17720
+ const have = version.split(".").map((part) => Number.parseInt(part, 10));
17721
+ const need = DUPEY_MIN_VERSION.split(".").map((part) => Number.parseInt(part, 10));
17722
+ for (let index = 0;index < need.length; index += 1) {
17723
+ const left = have[index] ?? 0;
17724
+ const right = need[index] ?? 0;
17725
+ if (left !== right)
17726
+ return left > right;
17727
+ }
17728
+ return true;
17729
+ }
17730
+ var DUPEY_MIN_VERSION = "0.2.0", VERSION_LINE2;
17731
+ var init_version2 = __esm(() => {
17732
+ VERSION_LINE2 = /^dupey\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
17733
+ });
17734
+
17683
17735
  // src/dupey/cli.ts
17684
17736
  import { spawn as spawn13 } from "node:child_process";
17685
17737
  import { resolve as resolve9 } from "node:path";
17686
- function runDupey(args, options) {
17687
- return options.run ? options.run(args) : spawnDupey(options.executable ?? "dupey", args, options);
17738
+ function checkDupeyVersion(executable, options) {
17739
+ const cached = versionChecks.get(executable);
17740
+ if (cached !== undefined)
17741
+ return cached;
17742
+ const check = spawnDupey(executable, ["--version"], options).then((stdout) => {
17743
+ const version = parseDupeyVersion(stdout);
17744
+ if (version !== undefined && !isSupportedDupeyVersion(version)) {
17745
+ throw new DupeyCliError(`dupey ${version} at ${executable} is older than the ${DUPEY_MIN_VERSION} AutoRAG requires; run \`cargo install dupey\` to update it.`);
17746
+ }
17747
+ });
17748
+ check.catch(() => versionChecks.delete(executable));
17749
+ versionChecks.set(executable, check);
17750
+ return check;
17751
+ }
17752
+ async function runDupey(args, options) {
17753
+ if (options.run)
17754
+ return options.run(args);
17755
+ const executable = options.executable ?? "dupey";
17756
+ await checkDupeyVersion(executable, options);
17757
+ return spawnDupey(executable, args, options);
17688
17758
  }
17689
17759
  async function scanWithDupey(dir, options = {}) {
17690
17760
  const absoluteDir = resolve9(dir);
@@ -17818,28 +17888,26 @@ function spawnDupey(executable, args, options) {
17818
17888
  });
17819
17889
  });
17820
17890
  }
17821
- var DupeyCliError, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
17891
+ var DupeyCliError, versionChecks, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
17822
17892
  var init_cli2 = __esm(() => {
17823
17893
  init_portable_spawn();
17894
+ init_version2();
17824
17895
  DupeyCliError = class DupeyCliError extends Error {
17825
17896
  constructor(message) {
17826
17897
  super(message);
17827
17898
  this.name = "DupeyCliError";
17828
17899
  }
17829
17900
  };
17901
+ versionChecks = new Map;
17830
17902
  });
17831
17903
 
17832
17904
  // src/dupey/filter.ts
17833
17905
  import { stat as stat2 } from "node:fs/promises";
17834
17906
  import { resolve as resolve10 } from "node:path";
17835
- function extractedNoText(file) {
17836
- const signals = file.signals;
17837
- return typeof signals === "object" && signals !== null && "chars" in signals && signals.chars === 0;
17838
- }
17839
17907
  async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerprints) {
17840
17908
  const byHash = new Map;
17841
17909
  for (const file of scan.files) {
17842
- if (typeof file.content_hash !== "string" || file.content_hash.length === 0 || extractedNoText(file))
17910
+ if (typeof file.content_hash !== "string" || file.content_hash.length === 0)
17843
17911
  continue;
17844
17912
  const path = resolve10(root, file.path);
17845
17913
  const group = byHash.get(file.content_hash) ?? [];
@@ -17848,7 +17916,7 @@ async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerp
17848
17916
  }
17849
17917
  const fingerprintModified = new Map;
17850
17918
  for (const [path, fingerprint] of fingerprints ?? []) {
17851
- if (fingerprint.exact.length === 0 || fingerprint.chars === 0)
17919
+ if (fingerprint.exact.length === 0)
17852
17920
  continue;
17853
17921
  fingerprintModified.set(path, fingerprint.modified);
17854
17922
  const group = byHash.get(fingerprint.exact) ?? [];
@@ -17915,1397 +17983,1368 @@ async function rankExactCandidates(paths, dupeyRanking, fingerprintModified, err
17915
17983
  }
17916
17984
  var init_filter = () => {};
17917
17985
 
17918
- // src/filesystem/cloud-placeholder.ts
17919
- import { execFile as execFile2 } from "node:child_process";
17920
- import { realpathSync as realpathSync3 } from "node:fs";
17921
- import { lstat, readdir } from "node:fs/promises";
17922
- import { homedir as homedir6 } from "node:os";
17923
- import { dirname as dirname14, join as join27 } from "node:path";
17924
- import { promisify } from "node:util";
17925
- function homeCloudRoots(home = homedir6()) {
17926
- const normalizedHome = home.replaceAll("\\", "/").replace(/\/+$/, "");
17927
- return [`${normalizedHome}/Library/CloudStorage`, `${normalizedHome}/Library/Mobile Documents`];
17928
- }
17929
- function datalessFromDarwinFlags(flags) {
17930
- return Number.isFinite(flags) && (flags >>> 0 & SF_DATALESS) !== 0;
17986
+ // src/dupey/index.ts
17987
+ var init_dupey = __esm(() => {
17988
+ init_cli2();
17989
+ init_filter();
17990
+ });
17991
+
17992
+ // src/convert/relate.ts
17993
+ async function convertAndRelate(request, options = {}) {
17994
+ const result = await convertWithFilee(request, options.filee);
17995
+ if (options.dupey === false)
17996
+ return { result, links: [] };
17997
+ const dupeyOptions = options.dupey ?? {};
17998
+ const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
17999
+ const links = await Promise.all(pairs.map(async ({ source, output }) => {
18000
+ try {
18001
+ return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
18002
+ } catch (error) {
18003
+ if (!(error instanceof DupeyCliError))
18004
+ throw error;
18005
+ return { source, output, dupey: { relation: "unavailable", reason: error.message } };
18006
+ }
18007
+ }));
18008
+ return { result, links };
17931
18009
  }
17932
- function datalessFromWindowsAttributes(attributes) {
17933
- return Number.isFinite(attributes) && (attributes & WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) !== 0;
18010
+ var init_relate = __esm(() => {
18011
+ init_dupey();
18012
+ init_cli();
18013
+ });
18014
+
18015
+ // src/convert/index.ts
18016
+ var init_convert = __esm(() => {
18017
+ init_cli();
18018
+ init_installer2();
18019
+ init_matrix();
18020
+ init_relate();
18021
+ });
18022
+
18023
+ // src/datasource/diagnostics.ts
18024
+ function mapDatasourceDiagnostic(ds) {
18025
+ return {
18026
+ code: datasourceCodeToRetrieval(ds.code),
18027
+ severity: ds.severity,
18028
+ message: `[datasource:${ds.code}] ${ds.message}`,
18029
+ source: ds.source ?? (ds.instanceId ? `datasource:${ds.instanceId}` : "datasource")
18030
+ };
17934
18031
  }
17935
- function cloudProviderFromPath(target) {
17936
- const lower = target.toLowerCase();
17937
- if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
17938
- return "google-drive";
17939
- if (lower.includes("onedrive"))
17940
- return "onedrive";
17941
- if (lower.includes("dropbox"))
17942
- return "dropbox";
17943
- if (lower.includes("mobile documents") || lower.includes("clouddocs"))
17944
- return "icloud";
17945
- return "file-provider";
18032
+ function mapDatasourceDiagnostics(diagnostics) {
18033
+ return diagnostics.map(mapDatasourceDiagnostic);
17946
18034
  }
17947
- function cloudProviderFromDomain(domain) {
17948
- const lower = domain.toLowerCase();
17949
- if (lower.includes("clouddocs") || lower.includes("icloud"))
17950
- return "icloud";
17951
- if (lower.includes("google"))
17952
- return "google-drive";
17953
- if (lower.includes("onedrive"))
17954
- return "onedrive";
17955
- if (lower.includes("dropbox"))
17956
- return "dropbox";
17957
- return "file-provider";
18035
+ function datasourceCodeToRetrieval(code) {
18036
+ switch (code) {
18037
+ case "datasource-unavailable":
18038
+ case "datasource-cli-error":
18039
+ case "datasource-empty":
18040
+ case "datasource-rate-limited":
18041
+ case "datasource-auth-error":
18042
+ case "datasource-embedding-egress-rejected":
18043
+ case "datasource-index-failed":
18044
+ case "datasource-index-partial":
18045
+ case "datasource-permission-denied":
18046
+ case "datasource-remote-embedding-rejected":
18047
+ return "retrieval-method-failed";
18048
+ }
17958
18049
  }
17959
- function normalizeCloudPath(value) {
17960
- let normalized = value.replaceAll("\\", "/");
17961
- if (normalized.length > 1)
17962
- normalized = normalized.replace(/\/+$/u, "");
17963
- return normalized;
18050
+
18051
+ // src/datasource/retry.ts
18052
+ import { setTimeout as delay2 } from "node:timers/promises";
18053
+ function datasourceRetryDelayMs(retry, policy) {
18054
+ const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
18055
+ return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
17964
18056
  }
17965
- function isPathInside(target, parent) {
17966
- const normalizedParent = normalizeCloudPath(parent);
17967
- if (normalizedParent === "")
17968
- return false;
17969
- const left = normalizeCloudPath(target);
17970
- const [a, b] = process.platform === "win32" ? [left.toLowerCase(), normalizedParent.toLowerCase()] : [left, normalizedParent];
17971
- return a === b || a.startsWith(`${b}/`);
18057
+ async function indexWithRetry(skill, options = {}, signal) {
18058
+ const policy = {
18059
+ maxRetries: options.maxRetries ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxRetries,
18060
+ baseDelayMs: options.baseDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.baseDelayMs,
18061
+ maxDelayMs: options.maxDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxDelayMs
18062
+ };
18063
+ const sleep = options.sleep ?? ((ms) => delay2(ms, undefined, signal === undefined ? undefined : { signal }));
18064
+ let lastError = "";
18065
+ for (let attempt = 1;; attempt += 1) {
18066
+ const result = await runIndex(skill, signal);
18067
+ signal?.throwIfAborted();
18068
+ if (result.ok) {
18069
+ if (attempt === 1)
18070
+ return result;
18071
+ const recovered = {
18072
+ code: "datasource-index-failed",
18073
+ severity: "info",
18074
+ message: `recovered after ${attempt} attempts; last error: ${lastError}`,
18075
+ instanceId: result.instanceId,
18076
+ source: result.skill
18077
+ };
18078
+ return { ...result, attempts: attempt, diagnostics: [...result.diagnostics, recovered] };
18079
+ }
18080
+ lastError = result.message ?? result.error ?? "datasource-index-failed";
18081
+ if (attempt > policy.maxRetries) {
18082
+ if (attempt === 1)
18083
+ return { ...result, attempts: attempt };
18084
+ const suffix = ` (gave up after ${attempt} attempts)`;
18085
+ const last = result.diagnostics.at(-1);
18086
+ return {
18087
+ ...result,
18088
+ attempts: attempt,
18089
+ message: `${lastError}${suffix}`,
18090
+ diagnostics: last === undefined ? [
18091
+ {
18092
+ code: result.code ?? "datasource-index-failed",
18093
+ severity: "error",
18094
+ message: `${lastError}${suffix}`,
18095
+ instanceId: result.instanceId,
18096
+ source: result.skill
18097
+ }
18098
+ ] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
18099
+ };
18100
+ }
18101
+ await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
18102
+ signal?.throwIfAborted();
18103
+ }
17972
18104
  }
17973
- function canonicalPath(value) {
18105
+ async function runIndex(skill, signal) {
17974
18106
  try {
17975
- return realpathSync3(value);
17976
- } catch {
17977
- return value;
18107
+ return await skill.index(signal === undefined ? undefined : { signal });
18108
+ } catch (error) {
18109
+ signal?.throwIfAborted();
18110
+ const descriptor = skill.describe();
18111
+ const message = error instanceof Error ? error.message : "Datasource indexing failed.";
18112
+ const failed = {
18113
+ ok: false,
18114
+ instanceId: descriptor.instanceId ?? "default",
18115
+ skill: descriptor.name,
18116
+ indexedAt: Date.now(),
18117
+ diagnostics: [
18118
+ {
18119
+ code: "datasource-index-failed",
18120
+ severity: "error",
18121
+ message,
18122
+ source: descriptor.name,
18123
+ instanceId: descriptor.instanceId
18124
+ }
18125
+ ],
18126
+ error: "datasource-index-failed",
18127
+ code: "datasource-index-failed",
18128
+ message
18129
+ };
18130
+ return failed;
17978
18131
  }
17979
18132
  }
17980
- function canonicalPathMapper(searchRoots) {
17981
- const forms = searchRoots.map((root) => ({ raw: normalizeCloudPath(root), canonical: normalizeCloudPath(canonicalPath(root)) })).filter(({ raw, canonical }) => raw !== "" && raw !== canonical).sort((left, right) => right.raw.length - left.raw.length);
17982
- if (forms.length === 0)
17983
- return (path) => path;
17984
- return (path) => {
17985
- const normalized = normalizeCloudPath(path);
17986
- for (const { raw, canonical } of forms) {
17987
- if (isPathInside(normalized, raw))
17988
- return `${canonical}${normalized.slice(raw.length)}`;
17989
- }
17990
- return path;
18133
+ var DEFAULT_DATASOURCE_RETRY_POLICY;
18134
+ var init_retry = __esm(() => {
18135
+ DEFAULT_DATASOURCE_RETRY_POLICY = {
18136
+ maxRetries: 2,
18137
+ baseDelayMs: 1000,
18138
+ maxDelayMs: 30000
17991
18139
  };
18140
+ });
18141
+
18142
+ // src/everything/bundle.ts
18143
+ import { createHash as createHash11 } from "node:crypto";
18144
+ import { existsSync as existsSync17, mkdirSync as mkdirSync14, readFileSync as readFileSync17, rmSync as rmSync7, writeFileSync as writeFileSync10 } from "node:fs";
18145
+ import { dirname as dirname14, join as join27 } from "node:path";
18146
+ import { fileURLToPath } from "node:url";
18147
+ import JSZip from "jszip";
18148
+ function resolveEverythingBundleDir() {
18149
+ let dir = dirname14(fileURLToPath(import.meta.url));
18150
+ for (let depth = 0;depth < 6; depth += 1) {
18151
+ const candidate = join27(dir, "vendor", "everything");
18152
+ if (existsSync17(join27(candidate, MANIFEST_FILENAME)))
18153
+ return candidate;
18154
+ const parent = dirname14(dir);
18155
+ if (parent === dir)
18156
+ break;
18157
+ dir = parent;
18158
+ }
18159
+ return;
17992
18160
  }
17993
- function cloudScope(searchRoot, cloudRoots) {
17994
- const canonicalSearchRoot = canonicalPath(searchRoot);
17995
- const roots = cloudRoots.map((root) => ({ path: canonicalPath(root.path), kind: root.kind }));
17996
- return {
17997
- insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
17998
- nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
17999
- };
18161
+ function everythingArch(arch) {
18162
+ if (arch === "x64" || arch === "arm64")
18163
+ return arch;
18164
+ return;
18000
18165
  }
18001
- function parseReleaseOutcome(stdout) {
18002
- let parsed;
18166
+ function everythingBinaryPaths(root, version, arch) {
18167
+ const dir = join27(root, ".autorag", "everything", version, arch);
18168
+ return { everythingPath: join27(dir, "everything.exe"), esPath: join27(dir, "es.exe") };
18169
+ }
18170
+ function loadEverythingBundleManifest(bundleDir) {
18171
+ return JSON.parse(readFileSync17(join27(bundleDir, MANIFEST_FILENAME), "utf8"));
18172
+ }
18173
+ async function ensureEverythingBinaries(options) {
18174
+ const platform = options.platform ?? process.platform;
18175
+ const rawArch = options.arch ?? process.arch;
18176
+ if (platform !== "win32") {
18177
+ return {
18178
+ ok: false,
18179
+ reason: "unsupported-platform",
18180
+ message: `Everything is Windows-only; this host is ${platform}.`
18181
+ };
18182
+ }
18183
+ const arch = everythingArch(rawArch);
18184
+ const bundleDir = options.bundleDir ?? resolveEverythingBundleDir();
18185
+ if (bundleDir === undefined) {
18186
+ return {
18187
+ ok: false,
18188
+ reason: "bundle-missing",
18189
+ message: `Bundled Everything assets (vendor/everything/${MANIFEST_FILENAME}) were not found above ${dirname14(fileURLToPath(import.meta.url))}.`
18190
+ };
18191
+ }
18192
+ let manifest;
18003
18193
  try {
18004
- parsed = JSON.parse(stdout.trim());
18005
- } catch {
18006
- return;
18194
+ manifest = options.manifest ?? loadEverythingBundleManifest(bundleDir);
18195
+ } catch (error) {
18196
+ return { ok: false, reason: "bundle-missing", message: errorMessage2(error) };
18007
18197
  }
18008
- if (typeof parsed !== "object" || parsed === null || !("ok" in parsed) || typeof parsed.ok !== "boolean") {
18009
- return;
18198
+ const assets = arch === undefined ? undefined : manifest.architectures[arch];
18199
+ if (arch === undefined || assets === undefined) {
18200
+ return {
18201
+ ok: false,
18202
+ reason: "unsupported-platform",
18203
+ message: `No bundled Everything build for Windows architecture ${rawArch}; bundled: ${Object.keys(manifest.architectures).join(", ")}.`
18204
+ };
18010
18205
  }
18011
- return {
18012
- ok: parsed.ok,
18013
- ..."isUploaded" in parsed ? { isUploaded: parsed.isUploaded } : {},
18014
- ..."isUploading" in parsed ? { isUploading: parsed.isUploading } : {},
18015
- ..."downloadingStatus" in parsed ? { downloadingStatus: parsed.downloadingStatus } : {}
18016
- };
18017
- }
18018
- function describeReleaseRefusal(outcome) {
18019
- if (outcome.isUploading === true)
18020
- return "The file is still uploading to the cloud, so it cannot be released yet.";
18021
- if (outcome.isUploaded === false)
18022
- return "The file has not finished uploading to the cloud, so it cannot be released yet.";
18023
- const status = typeof outcome.downloadingStatus === "string" ? outcome.downloadingStatus : "unknown";
18024
- return `The cloud provider declined to release the local copy (download status ${status}).`;
18206
+ const { everythingPath, esPath } = everythingBinaryPaths(options.root, manifest.everythingVersion, arch);
18207
+ const everythingCached = fileHasSha256(everythingPath, assets.everythingSha256);
18208
+ const esCached = fileHasSha256(esPath, assets.esSha256);
18209
+ if (everythingCached && esCached) {
18210
+ return { ok: true, everythingPath, esPath, source: "cached" };
18211
+ }
18212
+ if ((options.mode ?? "install") === "cached") {
18213
+ const state = everythingCached || esCached ? "are incomplete or do not match their pinned SHA-256 digests" : "are not installed";
18214
+ return {
18215
+ ok: false,
18216
+ reason: "not-installed",
18217
+ message: `Everything ${manifest.everythingVersion} (${arch}) binaries ${state} under ${dirname14(everythingPath)}; run \`autorag refresh\` to (re)install and index them.`
18218
+ };
18219
+ }
18220
+ try {
18221
+ mkdirSync14(dirname14(everythingPath), { recursive: true });
18222
+ const everythingBytes = await extractVerified(join27(bundleDir, assets.everythingArchive), assets.everythingArchiveSha256, assets.everythingMember, assets.everythingSha256);
18223
+ const esBytes = await extractVerified(join27(bundleDir, assets.esArchive), assets.esArchiveSha256, assets.esMember, assets.esSha256);
18224
+ writeAtomic(everythingPath, everythingBytes);
18225
+ writeAtomic(esPath, esBytes);
18226
+ } catch (error) {
18227
+ return { ok: false, reason: "install-failed", message: errorMessage2(error) };
18228
+ }
18229
+ return { ok: true, everythingPath, esPath, source: "installed" };
18025
18230
  }
18026
- function describeReleaseHelperFailure(error) {
18027
- const failure = error;
18028
- const parts = [];
18029
- if (failure.killed === true || typeof failure.signal === "string") {
18030
- parts.push(`the helper was killed${typeof failure.signal === "string" ? ` by ${failure.signal}` : ""}`);
18031
- } else if (typeof failure.code === "number") {
18032
- parts.push(`the helper exited with code ${failure.code}`);
18231
+ async function extractVerified(archivePath, archiveSha256, member, memberSha256) {
18232
+ const archive = readFileSync17(archivePath);
18233
+ const actualArchive = sha256(archive);
18234
+ if (actualArchive !== archiveSha256) {
18235
+ throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
18033
18236
  }
18034
- const stderr = typeof failure.stderr === "string" ? failure.stderr.trim() : "";
18035
- if (stderr !== "")
18036
- parts.push(stderr);
18037
- if (parts.length === 0)
18038
- parts.push(error instanceof Error ? error.message : String(error));
18039
- return `osascript could not release the local copy: ${parts.join("; ")}`;
18237
+ const zip = await JSZip.loadAsync(archive);
18238
+ const entry = zip.file(member);
18239
+ if (entry === null)
18240
+ throw new Error(`${archivePath} does not contain ${member}`);
18241
+ const bytes = await entry.async("nodebuffer");
18242
+ const actualMember = sha256(bytes);
18243
+ if (actualMember !== memberSha256) {
18244
+ throw new Error(`${member} from ${archivePath} SHA-256 ${actualMember} does not match pinned ${memberSha256}`);
18245
+ }
18246
+ return bytes;
18040
18247
  }
18041
- async function defaultDarwinReadDir(path) {
18042
- const entries = await readdir(path, { withFileTypes: true });
18043
- return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
18248
+ function fileHasSha256(path, expected) {
18249
+ return existsSync17(path) && sha256(readFileSync17(path)) === expected;
18044
18250
  }
18045
- function parseDarwinMounts(stdout) {
18046
- const mounts = [];
18047
- for (const line of stdout.split(`
18048
- `)) {
18049
- if (line.trim() === "")
18050
- continue;
18051
- const open = line.lastIndexOf(" (");
18052
- if (open <= 0 || !line.endsWith(")"))
18053
- continue;
18054
- const head = line.slice(0, open);
18055
- const on = head.indexOf(" on ");
18056
- if (on <= 0)
18057
- continue;
18058
- const mountPoint = head.slice(on + " on ".length);
18059
- if (mountPoint === "")
18060
- continue;
18061
- const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
18062
- mounts.push({
18063
- device: head.slice(0, on),
18064
- mountPoint,
18065
- fileSystem: fields[0] ?? "",
18066
- options: fields.slice(1)
18067
- });
18251
+ function writeAtomic(path, bytes) {
18252
+ const temp = `${path}.${process.pid}.tmp`;
18253
+ try {
18254
+ writeFileSync10(temp, bytes);
18255
+ renameWithRetry(temp, path);
18256
+ } finally {
18257
+ rmSync7(temp, { force: true });
18068
18258
  }
18069
- return mounts;
18070
18259
  }
18071
- function networkMountPoints(mounts) {
18072
- const points = [];
18073
- const seen = new Set;
18074
- for (const mount of mounts) {
18075
- if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
18076
- continue;
18077
- if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
18078
- continue;
18079
- if (seen.has(mount.mountPoint))
18080
- continue;
18081
- seen.add(mount.mountPoint);
18082
- points.push(mount.mountPoint);
18083
- }
18084
- return points;
18260
+ function sha256(bytes) {
18261
+ return createHash11("sha256").update(bytes).digest("hex");
18085
18262
  }
18086
- function parseXattrValues(stdout, candidates) {
18087
- const values = new Map;
18088
- if (candidates.length === 0)
18089
- return values;
18090
- const ordered = [...candidates].sort((left, right) => right.length - left.length);
18091
- for (const line of stdout.split(`
18092
- `)) {
18093
- if (line === "")
18094
- continue;
18095
- for (const candidate of ordered) {
18096
- if (line.startsWith(`${candidate}: `)) {
18097
- values.set(candidate, line.slice(candidate.length + 2));
18098
- break;
18099
- }
18100
- }
18101
- }
18102
- return values;
18263
+ function errorMessage2(error) {
18264
+ return error instanceof Error ? error.message : String(error);
18265
+ }
18266
+ var MANIFEST_FILENAME = "manifest.json";
18267
+ var init_bundle = __esm(() => {
18268
+ init_atomic_write();
18269
+ });
18270
+
18271
+ // src/filesystem/path-exclusions.ts
18272
+ import { posix as posix3, win32 as win322 } from "node:path";
18273
+ function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
18274
+ const paths = platform === "win32" ? win322 : posix3;
18275
+ const normalize = (path) => {
18276
+ const absolute = paths.resolve(path);
18277
+ return platform === "win32" ? absolute.toLowerCase() : absolute;
18278
+ };
18279
+ const excluded = excludedPaths.map((path) => {
18280
+ const absolute = normalize(path);
18281
+ return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
18282
+ });
18283
+ return (path) => {
18284
+ const absolute = normalize(path);
18285
+ return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
18286
+ };
18103
18287
  }
18288
+ var init_path_exclusions = () => {};
18104
18289
 
18105
- class DarwinCloudProbe {
18106
- exec;
18107
- readDir;
18108
- home;
18109
- constructor(options = {}) {
18110
- this.exec = options.exec ?? defaultCloudExec;
18111
- this.readDir = options.readDir ?? defaultDarwinReadDir;
18112
- this.home = options.home ?? homedir6();
18113
- }
18114
- async cloudRoots(searchRoots = []) {
18115
- const roots = [];
18116
- const seen = new Set;
18117
- const add = (path, kind) => {
18118
- const canonical = canonicalPath(path);
18119
- const key = normalizeCloudPath(canonical);
18120
- if (key === "" || seen.has(key))
18121
- return;
18122
- seen.add(key);
18123
- roots.push({ path: canonical, kind });
18124
- };
18125
- const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
18126
- for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
18127
- if (entry.isDirectory())
18128
- add(join27(cloudStorage, entry.name), "file-provider");
18129
- }
18130
- if (await this.tryReadDir(mobileDocuments) !== undefined)
18131
- add(mobileDocuments, "file-provider");
18132
- for (const mountPoint of await this.networkMountPoints())
18133
- add(mountPoint, "network");
18134
- for (const directory of await this.fileProviderDirectories(searchRoots))
18135
- add(directory, "file-provider");
18136
- return roots;
18290
+ // src/filesystem/shared-exclusive-queue.ts
18291
+ class SharedExclusiveQueue {
18292
+ sharedSinceExclusive = [];
18293
+ lastExclusive = Promise.resolve();
18294
+ shared(task) {
18295
+ const run = this.lastExclusive.then(task);
18296
+ const settled = run.then(() => {
18297
+ return;
18298
+ }, () => {
18299
+ return;
18300
+ });
18301
+ this.sharedSinceExclusive.push(settled);
18302
+ return run;
18137
18303
  }
18138
- async networkMountPoints() {
18139
- try {
18140
- const { stdout } = await this.exec("/sbin/mount", [], { timeoutMs: 1e4, maxBuffer: 4 * 1024 * 1024 });
18141
- return networkMountPoints(parseDarwinMounts(stdout));
18142
- } catch {
18143
- return [];
18144
- }
18145
- }
18146
- async fileProviderDirectories(searchRoots) {
18147
- if (searchRoots.length === 0)
18148
- return [];
18149
- const candidates = new Set;
18150
- for (const searchRoot of searchRoots) {
18151
- const canonical = canonicalPath(searchRoot);
18152
- for (let cursor = canonical;; cursor = dirname14(cursor)) {
18153
- candidates.add(cursor);
18154
- const parent = dirname14(cursor);
18155
- if (parent === cursor)
18156
- break;
18157
- }
18158
- for (const entry of await this.tryReadDir(canonical) ?? []) {
18159
- if (entry.isDirectory())
18160
- candidates.add(join27(canonical, entry.name));
18161
- }
18162
- }
18163
- const paths = [...candidates];
18164
- const managed = [];
18165
- for (let start = 0;start < paths.length; start += XATTR_BATCH) {
18166
- for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
18167
- managed.push(path);
18168
- }
18169
- return managed;
18170
- }
18171
- async xattrValues(paths) {
18172
- try {
18173
- const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
18174
- timeoutMs: 15000,
18175
- maxBuffer: 8 * 1024 * 1024
18176
- });
18177
- return parseXattrValues(stdout, paths);
18178
- } catch (error) {
18179
- const failure = error;
18180
- return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
18181
- }
18182
- }
18183
- async tryReadDir(path) {
18184
- try {
18185
- return await this.readDir(path);
18186
- } catch {
18304
+ exclusive(task) {
18305
+ const earlier = Promise.all([this.lastExclusive, ...this.sharedSinceExclusive]);
18306
+ const run = earlier.then(task);
18307
+ const settled = run.then(() => {
18187
18308
  return;
18188
- }
18189
- }
18190
- async datalessAmong(paths) {
18191
- const dataless = new Set;
18192
- for (let start = 0;start < paths.length; start += STAT_BATCH) {
18193
- const batch = paths.slice(start, start + STAT_BATCH);
18194
- const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
18195
- timeoutMs: 30000,
18196
- maxBuffer: 4 * 1024 * 1024
18197
- });
18198
- const flags = stdout.split(`
18199
- `).filter(Boolean);
18200
- for (const [index, path] of batch.entries()) {
18201
- if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
18202
- dataless.add(path);
18203
- }
18204
- }
18205
- return dataless;
18206
- }
18207
- async providerOf(path) {
18208
- const domain = await readFileProviderDomainUpwards(path);
18209
- return domain === undefined ? undefined : cloudProviderFromDomain(domain);
18210
- }
18211
- canRelease(provider) {
18212
- return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
18213
- }
18214
- async evict(path) {
18215
- let stdout;
18216
- try {
18217
- stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
18218
- timeoutMs: 30000
18219
- })).stdout;
18220
- } catch (error) {
18221
- return { ok: false, message: describeReleaseHelperFailure(error) };
18222
- }
18223
- const outcome = parseReleaseOutcome(stdout);
18224
- if (outcome === undefined) {
18225
- const detail = stdout.trim();
18226
- return {
18227
- ok: false,
18228
- message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
18229
- };
18230
- }
18231
- return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
18232
- }
18233
- }
18234
- function buildWindowsAttributeScript(paths) {
18235
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18236
- ` + `$ErrorActionPreference='SilentlyContinue'
18237
- ` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
18238
- `;
18239
- }
18240
- function buildWindowsSyncRootsScript() {
18241
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18242
- ` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
18243
- ` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
18244
- `;
18245
- }
18246
- function buildWindowsDehydrateScript(path) {
18247
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18248
- ` + `$ErrorActionPreference='Stop'
18249
- ` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
18250
- ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
18251
- ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
18252
- ` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
18253
- ` + `'@
18254
- ` + `$h=[IntPtr]::Zero
18255
- ` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
18256
- ` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
18257
- ` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
18258
- ` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
18259
- ` + `[Autorag.CfApi]::CfCloseHandle($h)
18260
- ` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
18261
- ` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
18262
- `;
18263
- }
18264
- function parseWindowsAttributeOutput(stdout, paths) {
18265
- const lines = stdout.split(/\r?\n/);
18266
- const attributes = new Map;
18267
- for (const [index, path] of paths.entries()) {
18268
- const value = Number.parseInt((lines[index] ?? "").trim(), 10);
18269
- if (Number.isFinite(value))
18270
- attributes.set(path, value);
18271
- }
18272
- return attributes;
18273
- }
18274
- function parseWindowsSyncRoots(stdout) {
18275
- const roots = [];
18276
- for (const line of stdout.split(/\r?\n/)) {
18277
- const tab = line.indexOf("\t");
18278
- if (tab <= 0)
18279
- continue;
18280
- const path = line.slice(0, tab).trim();
18281
- if (path !== "")
18282
- roots.push({ path, label: line.slice(tab + 1).trim() });
18309
+ }, () => {
18310
+ return;
18311
+ });
18312
+ this.sharedSinceExclusive = [];
18313
+ this.lastExclusive = settled;
18314
+ return run;
18283
18315
  }
18284
- return roots;
18285
18316
  }
18286
- function providerFromWindowsSyncRoot(label) {
18287
- const lower = label.toLowerCase();
18288
- if (lower.includes("onedrive") || lower.includes("one drive"))
18289
- return "onedrive";
18290
- if (lower.includes("google"))
18291
- return "google-drive";
18292
- return "file-provider";
18317
+
18318
+ // src/everything/client.ts
18319
+ import { spawn as spawn14 } from "node:child_process";
18320
+ import { createHash as createHash12 } from "node:crypto";
18321
+ import { mkdirSync as mkdirSync15, writeFileSync as writeFileSync11 } from "node:fs";
18322
+ import { join as join28, resolve as resolve11 } from "node:path";
18323
+ import { setTimeout as sleep2 } from "node:timers/promises";
18324
+ function everythingInstanceName(root) {
18325
+ return `autorag-${createHash12("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
18293
18326
  }
18294
- function buildWindowsDiskScript() {
18295
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18296
- ` + "Get-CimInstance -ClassName Win32_LogicalDisk | ForEach-Object {" + `$_.DeviceID + [char]9 + $_.DriveType + [char]9 + $_.FileSystem + [char]9 + $_.VolumeName}
18297
- `;
18327
+ function buildEverythingIni(input) {
18328
+ const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
18329
+ const perFolder = (value) => input.folders.map(() => value).join(",");
18330
+ return [
18331
+ "[Everything]",
18332
+ "app_data=0",
18333
+ "run_as_admin=0",
18334
+ "run_in_background=1",
18335
+ "show_tray_icon=0",
18336
+ "show_in_taskbar=0",
18337
+ "check_for_updates_on_startup=0",
18338
+ "beta_updates=0",
18339
+ "language=1033",
18340
+ "allow_http_server=0",
18341
+ "allow_etp_server=0",
18342
+ "search_history_enabled=0",
18343
+ "run_history_enabled=0",
18344
+ "auto_include_fixed_volumes=0",
18345
+ "auto_include_removable_volumes=0",
18346
+ "auto_include_fixed_refs_volumes=0",
18347
+ "auto_include_removable_refs_volumes=0",
18348
+ "ntfs_volume_paths=",
18349
+ "ntfs_volume_includes=",
18350
+ "refs_volume_paths=",
18351
+ "refs_volume_includes=",
18352
+ "index_size=1",
18353
+ "index_date_modified=1",
18354
+ `folders=${list(input.folders)}`,
18355
+ `folder_monitor_changes=${perFolder("1")}`,
18356
+ `folder_rescan_if_full_list=${perFolder("1")}`,
18357
+ `folder_update_types=${perFolder("0")}`,
18358
+ "exclude_list_enabled=1",
18359
+ `exclude_folders=${list(input.excludeFolders)}`,
18360
+ ""
18361
+ ].join(`\r
18362
+ `);
18298
18363
  }
18299
- function parseWindowsLogicalDisks(stdout) {
18300
- const disks = [];
18301
- for (const line of stdout.split(/\r?\n/)) {
18302
- if (line.trim() === "")
18303
- continue;
18304
- const [deviceId, driveType, fileSystem, ...rest] = line.split("\t");
18305
- const type = Number.parseInt((driveType ?? "").trim(), 10);
18306
- if (!Number.isFinite(type))
18307
- continue;
18308
- disks.push({
18309
- deviceId: (deviceId ?? "").trim(),
18310
- driveType: type,
18311
- fileSystem: (fileSystem ?? "").trim(),
18312
- volumeName: rest.join("\t").trim()
18313
- });
18314
- }
18315
- return disks;
18364
+ function buildEverythingSearchArgs(instance, request, timeoutMs) {
18365
+ const args = [
18366
+ "-argv",
18367
+ "-instance",
18368
+ instance,
18369
+ "-cp",
18370
+ "65001",
18371
+ "-json",
18372
+ "-full-path-and-name",
18373
+ "-size",
18374
+ "-date-modified",
18375
+ "-date-format",
18376
+ "1"
18377
+ ];
18378
+ if (timeoutMs !== undefined)
18379
+ args.push("-timeout", String(timeoutMs));
18380
+ if (request.matchCase)
18381
+ args.push("-case");
18382
+ if (request.matchPath)
18383
+ args.push("-match-path");
18384
+ if (request.wholeWord)
18385
+ args.push("-whole-word");
18386
+ if (request.kind === "files")
18387
+ args.push("/a-d");
18388
+ if (request.kind === "folders")
18389
+ args.push("/ad");
18390
+ if (request.path !== undefined)
18391
+ args.push("-path", request.path);
18392
+ if (request.sort !== undefined)
18393
+ args.push("-sort", request.sort);
18394
+ if (request.offset !== undefined && request.offset > 0)
18395
+ args.push("-offset", String(request.offset));
18396
+ args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
18397
+ if (request.regex)
18398
+ args.push("-regex", "--", request.query);
18399
+ else
18400
+ args.push("--", request.query);
18401
+ return args;
18316
18402
  }
18317
- function windowsRootsFromDisks(disks) {
18318
- const roots = [];
18319
- for (const disk of disks) {
18320
- if (disk.deviceId === "")
18321
- continue;
18322
- const path = `${disk.deviceId}\\`;
18323
- const fileSystem = disk.fileSystem.toLowerCase();
18324
- if (disk.driveType === WINDOWS_DRIVE_TYPE_NETWORK) {
18325
- roots.push({ path, kind: "network" });
18326
- continue;
18327
- }
18328
- if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
18329
- roots.push({ path, kind: "network" });
18330
- continue;
18331
- }
18332
- if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
18333
- roots.push({ path, kind: "cloud-files" });
18334
- }
18403
+ function parseEverythingJson(stdout) {
18404
+ const text = stdout.trim();
18405
+ if (text.length === 0)
18406
+ return [];
18407
+ let parsed;
18408
+ try {
18409
+ parsed = JSON.parse(text);
18410
+ } catch {
18411
+ throw new Error(`ES did not return JSON: ${text.slice(0, 2000)}`);
18335
18412
  }
18336
- return roots;
18337
- }
18338
- function uncShareRoot(target) {
18339
- const normalized = target.replaceAll("/", "\\");
18340
- if (!normalized.startsWith("\\\\") || normalized.startsWith("\\\\?\\") || normalized.startsWith("\\\\.\\"))
18341
- return;
18342
- const rest = normalized.slice(2);
18343
- const separator = rest.indexOf("\\");
18344
- if (separator <= 0)
18345
- return;
18346
- const server = rest.slice(0, separator);
18347
- const share = rest.slice(separator + 1).split("\\")[0] ?? "";
18348
- if (server === "" || share === "")
18349
- return;
18350
- return `\\\\${server}\\${share}`;
18351
- }
18352
- function parseWindowsDehydrateOutput(result) {
18353
- const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
18354
- if (open === null)
18355
- return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
18356
- if (Number.parseInt(open[1], 16) !== 0) {
18413
+ if (!Array.isArray(parsed))
18414
+ throw new Error(`ES did not return a JSON array: ${text.slice(0, 2000)}`);
18415
+ return parsed.map((item) => {
18416
+ const record = item;
18417
+ const filename = String(record.filename ?? "");
18418
+ const isFolder = filename.endsWith("\\") && filename.length > 3;
18357
18419
  return {
18358
- ok: false,
18359
- message: windowsFailureMessage(result, `could not open the file for release (0x${open[1]})`)
18420
+ path: isFolder ? filename.slice(0, -1) : filename,
18421
+ type: filename.endsWith("\\") ? "folder" : "file",
18422
+ size: typeof record.size === "number" ? record.size : undefined,
18423
+ dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
18360
18424
  };
18361
- }
18362
- const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
18363
- if (dehydrate === null) {
18364
- return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
18365
- }
18366
- const hresult = dehydrate[1];
18367
- if (Number.parseInt(hresult, 16) === 0)
18368
- return { ok: true };
18369
- const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
18370
- return {
18371
- ok: false,
18372
- message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
18373
- };
18374
- }
18375
- function windowsFailureMessage(result, fallback) {
18376
- const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
18377
- if (printed !== undefined && printed !== "")
18378
- return printed;
18379
- const stderr = result.stderr.trim();
18380
- return stderr !== "" ? stderr : fallback;
18425
+ });
18381
18426
  }
18382
18427
 
18383
- class WindowsCloudProbe {
18428
+ class EverythingClient {
18429
+ instanceName;
18430
+ options;
18431
+ platform;
18384
18432
  run;
18385
- syncRoots;
18386
- disks;
18387
- constructor(options = {}) {
18388
- this.run = options.run ?? runPowerShell;
18433
+ launch;
18434
+ stateDir;
18435
+ isExcludedPath;
18436
+ binaries;
18437
+ queue = new SharedExclusiveQueue;
18438
+ constructor(options) {
18439
+ this.options = options;
18440
+ this.platform = options.platform ?? process.platform;
18441
+ this.instanceName = everythingInstanceName(options.root);
18442
+ this.run = options.run ?? runProcess;
18443
+ this.launch = options.launch ?? launchDetached;
18444
+ this.stateDir = join28(options.root, ".autorag", "everything");
18445
+ this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
18389
18446
  }
18390
- async cloudRoots(searchRoots = []) {
18391
- const roots = [];
18392
- const seen = new Set;
18393
- const add = (path, kind) => {
18394
- const canonical = canonicalPath(path);
18395
- const key = normalizeCloudPath(canonical).toLowerCase();
18396
- if (key === "" || seen.has(key))
18397
- return;
18398
- seen.add(key);
18399
- roots.push({ path: canonical, kind });
18400
- };
18401
- for (const root of await this.loadSyncRoots())
18402
- add(root.path, "cloud-files");
18403
- for (const root of await this.loadDiskRoots())
18404
- add(root.path, root.kind);
18405
- for (const searchRoot of searchRoots) {
18406
- const share = uncShareRoot(searchRoot);
18407
- if (share !== undefined)
18408
- add(share, "network");
18409
- }
18410
- return roots;
18447
+ isSupported() {
18448
+ return this.platform === "win32";
18411
18449
  }
18412
- async datalessAmong(paths) {
18413
- const dataless = new Set;
18414
- for (let start = 0;start < paths.length; start += WINDOWS_ATTRIBUTE_BATCH) {
18415
- const batch = paths.slice(start, start + WINDOWS_ATTRIBUTE_BATCH);
18416
- const result = await this.run(buildWindowsAttributeScript(batch), 30000);
18417
- if (!result.ok) {
18418
- throw new Error(`Could not read Windows cloud file attributes: ${result.stderr.trim() || "powershell.exe failed"}`);
18450
+ search(request) {
18451
+ return this.queue.shared(async () => {
18452
+ const binaries = await this.resolveBinaries("cached");
18453
+ if (!binaries.ok)
18454
+ return binaries;
18455
+ const ping = await this.ping(binaries.esPath);
18456
+ if (ping.code !== 0) {
18457
+ return {
18458
+ ok: false,
18459
+ reason: "not-running",
18460
+ message: `Everything instance ${this.instanceName} is not running; run \`autorag refresh\` to start and index it.`
18461
+ };
18419
18462
  }
18420
- for (const [path, attributes] of parseWindowsAttributeOutput(result.stdout, batch)) {
18421
- if (datalessFromWindowsAttributes(attributes))
18422
- dataless.add(path);
18463
+ const es = { ok: true, esPath: binaries.esPath };
18464
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
18465
+ if ((this.options.excludeFolders?.length ?? 0) > 0) {
18466
+ return this.searchWithExclusions(es.esPath, request, timeoutMs);
18423
18467
  }
18424
- }
18425
- return dataless;
18426
- }
18427
- async providerOf(path) {
18428
- const root = longestSyncRoot(await this.loadSyncRoots(), path);
18429
- return root === undefined ? cloudProviderFromPath(path) : providerFromWindowsSyncRoot(root.label);
18468
+ const args = buildEverythingSearchArgs(this.instanceName, request, timeoutMs);
18469
+ const result = await this.run(es.esPath, args, timeoutMs + 5000);
18470
+ if (result.code !== 0)
18471
+ return this.failure("search-failed", es.esPath, args, result);
18472
+ try {
18473
+ return { ok: true, results: parseEverythingJson(result.stdout) };
18474
+ } catch (error) {
18475
+ return { ok: false, reason: "search-failed", message: error.message };
18476
+ }
18477
+ });
18430
18478
  }
18431
- canRelease(provider) {
18432
- return provider !== undefined;
18479
+ async searchWithExclusions(esPath, request, timeoutMs) {
18480
+ const deadline = Date.now() + timeoutMs;
18481
+ const offset = request.offset ?? 0;
18482
+ const maxResults = request.maxResults ?? DEFAULT_MAX_RESULTS;
18483
+ const results = [];
18484
+ let skipped = 0;
18485
+ let nativeOffset = 0;
18486
+ const timedOut = () => ({
18487
+ ok: false,
18488
+ reason: "search-failed",
18489
+ message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
18490
+ });
18491
+ for (;; ) {
18492
+ const remainingMs = deadline - Date.now();
18493
+ if (remainingMs <= 0)
18494
+ return timedOut();
18495
+ const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
18496
+ const response = await this.run(esPath, args, remainingMs + 5000);
18497
+ if (response.code !== 0)
18498
+ return this.failure("search-failed", esPath, args, response);
18499
+ if (Date.now() >= deadline)
18500
+ return timedOut();
18501
+ let entries;
18502
+ try {
18503
+ entries = parseEverythingJson(response.stdout);
18504
+ } catch (error) {
18505
+ return { ok: false, reason: "search-failed", message: error.message };
18506
+ }
18507
+ for (const entry of entries) {
18508
+ if (this.isExcludedPath(entry.path))
18509
+ continue;
18510
+ if (skipped < offset) {
18511
+ skipped += 1;
18512
+ continue;
18513
+ }
18514
+ results.push(entry);
18515
+ if (results.length >= maxResults)
18516
+ return { ok: true, results };
18517
+ }
18518
+ if (entries.length < MAX_FILTER_CANDIDATES)
18519
+ return { ok: true, results };
18520
+ nativeOffset += entries.length;
18521
+ }
18433
18522
  }
18434
- async evict(path) {
18435
- return parseWindowsDehydrateOutput(await this.run(buildWindowsDehydrateScript(path), 60000));
18523
+ index() {
18524
+ return this.queue.exclusive(async () => {
18525
+ const es = await this.ensureRunning(true);
18526
+ if (!es.ok)
18527
+ return es;
18528
+ const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
18529
+ const saveArgs = ["-instance", this.instanceName, "-save-db"];
18530
+ const saved = await this.run(es.esPath, saveArgs, timeoutMs);
18531
+ if (saved.code !== 0)
18532
+ return this.failure("index-failed", es.esPath, saveArgs, saved);
18533
+ const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
18534
+ const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
18535
+ if (counted.code !== 0)
18536
+ return this.failure("index-failed", es.esPath, countArgs, counted);
18537
+ return {
18538
+ ok: true,
18539
+ indexedFolders: [...this.options.folders],
18540
+ indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
18541
+ };
18542
+ });
18436
18543
  }
18437
- loadSyncRoots() {
18438
- this.syncRoots ??= this.run(buildWindowsSyncRootsScript(), 15000).then((result) => result.ok ? parseWindowsSyncRoots(result.stdout) : [], () => []);
18439
- return this.syncRoots;
18544
+ stop() {
18545
+ return this.queue.exclusive(async () => {
18546
+ const binaries = await this.resolveBinaries("cached");
18547
+ if (!binaries.ok)
18548
+ return;
18549
+ await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
18550
+ });
18440
18551
  }
18441
- loadDiskRoots() {
18442
- this.disks ??= this.run(buildWindowsDiskScript(), 30000).then((result) => result.ok ? windowsRootsFromDisks(parseWindowsLogicalDisks(result.stdout)) : [], () => []);
18443
- return this.disks;
18552
+ async resolveBinaries(mode) {
18553
+ if (!this.isSupported()) {
18554
+ return {
18555
+ ok: false,
18556
+ reason: "unsupported-platform",
18557
+ message: `Everything is Windows-only; this host is ${this.platform}.`
18558
+ };
18559
+ }
18560
+ if (this.binaries?.ok)
18561
+ return this.binaries;
18562
+ const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
18563
+ const result = await resolve(mode);
18564
+ if (result.ok)
18565
+ this.binaries = result;
18566
+ return result;
18444
18567
  }
18445
- }
18446
- function longestSyncRoot(roots, target) {
18447
- const normalized = target.replaceAll("/", "\\").toLowerCase();
18448
- let match;
18449
- let matchLength = -1;
18450
- for (const root of roots) {
18451
- const prefix = root.path.replaceAll("/", "\\").replace(/\\+$/u, "").toLowerCase();
18452
- if (prefix === "" || prefix.length <= matchLength)
18453
- continue;
18454
- if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
18455
- match = root;
18456
- matchLength = prefix.length;
18568
+ async ensureRunning(reconfigure) {
18569
+ const binaries = await this.resolveBinaries("install");
18570
+ if (!binaries.ok)
18571
+ return binaries;
18572
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
18573
+ let ping = await this.ping(binaries.esPath);
18574
+ if (ping.code === 0 && !reconfigure)
18575
+ return { ok: true, esPath: binaries.esPath };
18576
+ if (ping.code === 0) {
18577
+ const exitArgs = ["-instance", this.instanceName, "-exit"];
18578
+ const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
18579
+ if (exited.code !== 0)
18580
+ return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
18581
+ }
18582
+ mkdirSync15(this.stateDir, { recursive: true });
18583
+ const iniPath = join28(this.stateDir, "Everything.ini");
18584
+ writeFileSync11(iniPath, buildEverythingIni({
18585
+ folders: this.options.folders,
18586
+ excludeFolders: [
18587
+ ...new Set([
18588
+ join28(this.options.root, ".autorag"),
18589
+ ...this.options.folders.map((folder) => join28(folder, ".autorag")),
18590
+ ...this.options.excludeFolders ?? []
18591
+ ])
18592
+ ]
18593
+ }));
18594
+ const launchArgs = [
18595
+ "-instance",
18596
+ this.instanceName,
18597
+ "-config",
18598
+ iniPath,
18599
+ "-db",
18600
+ join28(this.stateDir, "Everything.db"),
18601
+ "-startup"
18602
+ ];
18603
+ try {
18604
+ this.launch(binaries.everythingPath, launchArgs);
18605
+ } catch (error) {
18606
+ return {
18607
+ ok: false,
18608
+ reason: "startup-failed",
18609
+ message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
18610
+ };
18611
+ }
18612
+ const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
18613
+ const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
18614
+ for (;; ) {
18615
+ ping = await this.ping(binaries.esPath);
18616
+ if (ping.code === 0)
18617
+ return { ok: true, esPath: binaries.esPath };
18618
+ if (Date.now() >= deadline)
18619
+ break;
18620
+ await sleep2(pollMs);
18457
18621
  }
18458
- }
18459
- return match;
18460
- }
18461
- function powerShellLaunch(script) {
18462
- return {
18463
- args: [
18464
- "-NoProfile",
18465
- "-NonInteractive",
18466
- "-Command",
18467
- `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; Invoke-Expression $env:${POWERSHELL_SCRIPT_VARIABLE}`
18468
- ],
18469
- env: { [POWERSHELL_SCRIPT_VARIABLE]: script }
18470
- };
18471
- }
18472
- async function runPowerShell(script, timeoutMs) {
18473
- const launch = powerShellLaunch(script);
18474
- try {
18475
- const { stdout, stderr } = await execFileAsync("powershell.exe", [...launch.args], {
18476
- timeout: timeoutMs,
18477
- maxBuffer: 16 * 1024 * 1024,
18478
- windowsHide: true,
18479
- env: { ...process.env, ...launch.env }
18480
- });
18481
- return { ok: true, stdout, stderr };
18482
- } catch (error) {
18483
- const record = typeof error === "object" && error !== null ? error : {};
18484
18622
  return {
18485
18623
  ok: false,
18486
- stdout: typeof record.stdout === "string" ? record.stdout : "",
18487
- stderr: typeof record.stderr === "string" ? record.stderr : String(error)
18624
+ reason: "startup-failed",
18625
+ message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
18488
18626
  };
18489
18627
  }
18490
- }
18491
- function defaultCloudProbe() {
18492
- if (process.platform === "darwin")
18493
- return new DarwinCloudProbe;
18494
- if (process.platform === "win32")
18495
- return new WindowsCloudProbe;
18496
- return;
18497
- }
18498
- async function readFileProviderDomain(target) {
18499
- try {
18500
- await lstat(target);
18501
- const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
18502
- timeout: 2000
18503
- });
18504
- const value = stdout.trim();
18505
- return value.length > 0 ? value : undefined;
18506
- } catch {
18507
- return;
18628
+ async ping(esPath) {
18629
+ const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
18630
+ return result.code === 0 && !VERSION_PATTERN.test(result.stdout.trim()) ? { ...result, code: -1 } : result;
18631
+ }
18632
+ failure(reason, esPath, args, result) {
18633
+ return {
18634
+ ok: false,
18635
+ reason,
18636
+ message: `${esPath} ${args.join(" ")} exit ${result.code}: ${(result.stderr || result.stdout).trim()}`
18637
+ };
18508
18638
  }
18509
18639
  }
18510
- async function readFileProviderDomainUpwards(target) {
18511
- let cursor = target;
18512
- for (let depth = 0;depth < 12; depth += 1) {
18513
- const domain = await readFileProviderDomain(cursor);
18514
- if (domain !== undefined)
18515
- return domain;
18516
- const parent = dirname14(cursor);
18517
- if (parent === cursor)
18640
+ function runProcess(command, args, timeoutMs) {
18641
+ let resolvePromise;
18642
+ const promise = new Promise((resolve) => {
18643
+ resolvePromise = resolve;
18644
+ });
18645
+ const child = spawn14(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
18646
+ const stdout = [];
18647
+ const stderr = [];
18648
+ let bytes = 0;
18649
+ let overflow = false;
18650
+ const collect = (target) => (chunk) => {
18651
+ bytes += chunk.length;
18652
+ if (bytes > MAX_OUTPUT_BYTES) {
18653
+ overflow = true;
18654
+ child.kill();
18518
18655
  return;
18519
- cursor = parent;
18520
- }
18521
- return;
18656
+ }
18657
+ target.push(chunk);
18658
+ };
18659
+ child.stdout.on("data", collect(stdout));
18660
+ child.stderr.on("data", collect(stderr));
18661
+ const timer = setTimeout(() => child.kill(), timeoutMs);
18662
+ child.on("error", (error) => {
18663
+ clearTimeout(timer);
18664
+ resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
18665
+ });
18666
+ child.on("close", (code, signal) => {
18667
+ clearTimeout(timer);
18668
+ const err = Buffer.concat(stderr).toString("utf8");
18669
+ resolvePromise({
18670
+ code,
18671
+ stdout: Buffer.concat(stdout).toString("utf8"),
18672
+ stderr: overflow ? `${err}
18673
+ output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
18674
+ killed by ${signal} after ${timeoutMs}ms` : err
18675
+ });
18676
+ });
18677
+ return promise;
18522
18678
  }
18523
- function quotePowerShell(value) {
18524
- return `'${value.replace(/['\u2018\u2019\u201A\u201B]/gu, (quote) => quote + quote)}'`;
18679
+ function launchDetached(command, args) {
18680
+ const child = spawn14(command, args, { detached: true, stdio: "ignore", windowsHide: true });
18681
+ child.on("error", () => {});
18682
+ child.unref();
18525
18683
  }
18526
- var execFileAsync, SF_DATALESS = 1073741824, FILE_PROVIDER_XATTR = "com.apple.file-provider-domain-id", WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 4194304, STAT_BATCH = 200, defaultCloudExec = (file, args, options) => execFileAsync(file, [...args], {
18527
- timeout: options?.timeoutMs ?? 30000,
18528
- ...options?.maxBuffer === undefined ? {} : { maxBuffer: options.maxBuffer }
18529
- }), JXA_RELEASE_PROGRAM = `ObjC.import("Foundation");
18530
- function run(argv) {
18531
- var url = $.NSURL.fileURLWithPath(argv[0]);
18532
- var errorRef = Ref();
18533
- var released = $.NSFileManager.defaultManager.evictUbiquitousItemAtURLError(url, errorRef);
18534
- if (released) return JSON.stringify({ ok: true });
18535
- function resourceValue(key) {
18536
- var ref = Ref();
18537
- if (!url.getResourceValueForKeyError(ref, key, null)) return null;
18538
- var value = ref[0];
18539
- return value === undefined || value === null ? null : ObjC.unwrap(value);
18540
- }
18541
- return JSON.stringify({
18542
- ok: false,
18543
- isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
18544
- isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
18545
- downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
18546
- });
18547
- }`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
18548
- var init_cloud_placeholder = __esm(() => {
18549
- execFileAsync = promisify(execFile2);
18550
- RELEASABLE_DARWIN_PROVIDERS = {
18551
- icloud: true,
18552
- "google-drive": true,
18553
- onedrive: true,
18554
- dropbox: true
18555
- };
18556
- DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
18557
- WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
18558
- WINDOWS_DEHYDRATE_REFUSALS = {
18559
- "80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
18560
- "80070188": "The file is pinned to stay on this device, so it is not released."
18561
- };
18684
+ var DEFAULT_TIMEOUT_MS8 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
18685
+ var init_client9 = __esm(() => {
18686
+ init_path_exclusions();
18687
+ init_bundle();
18688
+ DEFAULT_INDEX_TIMEOUT_MS3 = 10 * 60000;
18689
+ MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
18690
+ VERSION_PATTERN = /^\d+\.\d+\.\d+\.\d+$/;
18562
18691
  });
18563
18692
 
18564
- // src/dupey/scan-plan.ts
18565
- import { basename as basename7 } from "node:path";
18566
- async function listCloudRoots(probe, roots) {
18567
- if (probe === undefined)
18568
- return [];
18693
+ // src/everything/index.ts
18694
+ var init_everything = __esm(() => {
18695
+ init_bundle();
18696
+ init_client9();
18697
+ });
18698
+
18699
+ // src/filesystem/cloud-placeholder.ts
18700
+ import { execFile as execFile2 } from "node:child_process";
18701
+ import { realpathSync as realpathSync3 } from "node:fs";
18702
+ import { lstat, readdir } from "node:fs/promises";
18703
+ import { homedir as homedir6 } from "node:os";
18704
+ import { dirname as dirname15, join as join29 } from "node:path";
18705
+ import { promisify } from "node:util";
18706
+ function homeCloudRoots(home = homedir6()) {
18707
+ const normalizedHome = home.replaceAll("\\", "/").replace(/\/+$/, "");
18708
+ return [`${normalizedHome}/Library/CloudStorage`, `${normalizedHome}/Library/Mobile Documents`];
18709
+ }
18710
+ function datalessFromDarwinFlags(flags) {
18711
+ return Number.isFinite(flags) && (flags >>> 0 & SF_DATALESS) !== 0;
18712
+ }
18713
+ function datalessFromWindowsAttributes(attributes) {
18714
+ return Number.isFinite(attributes) && (attributes & WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) !== 0;
18715
+ }
18716
+ function cloudProviderFromPath(target) {
18717
+ const lower = target.toLowerCase();
18718
+ if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
18719
+ return "google-drive";
18720
+ if (lower.includes("onedrive"))
18721
+ return "onedrive";
18722
+ if (lower.includes("dropbox"))
18723
+ return "dropbox";
18724
+ if (lower.includes("mobile documents") || lower.includes("clouddocs"))
18725
+ return "icloud";
18726
+ return "file-provider";
18727
+ }
18728
+ function cloudProviderFromDomain(domain) {
18729
+ const lower = domain.toLowerCase();
18730
+ if (lower.includes("clouddocs") || lower.includes("icloud"))
18731
+ return "icloud";
18732
+ if (lower.includes("google"))
18733
+ return "google-drive";
18734
+ if (lower.includes("onedrive"))
18735
+ return "onedrive";
18736
+ if (lower.includes("dropbox"))
18737
+ return "dropbox";
18738
+ return "file-provider";
18739
+ }
18740
+ function normalizeCloudPath(value) {
18741
+ let normalized = value.replaceAll("\\", "/");
18742
+ if (normalized.length > 1)
18743
+ normalized = normalized.replace(/\/+$/u, "");
18744
+ return normalized;
18745
+ }
18746
+ function isPathInside(target, parent) {
18747
+ const normalizedParent = normalizeCloudPath(parent);
18748
+ if (normalizedParent === "")
18749
+ return false;
18750
+ const left = normalizeCloudPath(target);
18751
+ const [a, b] = process.platform === "win32" ? [left.toLowerCase(), normalizedParent.toLowerCase()] : [left, normalizedParent];
18752
+ return a === b || a.startsWith(`${b}/`);
18753
+ }
18754
+ function canonicalPath(value) {
18569
18755
  try {
18570
- return await probe.cloudRoots(roots);
18756
+ return realpathSync3(value);
18571
18757
  } catch {
18572
- return [];
18758
+ return value;
18573
18759
  }
18574
18760
  }
18575
- function planDupeyScans(roots, cloudRoots) {
18576
- const targets = [];
18577
- const skipped = [];
18578
- for (const path of roots) {
18579
- const scope = cloudScope(path, cloudRoots);
18580
- if (scope.insideCloudRoot) {
18581
- skipped.push(path);
18582
- continue;
18761
+ function canonicalPathMapper(searchRoots) {
18762
+ const forms = searchRoots.map((root) => ({ raw: normalizeCloudPath(root), canonical: normalizeCloudPath(canonicalPath(root)) })).filter(({ raw, canonical }) => raw !== "" && raw !== canonical).sort((left, right) => right.raw.length - left.raw.length);
18763
+ if (forms.length === 0)
18764
+ return (path) => path;
18765
+ return (path) => {
18766
+ const normalized = normalizeCloudPath(path);
18767
+ for (const { raw, canonical } of forms) {
18768
+ if (isPathInside(normalized, raw))
18769
+ return `${canonical}${normalized.slice(raw.length)}`;
18583
18770
  }
18584
- targets.push({ path, excludeDirs: scope.nestedCloudRoots.map((root) => basename7(root.path)) });
18585
- }
18586
- return { targets, skipped };
18771
+ return path;
18772
+ };
18587
18773
  }
18588
- var init_scan_plan = __esm(() => {
18589
- init_cloud_placeholder();
18590
- });
18591
-
18592
- // src/dupey/index.ts
18593
- var init_dupey = __esm(() => {
18594
- init_cli2();
18595
- init_filter();
18596
- init_scan_plan();
18597
- });
18598
-
18599
- // src/convert/relate.ts
18600
- async function convertAndRelate(request, options = {}) {
18601
- const result = await convertWithFilee(request, options.filee);
18602
- if (options.dupey === false)
18603
- return { result, links: [] };
18604
- const dupeyOptions = options.dupey ?? {};
18605
- const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
18606
- const links = await Promise.all(pairs.map(async ({ source, output }) => {
18607
- try {
18608
- return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
18609
- } catch (error) {
18610
- if (!(error instanceof DupeyCliError))
18611
- throw error;
18612
- return { source, output, dupey: { relation: "unavailable", reason: error.message } };
18613
- }
18614
- }));
18615
- return { result, links };
18774
+ function cloudScope(searchRoot, cloudRoots) {
18775
+ const canonicalSearchRoot = canonicalPath(searchRoot);
18776
+ const roots = cloudRoots.map((root) => ({ path: canonicalPath(root.path), kind: root.kind }));
18777
+ return {
18778
+ insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
18779
+ nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
18780
+ };
18616
18781
  }
18617
- var init_relate = __esm(() => {
18618
- init_dupey();
18619
- init_cli();
18620
- });
18621
-
18622
- // src/convert/index.ts
18623
- var init_convert = __esm(() => {
18624
- init_cli();
18625
- init_installer2();
18626
- init_matrix();
18627
- init_relate();
18628
- });
18629
-
18630
- // src/datasource/diagnostics.ts
18631
- function mapDatasourceDiagnostic(ds) {
18782
+ function parseReleaseOutcome(stdout) {
18783
+ let parsed;
18784
+ try {
18785
+ parsed = JSON.parse(stdout.trim());
18786
+ } catch {
18787
+ return;
18788
+ }
18789
+ if (typeof parsed !== "object" || parsed === null || !("ok" in parsed) || typeof parsed.ok !== "boolean") {
18790
+ return;
18791
+ }
18632
18792
  return {
18633
- code: datasourceCodeToRetrieval(ds.code),
18634
- severity: ds.severity,
18635
- message: `[datasource:${ds.code}] ${ds.message}`,
18636
- source: ds.source ?? (ds.instanceId ? `datasource:${ds.instanceId}` : "datasource")
18793
+ ok: parsed.ok,
18794
+ ..."isUploaded" in parsed ? { isUploaded: parsed.isUploaded } : {},
18795
+ ..."isUploading" in parsed ? { isUploading: parsed.isUploading } : {},
18796
+ ..."downloadingStatus" in parsed ? { downloadingStatus: parsed.downloadingStatus } : {}
18637
18797
  };
18638
18798
  }
18639
- function mapDatasourceDiagnostics(diagnostics) {
18640
- return diagnostics.map(mapDatasourceDiagnostic);
18799
+ function describeReleaseRefusal(outcome) {
18800
+ if (outcome.isUploading === true)
18801
+ return "The file is still uploading to the cloud, so it cannot be released yet.";
18802
+ if (outcome.isUploaded === false)
18803
+ return "The file has not finished uploading to the cloud, so it cannot be released yet.";
18804
+ const status = typeof outcome.downloadingStatus === "string" ? outcome.downloadingStatus : "unknown";
18805
+ return `The cloud provider declined to release the local copy (download status ${status}).`;
18641
18806
  }
18642
- function datasourceCodeToRetrieval(code) {
18643
- switch (code) {
18644
- case "datasource-unavailable":
18645
- case "datasource-cli-error":
18646
- case "datasource-empty":
18647
- case "datasource-rate-limited":
18648
- case "datasource-auth-error":
18649
- case "datasource-embedding-egress-rejected":
18650
- case "datasource-index-failed":
18651
- case "datasource-index-partial":
18652
- case "datasource-permission-denied":
18653
- case "datasource-remote-embedding-rejected":
18654
- return "retrieval-method-failed";
18807
+ function describeReleaseHelperFailure(error) {
18808
+ const failure = error;
18809
+ const parts = [];
18810
+ if (failure.killed === true || typeof failure.signal === "string") {
18811
+ parts.push(`the helper was killed${typeof failure.signal === "string" ? ` by ${failure.signal}` : ""}`);
18812
+ } else if (typeof failure.code === "number") {
18813
+ parts.push(`the helper exited with code ${failure.code}`);
18655
18814
  }
18815
+ const stderr = typeof failure.stderr === "string" ? failure.stderr.trim() : "";
18816
+ if (stderr !== "")
18817
+ parts.push(stderr);
18818
+ if (parts.length === 0)
18819
+ parts.push(error instanceof Error ? error.message : String(error));
18820
+ return `osascript could not release the local copy: ${parts.join("; ")}`;
18656
18821
  }
18657
-
18658
- // src/datasource/retry.ts
18659
- import { setTimeout as delay2 } from "node:timers/promises";
18660
- function datasourceRetryDelayMs(retry, policy) {
18661
- const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
18662
- return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
18822
+ async function defaultDarwinReadDir(path) {
18823
+ const entries = await readdir(path, { withFileTypes: true });
18824
+ return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
18663
18825
  }
18664
- async function indexWithRetry(skill, options = {}, signal) {
18665
- const policy = {
18666
- maxRetries: options.maxRetries ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxRetries,
18667
- baseDelayMs: options.baseDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.baseDelayMs,
18668
- maxDelayMs: options.maxDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxDelayMs
18669
- };
18670
- const sleep = options.sleep ?? ((ms) => delay2(ms, undefined, signal === undefined ? undefined : { signal }));
18671
- let lastError = "";
18672
- for (let attempt = 1;; attempt += 1) {
18673
- const result = await runIndex(skill, signal);
18674
- signal?.throwIfAborted();
18675
- if (result.ok) {
18676
- if (attempt === 1)
18677
- return result;
18678
- const recovered = {
18679
- code: "datasource-index-failed",
18680
- severity: "info",
18681
- message: `recovered after ${attempt} attempts; last error: ${lastError}`,
18682
- instanceId: result.instanceId,
18683
- source: result.skill
18684
- };
18685
- return { ...result, attempts: attempt, diagnostics: [...result.diagnostics, recovered] };
18686
- }
18687
- lastError = result.message ?? result.error ?? "datasource-index-failed";
18688
- if (attempt > policy.maxRetries) {
18689
- if (attempt === 1)
18690
- return { ...result, attempts: attempt };
18691
- const suffix = ` (gave up after ${attempt} attempts)`;
18692
- const last = result.diagnostics.at(-1);
18693
- return {
18694
- ...result,
18695
- attempts: attempt,
18696
- message: `${lastError}${suffix}`,
18697
- diagnostics: last === undefined ? [
18698
- {
18699
- code: result.code ?? "datasource-index-failed",
18700
- severity: "error",
18701
- message: `${lastError}${suffix}`,
18702
- instanceId: result.instanceId,
18703
- source: result.skill
18704
- }
18705
- ] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
18706
- };
18707
- }
18708
- await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
18709
- signal?.throwIfAborted();
18826
+ function parseDarwinMounts(stdout) {
18827
+ const mounts = [];
18828
+ for (const line of stdout.split(`
18829
+ `)) {
18830
+ if (line.trim() === "")
18831
+ continue;
18832
+ const open = line.lastIndexOf(" (");
18833
+ if (open <= 0 || !line.endsWith(")"))
18834
+ continue;
18835
+ const head = line.slice(0, open);
18836
+ const on = head.indexOf(" on ");
18837
+ if (on <= 0)
18838
+ continue;
18839
+ const mountPoint = head.slice(on + " on ".length);
18840
+ if (mountPoint === "")
18841
+ continue;
18842
+ const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
18843
+ mounts.push({
18844
+ device: head.slice(0, on),
18845
+ mountPoint,
18846
+ fileSystem: fields[0] ?? "",
18847
+ options: fields.slice(1)
18848
+ });
18710
18849
  }
18850
+ return mounts;
18711
18851
  }
18712
- async function runIndex(skill, signal) {
18713
- try {
18714
- return await skill.index(signal === undefined ? undefined : { signal });
18715
- } catch (error) {
18716
- signal?.throwIfAborted();
18717
- const descriptor = skill.describe();
18718
- const message = error instanceof Error ? error.message : "Datasource indexing failed.";
18719
- const failed = {
18720
- ok: false,
18721
- instanceId: descriptor.instanceId ?? "default",
18722
- skill: descriptor.name,
18723
- indexedAt: Date.now(),
18724
- diagnostics: [
18725
- {
18726
- code: "datasource-index-failed",
18727
- severity: "error",
18728
- message,
18729
- source: descriptor.name,
18730
- instanceId: descriptor.instanceId
18731
- }
18732
- ],
18733
- error: "datasource-index-failed",
18734
- code: "datasource-index-failed",
18735
- message
18736
- };
18737
- return failed;
18852
+ function networkMountPoints(mounts) {
18853
+ const points = [];
18854
+ const seen = new Set;
18855
+ for (const mount of mounts) {
18856
+ if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
18857
+ continue;
18858
+ if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
18859
+ continue;
18860
+ if (seen.has(mount.mountPoint))
18861
+ continue;
18862
+ seen.add(mount.mountPoint);
18863
+ points.push(mount.mountPoint);
18738
18864
  }
18865
+ return points;
18739
18866
  }
18740
- var DEFAULT_DATASOURCE_RETRY_POLICY;
18741
- var init_retry = __esm(() => {
18742
- DEFAULT_DATASOURCE_RETRY_POLICY = {
18743
- maxRetries: 2,
18744
- baseDelayMs: 1000,
18745
- maxDelayMs: 30000
18746
- };
18747
- });
18748
-
18749
- // src/everything/bundle.ts
18750
- import { createHash as createHash11 } from "node:crypto";
18751
- import { existsSync as existsSync17, mkdirSync as mkdirSync14, readFileSync as readFileSync17, rmSync as rmSync7, writeFileSync as writeFileSync10 } from "node:fs";
18752
- import { dirname as dirname15, join as join28 } from "node:path";
18753
- import { fileURLToPath } from "node:url";
18754
- import JSZip from "jszip";
18755
- function resolveEverythingBundleDir() {
18756
- let dir = dirname15(fileURLToPath(import.meta.url));
18757
- for (let depth = 0;depth < 6; depth += 1) {
18758
- const candidate = join28(dir, "vendor", "everything");
18759
- if (existsSync17(join28(candidate, MANIFEST_FILENAME)))
18760
- return candidate;
18761
- const parent = dirname15(dir);
18762
- if (parent === dir)
18763
- break;
18764
- dir = parent;
18867
+ function parseXattrValues(stdout, candidates) {
18868
+ const values = new Map;
18869
+ if (candidates.length === 0)
18870
+ return values;
18871
+ const ordered = [...candidates].sort((left, right) => right.length - left.length);
18872
+ for (const line of stdout.split(`
18873
+ `)) {
18874
+ if (line === "")
18875
+ continue;
18876
+ for (const candidate of ordered) {
18877
+ if (line.startsWith(`${candidate}: `)) {
18878
+ values.set(candidate, line.slice(candidate.length + 2));
18879
+ break;
18880
+ }
18881
+ }
18765
18882
  }
18766
- return;
18767
- }
18768
- function everythingArch(arch) {
18769
- if (arch === "x64" || arch === "arm64")
18770
- return arch;
18771
- return;
18772
- }
18773
- function everythingBinaryPaths(root, version, arch) {
18774
- const dir = join28(root, ".autorag", "everything", version, arch);
18775
- return { everythingPath: join28(dir, "everything.exe"), esPath: join28(dir, "es.exe") };
18776
- }
18777
- function loadEverythingBundleManifest(bundleDir) {
18778
- return JSON.parse(readFileSync17(join28(bundleDir, MANIFEST_FILENAME), "utf8"));
18883
+ return values;
18779
18884
  }
18780
- async function ensureEverythingBinaries(options) {
18781
- const platform = options.platform ?? process.platform;
18782
- const rawArch = options.arch ?? process.arch;
18783
- if (platform !== "win32") {
18784
- return {
18785
- ok: false,
18786
- reason: "unsupported-platform",
18787
- message: `Everything is Windows-only; this host is ${platform}.`
18788
- };
18885
+
18886
+ class DarwinCloudProbe {
18887
+ exec;
18888
+ readDir;
18889
+ home;
18890
+ constructor(options = {}) {
18891
+ this.exec = options.exec ?? defaultCloudExec;
18892
+ this.readDir = options.readDir ?? defaultDarwinReadDir;
18893
+ this.home = options.home ?? homedir6();
18789
18894
  }
18790
- const arch = everythingArch(rawArch);
18791
- const bundleDir = options.bundleDir ?? resolveEverythingBundleDir();
18792
- if (bundleDir === undefined) {
18793
- return {
18794
- ok: false,
18795
- reason: "bundle-missing",
18796
- message: `Bundled Everything assets (vendor/everything/${MANIFEST_FILENAME}) were not found above ${dirname15(fileURLToPath(import.meta.url))}.`
18895
+ async cloudRoots(searchRoots = []) {
18896
+ const roots = [];
18897
+ const seen = new Set;
18898
+ const add = (path, kind) => {
18899
+ const canonical = canonicalPath(path);
18900
+ const key = normalizeCloudPath(canonical);
18901
+ if (key === "" || seen.has(key))
18902
+ return;
18903
+ seen.add(key);
18904
+ roots.push({ path: canonical, kind });
18797
18905
  };
18906
+ const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
18907
+ for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
18908
+ if (entry.isDirectory())
18909
+ add(join29(cloudStorage, entry.name), "file-provider");
18910
+ }
18911
+ if (await this.tryReadDir(mobileDocuments) !== undefined)
18912
+ add(mobileDocuments, "file-provider");
18913
+ for (const mountPoint of await this.networkMountPoints())
18914
+ add(mountPoint, "network");
18915
+ for (const directory of await this.fileProviderDirectories(searchRoots))
18916
+ add(directory, "file-provider");
18917
+ return roots;
18798
18918
  }
18799
- let manifest;
18800
- try {
18801
- manifest = options.manifest ?? loadEverythingBundleManifest(bundleDir);
18802
- } catch (error) {
18803
- return { ok: false, reason: "bundle-missing", message: errorMessage2(error) };
18919
+ async networkMountPoints() {
18920
+ try {
18921
+ const { stdout } = await this.exec("/sbin/mount", [], { timeoutMs: 1e4, maxBuffer: 4 * 1024 * 1024 });
18922
+ return networkMountPoints(parseDarwinMounts(stdout));
18923
+ } catch {
18924
+ return [];
18925
+ }
18804
18926
  }
18805
- const assets = arch === undefined ? undefined : manifest.architectures[arch];
18806
- if (arch === undefined || assets === undefined) {
18807
- return {
18808
- ok: false,
18809
- reason: "unsupported-platform",
18810
- message: `No bundled Everything build for Windows architecture ${rawArch}; bundled: ${Object.keys(manifest.architectures).join(", ")}.`
18811
- };
18927
+ async fileProviderDirectories(searchRoots) {
18928
+ if (searchRoots.length === 0)
18929
+ return [];
18930
+ const candidates = new Set;
18931
+ for (const searchRoot of searchRoots) {
18932
+ const canonical = canonicalPath(searchRoot);
18933
+ for (let cursor = canonical;; cursor = dirname15(cursor)) {
18934
+ candidates.add(cursor);
18935
+ const parent = dirname15(cursor);
18936
+ if (parent === cursor)
18937
+ break;
18938
+ }
18939
+ for (const entry of await this.tryReadDir(canonical) ?? []) {
18940
+ if (entry.isDirectory())
18941
+ candidates.add(join29(canonical, entry.name));
18942
+ }
18943
+ }
18944
+ const paths = [...candidates];
18945
+ const managed = [];
18946
+ for (let start = 0;start < paths.length; start += XATTR_BATCH) {
18947
+ for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
18948
+ managed.push(path);
18949
+ }
18950
+ return managed;
18812
18951
  }
18813
- const { everythingPath, esPath } = everythingBinaryPaths(options.root, manifest.everythingVersion, arch);
18814
- const everythingCached = fileHasSha256(everythingPath, assets.everythingSha256);
18815
- const esCached = fileHasSha256(esPath, assets.esSha256);
18816
- if (everythingCached && esCached) {
18817
- return { ok: true, everythingPath, esPath, source: "cached" };
18952
+ async xattrValues(paths) {
18953
+ try {
18954
+ const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
18955
+ timeoutMs: 15000,
18956
+ maxBuffer: 8 * 1024 * 1024
18957
+ });
18958
+ return parseXattrValues(stdout, paths);
18959
+ } catch (error) {
18960
+ const failure = error;
18961
+ return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
18962
+ }
18818
18963
  }
18819
- if ((options.mode ?? "install") === "cached") {
18820
- const state = everythingCached || esCached ? "are incomplete or do not match their pinned SHA-256 digests" : "are not installed";
18821
- return {
18822
- ok: false,
18823
- reason: "not-installed",
18824
- message: `Everything ${manifest.everythingVersion} (${arch}) binaries ${state} under ${dirname15(everythingPath)}; run \`autorag refresh\` to (re)install and index them.`
18825
- };
18964
+ async tryReadDir(path) {
18965
+ try {
18966
+ return await this.readDir(path);
18967
+ } catch {
18968
+ return;
18969
+ }
18826
18970
  }
18827
- try {
18828
- mkdirSync14(dirname15(everythingPath), { recursive: true });
18829
- const everythingBytes = await extractVerified(join28(bundleDir, assets.everythingArchive), assets.everythingArchiveSha256, assets.everythingMember, assets.everythingSha256);
18830
- const esBytes = await extractVerified(join28(bundleDir, assets.esArchive), assets.esArchiveSha256, assets.esMember, assets.esSha256);
18831
- writeAtomic(everythingPath, everythingBytes);
18832
- writeAtomic(esPath, esBytes);
18833
- } catch (error) {
18834
- return { ok: false, reason: "install-failed", message: errorMessage2(error) };
18971
+ async datalessAmong(paths) {
18972
+ const dataless = new Set;
18973
+ for (let start = 0;start < paths.length; start += STAT_BATCH) {
18974
+ const batch = paths.slice(start, start + STAT_BATCH);
18975
+ const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
18976
+ timeoutMs: 30000,
18977
+ maxBuffer: 4 * 1024 * 1024
18978
+ });
18979
+ const flags = stdout.split(`
18980
+ `).filter(Boolean);
18981
+ for (const [index, path] of batch.entries()) {
18982
+ if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
18983
+ dataless.add(path);
18984
+ }
18985
+ }
18986
+ return dataless;
18835
18987
  }
18836
- return { ok: true, everythingPath, esPath, source: "installed" };
18837
- }
18838
- async function extractVerified(archivePath, archiveSha256, member, memberSha256) {
18839
- const archive = readFileSync17(archivePath);
18840
- const actualArchive = sha256(archive);
18841
- if (actualArchive !== archiveSha256) {
18842
- throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
18988
+ async providerOf(path) {
18989
+ const domain = await readFileProviderDomainUpwards(path);
18990
+ return domain === undefined ? undefined : cloudProviderFromDomain(domain);
18843
18991
  }
18844
- const zip = await JSZip.loadAsync(archive);
18845
- const entry = zip.file(member);
18846
- if (entry === null)
18847
- throw new Error(`${archivePath} does not contain ${member}`);
18848
- const bytes = await entry.async("nodebuffer");
18849
- const actualMember = sha256(bytes);
18850
- if (actualMember !== memberSha256) {
18851
- throw new Error(`${member} from ${archivePath} SHA-256 ${actualMember} does not match pinned ${memberSha256}`);
18992
+ canRelease(provider) {
18993
+ return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
18852
18994
  }
18853
- return bytes;
18854
- }
18855
- function fileHasSha256(path, expected) {
18856
- return existsSync17(path) && sha256(readFileSync17(path)) === expected;
18857
- }
18858
- function writeAtomic(path, bytes) {
18859
- const temp = `${path}.${process.pid}.tmp`;
18860
- try {
18861
- writeFileSync10(temp, bytes);
18862
- renameWithRetry(temp, path);
18863
- } finally {
18864
- rmSync7(temp, { force: true });
18995
+ async evict(path) {
18996
+ let stdout;
18997
+ try {
18998
+ stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
18999
+ timeoutMs: 30000
19000
+ })).stdout;
19001
+ } catch (error) {
19002
+ return { ok: false, message: describeReleaseHelperFailure(error) };
19003
+ }
19004
+ const outcome = parseReleaseOutcome(stdout);
19005
+ if (outcome === undefined) {
19006
+ const detail = stdout.trim();
19007
+ return {
19008
+ ok: false,
19009
+ message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
19010
+ };
19011
+ }
19012
+ return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
18865
19013
  }
18866
19014
  }
18867
- function sha256(bytes) {
18868
- return createHash11("sha256").update(bytes).digest("hex");
19015
+ function buildWindowsAttributeScript(paths) {
19016
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19017
+ ` + `$ErrorActionPreference='SilentlyContinue'
19018
+ ` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
19019
+ `;
18869
19020
  }
18870
- function errorMessage2(error) {
18871
- return error instanceof Error ? error.message : String(error);
19021
+ function buildWindowsSyncRootsScript() {
19022
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19023
+ ` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
19024
+ ` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
19025
+ `;
18872
19026
  }
18873
- var MANIFEST_FILENAME = "manifest.json";
18874
- var init_bundle = __esm(() => {
18875
- init_atomic_write();
18876
- });
18877
-
18878
- // src/filesystem/path-exclusions.ts
18879
- import { posix as posix3, win32 as win322 } from "node:path";
18880
- function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
18881
- const paths = platform === "win32" ? win322 : posix3;
18882
- const normalize = (path) => {
18883
- const absolute = paths.resolve(path);
18884
- return platform === "win32" ? absolute.toLowerCase() : absolute;
18885
- };
18886
- const excluded = excludedPaths.map((path) => {
18887
- const absolute = normalize(path);
18888
- return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
18889
- });
18890
- return (path) => {
18891
- const absolute = normalize(path);
18892
- return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
18893
- };
19027
+ function buildWindowsDehydrateScript(path) {
19028
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19029
+ ` + `$ErrorActionPreference='Stop'
19030
+ ` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
19031
+ ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
19032
+ ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
19033
+ ` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
19034
+ ` + `'@
19035
+ ` + `$h=[IntPtr]::Zero
19036
+ ` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
19037
+ ` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
19038
+ ` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
19039
+ ` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
19040
+ ` + `[Autorag.CfApi]::CfCloseHandle($h)
19041
+ ` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
19042
+ ` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
19043
+ `;
18894
19044
  }
18895
- var init_path_exclusions = () => {};
18896
-
18897
- // src/filesystem/shared-exclusive-queue.ts
18898
- class SharedExclusiveQueue {
18899
- sharedSinceExclusive = [];
18900
- lastExclusive = Promise.resolve();
18901
- shared(task) {
18902
- const run = this.lastExclusive.then(task);
18903
- const settled = run.then(() => {
18904
- return;
18905
- }, () => {
18906
- return;
18907
- });
18908
- this.sharedSinceExclusive.push(settled);
18909
- return run;
19045
+ function parseWindowsAttributeOutput(stdout, paths) {
19046
+ const lines = stdout.split(/\r?\n/);
19047
+ const attributes = new Map;
19048
+ for (const [index, path] of paths.entries()) {
19049
+ const value = Number.parseInt((lines[index] ?? "").trim(), 10);
19050
+ if (Number.isFinite(value))
19051
+ attributes.set(path, value);
18910
19052
  }
18911
- exclusive(task) {
18912
- const earlier = Promise.all([this.lastExclusive, ...this.sharedSinceExclusive]);
18913
- const run = earlier.then(task);
18914
- const settled = run.then(() => {
18915
- return;
18916
- }, () => {
18917
- return;
18918
- });
18919
- this.sharedSinceExclusive = [];
18920
- this.lastExclusive = settled;
18921
- return run;
19053
+ return attributes;
19054
+ }
19055
+ function parseWindowsSyncRoots(stdout) {
19056
+ const roots = [];
19057
+ for (const line of stdout.split(/\r?\n/)) {
19058
+ const tab = line.indexOf("\t");
19059
+ if (tab <= 0)
19060
+ continue;
19061
+ const path = line.slice(0, tab).trim();
19062
+ if (path !== "")
19063
+ roots.push({ path, label: line.slice(tab + 1).trim() });
18922
19064
  }
19065
+ return roots;
18923
19066
  }
18924
-
18925
- // src/everything/client.ts
18926
- import { spawn as spawn14 } from "node:child_process";
18927
- import { createHash as createHash12 } from "node:crypto";
18928
- import { mkdirSync as mkdirSync15, writeFileSync as writeFileSync11 } from "node:fs";
18929
- import { join as join29, resolve as resolve11 } from "node:path";
18930
- import { setTimeout as sleep2 } from "node:timers/promises";
18931
- function everythingInstanceName(root) {
18932
- return `autorag-${createHash12("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
19067
+ function providerFromWindowsSyncRoot(label) {
19068
+ const lower = label.toLowerCase();
19069
+ if (lower.includes("onedrive") || lower.includes("one drive"))
19070
+ return "onedrive";
19071
+ if (lower.includes("google"))
19072
+ return "google-drive";
19073
+ return "file-provider";
18933
19074
  }
18934
- function buildEverythingIni(input) {
18935
- const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
18936
- const perFolder = (value) => input.folders.map(() => value).join(",");
18937
- return [
18938
- "[Everything]",
18939
- "app_data=0",
18940
- "run_as_admin=0",
18941
- "run_in_background=1",
18942
- "show_tray_icon=0",
18943
- "show_in_taskbar=0",
18944
- "check_for_updates_on_startup=0",
18945
- "beta_updates=0",
18946
- "language=1033",
18947
- "allow_http_server=0",
18948
- "allow_etp_server=0",
18949
- "search_history_enabled=0",
18950
- "run_history_enabled=0",
18951
- "auto_include_fixed_volumes=0",
18952
- "auto_include_removable_volumes=0",
18953
- "auto_include_fixed_refs_volumes=0",
18954
- "auto_include_removable_refs_volumes=0",
18955
- "ntfs_volume_paths=",
18956
- "ntfs_volume_includes=",
18957
- "refs_volume_paths=",
18958
- "refs_volume_includes=",
18959
- "index_size=1",
18960
- "index_date_modified=1",
18961
- `folders=${list(input.folders)}`,
18962
- `folder_monitor_changes=${perFolder("1")}`,
18963
- `folder_rescan_if_full_list=${perFolder("1")}`,
18964
- `folder_update_types=${perFolder("0")}`,
18965
- "exclude_list_enabled=1",
18966
- `exclude_folders=${list(input.excludeFolders)}`,
18967
- ""
18968
- ].join(`\r
18969
- `);
19075
+ function buildWindowsDiskScript() {
19076
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19077
+ ` + "Get-CimInstance -ClassName Win32_LogicalDisk | ForEach-Object {" + `$_.DeviceID + [char]9 + $_.DriveType + [char]9 + $_.FileSystem + [char]9 + $_.VolumeName}
19078
+ `;
18970
19079
  }
18971
- function buildEverythingSearchArgs(instance, request, timeoutMs) {
18972
- const args = [
18973
- "-argv",
18974
- "-instance",
18975
- instance,
18976
- "-cp",
18977
- "65001",
18978
- "-json",
18979
- "-full-path-and-name",
18980
- "-size",
18981
- "-date-modified",
18982
- "-date-format",
18983
- "1"
18984
- ];
18985
- if (timeoutMs !== undefined)
18986
- args.push("-timeout", String(timeoutMs));
18987
- if (request.matchCase)
18988
- args.push("-case");
18989
- if (request.matchPath)
18990
- args.push("-match-path");
18991
- if (request.wholeWord)
18992
- args.push("-whole-word");
18993
- if (request.kind === "files")
18994
- args.push("/a-d");
18995
- if (request.kind === "folders")
18996
- args.push("/ad");
18997
- if (request.path !== undefined)
18998
- args.push("-path", request.path);
18999
- if (request.sort !== undefined)
19000
- args.push("-sort", request.sort);
19001
- if (request.offset !== undefined && request.offset > 0)
19002
- args.push("-offset", String(request.offset));
19003
- args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
19004
- if (request.regex)
19005
- args.push("-regex", "--", request.query);
19006
- else
19007
- args.push("--", request.query);
19008
- return args;
19080
+ function parseWindowsLogicalDisks(stdout) {
19081
+ const disks = [];
19082
+ for (const line of stdout.split(/\r?\n/)) {
19083
+ if (line.trim() === "")
19084
+ continue;
19085
+ const [deviceId, driveType, fileSystem, ...rest] = line.split("\t");
19086
+ const type = Number.parseInt((driveType ?? "").trim(), 10);
19087
+ if (!Number.isFinite(type))
19088
+ continue;
19089
+ disks.push({
19090
+ deviceId: (deviceId ?? "").trim(),
19091
+ driveType: type,
19092
+ fileSystem: (fileSystem ?? "").trim(),
19093
+ volumeName: rest.join("\t").trim()
19094
+ });
19095
+ }
19096
+ return disks;
19009
19097
  }
19010
- function parseEverythingJson(stdout) {
19011
- const text = stdout.trim();
19012
- if (text.length === 0)
19013
- return [];
19014
- let parsed;
19015
- try {
19016
- parsed = JSON.parse(text);
19017
- } catch {
19018
- throw new Error(`ES did not return JSON: ${text.slice(0, 2000)}`);
19098
+ function windowsRootsFromDisks(disks) {
19099
+ const roots = [];
19100
+ for (const disk of disks) {
19101
+ if (disk.deviceId === "")
19102
+ continue;
19103
+ const path = `${disk.deviceId}\\`;
19104
+ const fileSystem = disk.fileSystem.toLowerCase();
19105
+ if (disk.driveType === WINDOWS_DRIVE_TYPE_NETWORK) {
19106
+ roots.push({ path, kind: "network" });
19107
+ continue;
19108
+ }
19109
+ if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
19110
+ roots.push({ path, kind: "network" });
19111
+ continue;
19112
+ }
19113
+ if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
19114
+ roots.push({ path, kind: "cloud-files" });
19115
+ }
19019
19116
  }
19020
- if (!Array.isArray(parsed))
19021
- throw new Error(`ES did not return a JSON array: ${text.slice(0, 2000)}`);
19022
- return parsed.map((item) => {
19023
- const record = item;
19024
- const filename = String(record.filename ?? "");
19025
- const isFolder = filename.endsWith("\\") && filename.length > 3;
19117
+ return roots;
19118
+ }
19119
+ function uncShareRoot(target) {
19120
+ const normalized = target.replaceAll("/", "\\");
19121
+ if (!normalized.startsWith("\\\\") || normalized.startsWith("\\\\?\\") || normalized.startsWith("\\\\.\\"))
19122
+ return;
19123
+ const rest = normalized.slice(2);
19124
+ const separator = rest.indexOf("\\");
19125
+ if (separator <= 0)
19126
+ return;
19127
+ const server = rest.slice(0, separator);
19128
+ const share = rest.slice(separator + 1).split("\\")[0] ?? "";
19129
+ if (server === "" || share === "")
19130
+ return;
19131
+ return `\\\\${server}\\${share}`;
19132
+ }
19133
+ function parseWindowsDehydrateOutput(result) {
19134
+ const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
19135
+ if (open === null)
19136
+ return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
19137
+ if (Number.parseInt(open[1], 16) !== 0) {
19026
19138
  return {
19027
- path: isFolder ? filename.slice(0, -1) : filename,
19028
- type: filename.endsWith("\\") ? "folder" : "file",
19029
- size: typeof record.size === "number" ? record.size : undefined,
19030
- dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
19139
+ ok: false,
19140
+ message: windowsFailureMessage(result, `could not open the file for release (0x${open[1]})`)
19031
19141
  };
19032
- });
19142
+ }
19143
+ const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
19144
+ if (dehydrate === null) {
19145
+ return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
19146
+ }
19147
+ const hresult = dehydrate[1];
19148
+ if (Number.parseInt(hresult, 16) === 0)
19149
+ return { ok: true };
19150
+ const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
19151
+ return {
19152
+ ok: false,
19153
+ message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
19154
+ };
19155
+ }
19156
+ function windowsFailureMessage(result, fallback) {
19157
+ const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
19158
+ if (printed !== undefined && printed !== "")
19159
+ return printed;
19160
+ const stderr = result.stderr.trim();
19161
+ return stderr !== "" ? stderr : fallback;
19033
19162
  }
19034
19163
 
19035
- class EverythingClient {
19036
- instanceName;
19037
- options;
19038
- platform;
19164
+ class WindowsCloudProbe {
19039
19165
  run;
19040
- launch;
19041
- stateDir;
19042
- isExcludedPath;
19043
- binaries;
19044
- queue = new SharedExclusiveQueue;
19045
- constructor(options) {
19046
- this.options = options;
19047
- this.platform = options.platform ?? process.platform;
19048
- this.instanceName = everythingInstanceName(options.root);
19049
- this.run = options.run ?? runProcess;
19050
- this.launch = options.launch ?? launchDetached;
19051
- this.stateDir = join29(options.root, ".autorag", "everything");
19052
- this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
19053
- }
19054
- isSupported() {
19055
- return this.platform === "win32";
19166
+ syncRoots;
19167
+ disks;
19168
+ constructor(options = {}) {
19169
+ this.run = options.run ?? runPowerShell;
19056
19170
  }
19057
- search(request) {
19058
- return this.queue.shared(async () => {
19059
- const binaries = await this.resolveBinaries("cached");
19060
- if (!binaries.ok)
19061
- return binaries;
19062
- const ping = await this.ping(binaries.esPath);
19063
- if (ping.code !== 0) {
19064
- return {
19065
- ok: false,
19066
- reason: "not-running",
19067
- message: `Everything instance ${this.instanceName} is not running; run \`autorag refresh\` to start and index it.`
19068
- };
19069
- }
19070
- const es = { ok: true, esPath: binaries.esPath };
19071
- const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
19072
- if ((this.options.excludeFolders?.length ?? 0) > 0) {
19073
- return this.searchWithExclusions(es.esPath, request, timeoutMs);
19074
- }
19075
- const args = buildEverythingSearchArgs(this.instanceName, request, timeoutMs);
19076
- const result = await this.run(es.esPath, args, timeoutMs + 5000);
19077
- if (result.code !== 0)
19078
- return this.failure("search-failed", es.esPath, args, result);
19079
- try {
19080
- return { ok: true, results: parseEverythingJson(result.stdout) };
19081
- } catch (error) {
19082
- return { ok: false, reason: "search-failed", message: error.message };
19083
- }
19084
- });
19171
+ async cloudRoots(searchRoots = []) {
19172
+ const roots = [];
19173
+ const seen = new Set;
19174
+ const add = (path, kind) => {
19175
+ const canonical = canonicalPath(path);
19176
+ const key = normalizeCloudPath(canonical).toLowerCase();
19177
+ if (key === "" || seen.has(key))
19178
+ return;
19179
+ seen.add(key);
19180
+ roots.push({ path: canonical, kind });
19181
+ };
19182
+ for (const root of await this.loadSyncRoots())
19183
+ add(root.path, "cloud-files");
19184
+ for (const root of await this.loadDiskRoots())
19185
+ add(root.path, root.kind);
19186
+ for (const searchRoot of searchRoots) {
19187
+ const share = uncShareRoot(searchRoot);
19188
+ if (share !== undefined)
19189
+ add(share, "network");
19190
+ }
19191
+ return roots;
19085
19192
  }
19086
- async searchWithExclusions(esPath, request, timeoutMs) {
19087
- const deadline = Date.now() + timeoutMs;
19088
- const offset = request.offset ?? 0;
19089
- const maxResults = request.maxResults ?? DEFAULT_MAX_RESULTS;
19090
- const results = [];
19091
- let skipped = 0;
19092
- let nativeOffset = 0;
19093
- const timedOut = () => ({
19094
- ok: false,
19095
- reason: "search-failed",
19096
- message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
19097
- });
19098
- for (;; ) {
19099
- const remainingMs = deadline - Date.now();
19100
- if (remainingMs <= 0)
19101
- return timedOut();
19102
- const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
19103
- const response = await this.run(esPath, args, remainingMs + 5000);
19104
- if (response.code !== 0)
19105
- return this.failure("search-failed", esPath, args, response);
19106
- if (Date.now() >= deadline)
19107
- return timedOut();
19108
- let entries;
19109
- try {
19110
- entries = parseEverythingJson(response.stdout);
19111
- } catch (error) {
19112
- return { ok: false, reason: "search-failed", message: error.message };
19193
+ async datalessAmong(paths) {
19194
+ const dataless = new Set;
19195
+ for (let start = 0;start < paths.length; start += WINDOWS_ATTRIBUTE_BATCH) {
19196
+ const batch = paths.slice(start, start + WINDOWS_ATTRIBUTE_BATCH);
19197
+ const result = await this.run(buildWindowsAttributeScript(batch), 30000);
19198
+ if (!result.ok) {
19199
+ throw new Error(`Could not read Windows cloud file attributes: ${result.stderr.trim() || "powershell.exe failed"}`);
19113
19200
  }
19114
- for (const entry of entries) {
19115
- if (this.isExcludedPath(entry.path))
19116
- continue;
19117
- if (skipped < offset) {
19118
- skipped += 1;
19119
- continue;
19120
- }
19121
- results.push(entry);
19122
- if (results.length >= maxResults)
19123
- return { ok: true, results };
19201
+ for (const [path, attributes] of parseWindowsAttributeOutput(result.stdout, batch)) {
19202
+ if (datalessFromWindowsAttributes(attributes))
19203
+ dataless.add(path);
19124
19204
  }
19125
- if (entries.length < MAX_FILTER_CANDIDATES)
19126
- return { ok: true, results };
19127
- nativeOffset += entries.length;
19128
19205
  }
19206
+ return dataless;
19129
19207
  }
19130
- index() {
19131
- return this.queue.exclusive(async () => {
19132
- const es = await this.ensureRunning(true);
19133
- if (!es.ok)
19134
- return es;
19135
- const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
19136
- const saveArgs = ["-instance", this.instanceName, "-save-db"];
19137
- const saved = await this.run(es.esPath, saveArgs, timeoutMs);
19138
- if (saved.code !== 0)
19139
- return this.failure("index-failed", es.esPath, saveArgs, saved);
19140
- const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
19141
- const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
19142
- if (counted.code !== 0)
19143
- return this.failure("index-failed", es.esPath, countArgs, counted);
19144
- return {
19145
- ok: true,
19146
- indexedFolders: [...this.options.folders],
19147
- indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
19148
- };
19149
- });
19208
+ async providerOf(path) {
19209
+ const root = longestSyncRoot(await this.loadSyncRoots(), path);
19210
+ return root === undefined ? cloudProviderFromPath(path) : providerFromWindowsSyncRoot(root.label);
19150
19211
  }
19151
- stop() {
19152
- return this.queue.exclusive(async () => {
19153
- const binaries = await this.resolveBinaries("cached");
19154
- if (!binaries.ok)
19155
- return;
19156
- await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
19157
- });
19212
+ canRelease(provider) {
19213
+ return provider !== undefined;
19158
19214
  }
19159
- async resolveBinaries(mode) {
19160
- if (!this.isSupported()) {
19161
- return {
19162
- ok: false,
19163
- reason: "unsupported-platform",
19164
- message: `Everything is Windows-only; this host is ${this.platform}.`
19165
- };
19166
- }
19167
- if (this.binaries?.ok)
19168
- return this.binaries;
19169
- const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
19170
- const result = await resolve(mode);
19171
- if (result.ok)
19172
- this.binaries = result;
19173
- return result;
19215
+ async evict(path) {
19216
+ return parseWindowsDehydrateOutput(await this.run(buildWindowsDehydrateScript(path), 60000));
19174
19217
  }
19175
- async ensureRunning(reconfigure) {
19176
- const binaries = await this.resolveBinaries("install");
19177
- if (!binaries.ok)
19178
- return binaries;
19179
- const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
19180
- let ping = await this.ping(binaries.esPath);
19181
- if (ping.code === 0 && !reconfigure)
19182
- return { ok: true, esPath: binaries.esPath };
19183
- if (ping.code === 0) {
19184
- const exitArgs = ["-instance", this.instanceName, "-exit"];
19185
- const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
19186
- if (exited.code !== 0)
19187
- return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
19188
- }
19189
- mkdirSync15(this.stateDir, { recursive: true });
19190
- const iniPath = join29(this.stateDir, "Everything.ini");
19191
- writeFileSync11(iniPath, buildEverythingIni({
19192
- folders: this.options.folders,
19193
- excludeFolders: [
19194
- ...new Set([
19195
- join29(this.options.root, ".autorag"),
19196
- ...this.options.folders.map((folder) => join29(folder, ".autorag")),
19197
- ...this.options.excludeFolders ?? []
19198
- ])
19199
- ]
19200
- }));
19201
- const launchArgs = [
19202
- "-instance",
19203
- this.instanceName,
19204
- "-config",
19205
- iniPath,
19206
- "-db",
19207
- join29(this.stateDir, "Everything.db"),
19208
- "-startup"
19209
- ];
19210
- try {
19211
- this.launch(binaries.everythingPath, launchArgs);
19212
- } catch (error) {
19213
- return {
19214
- ok: false,
19215
- reason: "startup-failed",
19216
- message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
19217
- };
19218
- }
19219
- const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
19220
- const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
19221
- for (;; ) {
19222
- ping = await this.ping(binaries.esPath);
19223
- if (ping.code === 0)
19224
- return { ok: true, esPath: binaries.esPath };
19225
- if (Date.now() >= deadline)
19226
- break;
19227
- await sleep2(pollMs);
19228
- }
19229
- return {
19230
- ok: false,
19231
- reason: "startup-failed",
19232
- message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
19233
- };
19218
+ loadSyncRoots() {
19219
+ this.syncRoots ??= this.run(buildWindowsSyncRootsScript(), 15000).then((result) => result.ok ? parseWindowsSyncRoots(result.stdout) : [], () => []);
19220
+ return this.syncRoots;
19234
19221
  }
19235
- async ping(esPath) {
19236
- const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
19237
- return result.code === 0 && !VERSION_PATTERN.test(result.stdout.trim()) ? { ...result, code: -1 } : result;
19222
+ loadDiskRoots() {
19223
+ this.disks ??= this.run(buildWindowsDiskScript(), 30000).then((result) => result.ok ? windowsRootsFromDisks(parseWindowsLogicalDisks(result.stdout)) : [], () => []);
19224
+ return this.disks;
19238
19225
  }
19239
- failure(reason, esPath, args, result) {
19226
+ }
19227
+ function longestSyncRoot(roots, target) {
19228
+ const normalized = target.replaceAll("/", "\\").toLowerCase();
19229
+ let match;
19230
+ let matchLength = -1;
19231
+ for (const root of roots) {
19232
+ const prefix = root.path.replaceAll("/", "\\").replace(/\\+$/u, "").toLowerCase();
19233
+ if (prefix === "" || prefix.length <= matchLength)
19234
+ continue;
19235
+ if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
19236
+ match = root;
19237
+ matchLength = prefix.length;
19238
+ }
19239
+ }
19240
+ return match;
19241
+ }
19242
+ function powerShellLaunch(script) {
19243
+ return {
19244
+ args: [
19245
+ "-NoProfile",
19246
+ "-NonInteractive",
19247
+ "-Command",
19248
+ `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; Invoke-Expression $env:${POWERSHELL_SCRIPT_VARIABLE}`
19249
+ ],
19250
+ env: { [POWERSHELL_SCRIPT_VARIABLE]: script }
19251
+ };
19252
+ }
19253
+ async function runPowerShell(script, timeoutMs) {
19254
+ const launch = powerShellLaunch(script);
19255
+ try {
19256
+ const { stdout, stderr } = await execFileAsync("powershell.exe", [...launch.args], {
19257
+ timeout: timeoutMs,
19258
+ maxBuffer: 16 * 1024 * 1024,
19259
+ windowsHide: true,
19260
+ env: { ...process.env, ...launch.env }
19261
+ });
19262
+ return { ok: true, stdout, stderr };
19263
+ } catch (error) {
19264
+ const record = typeof error === "object" && error !== null ? error : {};
19240
19265
  return {
19241
19266
  ok: false,
19242
- reason,
19243
- message: `${esPath} ${args.join(" ")} exit ${result.code}: ${(result.stderr || result.stdout).trim()}`
19267
+ stdout: typeof record.stdout === "string" ? record.stdout : "",
19268
+ stderr: typeof record.stderr === "string" ? record.stderr : String(error)
19244
19269
  };
19245
19270
  }
19246
19271
  }
19247
- function runProcess(command, args, timeoutMs) {
19248
- let resolvePromise;
19249
- const promise = new Promise((resolve) => {
19250
- resolvePromise = resolve;
19251
- });
19252
- const child = spawn14(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
19253
- const stdout = [];
19254
- const stderr = [];
19255
- let bytes = 0;
19256
- let overflow = false;
19257
- const collect = (target) => (chunk) => {
19258
- bytes += chunk.length;
19259
- if (bytes > MAX_OUTPUT_BYTES) {
19260
- overflow = true;
19261
- child.kill();
19262
- return;
19263
- }
19264
- target.push(chunk);
19265
- };
19266
- child.stdout.on("data", collect(stdout));
19267
- child.stderr.on("data", collect(stderr));
19268
- const timer = setTimeout(() => child.kill(), timeoutMs);
19269
- child.on("error", (error) => {
19270
- clearTimeout(timer);
19271
- resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
19272
- });
19273
- child.on("close", (code, signal) => {
19274
- clearTimeout(timer);
19275
- const err = Buffer.concat(stderr).toString("utf8");
19276
- resolvePromise({
19277
- code,
19278
- stdout: Buffer.concat(stdout).toString("utf8"),
19279
- stderr: overflow ? `${err}
19280
- output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
19281
- killed by ${signal} after ${timeoutMs}ms` : err
19272
+ function defaultCloudProbe() {
19273
+ if (process.platform === "darwin")
19274
+ return new DarwinCloudProbe;
19275
+ if (process.platform === "win32")
19276
+ return new WindowsCloudProbe;
19277
+ return;
19278
+ }
19279
+ async function readFileProviderDomain(target) {
19280
+ try {
19281
+ await lstat(target);
19282
+ const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
19283
+ timeout: 2000
19282
19284
  });
19283
- });
19284
- return promise;
19285
+ const value = stdout.trim();
19286
+ return value.length > 0 ? value : undefined;
19287
+ } catch {
19288
+ return;
19289
+ }
19285
19290
  }
19286
- function launchDetached(command, args) {
19287
- const child = spawn14(command, args, { detached: true, stdio: "ignore", windowsHide: true });
19288
- child.on("error", () => {});
19289
- child.unref();
19291
+ async function readFileProviderDomainUpwards(target) {
19292
+ let cursor = target;
19293
+ for (let depth = 0;depth < 12; depth += 1) {
19294
+ const domain = await readFileProviderDomain(cursor);
19295
+ if (domain !== undefined)
19296
+ return domain;
19297
+ const parent = dirname15(cursor);
19298
+ if (parent === cursor)
19299
+ return;
19300
+ cursor = parent;
19301
+ }
19302
+ return;
19290
19303
  }
19291
- var DEFAULT_TIMEOUT_MS8 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
19292
- var init_client9 = __esm(() => {
19293
- init_path_exclusions();
19294
- init_bundle();
19295
- DEFAULT_INDEX_TIMEOUT_MS3 = 10 * 60000;
19296
- MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
19297
- VERSION_PATTERN = /^\d+\.\d+\.\d+\.\d+$/;
19298
- });
19299
-
19300
- // src/everything/index.ts
19301
- var init_everything = __esm(() => {
19302
- init_bundle();
19303
- init_client9();
19304
+ function quotePowerShell(value) {
19305
+ return `'${value.replace(/['\u2018\u2019\u201A\u201B]/gu, (quote) => quote + quote)}'`;
19306
+ }
19307
+ var execFileAsync, SF_DATALESS = 1073741824, FILE_PROVIDER_XATTR = "com.apple.file-provider-domain-id", WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 4194304, STAT_BATCH = 200, defaultCloudExec = (file, args, options) => execFileAsync(file, [...args], {
19308
+ timeout: options?.timeoutMs ?? 30000,
19309
+ ...options?.maxBuffer === undefined ? {} : { maxBuffer: options.maxBuffer }
19310
+ }), JXA_RELEASE_PROGRAM = `ObjC.import("Foundation");
19311
+ function run(argv) {
19312
+ var url = $.NSURL.fileURLWithPath(argv[0]);
19313
+ var errorRef = Ref();
19314
+ var released = $.NSFileManager.defaultManager.evictUbiquitousItemAtURLError(url, errorRef);
19315
+ if (released) return JSON.stringify({ ok: true });
19316
+ function resourceValue(key) {
19317
+ var ref = Ref();
19318
+ if (!url.getResourceValueForKeyError(ref, key, null)) return null;
19319
+ var value = ref[0];
19320
+ return value === undefined || value === null ? null : ObjC.unwrap(value);
19321
+ }
19322
+ return JSON.stringify({
19323
+ ok: false,
19324
+ isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
19325
+ isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
19326
+ downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
19327
+ });
19328
+ }`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
19329
+ var init_cloud_placeholder = __esm(() => {
19330
+ execFileAsync = promisify(execFile2);
19331
+ RELEASABLE_DARWIN_PROVIDERS = {
19332
+ icloud: true,
19333
+ "google-drive": true,
19334
+ onedrive: true,
19335
+ dropbox: true
19336
+ };
19337
+ DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
19338
+ WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
19339
+ WINDOWS_DEHYDRATE_REFUSALS = {
19340
+ "80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
19341
+ "80070188": "The file is pinned to stay on this device, so it is not released."
19342
+ };
19304
19343
  });
19305
19344
 
19306
19345
  // src/fsearch/walk.ts
19307
19346
  import { readdir as readdir2, stat as stat3 } from "node:fs/promises";
19308
- import { basename as basename8, join as join30, sep as sep5 } from "node:path";
19347
+ import { basename as basename7, join as join30, sep as sep5 } from "node:path";
19309
19348
  function compare(a, b) {
19310
19349
  return a < b ? -1 : a > b ? 1 : 0;
19311
19350
  }
@@ -19316,11 +19355,11 @@ async function walkFileSearch(folders, request, options = {}) {
19316
19355
  const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_DEADLINE_MS);
19317
19356
  const matches = request.regex ? (() => {
19318
19357
  const pattern = new RegExp(request.query, request.matchCase ? "" : "i");
19319
- return (path) => pattern.test(request.matchPath ? path : basename8(path));
19358
+ return (path) => pattern.test(request.matchPath ? path : basename7(path));
19320
19359
  })() : (() => {
19321
19360
  const needle = request.matchCase ? request.query : request.query.toLowerCase();
19322
19361
  return (path) => {
19323
- const haystack = request.matchPath ? path : basename8(path);
19362
+ const haystack = request.matchPath ? path : basename7(path);
19324
19363
  return (request.matchCase ? haystack : haystack.toLowerCase()).includes(needle);
19325
19364
  };
19326
19365
  })();
@@ -19375,7 +19414,7 @@ async function walkFileSearch(folders, request, options = {}) {
19375
19414
  continue;
19376
19415
  collected.push({
19377
19416
  path,
19378
- name: basename8(path),
19417
+ name: basename7(path),
19379
19418
  type: info.isDirectory() ? "folder" : "file",
19380
19419
  size: info.isDirectory() ? undefined : info.size,
19381
19420
  dateModified: info.mtime.toISOString()
@@ -19400,7 +19439,7 @@ async function walkFileSearch(folders, request, options = {}) {
19400
19439
  continue;
19401
19440
  collected.push({
19402
19441
  path: folder,
19403
- name: basename8(folder),
19442
+ name: basename7(folder),
19404
19443
  type: "folder",
19405
19444
  size: undefined,
19406
19445
  dateModified: info.mtime.toISOString()
@@ -19433,7 +19472,7 @@ var init_walk = __esm(() => {
19433
19472
  import { spawn as spawn15 } from "node:child_process";
19434
19473
  import { createHash as createHash13 } from "node:crypto";
19435
19474
  import { existsSync as existsSync18, mkdirSync as mkdirSync16, readFileSync as readFileSync18, realpathSync as realpathSync4, rmSync as rmSync8, writeFileSync as writeFileSync12 } from "node:fs";
19436
- import { basename as basename9, join as join31, resolve as resolve12, sep as sep6 } from "node:path";
19475
+ import { basename as basename8, join as join31, resolve as resolve12, sep as sep6 } from "node:path";
19437
19476
  import { setTimeout as sleep3 } from "node:timers/promises";
19438
19477
  function fsearchSocketPath(dbPath) {
19439
19478
  let canonical;
@@ -19497,7 +19536,7 @@ function parseFSearchJsonLines(stdout) {
19497
19536
  const path = String(record.path ?? "");
19498
19537
  entries.push({
19499
19538
  path,
19500
- name: basename9(path),
19539
+ name: basename8(path),
19501
19540
  type: record.type === "folder" ? "folder" : "file",
19502
19541
  size: typeof record.size === "number" ? record.size : undefined,
19503
19542
  dateModified: typeof record.mtime === "number" ? new Date(record.mtime * 1000).toISOString() : undefined
@@ -19915,7 +19954,7 @@ var init_diagnostics = __esm(() => {
19915
19954
  });
19916
19955
 
19917
19956
  // src/jikji/retry-command.ts
19918
- import { basename as basename10 } from "node:path";
19957
+ import { basename as basename9 } from "node:path";
19919
19958
  function shellWords(command) {
19920
19959
  const words = [];
19921
19960
  let current = "";
@@ -19982,7 +20021,7 @@ function parseJikjiRetryCommand(command, envelopeProof) {
19982
20021
  const words = shellWords(command);
19983
20022
  if (words === undefined || words.length < 4)
19984
20023
  return;
19985
- const program = basename10(words[0]).replace(/\.exe$/iu, "");
20024
+ const program = basename9(words[0]).replace(/\.exe$/iu, "");
19986
20025
  if (program !== "jikji" || words[1] !== "discover")
19987
20026
  return;
19988
20027
  const root = words[2];
@@ -20390,7 +20429,7 @@ class SerialQueue {
20390
20429
 
20391
20430
  // src/jikji/version.ts
20392
20431
  function parseJikjiVersion(output) {
20393
- return VERSION_LINE2.exec(output)?.[1];
20432
+ return VERSION_LINE3.exec(output)?.[1];
20394
20433
  }
20395
20434
  function parts(version) {
20396
20435
  return version.split(".").map((part) => Number.parseInt(part, 10));
@@ -20406,9 +20445,9 @@ function isSupportedJikjiVersion(version) {
20406
20445
  }
20407
20446
  return true;
20408
20447
  }
20409
- var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1", VERSION_LINE2;
20410
- var init_version2 = __esm(() => {
20411
- VERSION_LINE2 = /^jikji(?:-cli)?\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
20448
+ var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1", VERSION_LINE3;
20449
+ var init_version3 = __esm(() => {
20450
+ VERSION_LINE3 = /^jikji(?:-cli)?\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
20412
20451
  });
20413
20452
 
20414
20453
  // src/jikji/client.ts
@@ -20776,7 +20815,7 @@ var init_client11 = __esm(() => {
20776
20815
  init_answer_pack();
20777
20816
  init_installer3();
20778
20817
  init_prepare_budget();
20779
- init_version2();
20818
+ init_version3();
20780
20819
  ABORTED_BEFORE_START = { ok: false, reason: "aborted", stdout: "", stderr: "", code: null };
20781
20820
  });
20782
20821
 
@@ -21861,7 +21900,7 @@ import {
21861
21900
  rmSync as rmSync10,
21862
21901
  writeFileSync as writeFileSync14
21863
21902
  } from "node:fs";
21864
- import { basename as basename11, dirname as dirname18, isAbsolute as isAbsolute9, join as join36, relative as relative7, sep as sep8 } from "node:path";
21903
+ import { basename as basename10, dirname as dirname18, isAbsolute as isAbsolute9, join as join36, relative as relative7, sep as sep8 } from "node:path";
21865
21904
  function syncMinSyncWorkspace(root, options = {}) {
21866
21905
  const workspacePath = options.workspacePath ?? minSyncWorkspaceRoot(root);
21867
21906
  const filesRoot = join36(workspacePath, "files");
@@ -21902,7 +21941,7 @@ function syncMinSyncWorkspace(root, options = {}) {
21902
21941
  return { workspacePath, entries, excluded };
21903
21942
  }
21904
21943
  function stagingStatePath(workspacePath) {
21905
- return join36(dirname18(workspacePath), `.${basename11(workspacePath)}-autorag-staging.json`);
21944
+ return join36(dirname18(workspacePath), `.${basename10(workspacePath)}-autorag-staging.json`);
21906
21945
  }
21907
21946
  function loadStagingState2(workspacePath) {
21908
21947
  const path = stagingStatePath(workspacePath);
@@ -21953,7 +21992,7 @@ function ensureManagedDirectory(filesRoot, directory) {
21953
21992
  }
21954
21993
  }
21955
21994
  function copyFileAtomically(source, destination) {
21956
- const temporaryPath = join36(dirname18(destination), `.${basename11(destination)}.${randomUUID6()}.tmp`);
21995
+ const temporaryPath = join36(dirname18(destination), `.${basename10(destination)}.${randomUUID6()}.tmp`);
21957
21996
  try {
21958
21997
  copyFileSync4(source, temporaryPath);
21959
21998
  renameWithRetry(temporaryPath, destination);
@@ -22007,7 +22046,7 @@ var init_workspace = __esm(() => {
22007
22046
 
22008
22047
  // src/minsync/method.ts
22009
22048
  import { existsSync as existsSync23, readFileSync as readFileSync21, realpathSync as realpathSync7, statSync as statSync7 } from "node:fs";
22010
- import { basename as basename12, join as join37, normalize } from "node:path";
22049
+ import { basename as basename11, join as join37, normalize } from "node:path";
22011
22050
 
22012
22051
  class MinSyncVectorMethod {
22013
22052
  root;
@@ -22108,7 +22147,7 @@ class MinSyncVectorMethod {
22108
22147
  const entry = byPath.get(hit.path);
22109
22148
  if (!entry || !matchesVirtualPathScope(entry.virtualPath, options.scope))
22110
22149
  continue;
22111
- const chunkDiscriminator = hit.docId ?? basename12(hit.path);
22150
+ const chunkDiscriminator = hit.docId ?? basename11(hit.path);
22112
22151
  const chunkId = `minsync:${entry.virtualPath}:${chunkDiscriminator}`;
22113
22152
  const location = locateChunkInMirror(mirrorTexts, entry, hit.text);
22114
22153
  results.push({
@@ -22208,18 +22247,24 @@ var init_minsync = __esm(() => {
22208
22247
 
22209
22248
  // src/mirror/cloud-fingerprints.ts
22210
22249
  import { existsSync as existsSync24, readFileSync as readFileSync22 } from "node:fs";
22211
- import { join as join38 } from "node:path";
22250
+ import { extname as extname5, join as join38 } from "node:path";
22212
22251
  function cloudFingerprintStorePath(root) {
22213
22252
  return join38(root, ".autorag", CLOUD_FINGERPRINTS_FILE);
22214
22253
  }
22215
22254
  function emptyCloudFingerprints() {
22216
- return { version: 1, entries: {} };
22255
+ return { version: 2, entries: {} };
22217
22256
  }
22218
- function isCloudFingerprint(value) {
22257
+ function readCloudFingerprint(sourcePath, value, storeVersion) {
22219
22258
  if (typeof value !== "object" || value === null)
22220
- return false;
22259
+ return;
22221
22260
  const record = value;
22222
- return typeof record.sourceVersion === "string" && typeof record.exact === "string" && typeof record.chars === "number" && (record.modified === null || typeof record.modified === "string") && (record.revision === null || typeof record.revision === "string");
22261
+ const { sourceVersion, exact, modified, revision } = record;
22262
+ if (typeof sourceVersion !== "string" || typeof exact !== "string" || modified !== null && typeof modified !== "string" || revision !== null && typeof revision !== "string") {
22263
+ return;
22264
+ }
22265
+ if (storeVersion !== 2 && (record.chars === 0 || extname5(sourcePath).toLowerCase() === ".hwp"))
22266
+ return;
22267
+ return { sourceVersion, exact, modified, revision };
22223
22268
  }
22224
22269
  function loadCloudFingerprints(root) {
22225
22270
  const path = cloudFingerprintStorePath(root);
@@ -22232,24 +22277,26 @@ function loadCloudFingerprints(root) {
22232
22277
  const { entries } = parsed;
22233
22278
  if (typeof entries !== "object" || entries === null)
22234
22279
  return emptyCloudFingerprints();
22280
+ const storeVersion = "version" in parsed ? parsed.version : undefined;
22235
22281
  const valid = {};
22236
- for (const [sourcePath, fingerprint] of Object.entries(entries)) {
22237
- if (isCloudFingerprint(fingerprint))
22282
+ for (const [sourcePath, value] of Object.entries(entries)) {
22283
+ const fingerprint = readCloudFingerprint(sourcePath, value, storeVersion);
22284
+ if (fingerprint !== undefined)
22238
22285
  valid[sourcePath] = fingerprint;
22239
22286
  }
22240
- return { version: 1, entries: valid };
22287
+ return { version: 2, entries: valid };
22241
22288
  } catch {
22242
22289
  return emptyCloudFingerprints();
22243
22290
  }
22244
22291
  }
22245
22292
  function saveCloudFingerprints(root, entries) {
22246
- writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version: 1, entries }, null, 2)}
22293
+ writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version: 2, entries }, null, 2)}
22247
22294
  `);
22248
22295
  }
22249
22296
  function cloudFingerprintExclusions(store) {
22250
22297
  const map = new Map;
22251
22298
  for (const [sourcePath, fingerprint] of Object.entries(store.entries)) {
22252
- map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified, chars: fingerprint.chars });
22299
+ map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified });
22253
22300
  }
22254
22301
  return map;
22255
22302
  }
@@ -22641,7 +22688,7 @@ class Parser {
22641
22688
  }
22642
22689
 
22643
22690
  // src/parser/csv.ts
22644
- import { basename as basename13, extname as extname5 } from "node:path";
22691
+ import { basename as basename12, extname as extname6 } from "node:path";
22645
22692
  function toCell(text) {
22646
22693
  return text.trim() === "" ? undefined : { text, kind: classifyText(text) };
22647
22694
  }
@@ -22772,7 +22819,7 @@ var init_csv = __esm(() => {
22772
22819
  this.#maxRows = options.maxRows ?? DEFAULT_MAX_ROWS;
22773
22820
  }
22774
22821
  async parse(input) {
22775
- const extension = extname5(input.virtualPath);
22822
+ const extension = extname6(input.virtualPath);
22776
22823
  const format = extension.slice(1).toLowerCase();
22777
22824
  let text = decodeText(input.bytes).replace(/^\uFEFF/, "");
22778
22825
  const directive = SEPARATOR_DIRECTIVE.exec(text);
@@ -22784,10 +22831,10 @@ var init_csv = __esm(() => {
22784
22831
  {
22785
22832
  code: "parser-warning",
22786
22833
  severity: "warning",
22787
- message: `"${basename13(input.virtualPath)}": indexed the first ${this.#maxRows} of ${total} rows`
22834
+ message: `"${basename12(input.virtualPath)}": indexed the first ${this.#maxRows} of ${total} rows`
22788
22835
  }
22789
22836
  ] : [];
22790
- const grid = { name: basename13(input.virtualPath, extension), rows: rows.map((row) => row.map(toCell)) };
22837
+ const grid = { name: basename12(input.virtualPath, extension), rows: rows.map((row) => row.map(toCell)) };
22791
22838
  return {
22792
22839
  markdown: normalizeMarkdown(renderWorkbook([grid])),
22793
22840
  metadata: { parser: this.name, format },
@@ -23326,7 +23373,7 @@ var init_plain_text = __esm(() => {
23326
23373
  });
23327
23374
 
23328
23375
  // src/parser/registry.ts
23329
- import { extname as extname6 } from "node:path";
23376
+ import { extname as extname7 } from "node:path";
23330
23377
 
23331
23378
  class ParserRegistry {
23332
23379
  parsersByExtension = new Map;
@@ -23345,7 +23392,7 @@ class ParserRegistry {
23345
23392
  }
23346
23393
  }
23347
23394
  getForVirtualPath(virtualPath) {
23348
- return this.parsersByExtension.get(normalizeExtension(extname6(virtualPath)));
23395
+ return this.parsersByExtension.get(normalizeExtension(extname7(virtualPath)));
23349
23396
  }
23350
23397
  list() {
23351
23398
  return Array.from(new Set(this.parsersByExtension.values()));
@@ -23358,7 +23405,7 @@ function normalizeExtension(extension) {
23358
23405
  var init_registry = () => {};
23359
23406
 
23360
23407
  // src/parser/spreadsheet.ts
23361
- import { extname as extname7 } from "node:path";
23408
+ import { extname as extname8 } from "node:path";
23362
23409
  import * as XLSX from "xlsx";
23363
23410
  function assertWorkbookSignature(format, bytes) {
23364
23411
  if (startsWith(bytes, ZIP_SIGNATURE) || startsWith(bytes, OLE_SIGNATURE))
@@ -23457,7 +23504,7 @@ var init_spreadsheet = __esm(() => {
23457
23504
  this.#maxRowsPerSheet = options.maxRowsPerSheet ?? DEFAULT_MAX_ROWS_PER_SHEET;
23458
23505
  }
23459
23506
  async parse(input) {
23460
- const format = extname7(input.virtualPath).slice(1).toLowerCase();
23507
+ const format = extname8(input.virtualPath).slice(1).toLowerCase();
23461
23508
  let workbook;
23462
23509
  try {
23463
23510
  assertWorkbookSignature(format, input.bytes);
@@ -23731,6 +23778,10 @@ class CloudGate {
23731
23778
  isPlaceholder(path) {
23732
23779
  return this.placeholders.has(path);
23733
23780
  }
23781
+ startBatch() {
23782
+ this.downloadedFiles = 0;
23783
+ this.downloadedBytes = 0;
23784
+ }
23734
23785
  async decide(path, sizeBytes, directory) {
23735
23786
  if (this.probe === undefined || !this.placeholders.has(path))
23736
23787
  return LOCAL;
@@ -23740,7 +23791,7 @@ class CloudGate {
23740
23791
  const overFiles = this.downloadedFiles >= this.maxFiles;
23741
23792
  const overBytes = this.downloadedFiles > 0 && this.downloadedBytes + sizeBytes > this.maxBytes;
23742
23793
  if (overFiles || overBytes)
23743
- return { kind: "defer" };
23794
+ return { kind: "next-batch" };
23744
23795
  this.downloadedFiles += 1;
23745
23796
  this.downloadedBytes += sizeBytes;
23746
23797
  return { kind: "download" };
@@ -23785,7 +23836,7 @@ var init_cloud = __esm(() => {
23785
23836
  // src/mirror/sync.ts
23786
23837
  import { createHash as createHash17 } from "node:crypto";
23787
23838
  import { opendir as opendir2, readFile, stat as stat4 } from "node:fs/promises";
23788
- import { basename as basename14, dirname as dirname21, extname as extname8, join as join42, resolve as resolve15, sep as sep9 } from "node:path";
23839
+ import { basename as basename13, dirname as dirname21, extname as extname9, join as join42, resolve as resolve15, sep as sep9 } from "node:path";
23789
23840
  import { setTimeout as sleep4 } from "node:timers/promises";
23790
23841
  function errorCode(error) {
23791
23842
  if (!(error instanceof Error))
@@ -23989,195 +24040,221 @@ async function syncWithRegistry(options, registry, isolated) {
23989
24040
  });
23990
24041
  const pending = new Set(plan.decisions.filter((decision) => decision.action === "process" || decision.action === "retry").map((decision) => decision.key));
23991
24042
  const deferred = new Map(plan.decisions.filter((decision) => decision.deferred === true).map((decision) => [decision.key, decision]));
23992
- for (const decision of plan.decisions) {
23993
- if (decision.action !== "delete")
23994
- continue;
23995
- const path = pathsByKey.get(decision.key);
23996
- await session.delete(decision.key);
23997
- if (path === undefined)
23998
- continue;
23999
- if (previous.entries[path])
24000
- deleted += 1;
24001
- if (!currentPaths.has(path))
24002
- diagnostics.push({
24003
- code: "deleted-mirror",
24004
- severity: "info",
24005
- message: "A previously indexed document is gone; its parsed mirror was removed.",
24006
- source: path
24007
- });
24008
- }
24009
- await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
24010
- for (const [position, entry] of current.entries()) {
24011
- options.onProgress?.({
24012
- processed: position,
24013
- total: current.length,
24014
- written,
24015
- deleted,
24016
- skipped,
24017
- currentVirtualPath: entry.virtualPath
24018
- });
24019
- if (options.signal?.aborted)
24020
- throw abortReason(options.signal);
24021
- const omission = omitted.get(entry.virtualPath);
24022
- if (omission !== undefined) {
24023
- if (pending.has(mirrorStageKey(entry.virtualPath))) {
24024
- if (previous.entries[entry.virtualPath])
24025
- deleted += 1;
24026
- await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
24027
- }
24028
- skipped += 1;
24029
- if (omission.diagnostic)
24030
- diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
24031
- continue;
24032
- }
24033
- const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
24034
- if (deferredFailure !== undefined) {
24035
- skipped += 1;
24036
- const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
24037
- diagnostics.push({
24038
- code,
24039
- severity: "warning",
24040
- source: entry.virtualPath,
24041
- message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
24042
- });
24043
- continue;
24044
- }
24045
- const parser = registry.getForVirtualPath(entry.virtualPath);
24046
- const outputPath = parsedOutputPath(options.root, entry.virtualPath);
24047
- const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname21(entry.sourcePath)) : { kind: "local" };
24048
- if (cloudDecision.kind === "skip") {
24049
- skipped += 1;
24050
- diagnostics.push({
24051
- code: "cloud-placeholder-skipped",
24052
- severity: "info",
24053
- message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
24054
- source: entry.virtualPath,
24055
- ...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
24056
- });
24057
- continue;
24058
- }
24059
- if (cloudDecision.kind === "defer") {
24060
- skipped += 1;
24061
- diagnostics.push({
24062
- code: "cloud-download-deferred",
24063
- severity: "info",
24064
- message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
24065
- source: entry.virtualPath
24066
- });
24067
- continue;
24068
- }
24069
- if (pending.has(mirrorStageKey(entry.virtualPath))) {
24070
- let bytes;
24071
- try {
24072
- bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
24073
- } catch (error) {
24074
- const failure = sourceReadFailure(error);
24075
- await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
24076
- continue;
24077
- }
24078
- const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
24079
- if (tooSmallImage !== undefined) {
24080
- await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
24043
+ const applyDeleteDecisions = async () => {
24044
+ for (const decision of plan.decisions) {
24045
+ if (decision.action !== "delete")
24081
24046
  continue;
24082
- }
24083
- let parsed;
24084
- try {
24085
- const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
24086
- parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
24087
- } catch (error) {
24088
- if (options.signal?.aborted)
24089
- throw abortReason(options.signal);
24090
- if (error instanceof ParseError)
24091
- await skipFile(entry, {
24092
- code: "parser-failed",
24093
- severity: "warning",
24094
- message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
24095
- }, "parser-failed");
24096
- else
24097
- await skipFile(entry, {
24098
- code: "parser-crashed",
24099
- severity: "warning",
24100
- message: PARSER_CRASH_MESSAGE,
24101
- ...error instanceof Error ? { reason: error.name } : {}
24102
- });
24047
+ const path = pathsByKey.get(decision.key);
24048
+ await session.delete(decision.key);
24049
+ if (path === undefined)
24103
24050
  continue;
24104
- }
24105
- let info;
24106
- try {
24107
- info = await stat4(entry.sourcePath, { bigint: true });
24108
- } catch (error) {
24109
- await skipFile(entry, sourceReadFailure(error));
24051
+ if (previous.entries[path])
24052
+ deleted += 1;
24053
+ if (!currentPaths.has(path))
24054
+ diagnostics.push({
24055
+ code: "deleted-mirror",
24056
+ severity: "info",
24057
+ message: "A previously indexed document is gone; its parsed mirror was removed.",
24058
+ source: path
24059
+ });
24060
+ }
24061
+ };
24062
+ const deferDeletions = options.onBatch !== undefined;
24063
+ const batchSize = Math.max(1, Math.floor(options.batchSize ?? DEFAULT_MIRROR_BATCH_SIZE));
24064
+ let batchIndex = 0;
24065
+ let batchStopped = false;
24066
+ const runBatch = async (virtualPaths, final) => {
24067
+ if (options.onBatch === undefined)
24068
+ return;
24069
+ batchIndex += 1;
24070
+ const outcome = await options.onBatch({ index: batchIndex, virtualPaths, final });
24071
+ if (outcome?.stop === true)
24072
+ batchStopped = true;
24073
+ };
24074
+ if (!deferDeletions)
24075
+ await applyDeleteDecisions();
24076
+ await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
24077
+ let batchStart = 0;
24078
+ for (;; ) {
24079
+ const batchLimit = Math.min(batchStart + batchSize, current.length);
24080
+ let batchEnd = batchLimit;
24081
+ const changedBefore = written + deleted;
24082
+ gate.startBatch();
24083
+ for (let position = batchStart;position < batchLimit; position += 1) {
24084
+ const entry = current[position];
24085
+ options.onProgress?.({
24086
+ processed: position,
24087
+ total: current.length,
24088
+ written,
24089
+ deleted,
24090
+ skipped,
24091
+ currentVirtualPath: entry.virtualPath
24092
+ });
24093
+ if (options.signal?.aborted)
24094
+ throw abortReason(options.signal);
24095
+ const omission = omitted.get(entry.virtualPath);
24096
+ if (omission !== undefined) {
24097
+ if (pending.has(mirrorStageKey(entry.virtualPath))) {
24098
+ if (previous.entries[entry.virtualPath])
24099
+ deleted += 1;
24100
+ await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
24101
+ }
24102
+ skipped += 1;
24103
+ if (omission.diagnostic)
24104
+ diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
24110
24105
  continue;
24111
24106
  }
24112
- const unchanged = cloudDecision.kind === "download" ? contentIdentity(info) === entry.contentIdentity : statIdentity(info) === entry.statIdentity;
24113
- if (!unchanged) {
24107
+ const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
24108
+ if (deferredFailure !== undefined) {
24114
24109
  skipped += 1;
24110
+ const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
24115
24111
  diagnostics.push({
24116
- code: "source-changed",
24112
+ code,
24117
24113
  severity: "warning",
24118
- message: "The source changed while parsing; its last good mirror was kept and it will be retried on the next refresh.",
24119
- source: entry.virtualPath
24114
+ source: entry.virtualPath,
24115
+ message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
24120
24116
  });
24121
- await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
24122
24117
  continue;
24123
24118
  }
24124
- const markdown = normalizeMarkdown(parsed.markdown);
24125
- try {
24126
- await withTransientRetry(() => writeFileAtomic(outputPath, markdown), transientRetries, transientBackoffMs);
24127
- } catch (error) {
24128
- const code = errorCode(error);
24129
- if (code !== undefined && Object.hasOwn(WORKSPACE_ERROR_CODES, code))
24130
- throw new MirrorWorkspaceError(options.root, error);
24131
- await skipFile(entry, {
24132
- code: "mirror-write-failed",
24133
- severity: "warning",
24134
- message: error instanceof Error ? error.message : String(error),
24135
- ...code !== undefined ? { reason: code } : {}
24119
+ const parser = registry.getForVirtualPath(entry.virtualPath);
24120
+ const outputPath = parsedOutputPath(options.root, entry.virtualPath);
24121
+ const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname21(entry.sourcePath)) : { kind: "local" };
24122
+ if (cloudDecision.kind === "skip") {
24123
+ skipped += 1;
24124
+ diagnostics.push({
24125
+ code: "cloud-placeholder-skipped",
24126
+ severity: "info",
24127
+ message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
24128
+ source: entry.virtualPath,
24129
+ ...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
24136
24130
  });
24137
24131
  continue;
24138
24132
  }
24139
- await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
24140
- path: basename14(outputPath),
24141
- size: Buffer.byteLength(markdown)
24142
- });
24143
- for (const diagnostic of parsed.diagnostics ?? [])
24144
- diagnostics.push({ ...diagnostic, source: entry.virtualPath });
24145
- written += 1;
24146
- if (fingerprintCloud && cloudFilePaths.has(entry.sourcePath)) {
24147
- const existing = cloudFingerprintEntries[entry.sourcePath];
24148
- if (existing === undefined || existing.sourceVersion !== entry.sourceVersion) {
24149
- try {
24150
- const fingerprintResult = await fingerprintWithDupey(entry.sourcePath, options.cloudFingerprints?.dupey ?? {});
24151
- cloudFingerprintEntries[entry.sourcePath] = {
24152
- sourceVersion: entry.sourceVersion,
24153
- exact: fingerprintResult.exact,
24154
- chars: fingerprintResult.chars,
24155
- modified: fingerprintResult.modified,
24156
- revision: fingerprintResult.revision
24157
- };
24158
- cloudFingerprintsDirty = true;
24159
- } catch (error) {
24160
- diagnostics.push({
24161
- code: "cloud-fingerprint-failed",
24133
+ if (cloudDecision.kind === "next-batch") {
24134
+ batchEnd = position;
24135
+ break;
24136
+ }
24137
+ if (pending.has(mirrorStageKey(entry.virtualPath))) {
24138
+ let bytes;
24139
+ try {
24140
+ bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
24141
+ } catch (error) {
24142
+ const failure = sourceReadFailure(error);
24143
+ await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
24144
+ continue;
24145
+ }
24146
+ const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
24147
+ if (tooSmallImage !== undefined) {
24148
+ await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
24149
+ continue;
24150
+ }
24151
+ let parsed;
24152
+ try {
24153
+ const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
24154
+ parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
24155
+ } catch (error) {
24156
+ if (options.signal?.aborted)
24157
+ throw abortReason(options.signal);
24158
+ if (error instanceof ParseError)
24159
+ await skipFile(entry, {
24160
+ code: "parser-failed",
24162
24161
  severity: "warning",
24163
- message: `The cloud or network file was indexed but its duplicate fingerprint could not be read, so it will not take part in exact-duplicate exclusion: ${error instanceof Error ? error.message : String(error)}`,
24164
- source: entry.virtualPath
24162
+ message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
24163
+ }, "parser-failed");
24164
+ else
24165
+ await skipFile(entry, {
24166
+ code: "parser-crashed",
24167
+ severity: "warning",
24168
+ message: PARSER_CRASH_MESSAGE,
24169
+ ...error instanceof Error ? { reason: error.name } : {}
24165
24170
  });
24166
- }
24171
+ continue;
24167
24172
  }
24168
- }
24169
- if (cloudDecision.kind === "download" && gate.evictAfterParse) {
24170
- const released = await gate.release(entry.sourcePath);
24171
- if (!released.ok) {
24173
+ let info;
24174
+ try {
24175
+ info = await stat4(entry.sourcePath, { bigint: true });
24176
+ } catch (error) {
24177
+ await skipFile(entry, sourceReadFailure(error));
24178
+ continue;
24179
+ }
24180
+ const unchanged = cloudDecision.kind === "download" ? contentIdentity(info) === entry.contentIdentity : statIdentity(info) === entry.statIdentity;
24181
+ if (!unchanged) {
24182
+ skipped += 1;
24172
24183
  diagnostics.push({
24173
- code: "cloud-evict-failed",
24184
+ code: "source-changed",
24174
24185
  severity: "warning",
24175
- message: `The file was indexed but its local copy could not be released: ${released.message}`,
24186
+ message: "The source changed while parsing; its last good mirror was kept and it will be retried on the next refresh.",
24176
24187
  source: entry.virtualPath
24177
24188
  });
24189
+ await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
24190
+ continue;
24191
+ }
24192
+ const markdown = normalizeMarkdown(parsed.markdown);
24193
+ try {
24194
+ await withTransientRetry(() => writeFileAtomic(outputPath, markdown), transientRetries, transientBackoffMs);
24195
+ } catch (error) {
24196
+ const code = errorCode(error);
24197
+ if (code !== undefined && Object.hasOwn(WORKSPACE_ERROR_CODES, code))
24198
+ throw new MirrorWorkspaceError(options.root, error);
24199
+ await skipFile(entry, {
24200
+ code: "mirror-write-failed",
24201
+ severity: "warning",
24202
+ message: error instanceof Error ? error.message : String(error),
24203
+ ...code !== undefined ? { reason: code } : {}
24204
+ });
24205
+ continue;
24206
+ }
24207
+ await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
24208
+ path: basename13(outputPath),
24209
+ size: Buffer.byteLength(markdown)
24210
+ });
24211
+ for (const diagnostic of parsed.diagnostics ?? [])
24212
+ diagnostics.push({ ...diagnostic, source: entry.virtualPath });
24213
+ written += 1;
24214
+ if (fingerprintCloud && cloudFilePaths.has(entry.sourcePath)) {
24215
+ const existing = cloudFingerprintEntries[entry.sourcePath];
24216
+ if (existing === undefined || existing.sourceVersion !== entry.sourceVersion) {
24217
+ try {
24218
+ const fingerprintResult = await fingerprintWithDupey(entry.sourcePath, options.cloudFingerprints?.dupey ?? {});
24219
+ cloudFingerprintEntries[entry.sourcePath] = {
24220
+ sourceVersion: entry.sourceVersion,
24221
+ exact: fingerprintResult.exact,
24222
+ modified: fingerprintResult.modified,
24223
+ revision: fingerprintResult.revision
24224
+ };
24225
+ cloudFingerprintsDirty = true;
24226
+ } catch (error) {
24227
+ diagnostics.push({
24228
+ code: "cloud-fingerprint-failed",
24229
+ severity: "warning",
24230
+ message: `The cloud or network file was indexed but its duplicate fingerprint could not be read, so it will not take part in exact-duplicate exclusion: ${error instanceof Error ? error.message : String(error)}`,
24231
+ source: entry.virtualPath
24232
+ });
24233
+ }
24234
+ }
24235
+ }
24236
+ if (cloudDecision.kind === "download" && gate.evictAfterParse) {
24237
+ const released = await gate.release(entry.sourcePath);
24238
+ if (!released.ok) {
24239
+ diagnostics.push({
24240
+ code: "cloud-evict-failed",
24241
+ severity: "warning",
24242
+ message: `The file was indexed but its local copy could not be released: ${released.message}`,
24243
+ source: entry.virtualPath
24244
+ });
24245
+ }
24178
24246
  }
24179
24247
  }
24180
24248
  }
24249
+ const finalBatch = batchEnd >= current.length;
24250
+ if (finalBatch && deferDeletions)
24251
+ await applyDeleteDecisions();
24252
+ if (finalBatch || written + deleted !== changedBefore) {
24253
+ await runBatch(current.slice(batchStart, batchEnd).map((item) => item.virtualPath), finalBatch);
24254
+ }
24255
+ if (finalBatch || batchStopped)
24256
+ break;
24257
+ batchStart = batchEnd;
24181
24258
  }
24182
24259
  await session.finish();
24183
24260
  if (fingerprintCloud) {
@@ -24267,7 +24344,7 @@ async function detectMirrorStaleness(options) {
24267
24344
  }
24268
24345
  }
24269
24346
  function standaloneImageTooSmallMessage(virtualPath, bytes) {
24270
- if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname8(virtualPath))))
24347
+ if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname9(virtualPath))))
24271
24348
  return;
24272
24349
  return ocrImageTooSmallMessage(bytes);
24273
24350
  }
@@ -24435,7 +24512,7 @@ async function collectFiles(state, directory, depth) {
24435
24512
  }
24436
24513
  if (!entry.isFile())
24437
24514
  continue;
24438
- const extension = normalizeExtension2(extname8(entry.name));
24515
+ const extension = normalizeExtension2(extname9(entry.name));
24439
24516
  if (!state.supportedExtensions.has(extension))
24440
24517
  continue;
24441
24518
  let fileStat;
@@ -24469,7 +24546,7 @@ async function collectFiles(state, directory, depth) {
24469
24546
  warnDirectoryUnreadable(state, directory, code);
24470
24547
  }
24471
24548
  }
24472
- var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
24549
+ var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
24473
24550
  var init_sync = __esm(() => {
24474
24551
  init_cli2();
24475
24552
  init_cloud_placeholder();
@@ -29998,7 +30075,7 @@ var init_automation_tool = __esm(() => {
29998
30075
  });
29999
30076
 
30000
30077
  // src/agent/citation-links.ts
30001
- import { basename as basename15, isAbsolute as isAbsolute12 } from "node:path";
30078
+ import { basename as basename14, isAbsolute as isAbsolute12 } from "node:path";
30002
30079
  import { pathToFileURL as pathToFileURL3 } from "node:url";
30003
30080
  function openableTarget(source) {
30004
30081
  if (/^https?:\/\//iu.test(source)) {
@@ -30016,7 +30093,7 @@ function linkCitations(markdown, sourceOf, isFile) {
30016
30093
  for (const marker of answerMarkers(markdown, isFile)) {
30017
30094
  if (marker.kind === "number")
30018
30095
  continue;
30019
- const labels = marker.kind === "file" ? [{ label: basename15(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
30096
+ const labels = marker.kind === "file" ? [{ label: basename14(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
30020
30097
  let linked = false;
30021
30098
  const parts = labels.map(({ label, source }) => {
30022
30099
  const target = source === undefined ? undefined : openableTarget(source);
@@ -32142,10 +32219,7 @@ function renderScanReport(details) {
32142
32219
  const lines = [
32143
32220
  `dupey scanned ${details.scans.length} configured root(s).`,
32144
32221
  `families=${details.familyCount} exactDuplicates=${details.exactDuplicateCount} extractionErrors=${errors}`,
32145
- "Review exact families before cleanup; near/contains families are not safe deletion evidence.",
32146
- ...details.skippedRoots.length === 0 ? [] : [
32147
- `Not scanned (inside a cloud or network folder; scanning would download or re-read every file): ${details.skippedRoots.join(", ")}. Duplicates there are not reported, and an exact duplicate is only found against files an earlier refresh fingerprinted.`
32148
- ]
32222
+ "Review exact families before cleanup; near/contains families are not safe deletion evidence."
32149
32223
  ];
32150
32224
  let rendered = 0;
32151
32225
  let omitted = 0;
@@ -33123,6 +33197,56 @@ var init_packaged_skill = __esm(() => {
33123
33197
  PACKAGED_SKILL_ROOTS = ["skills", "internal-skills"];
33124
33198
  });
33125
33199
 
33200
+ // src/agent/pipelined-sync.ts
33201
+ class PipelinedSync {
33202
+ #sync;
33203
+ #covered = [];
33204
+ #dirty = false;
33205
+ #running;
33206
+ #failure;
33207
+ #stopped = false;
33208
+ #runs = 0;
33209
+ constructor(options) {
33210
+ this.#sync = options.sync;
33211
+ }
33212
+ async batch(batch) {
33213
+ this.#raise();
33214
+ this.#covered.push(...batch.virtualPaths);
33215
+ this.#dirty = true;
33216
+ if (!this.#stopped)
33217
+ this.#running ??= this.#loop();
33218
+ if (batch.final)
33219
+ await this.#running;
33220
+ this.#raise();
33221
+ return this.#stopped ? { stop: true } : undefined;
33222
+ }
33223
+ async settle() {
33224
+ this.#stopped = true;
33225
+ await this.#running;
33226
+ }
33227
+ #raise() {
33228
+ if (this.#failure !== undefined)
33229
+ throw this.#failure.error;
33230
+ }
33231
+ async#loop() {
33232
+ try {
33233
+ while (this.#dirty && !this.#stopped) {
33234
+ this.#dirty = false;
33235
+ const virtualPaths = this.#covered;
33236
+ this.#covered = [];
33237
+ this.#runs += 1;
33238
+ const outcome = await this.#sync({ index: this.#runs, virtualPaths });
33239
+ if (outcome?.stop === true)
33240
+ this.#stopped = true;
33241
+ }
33242
+ } catch (error) {
33243
+ this.#failure = { error };
33244
+ } finally {
33245
+ this.#running = undefined;
33246
+ }
33247
+ }
33248
+ }
33249
+
33126
33250
  // src/agent/query-decomposition.ts
33127
33251
  import { completeSimple as completeSimple2 } from "@earendil-works/pi-ai/compat";
33128
33252
  function buildDecompositionPrompt(query) {
@@ -33362,7 +33486,7 @@ var init_query_routing = __esm(() => {
33362
33486
  false: "Every part of the question is answered with a concrete, specific, consistent fact (a name, date, number, or definite statement)."
33363
33487
  });
33364
33488
  BASE_ROUTE_OPTIONS = {
33365
- local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. When unsure whether intrinsic knowledge suffices, choose local.",
33489
+ local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. A request to find, locate, open, or look up a document, file, slide deck, presentation, paper, note, message, or other material is local even when it names no owner ('find the slides about X', 'where is the Y presentation', 'Z 관련 자료 찾아줘'): such material lives in the user's own files and chats, and a well-known topic is not a reason to answer from general knowledge or the web. When unsure whether intrinsic knowledge suffices, choose local.",
33366
33490
  extra: "None of the other options fits and no search of the user's own records is needed first. Either the request is answerable without looking anything up — stable common knowledge, a definition, simple reasoning, or small talk — or it is a task for the assistant's tools that is not file organization and not AutoRAG configuration: committing code, pushing a branch, opening a pull request, running tests or a build, installing a package, running a script, writing or editing code. Programming work is extra even when the code names the user's files or folders ('write a script that lowercases every PNG', 'add this to my parser'): the assistant is asked to produce or change code, not to act on those files now. So is a request to run something (tests, a build, a script, a command) and report the outcome, even when what it runs lives in the user's own repository: running it is a task for the tool, not a search of the user's records. Never extra when the request refers to the user's own life, files, records, chats, calendar, or history (e.g. 'my', 'I', 'our', 'my girlfriend', 'my case', 'my routine'), and never extra when the answer depends on facts that may be recent, specific, or changing (current events, recent releases, prices, versions): search those locally instead. If any other option fits, do not choose extra.",
33367
33491
  organize: "The user asks the assistant to DO something to files or folders instead of finding information: move or copy files into folders, tidy or sort a folder, bulk or pattern rename (for example by date or number), rename files according to what their content says, convert files to another format, create folder structures, group files by type or topic, archive, or move to the trash ('tidy this up', 'move these into folders', 'bulk rename these files', 'rename the files to match their content', 'convert these to PDF', 'sort my Downloads folder'). It is about acting on files and folders, never about looking up what a document says or where something is. A question that only asks to find or read a file's content is local, not organize."
33368
33492
  };
@@ -34600,7 +34724,7 @@ var init_page_loader = __esm(() => {
34600
34724
  });
34601
34725
 
34602
34726
  // src/web/fetch/url-target.ts
34603
- import { extname as extname9 } from "node:path";
34727
+ import { extname as extname10 } from "node:path";
34604
34728
  function repairCollapsedScheme(value) {
34605
34729
  const m = value.match(/^(https?):\/(?!\/)/i);
34606
34730
  return m ? `${m[1]}://${value.slice(m[0].length)}` : value;
@@ -34619,7 +34743,7 @@ function getFilenameExtensionHint(filename) {
34619
34743
  const lower = filename.toLowerCase();
34620
34744
  if (lower.endsWith(".tar.gz"))
34621
34745
  return ".tar.gz";
34622
- return extname9(filename).toLowerCase();
34746
+ return extname10(filename).toLowerCase();
34623
34747
  }
34624
34748
  function getExtensionHint(url, contentDisposition) {
34625
34749
  if (contentDisposition) {
@@ -34796,7 +34920,7 @@ var init_convert2 = __esm(() => {
34796
34920
  });
34797
34921
 
34798
34922
  // src/web/fetch/feeds.ts
34799
- import { extname as extname10 } from "node:path";
34923
+ import { extname as extname11 } from "node:path";
34800
34924
  import { XMLParser as XMLParser3 } from "fast-xml-parser";
34801
34925
  function getHtmlAttribute(tag, attribute) {
34802
34926
  const pattern = new RegExp(`\\b${attribute}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'=<>]+))`, "i");
@@ -34859,7 +34983,7 @@ function extractDocumentLinks(html, baseUrl) {
34859
34983
  const href = getHtmlAttribute(tag, "href");
34860
34984
  if (!href)
34861
34985
  continue;
34862
- const ext = extname10(href).toLowerCase();
34986
+ const ext = extname11(href).toLowerCase();
34863
34987
  if (!CONVERTIBLE_EXTENSIONS2.has(ext))
34864
34988
  continue;
34865
34989
  const resolved = href.startsWith("http") ? href : new URL(href, baseUrl).href;
@@ -36527,7 +36651,7 @@ var init_web_search_tool = __esm(() => {
36527
36651
  // src/agent/agent.ts
36528
36652
  import { randomUUID as randomUUID7 } from "node:crypto";
36529
36653
  import { existsSync as existsSync34, watch as fsWatch, mkdirSync as mkdirSync24, realpathSync as realpathSync8, statSync as statSync11, writeFileSync as writeFileSync17 } from "node:fs";
36530
- import { basename as basename16, dirname as dirname27, join as join54, relative as relative8, resolve as resolve18, sep as sep10 } from "node:path";
36654
+ import { basename as basename15, dirname as dirname27, join as join54, relative as relative8, resolve as resolve18, sep as sep10 } from "node:path";
36531
36655
  import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
36532
36656
  import { getAgentDir as getAgentDir6, keyText } from "@earendil-works/pi-coding-agent";
36533
36657
  function describeDecisionModel(status) {
@@ -36962,10 +37086,20 @@ class AutoRAGAgent {
36962
37086
  }
36963
37087
  async scanDuplicateDocuments() {
36964
37088
  if (this.dupeyOptions === false) {
36965
- return { scans: [], skippedRoots: [], familyCount: 0, exactDuplicateCount: 0 };
37089
+ return { scans: [], familyCount: 0, exactDuplicateCount: 0 };
37090
+ }
37091
+ const cloudRoots = await this.cloudRoots();
37092
+ const scanRoots = [];
37093
+ for (const searchPath of this.searchPaths) {
37094
+ const scope = cloudScope(searchPath, cloudRoots);
37095
+ if (scope.insideCloudRoot)
37096
+ continue;
37097
+ scanRoots.push({
37098
+ path: searchPath,
37099
+ excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path))
37100
+ });
36966
37101
  }
36967
- const { targets, skipped } = planDupeyScans(this.searchPaths, await this.cloudRoots());
36968
- const scans = await Promise.all(targets.map(({ path, excludeDirs }) => scanWithDupey(path, {
37102
+ const scans = await Promise.all(scanRoots.map(({ path, excludeDirs }) => scanWithDupey(path, {
36969
37103
  ...this.dupeyOptions || {},
36970
37104
  ...excludeDirs.length === 0 ? {} : { excludeDirs }
36971
37105
  })));
@@ -36979,7 +37113,7 @@ class AutoRAGAgent {
36979
37113
  }
36980
37114
  return count + [...hashes.values()].reduce((sum, size) => sum + Math.max(0, size - 1), 0);
36981
37115
  }, 0);
36982
- return { scans, skippedRoots: skipped, familyCount, exactDuplicateCount };
37116
+ return { scans, familyCount, exactDuplicateCount };
36983
37117
  }
36984
37118
  loadMemoryContextFor(query) {
36985
37119
  if (this.memoryContextCache?.query === query)
@@ -38632,13 +38766,67 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38632
38766
  inFlight: true,
38633
38767
  lastStartedAt: startedAt
38634
38768
  };
38769
+ const syncAbort = new AbortController;
38770
+ const forwardAbort = () => syncAbort.abort(signal?.reason);
38771
+ signal?.addEventListener("abort", forwardAbort, { once: true });
38772
+ let minSyncPipeline;
38635
38773
  try {
38636
38774
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38775
+ const batchedMinSync = needsParsed && wants("minsync");
38776
+ const minSyncBatches = [];
38777
+ const minSyncBatchFailures = [];
38778
+ let minSyncSyncedBase = 0;
38779
+ let minSyncTotalBase = 0;
38780
+ minSyncPipeline = batchedMinSync ? new PipelinedSync({
38781
+ sync: async ({ index, virtualPaths }) => {
38782
+ syncAbort.signal.throwIfAborted();
38783
+ let syncTotal = minSyncTotalBase;
38784
+ const result = await this.syncMinSync(force && index === 1, syncAbort.signal, (minsyncProgress) => {
38785
+ syncTotal = minSyncTotalBase + minsyncProgress.total;
38786
+ progress = updateRefreshProgress(progress, {
38787
+ minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: syncTotal }
38788
+ });
38789
+ writeRefreshProgress(this.workspaceProjectRoot, progress);
38790
+ emitProgress({
38791
+ phase: "minsync",
38792
+ processed: minSyncSyncedBase + minsyncProgress.processed,
38793
+ total: syncTotal
38794
+ });
38795
+ });
38796
+ minSyncBatches.push(result);
38797
+ if (result.ok)
38798
+ minSyncBatchFailures.length = 0;
38799
+ else if (result.reason !== "lock-held") {
38800
+ for (const virtualPath of virtualPaths) {
38801
+ minSyncBatchFailures.push({
38802
+ code: "minsync-sync-failed",
38803
+ severity: "warning",
38804
+ message: `This file stayed at its previous index entry: the sync that covered it could not index it and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
38805
+ source: virtualPath
38806
+ });
38807
+ }
38808
+ }
38809
+ minSyncSyncedBase = syncTotal;
38810
+ minSyncTotalBase = syncTotal;
38811
+ return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
38812
+ }
38813
+ }) : undefined;
38814
+ const pipeline = minSyncPipeline;
38637
38815
  const summary = needsParsed ? await this.syncParsedMirrors(force, {
38638
38816
  ...opts?.retrySkipped === true ? { retrySkipped: true } : {},
38639
38817
  ...signal === undefined ? {} : { signal },
38818
+ ...pipeline === undefined ? {} : {
38819
+ onBatch: (batch) => {
38820
+ if (batch.final) {
38821
+ progress = updateRefreshProgress(progress, { phase: "minsync" });
38822
+ writeRefreshProgress(this.workspaceProjectRoot, progress);
38823
+ }
38824
+ return pipeline.batch(batch);
38825
+ }
38826
+ },
38640
38827
  onProgress: (parsedProgress) => {
38641
38828
  progress = updateRefreshProgress(progress, {
38829
+ phase: "parsed",
38642
38830
  sourceFiles: { total: parsedProgress.total },
38643
38831
  parsedCounts: {
38644
38832
  scanned: parsedProgress.processed,
@@ -38660,7 +38848,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38660
38848
  }) : await this.scanMirrorStaleness();
38661
38849
  signal?.throwIfAborted();
38662
38850
  progress = updateRefreshProgress(progress, {
38663
- phase: wants("minsync") ? "minsync" : wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
38851
+ phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
38664
38852
  sourceFiles: { total: summary.scanned },
38665
38853
  parsedCounts: {
38666
38854
  scanned: summary.scanned,
@@ -38670,24 +38858,12 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38670
38858
  }
38671
38859
  });
38672
38860
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38673
- const minsync = wants("minsync") ? await this.syncMinSync(force, signal, (minsyncProgress) => {
38674
- progress = updateRefreshProgress(progress, {
38675
- minsync: { synced: minsyncProgress.processed, total: minsyncProgress.total }
38676
- });
38677
- writeRefreshProgress(this.workspaceProjectRoot, progress);
38678
- emitProgress({
38679
- phase: "minsync",
38680
- processed: minsyncProgress.processed,
38681
- total: minsyncProgress.total
38682
- });
38683
- }) : undefined;
38684
- signal?.throwIfAborted();
38861
+ const minsync = batchedMinSync ? aggregateMinSyncBatches(minSyncBatches) : undefined;
38685
38862
  if (minsync !== undefined) {
38686
38863
  progress = updateRefreshProgress(progress, {
38687
- phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
38688
38864
  minsync: {
38689
- ...progress.minsync,
38690
38865
  synced: minsync.synced,
38866
+ total: Math.max(minSyncTotalBase, minsync.synced),
38691
38867
  ...minsync.filesFailed !== undefined && minsync.filesFailed.length > 0 ? { filesSkipped: minsync.filesFailed.length } : {}
38692
38868
  }
38693
38869
  });
@@ -38749,7 +38925,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38749
38925
  finishedAt: new Date().toISOString()
38750
38926
  });
38751
38927
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38752
- const minsyncDiagnostics = minSyncRefreshDiagnostics(minsync);
38928
+ const minsyncDiagnostics = [...minSyncRefreshDiagnostics(minsync), ...minSyncBatchFailures];
38753
38929
  const publicMinsync = minsync ? {
38754
38930
  ok: minsync.ok,
38755
38931
  synced: minsync.synced,
@@ -38783,6 +38959,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38783
38959
  ...fsearch !== undefined ? { fsearch } : {}
38784
38960
  };
38785
38961
  } catch (error) {
38962
+ syncAbort.abort(error);
38786
38963
  this.refreshState = {
38787
38964
  ...this.refreshState,
38788
38965
  lastOutcome: "failed",
@@ -38796,6 +38973,8 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38796
38973
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38797
38974
  throw error;
38798
38975
  } finally {
38976
+ signal?.removeEventListener("abort", forwardAbort);
38977
+ await minSyncPipeline?.settle();
38799
38978
  this.refreshState = {
38800
38979
  ...this.refreshState,
38801
38980
  inFlight: false,
@@ -39017,15 +39196,15 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
39017
39196
  const cloudRoots = await this.cloudRoots();
39018
39197
  const fingerprints = cloudFingerprintExclusions(loadCloudFingerprints(this.workspaceProjectRoot));
39019
39198
  for (const searchPath of this.searchPaths) {
39020
- const target = planDupeyScans([searchPath], cloudRoots).targets[0];
39199
+ const scope = cloudScope(searchPath, cloudRoots);
39021
39200
  let scan;
39022
- if (target === undefined) {
39201
+ if (scope.insideCloudRoot) {
39023
39202
  scan = { dir: searchPath, files: [], families: [], errors: [] };
39024
39203
  } else {
39025
39204
  try {
39026
39205
  scan = await scanWithDupey(searchPath, {
39027
39206
  ...this.dupeyOptions || {},
39028
- ...target.excludeDirs.length === 0 ? {} : { excludeDirs: target.excludeDirs }
39207
+ ...scope.nestedCloudRoots.length === 0 ? {} : { excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path)) }
39029
39208
  });
39030
39209
  } catch (error) {
39031
39210
  if (!(error instanceof DupeyCliError))
@@ -39554,6 +39733,12 @@ function fsearchRefreshDiagnostics(fsearch) {
39554
39733
  }
39555
39734
  ];
39556
39735
  }
39736
+ function aggregateMinSyncBatches(batches) {
39737
+ const last = batches.at(-1);
39738
+ if (last === undefined)
39739
+ return;
39740
+ return { ...last, synced: batches.reduce((total, batch) => total + batch.synced, 0) };
39741
+ }
39557
39742
  function minSyncRefreshDiagnostics(minsync) {
39558
39743
  if (!minsync)
39559
39744
  return [];
@@ -39584,7 +39769,7 @@ function toMinSyncReasonDiagnostic(reason) {
39584
39769
  function pinExcludedPath(path) {
39585
39770
  const resolvedPath = resolve18(path);
39586
39771
  try {
39587
- return join54(realpathSync8(dirname27(resolvedPath)), basename16(resolvedPath));
39772
+ return join54(realpathSync8(dirname27(resolvedPath)), basename15(resolvedPath));
39588
39773
  } catch {
39589
39774
  return resolvedPath;
39590
39775
  }