@autorag/librarian 2.13.6 → 2.13.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli/index.js CHANGED
@@ -5032,7 +5032,7 @@ function outdatedMinSyncVersion(binaryPath) {
5032
5032
  }
5033
5033
  return version !== undefined && !isSupportedMinSyncVersion(version) ? version : undefined;
5034
5034
  }
5035
- var MINSYNC_MIN_VERSION = "0.6.0", VERSION_PROBE_TIMEOUT_MS = 5000, VERSION_LINE, probeCache;
5035
+ var MINSYNC_MIN_VERSION = "0.7.1", VERSION_PROBE_TIMEOUT_MS = 5000, VERSION_LINE, probeCache;
5036
5036
  var init_version = __esm(() => {
5037
5037
  VERSION_LINE = /^minsync\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
5038
5038
  probeCache = new Map;
@@ -5276,7 +5276,7 @@ function isGitHubAsset(value) {
5276
5276
  function isRecord2(value) {
5277
5277
  return typeof value === "object" && value !== null;
5278
5278
  }
5279
- var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.6.0", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
5279
+ var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.7.3", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
5280
5280
  var init_installer = __esm(() => {
5281
5281
  init_atomic_write();
5282
5282
  init_process();
@@ -14567,7 +14567,8 @@ class WombatMethod {
14567
14567
  return matchesDatasourceScope(datasourceSourcePath("wombat", this.instanceId, hit.chunkId), options.scope);
14568
14568
  }).map((hit) => ({
14569
14569
  id: `wombat:${this.instanceId}:${hit.chunkId}`,
14570
- content: hit.snippet,
14570
+ content: `${sessionLabel(hit)}
14571
+ ${hit.snippet}`,
14571
14572
  source: datasourceSourcePath("wombat", this.instanceId, hit.chunkId),
14572
14573
  score: hit.score,
14573
14574
  metadata: {
@@ -14590,6 +14591,20 @@ class WombatMethod {
14590
14591
  })).slice(0, topK);
14591
14592
  }
14592
14593
  }
14594
+ function oneLine(value) {
14595
+ return value.replace(/\s+/gu, " ").trim();
14596
+ }
14597
+ function sessionLabel(hit) {
14598
+ const turns = hit.startTurn === undefined ? undefined : hit.endTurn === undefined || hit.endTurn === hit.startTurn ? `turn ${hit.startTurn}` : `turns ${hit.startTurn}-${hit.endTurn}`;
14599
+ const parts = [
14600
+ oneLine(hit.catalogKey),
14601
+ turns,
14602
+ hit.startedAt === undefined ? undefined : hit.startedAt.slice(0, 10),
14603
+ hit.title.length === 0 ? undefined : oneLine(hit.title),
14604
+ hit.cwd === undefined ? undefined : oneLine(hit.cwd)
14605
+ ].filter((part) => part !== undefined && part.length > 0);
14606
+ return `[${parts.join(" | ")}]`;
14607
+ }
14593
14608
  var WOMBAT_MODES, METHOD_TYPE;
14594
14609
  var init_methods7 = __esm(() => {
14595
14610
  init_errors2();
@@ -14702,7 +14717,7 @@ class WombatSkill {
14702
14717
  `);
14703
14718
  return {
14704
14719
  name: "datasource-wombat",
14705
- description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built.",
14720
+ description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built. Each hit names its session. Load this skill to open that session's full conversation or its original log with the tool calls that the search does not index.",
14706
14721
  content: [
14707
14722
  "# wombat coding-agent session datasource",
14708
14723
  "",
@@ -14716,10 +14731,10 @@ class WombatSkill {
14716
14731
  scopes,
14717
14732
  "",
14718
14733
  "The selectable retrieval modes are keyword (SPLADE), semantic, and hybrid. Scope arguments can only narrow trusted access. Source identifiers remain opaque and hierarchical.",
14719
- "Each result carries `agent`, `title`, `startedAt`, `startTurn`/`endTurn` and `sessionKey` (`<agent>:<session id>`).",
14734
+ "Every result starts with a label line, `[<sessionKey> | turns A-B | date | title | project]`, followed by the snippet. `sessionKey` is `<agent>:<session id>`.",
14720
14735
  "",
14721
14736
  "## Native CLI",
14722
- "The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the result's `sessionKey`:",
14737
+ "The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the `sessionKey` from the result's label:",
14723
14738
  "- `wombat show <sessionKey> [--turn N]`: the full cleaned conversation, or one turn.",
14724
14739
  "- `wombat trace <sessionKey> --turn N --context K`: the original session log around that turn, including the tool calls and outputs that the index leaves out.",
14725
14740
  "- `wombat chunk context <chunk id> --json`: the turns before and after a result.",
@@ -14873,6 +14888,7 @@ function common(config, workspaceRoot) {
14873
14888
  function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
14874
14889
  const skills = [];
14875
14890
  const unknown = [];
14891
+ const retired = [];
14876
14892
  for (const [name, raw] of Object.entries(config ?? {})) {
14877
14893
  if (raw === false)
14878
14894
  continue;
@@ -14884,6 +14900,10 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
14884
14900
  if (entry.enabled === false)
14885
14901
  continue;
14886
14902
  const templateName = entry.type ?? name;
14903
+ if (Object.hasOwn(RETIRED_TEMPLATES, templateName)) {
14904
+ retired.push(name);
14905
+ continue;
14906
+ }
14887
14907
  const builder = Object.hasOwn(BUILDERS, templateName) && datasourceSkillSupported(templateName) ? BUILDERS[templateName] : undefined;
14888
14908
  if (builder === undefined) {
14889
14909
  unknown.push(name);
@@ -14907,9 +14927,9 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
14907
14927
  }
14908
14928
  skills.push(typeof entry.description === "string" && entry.description.trim().length > 0 ? new DescribedDatasourceSkill(aliased, entry.description.trim()) : aliased);
14909
14929
  }
14910
- return { skills, unknown };
14930
+ return { skills, unknown, retired };
14911
14931
  }
14912
- var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES;
14932
+ var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES, RETIRED_TEMPLATES;
14913
14933
  var init_factory = __esm(() => {
14914
14934
  init_aliased_skill();
14915
14935
  init_config3();
@@ -15065,6 +15085,7 @@ var init_factory = __esm(() => {
15065
15085
  };
15066
15086
  PLATFORM_ONLY_SKILLS = { kakao: "darwin" };
15067
15087
  BUILTIN_DATASOURCE_SKILL_NAMES = builtinDatasourceSkillNames();
15088
+ RETIRED_TEMPLATES = { spotlight: true };
15068
15089
  });
15069
15090
 
15070
15091
  // src/language.ts
@@ -16297,7 +16318,7 @@ function buildAgentOptions(config, agentDir) {
16297
16318
  const embedder = createEmbedderFromConfig({ embedder: config.minSync?.embedder });
16298
16319
  opts.embedder = embedder;
16299
16320
  if (config.datasources !== undefined) {
16300
- const { skills, unknown } = buildDatasourceSkills(config.datasources, config.workspacePath, {
16321
+ const { skills, unknown, retired } = buildDatasourceSkills(config.datasources, config.workspacePath, {
16301
16322
  embedder,
16302
16323
  discrawl: async () => discrawlEmbeddingRuntimeFor(await embedder.describeEndpoint()),
16303
16324
  cliEndpoint: embeddingEndpointSource(embedder),
@@ -16310,16 +16331,27 @@ function buildAgentOptions(config, agentDir) {
16310
16331
  });
16311
16332
  if (skills.length > 0)
16312
16333
  opts.datasourceSkills = skills;
16334
+ const diagnostics = [];
16335
+ if (retired.length > 0) {
16336
+ const names = retired.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
16337
+ diagnostics.push({
16338
+ code: "retired-datasource-skill",
16339
+ severity: "warning",
16340
+ message: `Datasource config ignored because the Spotlight datasource was removed: ${names.join(", ")}. Delete these entries; use fsearch/mirror search (searchPaths) instead.`,
16341
+ source: "datasources"
16342
+ });
16343
+ }
16313
16344
  if (unknown.length > 0) {
16314
- const safeNames = unknown.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
16315
- const diagnostic = {
16345
+ const names = unknown.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
16346
+ diagnostics.push({
16316
16347
  code: "unknown-datasource-skill",
16317
16348
  severity: "warning",
16318
- message: `Unknown datasource skill(s) in config were skipped: ${safeNames.join(", ")}`,
16349
+ message: `Unknown datasource skill(s) in config were skipped: ${names.join(", ")}`,
16319
16350
  source: "datasources"
16320
- };
16321
- opts.startupDiagnostics = [diagnostic];
16351
+ });
16322
16352
  }
16353
+ if (diagnostics.length > 0)
16354
+ opts.startupDiagnostics = diagnostics;
16323
16355
  }
16324
16356
  return opts;
16325
16357
  }
@@ -17832,11 +17864,49 @@ var init_matrix = __esm(() => {
17832
17864
  };
17833
17865
  });
17834
17866
 
17867
+ // src/dupey/version.ts
17868
+ function parseDupeyVersion(output) {
17869
+ return VERSION_LINE2.exec(output)?.[1];
17870
+ }
17871
+ function isSupportedDupeyVersion(version) {
17872
+ const have = version.split(".").map((part) => Number.parseInt(part, 10));
17873
+ const need = DUPEY_MIN_VERSION.split(".").map((part) => Number.parseInt(part, 10));
17874
+ for (let index = 0;index < need.length; index += 1) {
17875
+ const left = have[index] ?? 0;
17876
+ const right = need[index] ?? 0;
17877
+ if (left !== right)
17878
+ return left > right;
17879
+ }
17880
+ return true;
17881
+ }
17882
+ var DUPEY_MIN_VERSION = "0.2.0", VERSION_LINE2;
17883
+ var init_version2 = __esm(() => {
17884
+ VERSION_LINE2 = /^dupey\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
17885
+ });
17886
+
17835
17887
  // src/dupey/cli.ts
17836
17888
  import { spawn as spawn14 } from "node:child_process";
17837
17889
  import { resolve as resolve9 } from "node:path";
17838
- function runDupey(args, options) {
17839
- return options.run ? options.run(args) : spawnDupey(options.executable ?? "dupey", args, options);
17890
+ function checkDupeyVersion(executable, options) {
17891
+ const cached = versionChecks.get(executable);
17892
+ if (cached !== undefined)
17893
+ return cached;
17894
+ const check = spawnDupey(executable, ["--version"], options).then((stdout) => {
17895
+ const version = parseDupeyVersion(stdout);
17896
+ if (version !== undefined && !isSupportedDupeyVersion(version)) {
17897
+ throw new DupeyCliError(`dupey ${version} at ${executable} is older than the ${DUPEY_MIN_VERSION} AutoRAG requires; run \`cargo install dupey\` to update it.`);
17898
+ }
17899
+ });
17900
+ check.catch(() => versionChecks.delete(executable));
17901
+ versionChecks.set(executable, check);
17902
+ return check;
17903
+ }
17904
+ async function runDupey(args, options) {
17905
+ if (options.run)
17906
+ return options.run(args);
17907
+ const executable = options.executable ?? "dupey";
17908
+ await checkDupeyVersion(executable, options);
17909
+ return spawnDupey(executable, args, options);
17840
17910
  }
17841
17911
  async function scanWithDupey(dir, options = {}) {
17842
17912
  const absoluteDir = resolve9(dir);
@@ -17970,28 +18040,26 @@ function spawnDupey(executable, args, options) {
17970
18040
  });
17971
18041
  });
17972
18042
  }
17973
- var DupeyCliError, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
18043
+ var DupeyCliError, versionChecks, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
17974
18044
  var init_cli2 = __esm(() => {
17975
18045
  init_portable_spawn();
18046
+ init_version2();
17976
18047
  DupeyCliError = class DupeyCliError extends Error {
17977
18048
  constructor(message) {
17978
18049
  super(message);
17979
18050
  this.name = "DupeyCliError";
17980
18051
  }
17981
18052
  };
18053
+ versionChecks = new Map;
17982
18054
  });
17983
18055
 
17984
18056
  // src/dupey/filter.ts
17985
18057
  import { stat as stat2 } from "node:fs/promises";
17986
18058
  import { resolve as resolve10 } from "node:path";
17987
- function extractedNoText(file) {
17988
- const signals = file.signals;
17989
- return typeof signals === "object" && signals !== null && "chars" in signals && signals.chars === 0;
17990
- }
17991
18059
  async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerprints) {
17992
18060
  const byHash = new Map;
17993
18061
  for (const file of scan.files) {
17994
- if (typeof file.content_hash !== "string" || file.content_hash.length === 0 || extractedNoText(file))
18062
+ if (typeof file.content_hash !== "string" || file.content_hash.length === 0)
17995
18063
  continue;
17996
18064
  const path = resolve10(root, file.path);
17997
18065
  const group = byHash.get(file.content_hash) ?? [];
@@ -18000,7 +18068,7 @@ async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerp
18000
18068
  }
18001
18069
  const fingerprintModified = new Map;
18002
18070
  for (const [path, fingerprint] of fingerprints ?? []) {
18003
- if (fingerprint.exact.length === 0 || fingerprint.chars === 0)
18071
+ if (fingerprint.exact.length === 0)
18004
18072
  continue;
18005
18073
  fingerprintModified.set(path, fingerprint.modified);
18006
18074
  const group = byHash.get(fingerprint.exact) ?? [];
@@ -18067,1397 +18135,1368 @@ async function rankExactCandidates(paths, dupeyRanking, fingerprintModified, err
18067
18135
  }
18068
18136
  var init_filter = () => {};
18069
18137
 
18070
- // src/filesystem/cloud-placeholder.ts
18071
- import { execFile as execFile2 } from "node:child_process";
18072
- import { realpathSync as realpathSync3 } from "node:fs";
18073
- import { lstat, readdir } from "node:fs/promises";
18074
- import { homedir as homedir6 } from "node:os";
18075
- import { dirname as dirname15, join as join28 } from "node:path";
18076
- import { promisify } from "node:util";
18077
- function homeCloudRoots(home = homedir6()) {
18078
- const normalizedHome = home.replaceAll("\\", "/").replace(/\/+$/, "");
18079
- return [`${normalizedHome}/Library/CloudStorage`, `${normalizedHome}/Library/Mobile Documents`];
18080
- }
18081
- function datalessFromDarwinFlags(flags) {
18082
- return Number.isFinite(flags) && (flags >>> 0 & SF_DATALESS) !== 0;
18138
+ // src/dupey/index.ts
18139
+ var init_dupey = __esm(() => {
18140
+ init_cli2();
18141
+ init_filter();
18142
+ });
18143
+
18144
+ // src/convert/relate.ts
18145
+ async function convertAndRelate(request, options = {}) {
18146
+ const result = await convertWithFilee(request, options.filee);
18147
+ if (options.dupey === false)
18148
+ return { result, links: [] };
18149
+ const dupeyOptions = options.dupey ?? {};
18150
+ const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
18151
+ const links = await Promise.all(pairs.map(async ({ source, output }) => {
18152
+ try {
18153
+ return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
18154
+ } catch (error) {
18155
+ if (!(error instanceof DupeyCliError))
18156
+ throw error;
18157
+ return { source, output, dupey: { relation: "unavailable", reason: error.message } };
18158
+ }
18159
+ }));
18160
+ return { result, links };
18083
18161
  }
18084
- function datalessFromWindowsAttributes(attributes) {
18085
- return Number.isFinite(attributes) && (attributes & WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) !== 0;
18162
+ var init_relate = __esm(() => {
18163
+ init_dupey();
18164
+ init_cli();
18165
+ });
18166
+
18167
+ // src/convert/index.ts
18168
+ var init_convert = __esm(() => {
18169
+ init_cli();
18170
+ init_installer2();
18171
+ init_matrix();
18172
+ init_relate();
18173
+ });
18174
+
18175
+ // src/datasource/diagnostics.ts
18176
+ function mapDatasourceDiagnostic(ds) {
18177
+ return {
18178
+ code: datasourceCodeToRetrieval(ds.code),
18179
+ severity: ds.severity,
18180
+ message: `[datasource:${ds.code}] ${ds.message}`,
18181
+ source: ds.source ?? (ds.instanceId ? `datasource:${ds.instanceId}` : "datasource")
18182
+ };
18086
18183
  }
18087
- function cloudProviderFromPath(target) {
18088
- const lower = target.toLowerCase();
18089
- if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
18090
- return "google-drive";
18091
- if (lower.includes("onedrive"))
18092
- return "onedrive";
18093
- if (lower.includes("dropbox"))
18094
- return "dropbox";
18095
- if (lower.includes("mobile documents") || lower.includes("clouddocs"))
18096
- return "icloud";
18097
- return "file-provider";
18184
+ function mapDatasourceDiagnostics(diagnostics) {
18185
+ return diagnostics.map(mapDatasourceDiagnostic);
18098
18186
  }
18099
- function cloudProviderFromDomain(domain) {
18100
- const lower = domain.toLowerCase();
18101
- if (lower.includes("clouddocs") || lower.includes("icloud"))
18102
- return "icloud";
18103
- if (lower.includes("google"))
18104
- return "google-drive";
18105
- if (lower.includes("onedrive"))
18106
- return "onedrive";
18107
- if (lower.includes("dropbox"))
18108
- return "dropbox";
18109
- return "file-provider";
18187
+ function datasourceCodeToRetrieval(code) {
18188
+ switch (code) {
18189
+ case "datasource-unavailable":
18190
+ case "datasource-cli-error":
18191
+ case "datasource-empty":
18192
+ case "datasource-rate-limited":
18193
+ case "datasource-auth-error":
18194
+ case "datasource-embedding-egress-rejected":
18195
+ case "datasource-index-failed":
18196
+ case "datasource-index-partial":
18197
+ case "datasource-permission-denied":
18198
+ case "datasource-remote-embedding-rejected":
18199
+ return "retrieval-method-failed";
18200
+ }
18110
18201
  }
18111
- function normalizeCloudPath(value) {
18112
- let normalized = value.replaceAll("\\", "/");
18113
- if (normalized.length > 1)
18114
- normalized = normalized.replace(/\/+$/u, "");
18115
- return normalized;
18202
+
18203
+ // src/datasource/retry.ts
18204
+ import { setTimeout as delay2 } from "node:timers/promises";
18205
+ function datasourceRetryDelayMs(retry, policy) {
18206
+ const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
18207
+ return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
18116
18208
  }
18117
- function isPathInside(target, parent) {
18118
- const normalizedParent = normalizeCloudPath(parent);
18119
- if (normalizedParent === "")
18120
- return false;
18121
- const left = normalizeCloudPath(target);
18122
- const [a, b] = process.platform === "win32" ? [left.toLowerCase(), normalizedParent.toLowerCase()] : [left, normalizedParent];
18123
- return a === b || a.startsWith(`${b}/`);
18209
+ async function indexWithRetry(skill, options = {}, signal) {
18210
+ const policy = {
18211
+ maxRetries: options.maxRetries ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxRetries,
18212
+ baseDelayMs: options.baseDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.baseDelayMs,
18213
+ maxDelayMs: options.maxDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxDelayMs
18214
+ };
18215
+ const sleep = options.sleep ?? ((ms) => delay2(ms, undefined, signal === undefined ? undefined : { signal }));
18216
+ let lastError = "";
18217
+ for (let attempt = 1;; attempt += 1) {
18218
+ const result = await runIndex(skill, signal);
18219
+ signal?.throwIfAborted();
18220
+ if (result.ok) {
18221
+ if (attempt === 1)
18222
+ return result;
18223
+ const recovered = {
18224
+ code: "datasource-index-failed",
18225
+ severity: "info",
18226
+ message: `recovered after ${attempt} attempts; last error: ${lastError}`,
18227
+ instanceId: result.instanceId,
18228
+ source: result.skill
18229
+ };
18230
+ return { ...result, attempts: attempt, diagnostics: [...result.diagnostics, recovered] };
18231
+ }
18232
+ lastError = result.message ?? result.error ?? "datasource-index-failed";
18233
+ if (attempt > policy.maxRetries) {
18234
+ if (attempt === 1)
18235
+ return { ...result, attempts: attempt };
18236
+ const suffix = ` (gave up after ${attempt} attempts)`;
18237
+ const last = result.diagnostics.at(-1);
18238
+ return {
18239
+ ...result,
18240
+ attempts: attempt,
18241
+ message: `${lastError}${suffix}`,
18242
+ diagnostics: last === undefined ? [
18243
+ {
18244
+ code: result.code ?? "datasource-index-failed",
18245
+ severity: "error",
18246
+ message: `${lastError}${suffix}`,
18247
+ instanceId: result.instanceId,
18248
+ source: result.skill
18249
+ }
18250
+ ] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
18251
+ };
18252
+ }
18253
+ await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
18254
+ signal?.throwIfAborted();
18255
+ }
18124
18256
  }
18125
- function canonicalPath(value) {
18257
+ async function runIndex(skill, signal) {
18126
18258
  try {
18127
- return realpathSync3(value);
18128
- } catch {
18129
- return value;
18259
+ return await skill.index(signal === undefined ? undefined : { signal });
18260
+ } catch (error) {
18261
+ signal?.throwIfAborted();
18262
+ const descriptor = skill.describe();
18263
+ const message = error instanceof Error ? error.message : "Datasource indexing failed.";
18264
+ const failed = {
18265
+ ok: false,
18266
+ instanceId: descriptor.instanceId ?? "default",
18267
+ skill: descriptor.name,
18268
+ indexedAt: Date.now(),
18269
+ diagnostics: [
18270
+ {
18271
+ code: "datasource-index-failed",
18272
+ severity: "error",
18273
+ message,
18274
+ source: descriptor.name,
18275
+ instanceId: descriptor.instanceId
18276
+ }
18277
+ ],
18278
+ error: "datasource-index-failed",
18279
+ code: "datasource-index-failed",
18280
+ message
18281
+ };
18282
+ return failed;
18130
18283
  }
18131
18284
  }
18132
- function canonicalPathMapper(searchRoots) {
18133
- const forms = searchRoots.map((root) => ({ raw: normalizeCloudPath(root), canonical: normalizeCloudPath(canonicalPath(root)) })).filter(({ raw, canonical }) => raw !== "" && raw !== canonical).sort((left, right) => right.raw.length - left.raw.length);
18134
- if (forms.length === 0)
18135
- return (path) => path;
18136
- return (path) => {
18137
- const normalized = normalizeCloudPath(path);
18138
- for (const { raw, canonical } of forms) {
18139
- if (isPathInside(normalized, raw))
18140
- return `${canonical}${normalized.slice(raw.length)}`;
18141
- }
18142
- return path;
18285
+ var DEFAULT_DATASOURCE_RETRY_POLICY;
18286
+ var init_retry = __esm(() => {
18287
+ DEFAULT_DATASOURCE_RETRY_POLICY = {
18288
+ maxRetries: 2,
18289
+ baseDelayMs: 1000,
18290
+ maxDelayMs: 30000
18143
18291
  };
18292
+ });
18293
+
18294
+ // src/everything/bundle.ts
18295
+ import { createHash as createHash11 } from "node:crypto";
18296
+ import { existsSync as existsSync18, mkdirSync as mkdirSync15, readFileSync as readFileSync18, rmSync as rmSync7, writeFileSync as writeFileSync10 } from "node:fs";
18297
+ import { dirname as dirname15, join as join28 } from "node:path";
18298
+ import { fileURLToPath as fileURLToPath2 } from "node:url";
18299
+ import JSZip from "jszip";
18300
+ function resolveEverythingBundleDir() {
18301
+ let dir = dirname15(fileURLToPath2(import.meta.url));
18302
+ for (let depth = 0;depth < 6; depth += 1) {
18303
+ const candidate = join28(dir, "vendor", "everything");
18304
+ if (existsSync18(join28(candidate, MANIFEST_FILENAME)))
18305
+ return candidate;
18306
+ const parent = dirname15(dir);
18307
+ if (parent === dir)
18308
+ break;
18309
+ dir = parent;
18310
+ }
18311
+ return;
18144
18312
  }
18145
- function cloudScope(searchRoot, cloudRoots) {
18146
- const canonicalSearchRoot = canonicalPath(searchRoot);
18147
- const roots = cloudRoots.map((root) => ({ path: canonicalPath(root.path), kind: root.kind }));
18148
- return {
18149
- insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
18150
- nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
18151
- };
18313
+ function everythingArch(arch) {
18314
+ if (arch === "x64" || arch === "arm64")
18315
+ return arch;
18316
+ return;
18152
18317
  }
18153
- function parseReleaseOutcome(stdout) {
18154
- let parsed;
18318
+ function everythingBinaryPaths(root, version, arch) {
18319
+ const dir = join28(root, ".autorag", "everything", version, arch);
18320
+ return { everythingPath: join28(dir, "everything.exe"), esPath: join28(dir, "es.exe") };
18321
+ }
18322
+ function loadEverythingBundleManifest(bundleDir) {
18323
+ return JSON.parse(readFileSync18(join28(bundleDir, MANIFEST_FILENAME), "utf8"));
18324
+ }
18325
+ async function ensureEverythingBinaries(options) {
18326
+ const platform = options.platform ?? process.platform;
18327
+ const rawArch = options.arch ?? process.arch;
18328
+ if (platform !== "win32") {
18329
+ return {
18330
+ ok: false,
18331
+ reason: "unsupported-platform",
18332
+ message: `Everything is Windows-only; this host is ${platform}.`
18333
+ };
18334
+ }
18335
+ const arch = everythingArch(rawArch);
18336
+ const bundleDir = options.bundleDir ?? resolveEverythingBundleDir();
18337
+ if (bundleDir === undefined) {
18338
+ return {
18339
+ ok: false,
18340
+ reason: "bundle-missing",
18341
+ message: `Bundled Everything assets (vendor/everything/${MANIFEST_FILENAME}) were not found above ${dirname15(fileURLToPath2(import.meta.url))}.`
18342
+ };
18343
+ }
18344
+ let manifest;
18155
18345
  try {
18156
- parsed = JSON.parse(stdout.trim());
18157
- } catch {
18158
- return;
18346
+ manifest = options.manifest ?? loadEverythingBundleManifest(bundleDir);
18347
+ } catch (error) {
18348
+ return { ok: false, reason: "bundle-missing", message: errorMessage2(error) };
18159
18349
  }
18160
- if (typeof parsed !== "object" || parsed === null || !("ok" in parsed) || typeof parsed.ok !== "boolean") {
18161
- return;
18350
+ const assets = arch === undefined ? undefined : manifest.architectures[arch];
18351
+ if (arch === undefined || assets === undefined) {
18352
+ return {
18353
+ ok: false,
18354
+ reason: "unsupported-platform",
18355
+ message: `No bundled Everything build for Windows architecture ${rawArch}; bundled: ${Object.keys(manifest.architectures).join(", ")}.`
18356
+ };
18162
18357
  }
18163
- return {
18164
- ok: parsed.ok,
18165
- ..."isUploaded" in parsed ? { isUploaded: parsed.isUploaded } : {},
18166
- ..."isUploading" in parsed ? { isUploading: parsed.isUploading } : {},
18167
- ..."downloadingStatus" in parsed ? { downloadingStatus: parsed.downloadingStatus } : {}
18168
- };
18169
- }
18170
- function describeReleaseRefusal(outcome) {
18171
- if (outcome.isUploading === true)
18172
- return "The file is still uploading to the cloud, so it cannot be released yet.";
18173
- if (outcome.isUploaded === false)
18174
- return "The file has not finished uploading to the cloud, so it cannot be released yet.";
18175
- const status = typeof outcome.downloadingStatus === "string" ? outcome.downloadingStatus : "unknown";
18176
- return `The cloud provider declined to release the local copy (download status ${status}).`;
18358
+ const { everythingPath, esPath } = everythingBinaryPaths(options.root, manifest.everythingVersion, arch);
18359
+ const everythingCached = fileHasSha256(everythingPath, assets.everythingSha256);
18360
+ const esCached = fileHasSha256(esPath, assets.esSha256);
18361
+ if (everythingCached && esCached) {
18362
+ return { ok: true, everythingPath, esPath, source: "cached" };
18363
+ }
18364
+ if ((options.mode ?? "install") === "cached") {
18365
+ const state = everythingCached || esCached ? "are incomplete or do not match their pinned SHA-256 digests" : "are not installed";
18366
+ return {
18367
+ ok: false,
18368
+ reason: "not-installed",
18369
+ message: `Everything ${manifest.everythingVersion} (${arch}) binaries ${state} under ${dirname15(everythingPath)}; run \`autorag refresh\` to (re)install and index them.`
18370
+ };
18371
+ }
18372
+ try {
18373
+ mkdirSync15(dirname15(everythingPath), { recursive: true });
18374
+ const everythingBytes = await extractVerified(join28(bundleDir, assets.everythingArchive), assets.everythingArchiveSha256, assets.everythingMember, assets.everythingSha256);
18375
+ const esBytes = await extractVerified(join28(bundleDir, assets.esArchive), assets.esArchiveSha256, assets.esMember, assets.esSha256);
18376
+ writeAtomic(everythingPath, everythingBytes);
18377
+ writeAtomic(esPath, esBytes);
18378
+ } catch (error) {
18379
+ return { ok: false, reason: "install-failed", message: errorMessage2(error) };
18380
+ }
18381
+ return { ok: true, everythingPath, esPath, source: "installed" };
18177
18382
  }
18178
- function describeReleaseHelperFailure(error) {
18179
- const failure = error;
18180
- const parts = [];
18181
- if (failure.killed === true || typeof failure.signal === "string") {
18182
- parts.push(`the helper was killed${typeof failure.signal === "string" ? ` by ${failure.signal}` : ""}`);
18183
- } else if (typeof failure.code === "number") {
18184
- parts.push(`the helper exited with code ${failure.code}`);
18383
+ async function extractVerified(archivePath, archiveSha256, member, memberSha256) {
18384
+ const archive = readFileSync18(archivePath);
18385
+ const actualArchive = sha256(archive);
18386
+ if (actualArchive !== archiveSha256) {
18387
+ throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
18185
18388
  }
18186
- const stderr = typeof failure.stderr === "string" ? failure.stderr.trim() : "";
18187
- if (stderr !== "")
18188
- parts.push(stderr);
18189
- if (parts.length === 0)
18190
- parts.push(error instanceof Error ? error.message : String(error));
18191
- return `osascript could not release the local copy: ${parts.join("; ")}`;
18389
+ const zip = await JSZip.loadAsync(archive);
18390
+ const entry = zip.file(member);
18391
+ if (entry === null)
18392
+ throw new Error(`${archivePath} does not contain ${member}`);
18393
+ const bytes = await entry.async("nodebuffer");
18394
+ const actualMember = sha256(bytes);
18395
+ if (actualMember !== memberSha256) {
18396
+ throw new Error(`${member} from ${archivePath} SHA-256 ${actualMember} does not match pinned ${memberSha256}`);
18397
+ }
18398
+ return bytes;
18192
18399
  }
18193
- async function defaultDarwinReadDir(path) {
18194
- const entries = await readdir(path, { withFileTypes: true });
18195
- return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
18400
+ function fileHasSha256(path, expected) {
18401
+ return existsSync18(path) && sha256(readFileSync18(path)) === expected;
18196
18402
  }
18197
- function parseDarwinMounts(stdout) {
18198
- const mounts = [];
18199
- for (const line of stdout.split(`
18200
- `)) {
18201
- if (line.trim() === "")
18202
- continue;
18203
- const open = line.lastIndexOf(" (");
18204
- if (open <= 0 || !line.endsWith(")"))
18205
- continue;
18206
- const head = line.slice(0, open);
18207
- const on = head.indexOf(" on ");
18208
- if (on <= 0)
18209
- continue;
18210
- const mountPoint = head.slice(on + " on ".length);
18211
- if (mountPoint === "")
18212
- continue;
18213
- const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
18214
- mounts.push({
18215
- device: head.slice(0, on),
18216
- mountPoint,
18217
- fileSystem: fields[0] ?? "",
18218
- options: fields.slice(1)
18219
- });
18403
+ function writeAtomic(path, bytes) {
18404
+ const temp = `${path}.${process.pid}.tmp`;
18405
+ try {
18406
+ writeFileSync10(temp, bytes);
18407
+ renameWithRetry(temp, path);
18408
+ } finally {
18409
+ rmSync7(temp, { force: true });
18220
18410
  }
18221
- return mounts;
18222
18411
  }
18223
- function networkMountPoints(mounts) {
18224
- const points = [];
18225
- const seen = new Set;
18226
- for (const mount of mounts) {
18227
- if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
18228
- continue;
18229
- if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
18230
- continue;
18231
- if (seen.has(mount.mountPoint))
18232
- continue;
18233
- seen.add(mount.mountPoint);
18234
- points.push(mount.mountPoint);
18235
- }
18236
- return points;
18412
+ function sha256(bytes) {
18413
+ return createHash11("sha256").update(bytes).digest("hex");
18237
18414
  }
18238
- function parseXattrValues(stdout, candidates) {
18239
- const values = new Map;
18240
- if (candidates.length === 0)
18241
- return values;
18242
- const ordered = [...candidates].sort((left, right) => right.length - left.length);
18243
- for (const line of stdout.split(`
18244
- `)) {
18245
- if (line === "")
18246
- continue;
18247
- for (const candidate of ordered) {
18248
- if (line.startsWith(`${candidate}: `)) {
18249
- values.set(candidate, line.slice(candidate.length + 2));
18250
- break;
18251
- }
18252
- }
18253
- }
18254
- return values;
18415
+ function errorMessage2(error) {
18416
+ return error instanceof Error ? error.message : String(error);
18417
+ }
18418
+ var MANIFEST_FILENAME = "manifest.json";
18419
+ var init_bundle = __esm(() => {
18420
+ init_atomic_write();
18421
+ });
18422
+
18423
+ // src/filesystem/path-exclusions.ts
18424
+ import { posix as posix3, win32 as win322 } from "node:path";
18425
+ function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
18426
+ const paths = platform === "win32" ? win322 : posix3;
18427
+ const normalize = (path) => {
18428
+ const absolute = paths.resolve(path);
18429
+ return platform === "win32" ? absolute.toLowerCase() : absolute;
18430
+ };
18431
+ const excluded = excludedPaths.map((path) => {
18432
+ const absolute = normalize(path);
18433
+ return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
18434
+ });
18435
+ return (path) => {
18436
+ const absolute = normalize(path);
18437
+ return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
18438
+ };
18255
18439
  }
18440
+ var init_path_exclusions = () => {};
18256
18441
 
18257
- class DarwinCloudProbe {
18258
- exec;
18259
- readDir;
18260
- home;
18261
- constructor(options = {}) {
18262
- this.exec = options.exec ?? defaultCloudExec;
18263
- this.readDir = options.readDir ?? defaultDarwinReadDir;
18264
- this.home = options.home ?? homedir6();
18442
+ // src/filesystem/shared-exclusive-queue.ts
18443
+ class SharedExclusiveQueue {
18444
+ sharedSinceExclusive = [];
18445
+ lastExclusive = Promise.resolve();
18446
+ shared(task) {
18447
+ const run = this.lastExclusive.then(task);
18448
+ const settled = run.then(() => {
18449
+ return;
18450
+ }, () => {
18451
+ return;
18452
+ });
18453
+ this.sharedSinceExclusive.push(settled);
18454
+ return run;
18265
18455
  }
18266
- async cloudRoots(searchRoots = []) {
18267
- const roots = [];
18268
- const seen = new Set;
18269
- const add = (path, kind) => {
18270
- const canonical = canonicalPath(path);
18271
- const key = normalizeCloudPath(canonical);
18272
- if (key === "" || seen.has(key))
18273
- return;
18274
- seen.add(key);
18275
- roots.push({ path: canonical, kind });
18276
- };
18277
- const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
18278
- for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
18279
- if (entry.isDirectory())
18280
- add(join28(cloudStorage, entry.name), "file-provider");
18281
- }
18282
- if (await this.tryReadDir(mobileDocuments) !== undefined)
18283
- add(mobileDocuments, "file-provider");
18284
- for (const mountPoint of await this.networkMountPoints())
18285
- add(mountPoint, "network");
18286
- for (const directory of await this.fileProviderDirectories(searchRoots))
18287
- add(directory, "file-provider");
18288
- return roots;
18456
+ exclusive(task) {
18457
+ const earlier = Promise.all([this.lastExclusive, ...this.sharedSinceExclusive]);
18458
+ const run = earlier.then(task);
18459
+ const settled = run.then(() => {
18460
+ return;
18461
+ }, () => {
18462
+ return;
18463
+ });
18464
+ this.sharedSinceExclusive = [];
18465
+ this.lastExclusive = settled;
18466
+ return run;
18289
18467
  }
18290
- async networkMountPoints() {
18291
- try {
18292
- const { stdout } = await this.exec("/sbin/mount", [], { timeoutMs: 1e4, maxBuffer: 4 * 1024 * 1024 });
18293
- return networkMountPoints(parseDarwinMounts(stdout));
18294
- } catch {
18295
- return [];
18296
- }
18468
+ }
18469
+
18470
+ // src/everything/client.ts
18471
+ import { spawn as spawn15 } from "node:child_process";
18472
+ import { createHash as createHash12 } from "node:crypto";
18473
+ import { mkdirSync as mkdirSync16, writeFileSync as writeFileSync11 } from "node:fs";
18474
+ import { join as join29, resolve as resolve11 } from "node:path";
18475
+ import { setTimeout as sleep2 } from "node:timers/promises";
18476
+ function everythingInstanceName(root) {
18477
+ return `autorag-${createHash12("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
18478
+ }
18479
+ function buildEverythingIni(input) {
18480
+ const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
18481
+ const perFolder = (value) => input.folders.map(() => value).join(",");
18482
+ return [
18483
+ "[Everything]",
18484
+ "app_data=0",
18485
+ "run_as_admin=0",
18486
+ "run_in_background=1",
18487
+ "show_tray_icon=0",
18488
+ "show_in_taskbar=0",
18489
+ "check_for_updates_on_startup=0",
18490
+ "beta_updates=0",
18491
+ "language=1033",
18492
+ "allow_http_server=0",
18493
+ "allow_etp_server=0",
18494
+ "search_history_enabled=0",
18495
+ "run_history_enabled=0",
18496
+ "auto_include_fixed_volumes=0",
18497
+ "auto_include_removable_volumes=0",
18498
+ "auto_include_fixed_refs_volumes=0",
18499
+ "auto_include_removable_refs_volumes=0",
18500
+ "ntfs_volume_paths=",
18501
+ "ntfs_volume_includes=",
18502
+ "refs_volume_paths=",
18503
+ "refs_volume_includes=",
18504
+ "index_size=1",
18505
+ "index_date_modified=1",
18506
+ `folders=${list(input.folders)}`,
18507
+ `folder_monitor_changes=${perFolder("1")}`,
18508
+ `folder_rescan_if_full_list=${perFolder("1")}`,
18509
+ `folder_update_types=${perFolder("0")}`,
18510
+ "exclude_list_enabled=1",
18511
+ `exclude_folders=${list(input.excludeFolders)}`,
18512
+ ""
18513
+ ].join(`\r
18514
+ `);
18515
+ }
18516
+ function buildEverythingSearchArgs(instance, request, timeoutMs) {
18517
+ const args = [
18518
+ "-argv",
18519
+ "-instance",
18520
+ instance,
18521
+ "-cp",
18522
+ "65001",
18523
+ "-json",
18524
+ "-full-path-and-name",
18525
+ "-size",
18526
+ "-date-modified",
18527
+ "-date-format",
18528
+ "1"
18529
+ ];
18530
+ if (timeoutMs !== undefined)
18531
+ args.push("-timeout", String(timeoutMs));
18532
+ if (request.matchCase)
18533
+ args.push("-case");
18534
+ if (request.matchPath)
18535
+ args.push("-match-path");
18536
+ if (request.wholeWord)
18537
+ args.push("-whole-word");
18538
+ if (request.kind === "files")
18539
+ args.push("/a-d");
18540
+ if (request.kind === "folders")
18541
+ args.push("/ad");
18542
+ if (request.path !== undefined)
18543
+ args.push("-path", request.path);
18544
+ if (request.sort !== undefined)
18545
+ args.push("-sort", request.sort);
18546
+ if (request.offset !== undefined && request.offset > 0)
18547
+ args.push("-offset", String(request.offset));
18548
+ args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
18549
+ if (request.regex)
18550
+ args.push("-regex", "--", request.query);
18551
+ else
18552
+ args.push("--", request.query);
18553
+ return args;
18554
+ }
18555
+ function parseEverythingJson(stdout) {
18556
+ const text = stdout.trim();
18557
+ if (text.length === 0)
18558
+ return [];
18559
+ let parsed;
18560
+ try {
18561
+ parsed = JSON.parse(text);
18562
+ } catch {
18563
+ throw new Error(`ES did not return JSON: ${text.slice(0, 2000)}`);
18297
18564
  }
18298
- async fileProviderDirectories(searchRoots) {
18299
- if (searchRoots.length === 0)
18300
- return [];
18301
- const candidates = new Set;
18302
- for (const searchRoot of searchRoots) {
18303
- const canonical = canonicalPath(searchRoot);
18304
- for (let cursor = canonical;; cursor = dirname15(cursor)) {
18305
- candidates.add(cursor);
18306
- const parent = dirname15(cursor);
18307
- if (parent === cursor)
18308
- break;
18309
- }
18310
- for (const entry of await this.tryReadDir(canonical) ?? []) {
18311
- if (entry.isDirectory())
18312
- candidates.add(join28(canonical, entry.name));
18313
- }
18314
- }
18315
- const paths = [...candidates];
18316
- const managed = [];
18317
- for (let start = 0;start < paths.length; start += XATTR_BATCH) {
18318
- for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
18319
- managed.push(path);
18320
- }
18321
- return managed;
18565
+ if (!Array.isArray(parsed))
18566
+ throw new Error(`ES did not return a JSON array: ${text.slice(0, 2000)}`);
18567
+ return parsed.map((item) => {
18568
+ const record = item;
18569
+ const filename = String(record.filename ?? "");
18570
+ const isFolder = filename.endsWith("\\") && filename.length > 3;
18571
+ return {
18572
+ path: isFolder ? filename.slice(0, -1) : filename,
18573
+ type: filename.endsWith("\\") ? "folder" : "file",
18574
+ size: typeof record.size === "number" ? record.size : undefined,
18575
+ dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
18576
+ };
18577
+ });
18578
+ }
18579
+
18580
+ class EverythingClient {
18581
+ instanceName;
18582
+ options;
18583
+ platform;
18584
+ run;
18585
+ launch;
18586
+ stateDir;
18587
+ isExcludedPath;
18588
+ binaries;
18589
+ queue = new SharedExclusiveQueue;
18590
+ constructor(options) {
18591
+ this.options = options;
18592
+ this.platform = options.platform ?? process.platform;
18593
+ this.instanceName = everythingInstanceName(options.root);
18594
+ this.run = options.run ?? runProcess;
18595
+ this.launch = options.launch ?? launchDetached;
18596
+ this.stateDir = join29(options.root, ".autorag", "everything");
18597
+ this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
18322
18598
  }
18323
- async xattrValues(paths) {
18324
- try {
18325
- const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
18326
- timeoutMs: 15000,
18327
- maxBuffer: 8 * 1024 * 1024
18328
- });
18329
- return parseXattrValues(stdout, paths);
18330
- } catch (error) {
18331
- const failure = error;
18332
- return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
18333
- }
18599
+ isSupported() {
18600
+ return this.platform === "win32";
18334
18601
  }
18335
- async tryReadDir(path) {
18336
- try {
18337
- return await this.readDir(path);
18338
- } catch {
18339
- return;
18340
- }
18602
+ search(request) {
18603
+ return this.queue.shared(async () => {
18604
+ const binaries = await this.resolveBinaries("cached");
18605
+ if (!binaries.ok)
18606
+ return binaries;
18607
+ const ping = await this.ping(binaries.esPath);
18608
+ if (ping.code !== 0) {
18609
+ return {
18610
+ ok: false,
18611
+ reason: "not-running",
18612
+ message: `Everything instance ${this.instanceName} is not running; run \`autorag refresh\` to start and index it.`
18613
+ };
18614
+ }
18615
+ const es = { ok: true, esPath: binaries.esPath };
18616
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
18617
+ if ((this.options.excludeFolders?.length ?? 0) > 0) {
18618
+ return this.searchWithExclusions(es.esPath, request, timeoutMs);
18619
+ }
18620
+ const args = buildEverythingSearchArgs(this.instanceName, request, timeoutMs);
18621
+ const result = await this.run(es.esPath, args, timeoutMs + 5000);
18622
+ if (result.code !== 0)
18623
+ return this.failure("search-failed", es.esPath, args, result);
18624
+ try {
18625
+ return { ok: true, results: parseEverythingJson(result.stdout) };
18626
+ } catch (error) {
18627
+ return { ok: false, reason: "search-failed", message: error.message };
18628
+ }
18629
+ });
18341
18630
  }
18342
- async datalessAmong(paths) {
18343
- const dataless = new Set;
18344
- for (let start = 0;start < paths.length; start += STAT_BATCH) {
18345
- const batch = paths.slice(start, start + STAT_BATCH);
18346
- const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
18347
- timeoutMs: 30000,
18348
- maxBuffer: 4 * 1024 * 1024
18349
- });
18350
- const flags = stdout.split(`
18351
- `).filter(Boolean);
18352
- for (const [index, path] of batch.entries()) {
18353
- if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
18354
- dataless.add(path);
18631
+ async searchWithExclusions(esPath, request, timeoutMs) {
18632
+ const deadline = Date.now() + timeoutMs;
18633
+ const offset = request.offset ?? 0;
18634
+ const maxResults = request.maxResults ?? DEFAULT_MAX_RESULTS;
18635
+ const results = [];
18636
+ let skipped = 0;
18637
+ let nativeOffset = 0;
18638
+ const timedOut = () => ({
18639
+ ok: false,
18640
+ reason: "search-failed",
18641
+ message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
18642
+ });
18643
+ for (;; ) {
18644
+ const remainingMs = deadline - Date.now();
18645
+ if (remainingMs <= 0)
18646
+ return timedOut();
18647
+ const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
18648
+ const response = await this.run(esPath, args, remainingMs + 5000);
18649
+ if (response.code !== 0)
18650
+ return this.failure("search-failed", esPath, args, response);
18651
+ if (Date.now() >= deadline)
18652
+ return timedOut();
18653
+ let entries;
18654
+ try {
18655
+ entries = parseEverythingJson(response.stdout);
18656
+ } catch (error) {
18657
+ return { ok: false, reason: "search-failed", message: error.message };
18658
+ }
18659
+ for (const entry of entries) {
18660
+ if (this.isExcludedPath(entry.path))
18661
+ continue;
18662
+ if (skipped < offset) {
18663
+ skipped += 1;
18664
+ continue;
18665
+ }
18666
+ results.push(entry);
18667
+ if (results.length >= maxResults)
18668
+ return { ok: true, results };
18355
18669
  }
18670
+ if (entries.length < MAX_FILTER_CANDIDATES)
18671
+ return { ok: true, results };
18672
+ nativeOffset += entries.length;
18356
18673
  }
18357
- return dataless;
18358
18674
  }
18359
- async providerOf(path) {
18360
- const domain = await readFileProviderDomainUpwards(path);
18361
- return domain === undefined ? undefined : cloudProviderFromDomain(domain);
18675
+ index() {
18676
+ return this.queue.exclusive(async () => {
18677
+ const es = await this.ensureRunning(true);
18678
+ if (!es.ok)
18679
+ return es;
18680
+ const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
18681
+ const saveArgs = ["-instance", this.instanceName, "-save-db"];
18682
+ const saved = await this.run(es.esPath, saveArgs, timeoutMs);
18683
+ if (saved.code !== 0)
18684
+ return this.failure("index-failed", es.esPath, saveArgs, saved);
18685
+ const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
18686
+ const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
18687
+ if (counted.code !== 0)
18688
+ return this.failure("index-failed", es.esPath, countArgs, counted);
18689
+ return {
18690
+ ok: true,
18691
+ indexedFolders: [...this.options.folders],
18692
+ indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
18693
+ };
18694
+ });
18362
18695
  }
18363
- canRelease(provider) {
18364
- return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
18696
+ stop() {
18697
+ return this.queue.exclusive(async () => {
18698
+ const binaries = await this.resolveBinaries("cached");
18699
+ if (!binaries.ok)
18700
+ return;
18701
+ await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
18702
+ });
18365
18703
  }
18366
- async evict(path) {
18367
- let stdout;
18368
- try {
18369
- stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
18370
- timeoutMs: 30000
18371
- })).stdout;
18372
- } catch (error) {
18373
- return { ok: false, message: describeReleaseHelperFailure(error) };
18374
- }
18375
- const outcome = parseReleaseOutcome(stdout);
18376
- if (outcome === undefined) {
18377
- const detail = stdout.trim();
18704
+ async resolveBinaries(mode) {
18705
+ if (!this.isSupported()) {
18378
18706
  return {
18379
18707
  ok: false,
18380
- message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
18708
+ reason: "unsupported-platform",
18709
+ message: `Everything is Windows-only; this host is ${this.platform}.`
18381
18710
  };
18382
18711
  }
18383
- return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
18384
- }
18385
- }
18386
- function buildWindowsAttributeScript(paths) {
18387
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18388
- ` + `$ErrorActionPreference='SilentlyContinue'
18389
- ` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
18390
- `;
18391
- }
18392
- function buildWindowsSyncRootsScript() {
18393
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18394
- ` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
18395
- ` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
18396
- `;
18397
- }
18398
- function buildWindowsDehydrateScript(path) {
18399
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18400
- ` + `$ErrorActionPreference='Stop'
18401
- ` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
18402
- ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
18403
- ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
18404
- ` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
18405
- ` + `'@
18406
- ` + `$h=[IntPtr]::Zero
18407
- ` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
18408
- ` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
18409
- ` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
18410
- ` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
18411
- ` + `[Autorag.CfApi]::CfCloseHandle($h)
18412
- ` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
18413
- ` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
18414
- `;
18415
- }
18416
- function parseWindowsAttributeOutput(stdout, paths) {
18417
- const lines = stdout.split(/\r?\n/);
18418
- const attributes = new Map;
18419
- for (const [index, path] of paths.entries()) {
18420
- const value = Number.parseInt((lines[index] ?? "").trim(), 10);
18421
- if (Number.isFinite(value))
18422
- attributes.set(path, value);
18423
- }
18424
- return attributes;
18425
- }
18426
- function parseWindowsSyncRoots(stdout) {
18427
- const roots = [];
18428
- for (const line of stdout.split(/\r?\n/)) {
18429
- const tab = line.indexOf("\t");
18430
- if (tab <= 0)
18431
- continue;
18432
- const path = line.slice(0, tab).trim();
18433
- if (path !== "")
18434
- roots.push({ path, label: line.slice(tab + 1).trim() });
18435
- }
18436
- return roots;
18437
- }
18438
- function providerFromWindowsSyncRoot(label) {
18439
- const lower = label.toLowerCase();
18440
- if (lower.includes("onedrive") || lower.includes("one drive"))
18441
- return "onedrive";
18442
- if (lower.includes("google"))
18443
- return "google-drive";
18444
- return "file-provider";
18445
- }
18446
- function buildWindowsDiskScript() {
18447
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18448
- ` + "Get-CimInstance -ClassName Win32_LogicalDisk | ForEach-Object {" + `$_.DeviceID + [char]9 + $_.DriveType + [char]9 + $_.FileSystem + [char]9 + $_.VolumeName}
18449
- `;
18450
- }
18451
- function parseWindowsLogicalDisks(stdout) {
18452
- const disks = [];
18453
- for (const line of stdout.split(/\r?\n/)) {
18454
- if (line.trim() === "")
18455
- continue;
18456
- const [deviceId, driveType, fileSystem, ...rest] = line.split("\t");
18457
- const type = Number.parseInt((driveType ?? "").trim(), 10);
18458
- if (!Number.isFinite(type))
18459
- continue;
18460
- disks.push({
18461
- deviceId: (deviceId ?? "").trim(),
18462
- driveType: type,
18463
- fileSystem: (fileSystem ?? "").trim(),
18464
- volumeName: rest.join("\t").trim()
18465
- });
18712
+ if (this.binaries?.ok)
18713
+ return this.binaries;
18714
+ const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
18715
+ const result = await resolve(mode);
18716
+ if (result.ok)
18717
+ this.binaries = result;
18718
+ return result;
18466
18719
  }
18467
- return disks;
18468
- }
18469
- function windowsRootsFromDisks(disks) {
18470
- const roots = [];
18471
- for (const disk of disks) {
18472
- if (disk.deviceId === "")
18473
- continue;
18474
- const path = `${disk.deviceId}\\`;
18475
- const fileSystem = disk.fileSystem.toLowerCase();
18476
- if (disk.driveType === WINDOWS_DRIVE_TYPE_NETWORK) {
18477
- roots.push({ path, kind: "network" });
18478
- continue;
18720
+ async ensureRunning(reconfigure) {
18721
+ const binaries = await this.resolveBinaries("install");
18722
+ if (!binaries.ok)
18723
+ return binaries;
18724
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
18725
+ let ping = await this.ping(binaries.esPath);
18726
+ if (ping.code === 0 && !reconfigure)
18727
+ return { ok: true, esPath: binaries.esPath };
18728
+ if (ping.code === 0) {
18729
+ const exitArgs = ["-instance", this.instanceName, "-exit"];
18730
+ const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
18731
+ if (exited.code !== 0)
18732
+ return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
18479
18733
  }
18480
- if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
18481
- roots.push({ path, kind: "network" });
18482
- continue;
18734
+ mkdirSync16(this.stateDir, { recursive: true });
18735
+ const iniPath = join29(this.stateDir, "Everything.ini");
18736
+ writeFileSync11(iniPath, buildEverythingIni({
18737
+ folders: this.options.folders,
18738
+ excludeFolders: [
18739
+ ...new Set([
18740
+ join29(this.options.root, ".autorag"),
18741
+ ...this.options.folders.map((folder) => join29(folder, ".autorag")),
18742
+ ...this.options.excludeFolders ?? []
18743
+ ])
18744
+ ]
18745
+ }));
18746
+ const launchArgs = [
18747
+ "-instance",
18748
+ this.instanceName,
18749
+ "-config",
18750
+ iniPath,
18751
+ "-db",
18752
+ join29(this.stateDir, "Everything.db"),
18753
+ "-startup"
18754
+ ];
18755
+ try {
18756
+ this.launch(binaries.everythingPath, launchArgs);
18757
+ } catch (error) {
18758
+ return {
18759
+ ok: false,
18760
+ reason: "startup-failed",
18761
+ message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
18762
+ };
18483
18763
  }
18484
- if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
18485
- roots.push({ path, kind: "cloud-files" });
18764
+ const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
18765
+ const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
18766
+ for (;; ) {
18767
+ ping = await this.ping(binaries.esPath);
18768
+ if (ping.code === 0)
18769
+ return { ok: true, esPath: binaries.esPath };
18770
+ if (Date.now() >= deadline)
18771
+ break;
18772
+ await sleep2(pollMs);
18486
18773
  }
18487
- }
18488
- return roots;
18489
- }
18490
- function uncShareRoot(target) {
18491
- const normalized = target.replaceAll("/", "\\");
18492
- if (!normalized.startsWith("\\\\") || normalized.startsWith("\\\\?\\") || normalized.startsWith("\\\\.\\"))
18493
- return;
18494
- const rest = normalized.slice(2);
18495
- const separator = rest.indexOf("\\");
18496
- if (separator <= 0)
18497
- return;
18498
- const server = rest.slice(0, separator);
18499
- const share = rest.slice(separator + 1).split("\\")[0] ?? "";
18500
- if (server === "" || share === "")
18501
- return;
18502
- return `\\\\${server}\\${share}`;
18503
- }
18504
- function parseWindowsDehydrateOutput(result) {
18505
- const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
18506
- if (open === null)
18507
- return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
18508
- if (Number.parseInt(open[1], 16) !== 0) {
18509
18774
  return {
18510
18775
  ok: false,
18511
- message: windowsFailureMessage(result, `could not open the file for release (0x${open[1]})`)
18776
+ reason: "startup-failed",
18777
+ message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
18512
18778
  };
18513
18779
  }
18514
- const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
18515
- if (dehydrate === null) {
18516
- return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
18780
+ async ping(esPath) {
18781
+ const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
18782
+ return result.code === 0 && !VERSION_PATTERN.test(result.stdout.trim()) ? { ...result, code: -1 } : result;
18517
18783
  }
18518
- const hresult = dehydrate[1];
18519
- if (Number.parseInt(hresult, 16) === 0)
18520
- return { ok: true };
18521
- const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
18522
- return {
18523
- ok: false,
18524
- message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
18784
+ failure(reason, esPath, args, result) {
18785
+ return {
18786
+ ok: false,
18787
+ reason,
18788
+ message: `${esPath} ${args.join(" ")} exit ${result.code}: ${(result.stderr || result.stdout).trim()}`
18789
+ };
18790
+ }
18791
+ }
18792
+ function runProcess(command, args, timeoutMs) {
18793
+ let resolvePromise;
18794
+ const promise = new Promise((resolve) => {
18795
+ resolvePromise = resolve;
18796
+ });
18797
+ const child = spawn15(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
18798
+ const stdout = [];
18799
+ const stderr = [];
18800
+ let bytes = 0;
18801
+ let overflow = false;
18802
+ const collect = (target) => (chunk) => {
18803
+ bytes += chunk.length;
18804
+ if (bytes > MAX_OUTPUT_BYTES) {
18805
+ overflow = true;
18806
+ child.kill();
18807
+ return;
18808
+ }
18809
+ target.push(chunk);
18525
18810
  };
18811
+ child.stdout.on("data", collect(stdout));
18812
+ child.stderr.on("data", collect(stderr));
18813
+ const timer = setTimeout(() => child.kill(), timeoutMs);
18814
+ child.on("error", (error) => {
18815
+ clearTimeout(timer);
18816
+ resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
18817
+ });
18818
+ child.on("close", (code, signal) => {
18819
+ clearTimeout(timer);
18820
+ const err = Buffer.concat(stderr).toString("utf8");
18821
+ resolvePromise({
18822
+ code,
18823
+ stdout: Buffer.concat(stdout).toString("utf8"),
18824
+ stderr: overflow ? `${err}
18825
+ output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
18826
+ killed by ${signal} after ${timeoutMs}ms` : err
18827
+ });
18828
+ });
18829
+ return promise;
18526
18830
  }
18527
- function windowsFailureMessage(result, fallback) {
18528
- const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
18529
- if (printed !== undefined && printed !== "")
18530
- return printed;
18531
- const stderr = result.stderr.trim();
18532
- return stderr !== "" ? stderr : fallback;
18831
+ function launchDetached(command, args) {
18832
+ const child = spawn15(command, args, { detached: true, stdio: "ignore", windowsHide: true });
18833
+ child.on("error", () => {});
18834
+ child.unref();
18533
18835
  }
18836
+ var DEFAULT_TIMEOUT_MS8 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
18837
+ var init_client9 = __esm(() => {
18838
+ init_path_exclusions();
18839
+ init_bundle();
18840
+ DEFAULT_INDEX_TIMEOUT_MS3 = 10 * 60000;
18841
+ MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
18842
+ VERSION_PATTERN = /^\d+\.\d+\.\d+\.\d+$/;
18843
+ });
18534
18844
 
18535
- class WindowsCloudProbe {
18536
- run;
18537
- syncRoots;
18538
- disks;
18539
- constructor(options = {}) {
18540
- this.run = options.run ?? runPowerShell;
18541
- }
18542
- async cloudRoots(searchRoots = []) {
18543
- const roots = [];
18544
- const seen = new Set;
18545
- const add = (path, kind) => {
18546
- const canonical = canonicalPath(path);
18547
- const key = normalizeCloudPath(canonical).toLowerCase();
18548
- if (key === "" || seen.has(key))
18549
- return;
18550
- seen.add(key);
18551
- roots.push({ path: canonical, kind });
18552
- };
18553
- for (const root of await this.loadSyncRoots())
18554
- add(root.path, "cloud-files");
18555
- for (const root of await this.loadDiskRoots())
18556
- add(root.path, root.kind);
18557
- for (const searchRoot of searchRoots) {
18558
- const share = uncShareRoot(searchRoot);
18559
- if (share !== undefined)
18560
- add(share, "network");
18561
- }
18562
- return roots;
18563
- }
18564
- async datalessAmong(paths) {
18565
- const dataless = new Set;
18566
- for (let start = 0;start < paths.length; start += WINDOWS_ATTRIBUTE_BATCH) {
18567
- const batch = paths.slice(start, start + WINDOWS_ATTRIBUTE_BATCH);
18568
- const result = await this.run(buildWindowsAttributeScript(batch), 30000);
18569
- if (!result.ok) {
18570
- throw new Error(`Could not read Windows cloud file attributes: ${result.stderr.trim() || "powershell.exe failed"}`);
18571
- }
18572
- for (const [path, attributes] of parseWindowsAttributeOutput(result.stdout, batch)) {
18573
- if (datalessFromWindowsAttributes(attributes))
18574
- dataless.add(path);
18575
- }
18576
- }
18577
- return dataless;
18578
- }
18579
- async providerOf(path) {
18580
- const root = longestSyncRoot(await this.loadSyncRoots(), path);
18581
- return root === undefined ? cloudProviderFromPath(path) : providerFromWindowsSyncRoot(root.label);
18582
- }
18583
- canRelease(provider) {
18584
- return provider !== undefined;
18585
- }
18586
- async evict(path) {
18587
- return parseWindowsDehydrateOutput(await this.run(buildWindowsDehydrateScript(path), 60000));
18588
- }
18589
- loadSyncRoots() {
18590
- this.syncRoots ??= this.run(buildWindowsSyncRootsScript(), 15000).then((result) => result.ok ? parseWindowsSyncRoots(result.stdout) : [], () => []);
18591
- return this.syncRoots;
18592
- }
18593
- loadDiskRoots() {
18594
- this.disks ??= this.run(buildWindowsDiskScript(), 30000).then((result) => result.ok ? windowsRootsFromDisks(parseWindowsLogicalDisks(result.stdout)) : [], () => []);
18595
- return this.disks;
18596
- }
18845
+ // src/everything/index.ts
18846
+ var init_everything = __esm(() => {
18847
+ init_bundle();
18848
+ init_client9();
18849
+ });
18850
+
18851
+ // src/filesystem/cloud-placeholder.ts
18852
+ import { execFile as execFile2 } from "node:child_process";
18853
+ import { realpathSync as realpathSync3 } from "node:fs";
18854
+ import { lstat, readdir } from "node:fs/promises";
18855
+ import { homedir as homedir6 } from "node:os";
18856
+ import { dirname as dirname16, join as join30 } from "node:path";
18857
+ import { promisify } from "node:util";
18858
+ function homeCloudRoots(home = homedir6()) {
18859
+ const normalizedHome = home.replaceAll("\\", "/").replace(/\/+$/, "");
18860
+ return [`${normalizedHome}/Library/CloudStorage`, `${normalizedHome}/Library/Mobile Documents`];
18597
18861
  }
18598
- function longestSyncRoot(roots, target) {
18599
- const normalized = target.replaceAll("/", "\\").toLowerCase();
18600
- let match;
18601
- let matchLength = -1;
18602
- for (const root of roots) {
18603
- const prefix = root.path.replaceAll("/", "\\").replace(/\\+$/u, "").toLowerCase();
18604
- if (prefix === "" || prefix.length <= matchLength)
18605
- continue;
18606
- if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
18607
- match = root;
18608
- matchLength = prefix.length;
18609
- }
18610
- }
18611
- return match;
18862
+ function datalessFromDarwinFlags(flags) {
18863
+ return Number.isFinite(flags) && (flags >>> 0 & SF_DATALESS) !== 0;
18612
18864
  }
18613
- function powerShellLaunch(script) {
18614
- return {
18615
- args: [
18616
- "-NoProfile",
18617
- "-NonInteractive",
18618
- "-Command",
18619
- `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; Invoke-Expression $env:${POWERSHELL_SCRIPT_VARIABLE}`
18620
- ],
18621
- env: { [POWERSHELL_SCRIPT_VARIABLE]: script }
18622
- };
18865
+ function datalessFromWindowsAttributes(attributes) {
18866
+ return Number.isFinite(attributes) && (attributes & WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) !== 0;
18623
18867
  }
18624
- async function runPowerShell(script, timeoutMs) {
18625
- const launch = powerShellLaunch(script);
18626
- try {
18627
- const { stdout, stderr } = await execFileAsync("powershell.exe", [...launch.args], {
18628
- timeout: timeoutMs,
18629
- maxBuffer: 16 * 1024 * 1024,
18630
- windowsHide: true,
18631
- env: { ...process.env, ...launch.env }
18632
- });
18633
- return { ok: true, stdout, stderr };
18634
- } catch (error) {
18635
- const record = typeof error === "object" && error !== null ? error : {};
18636
- return {
18637
- ok: false,
18638
- stdout: typeof record.stdout === "string" ? record.stdout : "",
18639
- stderr: typeof record.stderr === "string" ? record.stderr : String(error)
18640
- };
18641
- }
18868
+ function cloudProviderFromPath(target) {
18869
+ const lower = target.toLowerCase();
18870
+ if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
18871
+ return "google-drive";
18872
+ if (lower.includes("onedrive"))
18873
+ return "onedrive";
18874
+ if (lower.includes("dropbox"))
18875
+ return "dropbox";
18876
+ if (lower.includes("mobile documents") || lower.includes("clouddocs"))
18877
+ return "icloud";
18878
+ return "file-provider";
18642
18879
  }
18643
- function defaultCloudProbe() {
18644
- if (process.platform === "darwin")
18645
- return new DarwinCloudProbe;
18646
- if (process.platform === "win32")
18647
- return new WindowsCloudProbe;
18648
- return;
18880
+ function cloudProviderFromDomain(domain) {
18881
+ const lower = domain.toLowerCase();
18882
+ if (lower.includes("clouddocs") || lower.includes("icloud"))
18883
+ return "icloud";
18884
+ if (lower.includes("google"))
18885
+ return "google-drive";
18886
+ if (lower.includes("onedrive"))
18887
+ return "onedrive";
18888
+ if (lower.includes("dropbox"))
18889
+ return "dropbox";
18890
+ return "file-provider";
18649
18891
  }
18650
- async function readFileProviderDomain(target) {
18892
+ function normalizeCloudPath(value) {
18893
+ let normalized = value.replaceAll("\\", "/");
18894
+ if (normalized.length > 1)
18895
+ normalized = normalized.replace(/\/+$/u, "");
18896
+ return normalized;
18897
+ }
18898
+ function isPathInside(target, parent) {
18899
+ const normalizedParent = normalizeCloudPath(parent);
18900
+ if (normalizedParent === "")
18901
+ return false;
18902
+ const left = normalizeCloudPath(target);
18903
+ const [a, b] = process.platform === "win32" ? [left.toLowerCase(), normalizedParent.toLowerCase()] : [left, normalizedParent];
18904
+ return a === b || a.startsWith(`${b}/`);
18905
+ }
18906
+ function canonicalPath(value) {
18651
18907
  try {
18652
- await lstat(target);
18653
- const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
18654
- timeout: 2000
18655
- });
18656
- const value = stdout.trim();
18657
- return value.length > 0 ? value : undefined;
18908
+ return realpathSync3(value);
18658
18909
  } catch {
18659
- return;
18660
- }
18661
- }
18662
- async function readFileProviderDomainUpwards(target) {
18663
- let cursor = target;
18664
- for (let depth = 0;depth < 12; depth += 1) {
18665
- const domain = await readFileProviderDomain(cursor);
18666
- if (domain !== undefined)
18667
- return domain;
18668
- const parent = dirname15(cursor);
18669
- if (parent === cursor)
18670
- return;
18671
- cursor = parent;
18910
+ return value;
18672
18911
  }
18673
- return;
18674
18912
  }
18675
- function quotePowerShell(value) {
18676
- return `'${value.replace(/['\u2018\u2019\u201A\u201B]/gu, (quote) => quote + quote)}'`;
18677
- }
18678
- var execFileAsync, SF_DATALESS = 1073741824, FILE_PROVIDER_XATTR = "com.apple.file-provider-domain-id", WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 4194304, STAT_BATCH = 200, defaultCloudExec = (file, args, options) => execFileAsync(file, [...args], {
18679
- timeout: options?.timeoutMs ?? 30000,
18680
- ...options?.maxBuffer === undefined ? {} : { maxBuffer: options.maxBuffer }
18681
- }), JXA_RELEASE_PROGRAM = `ObjC.import("Foundation");
18682
- function run(argv) {
18683
- var url = $.NSURL.fileURLWithPath(argv[0]);
18684
- var errorRef = Ref();
18685
- var released = $.NSFileManager.defaultManager.evictUbiquitousItemAtURLError(url, errorRef);
18686
- if (released) return JSON.stringify({ ok: true });
18687
- function resourceValue(key) {
18688
- var ref = Ref();
18689
- if (!url.getResourceValueForKeyError(ref, key, null)) return null;
18690
- var value = ref[0];
18691
- return value === undefined || value === null ? null : ObjC.unwrap(value);
18692
- }
18693
- return JSON.stringify({
18694
- ok: false,
18695
- isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
18696
- isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
18697
- downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
18698
- });
18699
- }`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
18700
- var init_cloud_placeholder = __esm(() => {
18701
- execFileAsync = promisify(execFile2);
18702
- RELEASABLE_DARWIN_PROVIDERS = {
18703
- icloud: true,
18704
- "google-drive": true,
18705
- onedrive: true,
18706
- dropbox: true
18913
+ function canonicalPathMapper(searchRoots) {
18914
+ const forms = searchRoots.map((root) => ({ raw: normalizeCloudPath(root), canonical: normalizeCloudPath(canonicalPath(root)) })).filter(({ raw, canonical }) => raw !== "" && raw !== canonical).sort((left, right) => right.raw.length - left.raw.length);
18915
+ if (forms.length === 0)
18916
+ return (path) => path;
18917
+ return (path) => {
18918
+ const normalized = normalizeCloudPath(path);
18919
+ for (const { raw, canonical } of forms) {
18920
+ if (isPathInside(normalized, raw))
18921
+ return `${canonical}${normalized.slice(raw.length)}`;
18922
+ }
18923
+ return path;
18707
18924
  };
18708
- DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
18709
- WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
18710
- WINDOWS_DEHYDRATE_REFUSALS = {
18711
- "80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
18712
- "80070188": "The file is pinned to stay on this device, so it is not released."
18925
+ }
18926
+ function cloudScope(searchRoot, cloudRoots) {
18927
+ const canonicalSearchRoot = canonicalPath(searchRoot);
18928
+ const roots = cloudRoots.map((root) => ({ path: canonicalPath(root.path), kind: root.kind }));
18929
+ return {
18930
+ insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
18931
+ nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
18713
18932
  };
18714
- });
18715
-
18716
- // src/dupey/scan-plan.ts
18717
- import { basename as basename7 } from "node:path";
18718
- async function listCloudRoots(probe, roots) {
18719
- if (probe === undefined)
18720
- return [];
18933
+ }
18934
+ function parseReleaseOutcome(stdout) {
18935
+ let parsed;
18721
18936
  try {
18722
- return await probe.cloudRoots(roots);
18937
+ parsed = JSON.parse(stdout.trim());
18723
18938
  } catch {
18724
- return [];
18939
+ return;
18725
18940
  }
18726
- }
18727
- function planDupeyScans(roots, cloudRoots) {
18728
- const targets = [];
18729
- const skipped = [];
18730
- for (const path of roots) {
18731
- const scope = cloudScope(path, cloudRoots);
18732
- if (scope.insideCloudRoot) {
18733
- skipped.push(path);
18734
- continue;
18735
- }
18736
- targets.push({ path, excludeDirs: scope.nestedCloudRoots.map((root) => basename7(root.path)) });
18941
+ if (typeof parsed !== "object" || parsed === null || !("ok" in parsed) || typeof parsed.ok !== "boolean") {
18942
+ return;
18737
18943
  }
18738
- return { targets, skipped };
18739
- }
18740
- var init_scan_plan = __esm(() => {
18741
- init_cloud_placeholder();
18742
- });
18743
-
18744
- // src/dupey/index.ts
18745
- var init_dupey = __esm(() => {
18746
- init_cli2();
18747
- init_filter();
18748
- init_scan_plan();
18749
- });
18750
-
18751
- // src/convert/relate.ts
18752
- async function convertAndRelate(request, options = {}) {
18753
- const result = await convertWithFilee(request, options.filee);
18754
- if (options.dupey === false)
18755
- return { result, links: [] };
18756
- const dupeyOptions = options.dupey ?? {};
18757
- const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
18758
- const links = await Promise.all(pairs.map(async ({ source, output }) => {
18759
- try {
18760
- return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
18761
- } catch (error) {
18762
- if (!(error instanceof DupeyCliError))
18763
- throw error;
18764
- return { source, output, dupey: { relation: "unavailable", reason: error.message } };
18765
- }
18766
- }));
18767
- return { result, links };
18768
- }
18769
- var init_relate = __esm(() => {
18770
- init_dupey();
18771
- init_cli();
18772
- });
18773
-
18774
- // src/convert/index.ts
18775
- var init_convert = __esm(() => {
18776
- init_cli();
18777
- init_installer2();
18778
- init_matrix();
18779
- init_relate();
18780
- });
18781
-
18782
- // src/datasource/diagnostics.ts
18783
- function mapDatasourceDiagnostic(ds) {
18784
18944
  return {
18785
- code: datasourceCodeToRetrieval(ds.code),
18786
- severity: ds.severity,
18787
- message: `[datasource:${ds.code}] ${ds.message}`,
18788
- source: ds.source ?? (ds.instanceId ? `datasource:${ds.instanceId}` : "datasource")
18945
+ ok: parsed.ok,
18946
+ ..."isUploaded" in parsed ? { isUploaded: parsed.isUploaded } : {},
18947
+ ..."isUploading" in parsed ? { isUploading: parsed.isUploading } : {},
18948
+ ..."downloadingStatus" in parsed ? { downloadingStatus: parsed.downloadingStatus } : {}
18789
18949
  };
18790
18950
  }
18791
- function mapDatasourceDiagnostics(diagnostics) {
18792
- return diagnostics.map(mapDatasourceDiagnostic);
18951
+ function describeReleaseRefusal(outcome) {
18952
+ if (outcome.isUploading === true)
18953
+ return "The file is still uploading to the cloud, so it cannot be released yet.";
18954
+ if (outcome.isUploaded === false)
18955
+ return "The file has not finished uploading to the cloud, so it cannot be released yet.";
18956
+ const status = typeof outcome.downloadingStatus === "string" ? outcome.downloadingStatus : "unknown";
18957
+ return `The cloud provider declined to release the local copy (download status ${status}).`;
18793
18958
  }
18794
- function datasourceCodeToRetrieval(code) {
18795
- switch (code) {
18796
- case "datasource-unavailable":
18797
- case "datasource-cli-error":
18798
- case "datasource-empty":
18799
- case "datasource-rate-limited":
18800
- case "datasource-auth-error":
18801
- case "datasource-embedding-egress-rejected":
18802
- case "datasource-index-failed":
18803
- case "datasource-index-partial":
18804
- case "datasource-permission-denied":
18805
- case "datasource-remote-embedding-rejected":
18806
- return "retrieval-method-failed";
18959
+ function describeReleaseHelperFailure(error) {
18960
+ const failure = error;
18961
+ const parts = [];
18962
+ if (failure.killed === true || typeof failure.signal === "string") {
18963
+ parts.push(`the helper was killed${typeof failure.signal === "string" ? ` by ${failure.signal}` : ""}`);
18964
+ } else if (typeof failure.code === "number") {
18965
+ parts.push(`the helper exited with code ${failure.code}`);
18807
18966
  }
18967
+ const stderr = typeof failure.stderr === "string" ? failure.stderr.trim() : "";
18968
+ if (stderr !== "")
18969
+ parts.push(stderr);
18970
+ if (parts.length === 0)
18971
+ parts.push(error instanceof Error ? error.message : String(error));
18972
+ return `osascript could not release the local copy: ${parts.join("; ")}`;
18808
18973
  }
18809
-
18810
- // src/datasource/retry.ts
18811
- import { setTimeout as delay2 } from "node:timers/promises";
18812
- function datasourceRetryDelayMs(retry, policy) {
18813
- const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
18814
- return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
18974
+ async function defaultDarwinReadDir(path) {
18975
+ const entries = await readdir(path, { withFileTypes: true });
18976
+ return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
18815
18977
  }
18816
- async function indexWithRetry(skill, options = {}, signal) {
18817
- const policy = {
18818
- maxRetries: options.maxRetries ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxRetries,
18819
- baseDelayMs: options.baseDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.baseDelayMs,
18820
- maxDelayMs: options.maxDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxDelayMs
18821
- };
18822
- const sleep = options.sleep ?? ((ms) => delay2(ms, undefined, signal === undefined ? undefined : { signal }));
18823
- let lastError = "";
18824
- for (let attempt = 1;; attempt += 1) {
18825
- const result = await runIndex(skill, signal);
18826
- signal?.throwIfAborted();
18827
- if (result.ok) {
18828
- if (attempt === 1)
18829
- return result;
18830
- const recovered = {
18831
- code: "datasource-index-failed",
18832
- severity: "info",
18833
- message: `recovered after ${attempt} attempts; last error: ${lastError}`,
18834
- instanceId: result.instanceId,
18835
- source: result.skill
18836
- };
18837
- return { ...result, attempts: attempt, diagnostics: [...result.diagnostics, recovered] };
18838
- }
18839
- lastError = result.message ?? result.error ?? "datasource-index-failed";
18840
- if (attempt > policy.maxRetries) {
18841
- if (attempt === 1)
18842
- return { ...result, attempts: attempt };
18843
- const suffix = ` (gave up after ${attempt} attempts)`;
18844
- const last = result.diagnostics.at(-1);
18845
- return {
18846
- ...result,
18847
- attempts: attempt,
18848
- message: `${lastError}${suffix}`,
18849
- diagnostics: last === undefined ? [
18850
- {
18851
- code: result.code ?? "datasource-index-failed",
18852
- severity: "error",
18853
- message: `${lastError}${suffix}`,
18854
- instanceId: result.instanceId,
18855
- source: result.skill
18856
- }
18857
- ] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
18858
- };
18859
- }
18860
- await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
18861
- signal?.throwIfAborted();
18978
+ function parseDarwinMounts(stdout) {
18979
+ const mounts = [];
18980
+ for (const line of stdout.split(`
18981
+ `)) {
18982
+ if (line.trim() === "")
18983
+ continue;
18984
+ const open = line.lastIndexOf(" (");
18985
+ if (open <= 0 || !line.endsWith(")"))
18986
+ continue;
18987
+ const head = line.slice(0, open);
18988
+ const on = head.indexOf(" on ");
18989
+ if (on <= 0)
18990
+ continue;
18991
+ const mountPoint = head.slice(on + " on ".length);
18992
+ if (mountPoint === "")
18993
+ continue;
18994
+ const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
18995
+ mounts.push({
18996
+ device: head.slice(0, on),
18997
+ mountPoint,
18998
+ fileSystem: fields[0] ?? "",
18999
+ options: fields.slice(1)
19000
+ });
18862
19001
  }
19002
+ return mounts;
18863
19003
  }
18864
- async function runIndex(skill, signal) {
18865
- try {
18866
- return await skill.index(signal === undefined ? undefined : { signal });
18867
- } catch (error) {
18868
- signal?.throwIfAborted();
18869
- const descriptor = skill.describe();
18870
- const message = error instanceof Error ? error.message : "Datasource indexing failed.";
18871
- const failed = {
18872
- ok: false,
18873
- instanceId: descriptor.instanceId ?? "default",
18874
- skill: descriptor.name,
18875
- indexedAt: Date.now(),
18876
- diagnostics: [
18877
- {
18878
- code: "datasource-index-failed",
18879
- severity: "error",
18880
- message,
18881
- source: descriptor.name,
18882
- instanceId: descriptor.instanceId
18883
- }
18884
- ],
18885
- error: "datasource-index-failed",
18886
- code: "datasource-index-failed",
18887
- message
18888
- };
18889
- return failed;
19004
+ function networkMountPoints(mounts) {
19005
+ const points = [];
19006
+ const seen = new Set;
19007
+ for (const mount of mounts) {
19008
+ if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
19009
+ continue;
19010
+ if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
19011
+ continue;
19012
+ if (seen.has(mount.mountPoint))
19013
+ continue;
19014
+ seen.add(mount.mountPoint);
19015
+ points.push(mount.mountPoint);
18890
19016
  }
19017
+ return points;
18891
19018
  }
18892
- var DEFAULT_DATASOURCE_RETRY_POLICY;
18893
- var init_retry = __esm(() => {
18894
- DEFAULT_DATASOURCE_RETRY_POLICY = {
18895
- maxRetries: 2,
18896
- baseDelayMs: 1000,
18897
- maxDelayMs: 30000
18898
- };
18899
- });
18900
-
18901
- // src/everything/bundle.ts
18902
- import { createHash as createHash11 } from "node:crypto";
18903
- import { existsSync as existsSync18, mkdirSync as mkdirSync15, readFileSync as readFileSync18, rmSync as rmSync7, writeFileSync as writeFileSync10 } from "node:fs";
18904
- import { dirname as dirname16, join as join29 } from "node:path";
18905
- import { fileURLToPath as fileURLToPath2 } from "node:url";
18906
- import JSZip from "jszip";
18907
- function resolveEverythingBundleDir() {
18908
- let dir = dirname16(fileURLToPath2(import.meta.url));
18909
- for (let depth = 0;depth < 6; depth += 1) {
18910
- const candidate = join29(dir, "vendor", "everything");
18911
- if (existsSync18(join29(candidate, MANIFEST_FILENAME)))
18912
- return candidate;
18913
- const parent = dirname16(dir);
18914
- if (parent === dir)
18915
- break;
18916
- dir = parent;
19019
+ function parseXattrValues(stdout, candidates) {
19020
+ const values = new Map;
19021
+ if (candidates.length === 0)
19022
+ return values;
19023
+ const ordered = [...candidates].sort((left, right) => right.length - left.length);
19024
+ for (const line of stdout.split(`
19025
+ `)) {
19026
+ if (line === "")
19027
+ continue;
19028
+ for (const candidate of ordered) {
19029
+ if (line.startsWith(`${candidate}: `)) {
19030
+ values.set(candidate, line.slice(candidate.length + 2));
19031
+ break;
19032
+ }
19033
+ }
18917
19034
  }
18918
- return;
18919
- }
18920
- function everythingArch(arch) {
18921
- if (arch === "x64" || arch === "arm64")
18922
- return arch;
18923
- return;
18924
- }
18925
- function everythingBinaryPaths(root, version, arch) {
18926
- const dir = join29(root, ".autorag", "everything", version, arch);
18927
- return { everythingPath: join29(dir, "everything.exe"), esPath: join29(dir, "es.exe") };
18928
- }
18929
- function loadEverythingBundleManifest(bundleDir) {
18930
- return JSON.parse(readFileSync18(join29(bundleDir, MANIFEST_FILENAME), "utf8"));
19035
+ return values;
18931
19036
  }
18932
- async function ensureEverythingBinaries(options) {
18933
- const platform = options.platform ?? process.platform;
18934
- const rawArch = options.arch ?? process.arch;
18935
- if (platform !== "win32") {
18936
- return {
18937
- ok: false,
18938
- reason: "unsupported-platform",
18939
- message: `Everything is Windows-only; this host is ${platform}.`
18940
- };
19037
+
19038
+ class DarwinCloudProbe {
19039
+ exec;
19040
+ readDir;
19041
+ home;
19042
+ constructor(options = {}) {
19043
+ this.exec = options.exec ?? defaultCloudExec;
19044
+ this.readDir = options.readDir ?? defaultDarwinReadDir;
19045
+ this.home = options.home ?? homedir6();
18941
19046
  }
18942
- const arch = everythingArch(rawArch);
18943
- const bundleDir = options.bundleDir ?? resolveEverythingBundleDir();
18944
- if (bundleDir === undefined) {
18945
- return {
18946
- ok: false,
18947
- reason: "bundle-missing",
18948
- message: `Bundled Everything assets (vendor/everything/${MANIFEST_FILENAME}) were not found above ${dirname16(fileURLToPath2(import.meta.url))}.`
19047
+ async cloudRoots(searchRoots = []) {
19048
+ const roots = [];
19049
+ const seen = new Set;
19050
+ const add = (path, kind) => {
19051
+ const canonical = canonicalPath(path);
19052
+ const key = normalizeCloudPath(canonical);
19053
+ if (key === "" || seen.has(key))
19054
+ return;
19055
+ seen.add(key);
19056
+ roots.push({ path: canonical, kind });
18949
19057
  };
19058
+ const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
19059
+ for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
19060
+ if (entry.isDirectory())
19061
+ add(join30(cloudStorage, entry.name), "file-provider");
19062
+ }
19063
+ if (await this.tryReadDir(mobileDocuments) !== undefined)
19064
+ add(mobileDocuments, "file-provider");
19065
+ for (const mountPoint of await this.networkMountPoints())
19066
+ add(mountPoint, "network");
19067
+ for (const directory of await this.fileProviderDirectories(searchRoots))
19068
+ add(directory, "file-provider");
19069
+ return roots;
18950
19070
  }
18951
- let manifest;
18952
- try {
18953
- manifest = options.manifest ?? loadEverythingBundleManifest(bundleDir);
18954
- } catch (error) {
18955
- return { ok: false, reason: "bundle-missing", message: errorMessage2(error) };
19071
+ async networkMountPoints() {
19072
+ try {
19073
+ const { stdout } = await this.exec("/sbin/mount", [], { timeoutMs: 1e4, maxBuffer: 4 * 1024 * 1024 });
19074
+ return networkMountPoints(parseDarwinMounts(stdout));
19075
+ } catch {
19076
+ return [];
19077
+ }
18956
19078
  }
18957
- const assets = arch === undefined ? undefined : manifest.architectures[arch];
18958
- if (arch === undefined || assets === undefined) {
18959
- return {
18960
- ok: false,
18961
- reason: "unsupported-platform",
18962
- message: `No bundled Everything build for Windows architecture ${rawArch}; bundled: ${Object.keys(manifest.architectures).join(", ")}.`
18963
- };
19079
+ async fileProviderDirectories(searchRoots) {
19080
+ if (searchRoots.length === 0)
19081
+ return [];
19082
+ const candidates = new Set;
19083
+ for (const searchRoot of searchRoots) {
19084
+ const canonical = canonicalPath(searchRoot);
19085
+ for (let cursor = canonical;; cursor = dirname16(cursor)) {
19086
+ candidates.add(cursor);
19087
+ const parent = dirname16(cursor);
19088
+ if (parent === cursor)
19089
+ break;
19090
+ }
19091
+ for (const entry of await this.tryReadDir(canonical) ?? []) {
19092
+ if (entry.isDirectory())
19093
+ candidates.add(join30(canonical, entry.name));
19094
+ }
19095
+ }
19096
+ const paths = [...candidates];
19097
+ const managed = [];
19098
+ for (let start = 0;start < paths.length; start += XATTR_BATCH) {
19099
+ for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
19100
+ managed.push(path);
19101
+ }
19102
+ return managed;
18964
19103
  }
18965
- const { everythingPath, esPath } = everythingBinaryPaths(options.root, manifest.everythingVersion, arch);
18966
- const everythingCached = fileHasSha256(everythingPath, assets.everythingSha256);
18967
- const esCached = fileHasSha256(esPath, assets.esSha256);
18968
- if (everythingCached && esCached) {
18969
- return { ok: true, everythingPath, esPath, source: "cached" };
19104
+ async xattrValues(paths) {
19105
+ try {
19106
+ const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
19107
+ timeoutMs: 15000,
19108
+ maxBuffer: 8 * 1024 * 1024
19109
+ });
19110
+ return parseXattrValues(stdout, paths);
19111
+ } catch (error) {
19112
+ const failure = error;
19113
+ return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
19114
+ }
18970
19115
  }
18971
- if ((options.mode ?? "install") === "cached") {
18972
- const state = everythingCached || esCached ? "are incomplete or do not match their pinned SHA-256 digests" : "are not installed";
18973
- return {
18974
- ok: false,
18975
- reason: "not-installed",
18976
- message: `Everything ${manifest.everythingVersion} (${arch}) binaries ${state} under ${dirname16(everythingPath)}; run \`autorag refresh\` to (re)install and index them.`
18977
- };
19116
+ async tryReadDir(path) {
19117
+ try {
19118
+ return await this.readDir(path);
19119
+ } catch {
19120
+ return;
19121
+ }
18978
19122
  }
18979
- try {
18980
- mkdirSync15(dirname16(everythingPath), { recursive: true });
18981
- const everythingBytes = await extractVerified(join29(bundleDir, assets.everythingArchive), assets.everythingArchiveSha256, assets.everythingMember, assets.everythingSha256);
18982
- const esBytes = await extractVerified(join29(bundleDir, assets.esArchive), assets.esArchiveSha256, assets.esMember, assets.esSha256);
18983
- writeAtomic(everythingPath, everythingBytes);
18984
- writeAtomic(esPath, esBytes);
18985
- } catch (error) {
18986
- return { ok: false, reason: "install-failed", message: errorMessage2(error) };
19123
+ async datalessAmong(paths) {
19124
+ const dataless = new Set;
19125
+ for (let start = 0;start < paths.length; start += STAT_BATCH) {
19126
+ const batch = paths.slice(start, start + STAT_BATCH);
19127
+ const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
19128
+ timeoutMs: 30000,
19129
+ maxBuffer: 4 * 1024 * 1024
19130
+ });
19131
+ const flags = stdout.split(`
19132
+ `).filter(Boolean);
19133
+ for (const [index, path] of batch.entries()) {
19134
+ if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
19135
+ dataless.add(path);
19136
+ }
19137
+ }
19138
+ return dataless;
18987
19139
  }
18988
- return { ok: true, everythingPath, esPath, source: "installed" };
18989
- }
18990
- async function extractVerified(archivePath, archiveSha256, member, memberSha256) {
18991
- const archive = readFileSync18(archivePath);
18992
- const actualArchive = sha256(archive);
18993
- if (actualArchive !== archiveSha256) {
18994
- throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
19140
+ async providerOf(path) {
19141
+ const domain = await readFileProviderDomainUpwards(path);
19142
+ return domain === undefined ? undefined : cloudProviderFromDomain(domain);
18995
19143
  }
18996
- const zip = await JSZip.loadAsync(archive);
18997
- const entry = zip.file(member);
18998
- if (entry === null)
18999
- throw new Error(`${archivePath} does not contain ${member}`);
19000
- const bytes = await entry.async("nodebuffer");
19001
- const actualMember = sha256(bytes);
19002
- if (actualMember !== memberSha256) {
19003
- throw new Error(`${member} from ${archivePath} SHA-256 ${actualMember} does not match pinned ${memberSha256}`);
19144
+ canRelease(provider) {
19145
+ return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
19146
+ }
19147
+ async evict(path) {
19148
+ let stdout;
19149
+ try {
19150
+ stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
19151
+ timeoutMs: 30000
19152
+ })).stdout;
19153
+ } catch (error) {
19154
+ return { ok: false, message: describeReleaseHelperFailure(error) };
19155
+ }
19156
+ const outcome = parseReleaseOutcome(stdout);
19157
+ if (outcome === undefined) {
19158
+ const detail = stdout.trim();
19159
+ return {
19160
+ ok: false,
19161
+ message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
19162
+ };
19163
+ }
19164
+ return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
19004
19165
  }
19005
- return bytes;
19006
19166
  }
19007
- function fileHasSha256(path, expected) {
19008
- return existsSync18(path) && sha256(readFileSync18(path)) === expected;
19167
+ function buildWindowsAttributeScript(paths) {
19168
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19169
+ ` + `$ErrorActionPreference='SilentlyContinue'
19170
+ ` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
19171
+ `;
19009
19172
  }
19010
- function writeAtomic(path, bytes) {
19011
- const temp = `${path}.${process.pid}.tmp`;
19012
- try {
19013
- writeFileSync10(temp, bytes);
19014
- renameWithRetry(temp, path);
19015
- } finally {
19016
- rmSync7(temp, { force: true });
19173
+ function buildWindowsSyncRootsScript() {
19174
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19175
+ ` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
19176
+ ` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
19177
+ `;
19178
+ }
19179
+ function buildWindowsDehydrateScript(path) {
19180
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19181
+ ` + `$ErrorActionPreference='Stop'
19182
+ ` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
19183
+ ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
19184
+ ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
19185
+ ` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
19186
+ ` + `'@
19187
+ ` + `$h=[IntPtr]::Zero
19188
+ ` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
19189
+ ` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
19190
+ ` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
19191
+ ` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
19192
+ ` + `[Autorag.CfApi]::CfCloseHandle($h)
19193
+ ` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
19194
+ ` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
19195
+ `;
19196
+ }
19197
+ function parseWindowsAttributeOutput(stdout, paths) {
19198
+ const lines = stdout.split(/\r?\n/);
19199
+ const attributes = new Map;
19200
+ for (const [index, path] of paths.entries()) {
19201
+ const value = Number.parseInt((lines[index] ?? "").trim(), 10);
19202
+ if (Number.isFinite(value))
19203
+ attributes.set(path, value);
19017
19204
  }
19205
+ return attributes;
19018
19206
  }
19019
- function sha256(bytes) {
19020
- return createHash11("sha256").update(bytes).digest("hex");
19207
+ function parseWindowsSyncRoots(stdout) {
19208
+ const roots = [];
19209
+ for (const line of stdout.split(/\r?\n/)) {
19210
+ const tab = line.indexOf("\t");
19211
+ if (tab <= 0)
19212
+ continue;
19213
+ const path = line.slice(0, tab).trim();
19214
+ if (path !== "")
19215
+ roots.push({ path, label: line.slice(tab + 1).trim() });
19216
+ }
19217
+ return roots;
19021
19218
  }
19022
- function errorMessage2(error) {
19023
- return error instanceof Error ? error.message : String(error);
19219
+ function providerFromWindowsSyncRoot(label) {
19220
+ const lower = label.toLowerCase();
19221
+ if (lower.includes("onedrive") || lower.includes("one drive"))
19222
+ return "onedrive";
19223
+ if (lower.includes("google"))
19224
+ return "google-drive";
19225
+ return "file-provider";
19024
19226
  }
19025
- var MANIFEST_FILENAME = "manifest.json";
19026
- var init_bundle = __esm(() => {
19027
- init_atomic_write();
19028
- });
19029
-
19030
- // src/filesystem/path-exclusions.ts
19031
- import { posix as posix3, win32 as win322 } from "node:path";
19032
- function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
19033
- const paths = platform === "win32" ? win322 : posix3;
19034
- const normalize = (path) => {
19035
- const absolute = paths.resolve(path);
19036
- return platform === "win32" ? absolute.toLowerCase() : absolute;
19037
- };
19038
- const excluded = excludedPaths.map((path) => {
19039
- const absolute = normalize(path);
19040
- return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
19041
- });
19042
- return (path) => {
19043
- const absolute = normalize(path);
19044
- return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
19045
- };
19227
+ function buildWindowsDiskScript() {
19228
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19229
+ ` + "Get-CimInstance -ClassName Win32_LogicalDisk | ForEach-Object {" + `$_.DeviceID + [char]9 + $_.DriveType + [char]9 + $_.FileSystem + [char]9 + $_.VolumeName}
19230
+ `;
19046
19231
  }
19047
- var init_path_exclusions = () => {};
19048
-
19049
- // src/filesystem/shared-exclusive-queue.ts
19050
- class SharedExclusiveQueue {
19051
- sharedSinceExclusive = [];
19052
- lastExclusive = Promise.resolve();
19053
- shared(task) {
19054
- const run = this.lastExclusive.then(task);
19055
- const settled = run.then(() => {
19056
- return;
19057
- }, () => {
19058
- return;
19059
- });
19060
- this.sharedSinceExclusive.push(settled);
19061
- return run;
19062
- }
19063
- exclusive(task) {
19064
- const earlier = Promise.all([this.lastExclusive, ...this.sharedSinceExclusive]);
19065
- const run = earlier.then(task);
19066
- const settled = run.then(() => {
19067
- return;
19068
- }, () => {
19069
- return;
19232
+ function parseWindowsLogicalDisks(stdout) {
19233
+ const disks = [];
19234
+ for (const line of stdout.split(/\r?\n/)) {
19235
+ if (line.trim() === "")
19236
+ continue;
19237
+ const [deviceId, driveType, fileSystem, ...rest] = line.split("\t");
19238
+ const type = Number.parseInt((driveType ?? "").trim(), 10);
19239
+ if (!Number.isFinite(type))
19240
+ continue;
19241
+ disks.push({
19242
+ deviceId: (deviceId ?? "").trim(),
19243
+ driveType: type,
19244
+ fileSystem: (fileSystem ?? "").trim(),
19245
+ volumeName: rest.join("\t").trim()
19070
19246
  });
19071
- this.sharedSinceExclusive = [];
19072
- this.lastExclusive = settled;
19073
- return run;
19074
19247
  }
19248
+ return disks;
19075
19249
  }
19076
-
19077
- // src/everything/client.ts
19078
- import { spawn as spawn15 } from "node:child_process";
19079
- import { createHash as createHash12 } from "node:crypto";
19080
- import { mkdirSync as mkdirSync16, writeFileSync as writeFileSync11 } from "node:fs";
19081
- import { join as join30, resolve as resolve11 } from "node:path";
19082
- import { setTimeout as sleep2 } from "node:timers/promises";
19083
- function everythingInstanceName(root) {
19084
- return `autorag-${createHash12("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
19085
- }
19086
- function buildEverythingIni(input) {
19087
- const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
19088
- const perFolder = (value) => input.folders.map(() => value).join(",");
19089
- return [
19090
- "[Everything]",
19091
- "app_data=0",
19092
- "run_as_admin=0",
19093
- "run_in_background=1",
19094
- "show_tray_icon=0",
19095
- "show_in_taskbar=0",
19096
- "check_for_updates_on_startup=0",
19097
- "beta_updates=0",
19098
- "language=1033",
19099
- "allow_http_server=0",
19100
- "allow_etp_server=0",
19101
- "search_history_enabled=0",
19102
- "run_history_enabled=0",
19103
- "auto_include_fixed_volumes=0",
19104
- "auto_include_removable_volumes=0",
19105
- "auto_include_fixed_refs_volumes=0",
19106
- "auto_include_removable_refs_volumes=0",
19107
- "ntfs_volume_paths=",
19108
- "ntfs_volume_includes=",
19109
- "refs_volume_paths=",
19110
- "refs_volume_includes=",
19111
- "index_size=1",
19112
- "index_date_modified=1",
19113
- `folders=${list(input.folders)}`,
19114
- `folder_monitor_changes=${perFolder("1")}`,
19115
- `folder_rescan_if_full_list=${perFolder("1")}`,
19116
- `folder_update_types=${perFolder("0")}`,
19117
- "exclude_list_enabled=1",
19118
- `exclude_folders=${list(input.excludeFolders)}`,
19119
- ""
19120
- ].join(`\r
19121
- `);
19250
+ function windowsRootsFromDisks(disks) {
19251
+ const roots = [];
19252
+ for (const disk of disks) {
19253
+ if (disk.deviceId === "")
19254
+ continue;
19255
+ const path = `${disk.deviceId}\\`;
19256
+ const fileSystem = disk.fileSystem.toLowerCase();
19257
+ if (disk.driveType === WINDOWS_DRIVE_TYPE_NETWORK) {
19258
+ roots.push({ path, kind: "network" });
19259
+ continue;
19260
+ }
19261
+ if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
19262
+ roots.push({ path, kind: "network" });
19263
+ continue;
19264
+ }
19265
+ if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
19266
+ roots.push({ path, kind: "cloud-files" });
19267
+ }
19268
+ }
19269
+ return roots;
19122
19270
  }
19123
- function buildEverythingSearchArgs(instance, request, timeoutMs) {
19124
- const args = [
19125
- "-argv",
19126
- "-instance",
19127
- instance,
19128
- "-cp",
19129
- "65001",
19130
- "-json",
19131
- "-full-path-and-name",
19132
- "-size",
19133
- "-date-modified",
19134
- "-date-format",
19135
- "1"
19136
- ];
19137
- if (timeoutMs !== undefined)
19138
- args.push("-timeout", String(timeoutMs));
19139
- if (request.matchCase)
19140
- args.push("-case");
19141
- if (request.matchPath)
19142
- args.push("-match-path");
19143
- if (request.wholeWord)
19144
- args.push("-whole-word");
19145
- if (request.kind === "files")
19146
- args.push("/a-d");
19147
- if (request.kind === "folders")
19148
- args.push("/ad");
19149
- if (request.path !== undefined)
19150
- args.push("-path", request.path);
19151
- if (request.sort !== undefined)
19152
- args.push("-sort", request.sort);
19153
- if (request.offset !== undefined && request.offset > 0)
19154
- args.push("-offset", String(request.offset));
19155
- args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
19156
- if (request.regex)
19157
- args.push("-regex", "--", request.query);
19158
- else
19159
- args.push("--", request.query);
19160
- return args;
19271
+ function uncShareRoot(target) {
19272
+ const normalized = target.replaceAll("/", "\\");
19273
+ if (!normalized.startsWith("\\\\") || normalized.startsWith("\\\\?\\") || normalized.startsWith("\\\\.\\"))
19274
+ return;
19275
+ const rest = normalized.slice(2);
19276
+ const separator = rest.indexOf("\\");
19277
+ if (separator <= 0)
19278
+ return;
19279
+ const server = rest.slice(0, separator);
19280
+ const share = rest.slice(separator + 1).split("\\")[0] ?? "";
19281
+ if (server === "" || share === "")
19282
+ return;
19283
+ return `\\\\${server}\\${share}`;
19161
19284
  }
19162
- function parseEverythingJson(stdout) {
19163
- const text = stdout.trim();
19164
- if (text.length === 0)
19165
- return [];
19166
- let parsed;
19167
- try {
19168
- parsed = JSON.parse(text);
19169
- } catch {
19170
- throw new Error(`ES did not return JSON: ${text.slice(0, 2000)}`);
19171
- }
19172
- if (!Array.isArray(parsed))
19173
- throw new Error(`ES did not return a JSON array: ${text.slice(0, 2000)}`);
19174
- return parsed.map((item) => {
19175
- const record = item;
19176
- const filename = String(record.filename ?? "");
19177
- const isFolder = filename.endsWith("\\") && filename.length > 3;
19285
+ function parseWindowsDehydrateOutput(result) {
19286
+ const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
19287
+ if (open === null)
19288
+ return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
19289
+ if (Number.parseInt(open[1], 16) !== 0) {
19178
19290
  return {
19179
- path: isFolder ? filename.slice(0, -1) : filename,
19180
- type: filename.endsWith("\\") ? "folder" : "file",
19181
- size: typeof record.size === "number" ? record.size : undefined,
19182
- dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
19291
+ ok: false,
19292
+ message: windowsFailureMessage(result, `could not open the file for release (0x${open[1]})`)
19183
19293
  };
19184
- });
19294
+ }
19295
+ const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
19296
+ if (dehydrate === null) {
19297
+ return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
19298
+ }
19299
+ const hresult = dehydrate[1];
19300
+ if (Number.parseInt(hresult, 16) === 0)
19301
+ return { ok: true };
19302
+ const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
19303
+ return {
19304
+ ok: false,
19305
+ message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
19306
+ };
19307
+ }
19308
+ function windowsFailureMessage(result, fallback) {
19309
+ const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
19310
+ if (printed !== undefined && printed !== "")
19311
+ return printed;
19312
+ const stderr = result.stderr.trim();
19313
+ return stderr !== "" ? stderr : fallback;
19185
19314
  }
19186
19315
 
19187
- class EverythingClient {
19188
- instanceName;
19189
- options;
19190
- platform;
19316
+ class WindowsCloudProbe {
19191
19317
  run;
19192
- launch;
19193
- stateDir;
19194
- isExcludedPath;
19195
- binaries;
19196
- queue = new SharedExclusiveQueue;
19197
- constructor(options) {
19198
- this.options = options;
19199
- this.platform = options.platform ?? process.platform;
19200
- this.instanceName = everythingInstanceName(options.root);
19201
- this.run = options.run ?? runProcess;
19202
- this.launch = options.launch ?? launchDetached;
19203
- this.stateDir = join30(options.root, ".autorag", "everything");
19204
- this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
19205
- }
19206
- isSupported() {
19207
- return this.platform === "win32";
19318
+ syncRoots;
19319
+ disks;
19320
+ constructor(options = {}) {
19321
+ this.run = options.run ?? runPowerShell;
19208
19322
  }
19209
- search(request) {
19210
- return this.queue.shared(async () => {
19211
- const binaries = await this.resolveBinaries("cached");
19212
- if (!binaries.ok)
19213
- return binaries;
19214
- const ping = await this.ping(binaries.esPath);
19215
- if (ping.code !== 0) {
19216
- return {
19217
- ok: false,
19218
- reason: "not-running",
19219
- message: `Everything instance ${this.instanceName} is not running; run \`autorag refresh\` to start and index it.`
19220
- };
19221
- }
19222
- const es = { ok: true, esPath: binaries.esPath };
19223
- const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
19224
- if ((this.options.excludeFolders?.length ?? 0) > 0) {
19225
- return this.searchWithExclusions(es.esPath, request, timeoutMs);
19226
- }
19227
- const args = buildEverythingSearchArgs(this.instanceName, request, timeoutMs);
19228
- const result = await this.run(es.esPath, args, timeoutMs + 5000);
19229
- if (result.code !== 0)
19230
- return this.failure("search-failed", es.esPath, args, result);
19231
- try {
19232
- return { ok: true, results: parseEverythingJson(result.stdout) };
19233
- } catch (error) {
19234
- return { ok: false, reason: "search-failed", message: error.message };
19235
- }
19236
- });
19323
+ async cloudRoots(searchRoots = []) {
19324
+ const roots = [];
19325
+ const seen = new Set;
19326
+ const add = (path, kind) => {
19327
+ const canonical = canonicalPath(path);
19328
+ const key = normalizeCloudPath(canonical).toLowerCase();
19329
+ if (key === "" || seen.has(key))
19330
+ return;
19331
+ seen.add(key);
19332
+ roots.push({ path: canonical, kind });
19333
+ };
19334
+ for (const root of await this.loadSyncRoots())
19335
+ add(root.path, "cloud-files");
19336
+ for (const root of await this.loadDiskRoots())
19337
+ add(root.path, root.kind);
19338
+ for (const searchRoot of searchRoots) {
19339
+ const share = uncShareRoot(searchRoot);
19340
+ if (share !== undefined)
19341
+ add(share, "network");
19342
+ }
19343
+ return roots;
19237
19344
  }
19238
- async searchWithExclusions(esPath, request, timeoutMs) {
19239
- const deadline = Date.now() + timeoutMs;
19240
- const offset = request.offset ?? 0;
19241
- const maxResults = request.maxResults ?? DEFAULT_MAX_RESULTS;
19242
- const results = [];
19243
- let skipped = 0;
19244
- let nativeOffset = 0;
19245
- const timedOut = () => ({
19246
- ok: false,
19247
- reason: "search-failed",
19248
- message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
19249
- });
19250
- for (;; ) {
19251
- const remainingMs = deadline - Date.now();
19252
- if (remainingMs <= 0)
19253
- return timedOut();
19254
- const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
19255
- const response = await this.run(esPath, args, remainingMs + 5000);
19256
- if (response.code !== 0)
19257
- return this.failure("search-failed", esPath, args, response);
19258
- if (Date.now() >= deadline)
19259
- return timedOut();
19260
- let entries;
19261
- try {
19262
- entries = parseEverythingJson(response.stdout);
19263
- } catch (error) {
19264
- return { ok: false, reason: "search-failed", message: error.message };
19345
+ async datalessAmong(paths) {
19346
+ const dataless = new Set;
19347
+ for (let start = 0;start < paths.length; start += WINDOWS_ATTRIBUTE_BATCH) {
19348
+ const batch = paths.slice(start, start + WINDOWS_ATTRIBUTE_BATCH);
19349
+ const result = await this.run(buildWindowsAttributeScript(batch), 30000);
19350
+ if (!result.ok) {
19351
+ throw new Error(`Could not read Windows cloud file attributes: ${result.stderr.trim() || "powershell.exe failed"}`);
19265
19352
  }
19266
- for (const entry of entries) {
19267
- if (this.isExcludedPath(entry.path))
19268
- continue;
19269
- if (skipped < offset) {
19270
- skipped += 1;
19271
- continue;
19272
- }
19273
- results.push(entry);
19274
- if (results.length >= maxResults)
19275
- return { ok: true, results };
19353
+ for (const [path, attributes] of parseWindowsAttributeOutput(result.stdout, batch)) {
19354
+ if (datalessFromWindowsAttributes(attributes))
19355
+ dataless.add(path);
19276
19356
  }
19277
- if (entries.length < MAX_FILTER_CANDIDATES)
19278
- return { ok: true, results };
19279
- nativeOffset += entries.length;
19280
19357
  }
19358
+ return dataless;
19281
19359
  }
19282
- index() {
19283
- return this.queue.exclusive(async () => {
19284
- const es = await this.ensureRunning(true);
19285
- if (!es.ok)
19286
- return es;
19287
- const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
19288
- const saveArgs = ["-instance", this.instanceName, "-save-db"];
19289
- const saved = await this.run(es.esPath, saveArgs, timeoutMs);
19290
- if (saved.code !== 0)
19291
- return this.failure("index-failed", es.esPath, saveArgs, saved);
19292
- const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
19293
- const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
19294
- if (counted.code !== 0)
19295
- return this.failure("index-failed", es.esPath, countArgs, counted);
19296
- return {
19297
- ok: true,
19298
- indexedFolders: [...this.options.folders],
19299
- indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
19300
- };
19301
- });
19360
+ async providerOf(path) {
19361
+ const root = longestSyncRoot(await this.loadSyncRoots(), path);
19362
+ return root === undefined ? cloudProviderFromPath(path) : providerFromWindowsSyncRoot(root.label);
19302
19363
  }
19303
- stop() {
19304
- return this.queue.exclusive(async () => {
19305
- const binaries = await this.resolveBinaries("cached");
19306
- if (!binaries.ok)
19307
- return;
19308
- await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
19309
- });
19364
+ canRelease(provider) {
19365
+ return provider !== undefined;
19310
19366
  }
19311
- async resolveBinaries(mode) {
19312
- if (!this.isSupported()) {
19313
- return {
19314
- ok: false,
19315
- reason: "unsupported-platform",
19316
- message: `Everything is Windows-only; this host is ${this.platform}.`
19317
- };
19318
- }
19319
- if (this.binaries?.ok)
19320
- return this.binaries;
19321
- const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
19322
- const result = await resolve(mode);
19323
- if (result.ok)
19324
- this.binaries = result;
19325
- return result;
19367
+ async evict(path) {
19368
+ return parseWindowsDehydrateOutput(await this.run(buildWindowsDehydrateScript(path), 60000));
19326
19369
  }
19327
- async ensureRunning(reconfigure) {
19328
- const binaries = await this.resolveBinaries("install");
19329
- if (!binaries.ok)
19330
- return binaries;
19331
- const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
19332
- let ping = await this.ping(binaries.esPath);
19333
- if (ping.code === 0 && !reconfigure)
19334
- return { ok: true, esPath: binaries.esPath };
19335
- if (ping.code === 0) {
19336
- const exitArgs = ["-instance", this.instanceName, "-exit"];
19337
- const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
19338
- if (exited.code !== 0)
19339
- return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
19340
- }
19341
- mkdirSync16(this.stateDir, { recursive: true });
19342
- const iniPath = join30(this.stateDir, "Everything.ini");
19343
- writeFileSync11(iniPath, buildEverythingIni({
19344
- folders: this.options.folders,
19345
- excludeFolders: [
19346
- ...new Set([
19347
- join30(this.options.root, ".autorag"),
19348
- ...this.options.folders.map((folder) => join30(folder, ".autorag")),
19349
- ...this.options.excludeFolders ?? []
19350
- ])
19351
- ]
19352
- }));
19353
- const launchArgs = [
19354
- "-instance",
19355
- this.instanceName,
19356
- "-config",
19357
- iniPath,
19358
- "-db",
19359
- join30(this.stateDir, "Everything.db"),
19360
- "-startup"
19361
- ];
19362
- try {
19363
- this.launch(binaries.everythingPath, launchArgs);
19364
- } catch (error) {
19365
- return {
19366
- ok: false,
19367
- reason: "startup-failed",
19368
- message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
19369
- };
19370
- }
19371
- const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
19372
- const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
19373
- for (;; ) {
19374
- ping = await this.ping(binaries.esPath);
19375
- if (ping.code === 0)
19376
- return { ok: true, esPath: binaries.esPath };
19377
- if (Date.now() >= deadline)
19378
- break;
19379
- await sleep2(pollMs);
19380
- }
19381
- return {
19382
- ok: false,
19383
- reason: "startup-failed",
19384
- message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
19385
- };
19370
+ loadSyncRoots() {
19371
+ this.syncRoots ??= this.run(buildWindowsSyncRootsScript(), 15000).then((result) => result.ok ? parseWindowsSyncRoots(result.stdout) : [], () => []);
19372
+ return this.syncRoots;
19386
19373
  }
19387
- async ping(esPath) {
19388
- const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
19389
- return result.code === 0 && !VERSION_PATTERN.test(result.stdout.trim()) ? { ...result, code: -1 } : result;
19374
+ loadDiskRoots() {
19375
+ this.disks ??= this.run(buildWindowsDiskScript(), 30000).then((result) => result.ok ? windowsRootsFromDisks(parseWindowsLogicalDisks(result.stdout)) : [], () => []);
19376
+ return this.disks;
19390
19377
  }
19391
- failure(reason, esPath, args, result) {
19378
+ }
19379
+ function longestSyncRoot(roots, target) {
19380
+ const normalized = target.replaceAll("/", "\\").toLowerCase();
19381
+ let match;
19382
+ let matchLength = -1;
19383
+ for (const root of roots) {
19384
+ const prefix = root.path.replaceAll("/", "\\").replace(/\\+$/u, "").toLowerCase();
19385
+ if (prefix === "" || prefix.length <= matchLength)
19386
+ continue;
19387
+ if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
19388
+ match = root;
19389
+ matchLength = prefix.length;
19390
+ }
19391
+ }
19392
+ return match;
19393
+ }
19394
+ function powerShellLaunch(script) {
19395
+ return {
19396
+ args: [
19397
+ "-NoProfile",
19398
+ "-NonInteractive",
19399
+ "-Command",
19400
+ `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; Invoke-Expression $env:${POWERSHELL_SCRIPT_VARIABLE}`
19401
+ ],
19402
+ env: { [POWERSHELL_SCRIPT_VARIABLE]: script }
19403
+ };
19404
+ }
19405
+ async function runPowerShell(script, timeoutMs) {
19406
+ const launch = powerShellLaunch(script);
19407
+ try {
19408
+ const { stdout, stderr } = await execFileAsync("powershell.exe", [...launch.args], {
19409
+ timeout: timeoutMs,
19410
+ maxBuffer: 16 * 1024 * 1024,
19411
+ windowsHide: true,
19412
+ env: { ...process.env, ...launch.env }
19413
+ });
19414
+ return { ok: true, stdout, stderr };
19415
+ } catch (error) {
19416
+ const record = typeof error === "object" && error !== null ? error : {};
19392
19417
  return {
19393
19418
  ok: false,
19394
- reason,
19395
- message: `${esPath} ${args.join(" ")} exit ${result.code}: ${(result.stderr || result.stdout).trim()}`
19419
+ stdout: typeof record.stdout === "string" ? record.stdout : "",
19420
+ stderr: typeof record.stderr === "string" ? record.stderr : String(error)
19396
19421
  };
19397
19422
  }
19398
19423
  }
19399
- function runProcess(command, args, timeoutMs) {
19400
- let resolvePromise;
19401
- const promise = new Promise((resolve) => {
19402
- resolvePromise = resolve;
19403
- });
19404
- const child = spawn15(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
19405
- const stdout = [];
19406
- const stderr = [];
19407
- let bytes = 0;
19408
- let overflow = false;
19409
- const collect = (target) => (chunk) => {
19410
- bytes += chunk.length;
19411
- if (bytes > MAX_OUTPUT_BYTES) {
19412
- overflow = true;
19413
- child.kill();
19414
- return;
19415
- }
19416
- target.push(chunk);
19417
- };
19418
- child.stdout.on("data", collect(stdout));
19419
- child.stderr.on("data", collect(stderr));
19420
- const timer = setTimeout(() => child.kill(), timeoutMs);
19421
- child.on("error", (error) => {
19422
- clearTimeout(timer);
19423
- resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
19424
- });
19425
- child.on("close", (code, signal) => {
19426
- clearTimeout(timer);
19427
- const err = Buffer.concat(stderr).toString("utf8");
19428
- resolvePromise({
19429
- code,
19430
- stdout: Buffer.concat(stdout).toString("utf8"),
19431
- stderr: overflow ? `${err}
19432
- output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
19433
- killed by ${signal} after ${timeoutMs}ms` : err
19424
+ function defaultCloudProbe() {
19425
+ if (process.platform === "darwin")
19426
+ return new DarwinCloudProbe;
19427
+ if (process.platform === "win32")
19428
+ return new WindowsCloudProbe;
19429
+ return;
19430
+ }
19431
+ async function readFileProviderDomain(target) {
19432
+ try {
19433
+ await lstat(target);
19434
+ const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
19435
+ timeout: 2000
19434
19436
  });
19435
- });
19436
- return promise;
19437
+ const value = stdout.trim();
19438
+ return value.length > 0 ? value : undefined;
19439
+ } catch {
19440
+ return;
19441
+ }
19437
19442
  }
19438
- function launchDetached(command, args) {
19439
- const child = spawn15(command, args, { detached: true, stdio: "ignore", windowsHide: true });
19440
- child.on("error", () => {});
19441
- child.unref();
19443
+ async function readFileProviderDomainUpwards(target) {
19444
+ let cursor = target;
19445
+ for (let depth = 0;depth < 12; depth += 1) {
19446
+ const domain = await readFileProviderDomain(cursor);
19447
+ if (domain !== undefined)
19448
+ return domain;
19449
+ const parent = dirname16(cursor);
19450
+ if (parent === cursor)
19451
+ return;
19452
+ cursor = parent;
19453
+ }
19454
+ return;
19442
19455
  }
19443
- var DEFAULT_TIMEOUT_MS8 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
19444
- var init_client9 = __esm(() => {
19445
- init_path_exclusions();
19446
- init_bundle();
19447
- DEFAULT_INDEX_TIMEOUT_MS3 = 10 * 60000;
19448
- MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
19449
- VERSION_PATTERN = /^\d+\.\d+\.\d+\.\d+$/;
19450
- });
19451
-
19452
- // src/everything/index.ts
19453
- var init_everything = __esm(() => {
19454
- init_bundle();
19455
- init_client9();
19456
+ function quotePowerShell(value) {
19457
+ return `'${value.replace(/['\u2018\u2019\u201A\u201B]/gu, (quote) => quote + quote)}'`;
19458
+ }
19459
+ var execFileAsync, SF_DATALESS = 1073741824, FILE_PROVIDER_XATTR = "com.apple.file-provider-domain-id", WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 4194304, STAT_BATCH = 200, defaultCloudExec = (file, args, options) => execFileAsync(file, [...args], {
19460
+ timeout: options?.timeoutMs ?? 30000,
19461
+ ...options?.maxBuffer === undefined ? {} : { maxBuffer: options.maxBuffer }
19462
+ }), JXA_RELEASE_PROGRAM = `ObjC.import("Foundation");
19463
+ function run(argv) {
19464
+ var url = $.NSURL.fileURLWithPath(argv[0]);
19465
+ var errorRef = Ref();
19466
+ var released = $.NSFileManager.defaultManager.evictUbiquitousItemAtURLError(url, errorRef);
19467
+ if (released) return JSON.stringify({ ok: true });
19468
+ function resourceValue(key) {
19469
+ var ref = Ref();
19470
+ if (!url.getResourceValueForKeyError(ref, key, null)) return null;
19471
+ var value = ref[0];
19472
+ return value === undefined || value === null ? null : ObjC.unwrap(value);
19473
+ }
19474
+ return JSON.stringify({
19475
+ ok: false,
19476
+ isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
19477
+ isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
19478
+ downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
19479
+ });
19480
+ }`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
19481
+ var init_cloud_placeholder = __esm(() => {
19482
+ execFileAsync = promisify(execFile2);
19483
+ RELEASABLE_DARWIN_PROVIDERS = {
19484
+ icloud: true,
19485
+ "google-drive": true,
19486
+ onedrive: true,
19487
+ dropbox: true
19488
+ };
19489
+ DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
19490
+ WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
19491
+ WINDOWS_DEHYDRATE_REFUSALS = {
19492
+ "80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
19493
+ "80070188": "The file is pinned to stay on this device, so it is not released."
19494
+ };
19456
19495
  });
19457
19496
 
19458
19497
  // src/fsearch/walk.ts
19459
19498
  import { readdir as readdir2, stat as stat3 } from "node:fs/promises";
19460
- import { basename as basename8, join as join31, sep as sep5 } from "node:path";
19499
+ import { basename as basename7, join as join31, sep as sep5 } from "node:path";
19461
19500
  function compare(a, b) {
19462
19501
  return a < b ? -1 : a > b ? 1 : 0;
19463
19502
  }
@@ -19468,11 +19507,11 @@ async function walkFileSearch(folders, request, options = {}) {
19468
19507
  const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_DEADLINE_MS);
19469
19508
  const matches = request.regex ? (() => {
19470
19509
  const pattern = new RegExp(request.query, request.matchCase ? "" : "i");
19471
- return (path) => pattern.test(request.matchPath ? path : basename8(path));
19510
+ return (path) => pattern.test(request.matchPath ? path : basename7(path));
19472
19511
  })() : (() => {
19473
19512
  const needle = request.matchCase ? request.query : request.query.toLowerCase();
19474
19513
  return (path) => {
19475
- const haystack = request.matchPath ? path : basename8(path);
19514
+ const haystack = request.matchPath ? path : basename7(path);
19476
19515
  return (request.matchCase ? haystack : haystack.toLowerCase()).includes(needle);
19477
19516
  };
19478
19517
  })();
@@ -19527,7 +19566,7 @@ async function walkFileSearch(folders, request, options = {}) {
19527
19566
  continue;
19528
19567
  collected.push({
19529
19568
  path,
19530
- name: basename8(path),
19569
+ name: basename7(path),
19531
19570
  type: info.isDirectory() ? "folder" : "file",
19532
19571
  size: info.isDirectory() ? undefined : info.size,
19533
19572
  dateModified: info.mtime.toISOString()
@@ -19552,7 +19591,7 @@ async function walkFileSearch(folders, request, options = {}) {
19552
19591
  continue;
19553
19592
  collected.push({
19554
19593
  path: folder,
19555
- name: basename8(folder),
19594
+ name: basename7(folder),
19556
19595
  type: "folder",
19557
19596
  size: undefined,
19558
19597
  dateModified: info.mtime.toISOString()
@@ -19585,7 +19624,7 @@ var init_walk = __esm(() => {
19585
19624
  import { spawn as spawn16 } from "node:child_process";
19586
19625
  import { createHash as createHash13 } from "node:crypto";
19587
19626
  import { existsSync as existsSync19, mkdirSync as mkdirSync17, readFileSync as readFileSync19, realpathSync as realpathSync4, rmSync as rmSync8, writeFileSync as writeFileSync12 } from "node:fs";
19588
- import { basename as basename9, join as join32, resolve as resolve12, sep as sep6 } from "node:path";
19627
+ import { basename as basename8, join as join32, resolve as resolve12, sep as sep6 } from "node:path";
19589
19628
  import { setTimeout as sleep3 } from "node:timers/promises";
19590
19629
  function fsearchSocketPath(dbPath) {
19591
19630
  let canonical;
@@ -19649,7 +19688,7 @@ function parseFSearchJsonLines(stdout) {
19649
19688
  const path = String(record.path ?? "");
19650
19689
  entries.push({
19651
19690
  path,
19652
- name: basename9(path),
19691
+ name: basename8(path),
19653
19692
  type: record.type === "folder" ? "folder" : "file",
19654
19693
  size: typeof record.size === "number" ? record.size : undefined,
19655
19694
  dateModified: typeof record.mtime === "number" ? new Date(record.mtime * 1000).toISOString() : undefined
@@ -20067,7 +20106,7 @@ var init_diagnostics = __esm(() => {
20067
20106
  });
20068
20107
 
20069
20108
  // src/jikji/retry-command.ts
20070
- import { basename as basename10 } from "node:path";
20109
+ import { basename as basename9 } from "node:path";
20071
20110
  function shellWords(command) {
20072
20111
  const words = [];
20073
20112
  let current = "";
@@ -20134,7 +20173,7 @@ function parseJikjiRetryCommand(command, envelopeProof) {
20134
20173
  const words = shellWords(command);
20135
20174
  if (words === undefined || words.length < 4)
20136
20175
  return;
20137
- const program = basename10(words[0]).replace(/\.exe$/iu, "");
20176
+ const program = basename9(words[0]).replace(/\.exe$/iu, "");
20138
20177
  if (program !== "jikji" || words[1] !== "discover")
20139
20178
  return;
20140
20179
  const root = words[2];
@@ -20542,7 +20581,7 @@ class SerialQueue {
20542
20581
 
20543
20582
  // src/jikji/version.ts
20544
20583
  function parseJikjiVersion(output) {
20545
- return VERSION_LINE2.exec(output)?.[1];
20584
+ return VERSION_LINE3.exec(output)?.[1];
20546
20585
  }
20547
20586
  function parts(version) {
20548
20587
  return version.split(".").map((part) => Number.parseInt(part, 10));
@@ -20558,9 +20597,9 @@ function isSupportedJikjiVersion(version) {
20558
20597
  }
20559
20598
  return true;
20560
20599
  }
20561
- var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1", VERSION_LINE2;
20562
- var init_version2 = __esm(() => {
20563
- VERSION_LINE2 = /^jikji(?:-cli)?\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
20600
+ var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1", VERSION_LINE3;
20601
+ var init_version3 = __esm(() => {
20602
+ VERSION_LINE3 = /^jikji(?:-cli)?\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
20564
20603
  });
20565
20604
 
20566
20605
  // src/jikji/client.ts
@@ -20928,7 +20967,7 @@ var init_client11 = __esm(() => {
20928
20967
  init_answer_pack();
20929
20968
  init_installer3();
20930
20969
  init_prepare_budget();
20931
- init_version2();
20970
+ init_version3();
20932
20971
  ABORTED_BEFORE_START = { ok: false, reason: "aborted", stdout: "", stderr: "", code: null };
20933
20972
  });
20934
20973
 
@@ -22013,7 +22052,7 @@ import {
22013
22052
  rmSync as rmSync10,
22014
22053
  writeFileSync as writeFileSync14
22015
22054
  } from "node:fs";
22016
- import { basename as basename11, dirname as dirname19, isAbsolute as isAbsolute9, join as join37, relative as relative7, sep as sep8 } from "node:path";
22055
+ import { basename as basename10, dirname as dirname19, isAbsolute as isAbsolute9, join as join37, relative as relative7, sep as sep8 } from "node:path";
22017
22056
  function syncMinSyncWorkspace(root, options = {}) {
22018
22057
  const workspacePath = options.workspacePath ?? minSyncWorkspaceRoot(root);
22019
22058
  const filesRoot = join37(workspacePath, "files");
@@ -22054,7 +22093,7 @@ function syncMinSyncWorkspace(root, options = {}) {
22054
22093
  return { workspacePath, entries, excluded };
22055
22094
  }
22056
22095
  function stagingStatePath(workspacePath) {
22057
- return join37(dirname19(workspacePath), `.${basename11(workspacePath)}-autorag-staging.json`);
22096
+ return join37(dirname19(workspacePath), `.${basename10(workspacePath)}-autorag-staging.json`);
22058
22097
  }
22059
22098
  function loadStagingState2(workspacePath) {
22060
22099
  const path = stagingStatePath(workspacePath);
@@ -22105,7 +22144,7 @@ function ensureManagedDirectory(filesRoot, directory) {
22105
22144
  }
22106
22145
  }
22107
22146
  function copyFileAtomically(source, destination) {
22108
- const temporaryPath = join37(dirname19(destination), `.${basename11(destination)}.${randomUUID6()}.tmp`);
22147
+ const temporaryPath = join37(dirname19(destination), `.${basename10(destination)}.${randomUUID6()}.tmp`);
22109
22148
  try {
22110
22149
  copyFileSync4(source, temporaryPath);
22111
22150
  renameWithRetry(temporaryPath, destination);
@@ -22159,7 +22198,7 @@ var init_workspace = __esm(() => {
22159
22198
 
22160
22199
  // src/minsync/method.ts
22161
22200
  import { existsSync as existsSync24, readFileSync as readFileSync22, realpathSync as realpathSync7, statSync as statSync8 } from "node:fs";
22162
- import { basename as basename12, join as join38, normalize } from "node:path";
22201
+ import { basename as basename11, join as join38, normalize } from "node:path";
22163
22202
 
22164
22203
  class MinSyncVectorMethod {
22165
22204
  root;
@@ -22260,7 +22299,7 @@ class MinSyncVectorMethod {
22260
22299
  const entry = byPath.get(hit.path);
22261
22300
  if (!entry || !matchesVirtualPathScope(entry.virtualPath, options.scope))
22262
22301
  continue;
22263
- const chunkDiscriminator = hit.docId ?? basename12(hit.path);
22302
+ const chunkDiscriminator = hit.docId ?? basename11(hit.path);
22264
22303
  const chunkId = `minsync:${entry.virtualPath}:${chunkDiscriminator}`;
22265
22304
  const location = locateChunkInMirror(mirrorTexts, entry, hit.text);
22266
22305
  results.push({
@@ -22360,18 +22399,24 @@ var init_minsync = __esm(() => {
22360
22399
 
22361
22400
  // src/mirror/cloud-fingerprints.ts
22362
22401
  import { existsSync as existsSync25, readFileSync as readFileSync23 } from "node:fs";
22363
- import { join as join39 } from "node:path";
22402
+ import { extname as extname5, join as join39 } from "node:path";
22364
22403
  function cloudFingerprintStorePath(root) {
22365
22404
  return join39(root, ".autorag", CLOUD_FINGERPRINTS_FILE);
22366
22405
  }
22367
22406
  function emptyCloudFingerprints() {
22368
- return { version: 1, entries: {} };
22407
+ return { version: 2, entries: {} };
22369
22408
  }
22370
- function isCloudFingerprint(value) {
22409
+ function readCloudFingerprint(sourcePath, value, storeVersion) {
22371
22410
  if (typeof value !== "object" || value === null)
22372
- return false;
22411
+ return;
22373
22412
  const record = value;
22374
- return typeof record.sourceVersion === "string" && typeof record.exact === "string" && typeof record.chars === "number" && (record.modified === null || typeof record.modified === "string") && (record.revision === null || typeof record.revision === "string");
22413
+ const { sourceVersion, exact, modified, revision } = record;
22414
+ if (typeof sourceVersion !== "string" || typeof exact !== "string" || modified !== null && typeof modified !== "string" || revision !== null && typeof revision !== "string") {
22415
+ return;
22416
+ }
22417
+ if (storeVersion !== 2 && (record.chars === 0 || extname5(sourcePath).toLowerCase() === ".hwp"))
22418
+ return;
22419
+ return { sourceVersion, exact, modified, revision };
22375
22420
  }
22376
22421
  function loadCloudFingerprints(root) {
22377
22422
  const path = cloudFingerprintStorePath(root);
@@ -22384,24 +22429,26 @@ function loadCloudFingerprints(root) {
22384
22429
  const { entries } = parsed;
22385
22430
  if (typeof entries !== "object" || entries === null)
22386
22431
  return emptyCloudFingerprints();
22432
+ const storeVersion = "version" in parsed ? parsed.version : undefined;
22387
22433
  const valid = {};
22388
- for (const [sourcePath, fingerprint] of Object.entries(entries)) {
22389
- if (isCloudFingerprint(fingerprint))
22434
+ for (const [sourcePath, value] of Object.entries(entries)) {
22435
+ const fingerprint = readCloudFingerprint(sourcePath, value, storeVersion);
22436
+ if (fingerprint !== undefined)
22390
22437
  valid[sourcePath] = fingerprint;
22391
22438
  }
22392
- return { version: 1, entries: valid };
22439
+ return { version: 2, entries: valid };
22393
22440
  } catch {
22394
22441
  return emptyCloudFingerprints();
22395
22442
  }
22396
22443
  }
22397
22444
  function saveCloudFingerprints(root, entries) {
22398
- writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version: 1, entries }, null, 2)}
22445
+ writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version: 2, entries }, null, 2)}
22399
22446
  `);
22400
22447
  }
22401
22448
  function cloudFingerprintExclusions(store) {
22402
22449
  const map = new Map;
22403
22450
  for (const [sourcePath, fingerprint] of Object.entries(store.entries)) {
22404
- map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified, chars: fingerprint.chars });
22451
+ map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified });
22405
22452
  }
22406
22453
  return map;
22407
22454
  }
@@ -22793,7 +22840,7 @@ class Parser {
22793
22840
  }
22794
22841
 
22795
22842
  // src/parser/csv.ts
22796
- import { basename as basename13, extname as extname5 } from "node:path";
22843
+ import { basename as basename12, extname as extname6 } from "node:path";
22797
22844
  function toCell(text) {
22798
22845
  return text.trim() === "" ? undefined : { text, kind: classifyText(text) };
22799
22846
  }
@@ -22924,7 +22971,7 @@ var init_csv = __esm(() => {
22924
22971
  this.#maxRows = options.maxRows ?? DEFAULT_MAX_ROWS;
22925
22972
  }
22926
22973
  async parse(input) {
22927
- const extension = extname5(input.virtualPath);
22974
+ const extension = extname6(input.virtualPath);
22928
22975
  const format = extension.slice(1).toLowerCase();
22929
22976
  let text = decodeText(input.bytes).replace(/^\uFEFF/, "");
22930
22977
  const directive = SEPARATOR_DIRECTIVE.exec(text);
@@ -22936,10 +22983,10 @@ var init_csv = __esm(() => {
22936
22983
  {
22937
22984
  code: "parser-warning",
22938
22985
  severity: "warning",
22939
- message: `"${basename13(input.virtualPath)}": indexed the first ${this.#maxRows} of ${total} rows`
22986
+ message: `"${basename12(input.virtualPath)}": indexed the first ${this.#maxRows} of ${total} rows`
22940
22987
  }
22941
22988
  ] : [];
22942
- const grid = { name: basename13(input.virtualPath, extension), rows: rows.map((row) => row.map(toCell)) };
22989
+ const grid = { name: basename12(input.virtualPath, extension), rows: rows.map((row) => row.map(toCell)) };
22943
22990
  return {
22944
22991
  markdown: normalizeMarkdown(renderWorkbook([grid])),
22945
22992
  metadata: { parser: this.name, format },
@@ -23478,7 +23525,7 @@ var init_plain_text = __esm(() => {
23478
23525
  });
23479
23526
 
23480
23527
  // src/parser/registry.ts
23481
- import { extname as extname6 } from "node:path";
23528
+ import { extname as extname7 } from "node:path";
23482
23529
 
23483
23530
  class ParserRegistry {
23484
23531
  parsersByExtension = new Map;
@@ -23497,7 +23544,7 @@ class ParserRegistry {
23497
23544
  }
23498
23545
  }
23499
23546
  getForVirtualPath(virtualPath) {
23500
- return this.parsersByExtension.get(normalizeExtension(extname6(virtualPath)));
23547
+ return this.parsersByExtension.get(normalizeExtension(extname7(virtualPath)));
23501
23548
  }
23502
23549
  list() {
23503
23550
  return Array.from(new Set(this.parsersByExtension.values()));
@@ -23510,7 +23557,7 @@ function normalizeExtension(extension) {
23510
23557
  var init_registry = () => {};
23511
23558
 
23512
23559
  // src/parser/spreadsheet.ts
23513
- import { extname as extname7 } from "node:path";
23560
+ import { extname as extname8 } from "node:path";
23514
23561
  import * as XLSX from "xlsx";
23515
23562
  function assertWorkbookSignature(format, bytes) {
23516
23563
  if (startsWith(bytes, ZIP_SIGNATURE) || startsWith(bytes, OLE_SIGNATURE))
@@ -23609,7 +23656,7 @@ var init_spreadsheet = __esm(() => {
23609
23656
  this.#maxRowsPerSheet = options.maxRowsPerSheet ?? DEFAULT_MAX_ROWS_PER_SHEET;
23610
23657
  }
23611
23658
  async parse(input) {
23612
- const format = extname7(input.virtualPath).slice(1).toLowerCase();
23659
+ const format = extname8(input.virtualPath).slice(1).toLowerCase();
23613
23660
  let workbook;
23614
23661
  try {
23615
23662
  assertWorkbookSignature(format, input.bytes);
@@ -23883,6 +23930,10 @@ class CloudGate {
23883
23930
  isPlaceholder(path) {
23884
23931
  return this.placeholders.has(path);
23885
23932
  }
23933
+ startBatch() {
23934
+ this.downloadedFiles = 0;
23935
+ this.downloadedBytes = 0;
23936
+ }
23886
23937
  async decide(path, sizeBytes, directory) {
23887
23938
  if (this.probe === undefined || !this.placeholders.has(path))
23888
23939
  return LOCAL;
@@ -23892,7 +23943,7 @@ class CloudGate {
23892
23943
  const overFiles = this.downloadedFiles >= this.maxFiles;
23893
23944
  const overBytes = this.downloadedFiles > 0 && this.downloadedBytes + sizeBytes > this.maxBytes;
23894
23945
  if (overFiles || overBytes)
23895
- return { kind: "defer" };
23946
+ return { kind: "next-batch" };
23896
23947
  this.downloadedFiles += 1;
23897
23948
  this.downloadedBytes += sizeBytes;
23898
23949
  return { kind: "download" };
@@ -23937,7 +23988,7 @@ var init_cloud = __esm(() => {
23937
23988
  // src/mirror/sync.ts
23938
23989
  import { createHash as createHash17 } from "node:crypto";
23939
23990
  import { opendir as opendir2, readFile, stat as stat4 } from "node:fs/promises";
23940
- import { basename as basename14, dirname as dirname22, extname as extname8, join as join43, resolve as resolve15, sep as sep9 } from "node:path";
23991
+ import { basename as basename13, dirname as dirname22, extname as extname9, join as join43, resolve as resolve15, sep as sep9 } from "node:path";
23941
23992
  import { setTimeout as sleep4 } from "node:timers/promises";
23942
23993
  function errorCode(error) {
23943
23994
  if (!(error instanceof Error))
@@ -24162,25 +24213,26 @@ async function syncWithRegistry(options, registry, isolated) {
24162
24213
  };
24163
24214
  const deferDeletions = options.onBatch !== undefined;
24164
24215
  const batchSize = Math.max(1, Math.floor(options.batchSize ?? DEFAULT_MIRROR_BATCH_SIZE));
24165
- const batchCount = Math.max(1, Math.ceil(current.length / batchSize));
24166
24216
  let batchIndex = 0;
24167
24217
  let batchStopped = false;
24168
24218
  const runBatch = async (virtualPaths, final) => {
24169
24219
  if (options.onBatch === undefined)
24170
24220
  return;
24171
24221
  batchIndex += 1;
24172
- const outcome = await options.onBatch({ index: batchIndex, total: batchCount, virtualPaths, final });
24222
+ const outcome = await options.onBatch({ index: batchIndex, virtualPaths, final });
24173
24223
  if (outcome?.stop === true)
24174
24224
  batchStopped = true;
24175
24225
  };
24176
24226
  if (!deferDeletions)
24177
24227
  await applyDeleteDecisions();
24178
24228
  await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
24179
- for (let batchStart = 0;batchStart < Math.max(current.length, 1); batchStart += batchSize) {
24180
- const batchEnd = Math.min(batchStart + batchSize, current.length);
24181
- const finalBatch = batchEnd >= current.length;
24229
+ let batchStart = 0;
24230
+ for (;; ) {
24231
+ const batchLimit = Math.min(batchStart + batchSize, current.length);
24232
+ let batchEnd = batchLimit;
24182
24233
  const changedBefore = written + deleted;
24183
- for (let position = batchStart;position < batchEnd; position += 1) {
24234
+ gate.startBatch();
24235
+ for (let position = batchStart;position < batchLimit; position += 1) {
24184
24236
  const entry = current[position];
24185
24237
  options.onProgress?.({
24186
24238
  processed: position,
@@ -24230,15 +24282,9 @@ async function syncWithRegistry(options, registry, isolated) {
24230
24282
  });
24231
24283
  continue;
24232
24284
  }
24233
- if (cloudDecision.kind === "defer") {
24234
- skipped += 1;
24235
- diagnostics.push({
24236
- code: "cloud-download-deferred",
24237
- severity: "info",
24238
- message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
24239
- source: entry.virtualPath
24240
- });
24241
- continue;
24285
+ if (cloudDecision.kind === "next-batch") {
24286
+ batchEnd = position;
24287
+ break;
24242
24288
  }
24243
24289
  if (pending.has(mirrorStageKey(entry.virtualPath))) {
24244
24290
  let bytes;
@@ -24311,7 +24357,7 @@ async function syncWithRegistry(options, registry, isolated) {
24311
24357
  continue;
24312
24358
  }
24313
24359
  await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
24314
- path: basename14(outputPath),
24360
+ path: basename13(outputPath),
24315
24361
  size: Buffer.byteLength(markdown)
24316
24362
  });
24317
24363
  for (const diagnostic of parsed.diagnostics ?? [])
@@ -24325,7 +24371,6 @@ async function syncWithRegistry(options, registry, isolated) {
24325
24371
  cloudFingerprintEntries[entry.sourcePath] = {
24326
24372
  sourceVersion: entry.sourceVersion,
24327
24373
  exact: fingerprintResult.exact,
24328
- chars: fingerprintResult.chars,
24329
24374
  modified: fingerprintResult.modified,
24330
24375
  revision: fingerprintResult.revision
24331
24376
  };
@@ -24353,13 +24398,15 @@ async function syncWithRegistry(options, registry, isolated) {
24353
24398
  }
24354
24399
  }
24355
24400
  }
24401
+ const finalBatch = batchEnd >= current.length;
24356
24402
  if (finalBatch && deferDeletions)
24357
24403
  await applyDeleteDecisions();
24358
24404
  if (finalBatch || written + deleted !== changedBefore) {
24359
24405
  await runBatch(current.slice(batchStart, batchEnd).map((item) => item.virtualPath), finalBatch);
24360
24406
  }
24361
- if (batchStopped)
24407
+ if (finalBatch || batchStopped)
24362
24408
  break;
24409
+ batchStart = batchEnd;
24363
24410
  }
24364
24411
  await session.finish();
24365
24412
  if (fingerprintCloud) {
@@ -24449,7 +24496,7 @@ async function detectMirrorStaleness(options) {
24449
24496
  }
24450
24497
  }
24451
24498
  function standaloneImageTooSmallMessage(virtualPath, bytes) {
24452
- if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname8(virtualPath))))
24499
+ if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname9(virtualPath))))
24453
24500
  return;
24454
24501
  return ocrImageTooSmallMessage(bytes);
24455
24502
  }
@@ -24617,7 +24664,7 @@ async function collectFiles(state, directory, depth) {
24617
24664
  }
24618
24665
  if (!entry.isFile())
24619
24666
  continue;
24620
- const extension = normalizeExtension2(extname8(entry.name));
24667
+ const extension = normalizeExtension2(extname9(entry.name));
24621
24668
  if (!state.supportedExtensions.has(extension))
24622
24669
  continue;
24623
24670
  let fileStat;
@@ -29662,7 +29709,7 @@ var init_automation_tool = __esm(() => {
29662
29709
  });
29663
29710
 
29664
29711
  // src/agent/citation-links.ts
29665
- import { basename as basename15, isAbsolute as isAbsolute12 } from "node:path";
29712
+ import { basename as basename14, isAbsolute as isAbsolute12 } from "node:path";
29666
29713
  import { pathToFileURL as pathToFileURL3 } from "node:url";
29667
29714
  function openableTarget(source) {
29668
29715
  if (/^https?:\/\//iu.test(source)) {
@@ -29680,7 +29727,7 @@ function linkCitations(markdown, sourceOf, isFile) {
29680
29727
  for (const marker of answerMarkers(markdown, isFile)) {
29681
29728
  if (marker.kind === "number")
29682
29729
  continue;
29683
- const labels = marker.kind === "file" ? [{ label: basename15(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
29730
+ const labels = marker.kind === "file" ? [{ label: basename14(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
29684
29731
  let linked = false;
29685
29732
  const parts = labels.map(({ label, source }) => {
29686
29733
  const target = source === undefined ? undefined : openableTarget(source);
@@ -31806,10 +31853,7 @@ function renderScanReport(details) {
31806
31853
  const lines = [
31807
31854
  `dupey scanned ${details.scans.length} configured root(s).`,
31808
31855
  `families=${details.familyCount} exactDuplicates=${details.exactDuplicateCount} extractionErrors=${errors}`,
31809
- "Review exact families before cleanup; near/contains families are not safe deletion evidence.",
31810
- ...details.skippedRoots.length === 0 ? [] : [
31811
- `Not scanned (inside a cloud or network folder; scanning would download or re-read every file): ${details.skippedRoots.join(", ")}. Duplicates there are not reported, and an exact duplicate is only found against files an earlier refresh fingerprinted.`
31812
- ]
31856
+ "Review exact families before cleanup; near/contains families are not safe deletion evidence."
31813
31857
  ];
31814
31858
  let rendered = 0;
31815
31859
  let omitted = 0;
@@ -32787,6 +32831,56 @@ var init_packaged_skill = __esm(() => {
32787
32831
  PACKAGED_SKILL_ROOTS = ["skills", "internal-skills"];
32788
32832
  });
32789
32833
 
32834
+ // src/agent/pipelined-sync.ts
32835
+ class PipelinedSync {
32836
+ #sync;
32837
+ #covered = [];
32838
+ #dirty = false;
32839
+ #running;
32840
+ #failure;
32841
+ #stopped = false;
32842
+ #runs = 0;
32843
+ constructor(options) {
32844
+ this.#sync = options.sync;
32845
+ }
32846
+ async batch(batch) {
32847
+ this.#raise();
32848
+ this.#covered.push(...batch.virtualPaths);
32849
+ this.#dirty = true;
32850
+ if (!this.#stopped)
32851
+ this.#running ??= this.#loop();
32852
+ if (batch.final)
32853
+ await this.#running;
32854
+ this.#raise();
32855
+ return this.#stopped ? { stop: true } : undefined;
32856
+ }
32857
+ async settle() {
32858
+ this.#stopped = true;
32859
+ await this.#running;
32860
+ }
32861
+ #raise() {
32862
+ if (this.#failure !== undefined)
32863
+ throw this.#failure.error;
32864
+ }
32865
+ async#loop() {
32866
+ try {
32867
+ while (this.#dirty && !this.#stopped) {
32868
+ this.#dirty = false;
32869
+ const virtualPaths = this.#covered;
32870
+ this.#covered = [];
32871
+ this.#runs += 1;
32872
+ const outcome = await this.#sync({ index: this.#runs, virtualPaths });
32873
+ if (outcome?.stop === true)
32874
+ this.#stopped = true;
32875
+ }
32876
+ } catch (error) {
32877
+ this.#failure = { error };
32878
+ } finally {
32879
+ this.#running = undefined;
32880
+ }
32881
+ }
32882
+ }
32883
+
32790
32884
  // src/agent/query-decomposition.ts
32791
32885
  import { completeSimple as completeSimple2 } from "@earendil-works/pi-ai/compat";
32792
32886
  function buildDecompositionPrompt(query) {
@@ -33026,7 +33120,7 @@ var init_query_routing = __esm(() => {
33026
33120
  false: "Every part of the question is answered with a concrete, specific, consistent fact (a name, date, number, or definite statement)."
33027
33121
  });
33028
33122
  BASE_ROUTE_OPTIONS = {
33029
- local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. When unsure whether intrinsic knowledge suffices, choose local.",
33123
+ local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. A request to find, locate, open, or look up a document, file, slide deck, presentation, paper, note, message, or other material is local even when it names no owner ('find the slides about X', 'where is the Y presentation', 'Z 관련 자료 찾아줘'): such material lives in the user's own files and chats, and a well-known topic is not a reason to answer from general knowledge or the web. When unsure whether intrinsic knowledge suffices, choose local.",
33030
33124
  extra: "None of the other options fits and no search of the user's own records is needed first. Either the request is answerable without looking anything up — stable common knowledge, a definition, simple reasoning, or small talk — or it is a task for the assistant's tools that is not file organization and not AutoRAG configuration: committing code, pushing a branch, opening a pull request, running tests or a build, installing a package, running a script, writing or editing code. Programming work is extra even when the code names the user's files or folders ('write a script that lowercases every PNG', 'add this to my parser'): the assistant is asked to produce or change code, not to act on those files now. So is a request to run something (tests, a build, a script, a command) and report the outcome, even when what it runs lives in the user's own repository: running it is a task for the tool, not a search of the user's records. Never extra when the request refers to the user's own life, files, records, chats, calendar, or history (e.g. 'my', 'I', 'our', 'my girlfriend', 'my case', 'my routine'), and never extra when the answer depends on facts that may be recent, specific, or changing (current events, recent releases, prices, versions): search those locally instead. If any other option fits, do not choose extra.",
33031
33125
  organize: "The user asks the assistant to DO something to files or folders instead of finding information: move or copy files into folders, tidy or sort a folder, bulk or pattern rename (for example by date or number), rename files according to what their content says, convert files to another format, create folder structures, group files by type or topic, archive, or move to the trash ('tidy this up', 'move these into folders', 'bulk rename these files', 'rename the files to match their content', 'convert these to PDF', 'sort my Downloads folder'). It is about acting on files and folders, never about looking up what a document says or where something is. A question that only asks to find or read a file's content is local, not organize."
33032
33126
  };
@@ -34264,7 +34358,7 @@ var init_page_loader = __esm(() => {
34264
34358
  });
34265
34359
 
34266
34360
  // src/web/fetch/url-target.ts
34267
- import { extname as extname9 } from "node:path";
34361
+ import { extname as extname10 } from "node:path";
34268
34362
  function repairCollapsedScheme(value) {
34269
34363
  const m = value.match(/^(https?):\/(?!\/)/i);
34270
34364
  return m ? `${m[1]}://${value.slice(m[0].length)}` : value;
@@ -34283,7 +34377,7 @@ function getFilenameExtensionHint(filename) {
34283
34377
  const lower = filename.toLowerCase();
34284
34378
  if (lower.endsWith(".tar.gz"))
34285
34379
  return ".tar.gz";
34286
- return extname9(filename).toLowerCase();
34380
+ return extname10(filename).toLowerCase();
34287
34381
  }
34288
34382
  function getExtensionHint(url, contentDisposition) {
34289
34383
  if (contentDisposition) {
@@ -34460,7 +34554,7 @@ var init_convert2 = __esm(() => {
34460
34554
  });
34461
34555
 
34462
34556
  // src/web/fetch/feeds.ts
34463
- import { extname as extname10 } from "node:path";
34557
+ import { extname as extname11 } from "node:path";
34464
34558
  import { XMLParser as XMLParser3 } from "fast-xml-parser";
34465
34559
  function getHtmlAttribute(tag, attribute) {
34466
34560
  const pattern = new RegExp(`\\b${attribute}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'=<>]+))`, "i");
@@ -34523,7 +34617,7 @@ function extractDocumentLinks(html, baseUrl) {
34523
34617
  const href = getHtmlAttribute(tag, "href");
34524
34618
  if (!href)
34525
34619
  continue;
34526
- const ext = extname10(href).toLowerCase();
34620
+ const ext = extname11(href).toLowerCase();
34527
34621
  if (!CONVERTIBLE_EXTENSIONS2.has(ext))
34528
34622
  continue;
34529
34623
  const resolved = href.startsWith("http") ? href : new URL(href, baseUrl).href;
@@ -36191,7 +36285,7 @@ var init_web_search_tool = __esm(() => {
36191
36285
  // src/agent/agent.ts
36192
36286
  import { randomUUID as randomUUID7 } from "node:crypto";
36193
36287
  import { existsSync as existsSync32, watch as fsWatch, mkdirSync as mkdirSync23, realpathSync as realpathSync8, statSync as statSync11, writeFileSync as writeFileSync17 } from "node:fs";
36194
- import { basename as basename16, dirname as dirname27, join as join53, relative as relative8, resolve as resolve17, sep as sep10 } from "node:path";
36288
+ import { basename as basename15, dirname as dirname27, join as join53, relative as relative8, resolve as resolve17, sep as sep10 } from "node:path";
36195
36289
  import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
36196
36290
  import { getAgentDir as getAgentDir5, keyText } from "@earendil-works/pi-coding-agent";
36197
36291
  function describeDecisionModel(status) {
@@ -36626,10 +36720,20 @@ class AutoRAGAgent {
36626
36720
  }
36627
36721
  async scanDuplicateDocuments() {
36628
36722
  if (this.dupeyOptions === false) {
36629
- return { scans: [], skippedRoots: [], familyCount: 0, exactDuplicateCount: 0 };
36723
+ return { scans: [], familyCount: 0, exactDuplicateCount: 0 };
36724
+ }
36725
+ const cloudRoots = await this.cloudRoots();
36726
+ const scanRoots = [];
36727
+ for (const searchPath of this.searchPaths) {
36728
+ const scope = cloudScope(searchPath, cloudRoots);
36729
+ if (scope.insideCloudRoot)
36730
+ continue;
36731
+ scanRoots.push({
36732
+ path: searchPath,
36733
+ excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path))
36734
+ });
36630
36735
  }
36631
- const { targets, skipped } = planDupeyScans(this.searchPaths, await this.cloudRoots());
36632
- const scans = await Promise.all(targets.map(({ path, excludeDirs }) => scanWithDupey(path, {
36736
+ const scans = await Promise.all(scanRoots.map(({ path, excludeDirs }) => scanWithDupey(path, {
36633
36737
  ...this.dupeyOptions || {},
36634
36738
  ...excludeDirs.length === 0 ? {} : { excludeDirs }
36635
36739
  })));
@@ -36643,7 +36747,7 @@ class AutoRAGAgent {
36643
36747
  }
36644
36748
  return count + [...hashes.values()].reduce((sum, size) => sum + Math.max(0, size - 1), 0);
36645
36749
  }, 0);
36646
- return { scans, skippedRoots: skipped, familyCount, exactDuplicateCount };
36750
+ return { scans, familyCount, exactDuplicateCount };
36647
36751
  }
36648
36752
  loadMemoryContextFor(query) {
36649
36753
  if (this.memoryContextCache?.query === query)
@@ -38296,6 +38400,10 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38296
38400
  inFlight: true,
38297
38401
  lastStartedAt: startedAt
38298
38402
  };
38403
+ const syncAbort = new AbortController;
38404
+ const forwardAbort = () => syncAbort.abort(signal?.reason);
38405
+ signal?.addEventListener("abort", forwardAbort, { once: true });
38406
+ let minSyncPipeline;
38299
38407
  try {
38300
38408
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38301
38409
  const batchedMinSync = needsParsed && wants("minsync");
@@ -38303,44 +38411,53 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38303
38411
  const minSyncBatchFailures = [];
38304
38412
  let minSyncSyncedBase = 0;
38305
38413
  let minSyncTotalBase = 0;
38306
- const minSyncBatch = async (batch) => {
38307
- signal?.throwIfAborted();
38308
- progress = updateRefreshProgress(progress, { phase: "minsync" });
38309
- writeRefreshProgress(this.workspaceProjectRoot, progress);
38310
- let batchTotal = minSyncTotalBase;
38311
- const result = await this.syncMinSync(force && batch.index === 1, signal, (minsyncProgress) => {
38312
- batchTotal = minSyncTotalBase + minsyncProgress.total;
38313
- progress = updateRefreshProgress(progress, {
38314
- minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: batchTotal }
38315
- });
38316
- writeRefreshProgress(this.workspaceProjectRoot, progress);
38317
- emitProgress({
38318
- phase: "minsync",
38319
- processed: minSyncSyncedBase + minsyncProgress.processed,
38320
- total: batchTotal
38321
- });
38322
- });
38323
- minSyncBatches.push(result);
38324
- if (result.ok)
38325
- minSyncBatchFailures.length = 0;
38326
- else if (result.reason !== "lock-held") {
38327
- for (const virtualPath of batch.virtualPaths) {
38328
- minSyncBatchFailures.push({
38329
- code: "minsync-sync-failed",
38330
- severity: "warning",
38331
- message: `This file stayed at its previous index entry: its batch could not be indexed and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
38332
- source: virtualPath
38414
+ minSyncPipeline = batchedMinSync ? new PipelinedSync({
38415
+ sync: async ({ index, virtualPaths }) => {
38416
+ syncAbort.signal.throwIfAborted();
38417
+ let syncTotal = minSyncTotalBase;
38418
+ const result = await this.syncMinSync(force && index === 1, syncAbort.signal, (minsyncProgress) => {
38419
+ syncTotal = minSyncTotalBase + minsyncProgress.total;
38420
+ progress = updateRefreshProgress(progress, {
38421
+ minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: syncTotal }
38422
+ });
38423
+ writeRefreshProgress(this.workspaceProjectRoot, progress);
38424
+ emitProgress({
38425
+ phase: "minsync",
38426
+ processed: minSyncSyncedBase + minsyncProgress.processed,
38427
+ total: syncTotal
38333
38428
  });
38429
+ });
38430
+ minSyncBatches.push(result);
38431
+ if (result.ok)
38432
+ minSyncBatchFailures.length = 0;
38433
+ else if (result.reason !== "lock-held") {
38434
+ for (const virtualPath of virtualPaths) {
38435
+ minSyncBatchFailures.push({
38436
+ code: "minsync-sync-failed",
38437
+ severity: "warning",
38438
+ message: `This file stayed at its previous index entry: the sync that covered it could not index it and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
38439
+ source: virtualPath
38440
+ });
38441
+ }
38334
38442
  }
38443
+ minSyncSyncedBase = syncTotal;
38444
+ minSyncTotalBase = syncTotal;
38445
+ return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
38335
38446
  }
38336
- minSyncSyncedBase = batchTotal;
38337
- minSyncTotalBase = batchTotal;
38338
- return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
38339
- };
38447
+ }) : undefined;
38448
+ const pipeline = minSyncPipeline;
38340
38449
  const summary = needsParsed ? await this.syncParsedMirrors(force, {
38341
38450
  ...opts?.retrySkipped === true ? { retrySkipped: true } : {},
38342
38451
  ...signal === undefined ? {} : { signal },
38343
- ...batchedMinSync ? { onBatch: minSyncBatch } : {},
38452
+ ...pipeline === undefined ? {} : {
38453
+ onBatch: (batch) => {
38454
+ if (batch.final) {
38455
+ progress = updateRefreshProgress(progress, { phase: "minsync" });
38456
+ writeRefreshProgress(this.workspaceProjectRoot, progress);
38457
+ }
38458
+ return pipeline.batch(batch);
38459
+ }
38460
+ },
38344
38461
  onProgress: (parsedProgress) => {
38345
38462
  progress = updateRefreshProgress(progress, {
38346
38463
  phase: "parsed",
@@ -38476,6 +38593,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38476
38593
  ...fsearch !== undefined ? { fsearch } : {}
38477
38594
  };
38478
38595
  } catch (error) {
38596
+ syncAbort.abort(error);
38479
38597
  this.refreshState = {
38480
38598
  ...this.refreshState,
38481
38599
  lastOutcome: "failed",
@@ -38489,6 +38607,8 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38489
38607
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38490
38608
  throw error;
38491
38609
  } finally {
38610
+ signal?.removeEventListener("abort", forwardAbort);
38611
+ await minSyncPipeline?.settle();
38492
38612
  this.refreshState = {
38493
38613
  ...this.refreshState,
38494
38614
  inFlight: false,
@@ -38710,15 +38830,15 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38710
38830
  const cloudRoots = await this.cloudRoots();
38711
38831
  const fingerprints = cloudFingerprintExclusions(loadCloudFingerprints(this.workspaceProjectRoot));
38712
38832
  for (const searchPath of this.searchPaths) {
38713
- const target = planDupeyScans([searchPath], cloudRoots).targets[0];
38833
+ const scope = cloudScope(searchPath, cloudRoots);
38714
38834
  let scan;
38715
- if (target === undefined) {
38835
+ if (scope.insideCloudRoot) {
38716
38836
  scan = { dir: searchPath, files: [], families: [], errors: [] };
38717
38837
  } else {
38718
38838
  try {
38719
38839
  scan = await scanWithDupey(searchPath, {
38720
38840
  ...this.dupeyOptions || {},
38721
- ...target.excludeDirs.length === 0 ? {} : { excludeDirs: target.excludeDirs }
38841
+ ...scope.nestedCloudRoots.length === 0 ? {} : { excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path)) }
38722
38842
  });
38723
38843
  } catch (error) {
38724
38844
  if (!(error instanceof DupeyCliError))
@@ -39283,7 +39403,7 @@ function toMinSyncReasonDiagnostic(reason) {
39283
39403
  function pinExcludedPath(path) {
39284
39404
  const resolvedPath = resolve17(path);
39285
39405
  try {
39286
- return join53(realpathSync8(dirname27(resolvedPath)), basename16(resolvedPath));
39406
+ return join53(realpathSync8(dirname27(resolvedPath)), basename15(resolvedPath));
39287
39407
  } catch {
39288
39408
  return resolvedPath;
39289
39409
  }
@@ -40272,7 +40392,7 @@ function readPackageVersion() {
40272
40392
  return "0.0.0";
40273
40393
  }
40274
40394
  }
40275
- var init_version3 = () => {};
40395
+ var init_version4 = () => {};
40276
40396
 
40277
40397
  // src/cli/commands/init.ts
40278
40398
  import { existsSync as existsSync35, readFileSync as readFileSync30 } from "node:fs";
@@ -42104,20 +42224,18 @@ var init_health = __esm(() => {
42104
42224
 
42105
42225
  // src/cli/commands/duplicates.ts
42106
42226
  import { resolve as resolve23 } from "node:path";
42107
- async function runDuplicates(ctx, scanner = (path, excludeDirs) => scanWithDupey(path, { excludeDirs }), cloudProbe = defaultCloudProbe()) {
42227
+ async function runDuplicates(ctx, scanner = (path) => scanWithDupey(path)) {
42108
42228
  try {
42109
42229
  const config = resolveConfig({ flags: ctx.flags, cwd: ctx.cwd });
42110
42230
  const requested = ctx.positionals[0];
42111
42231
  const roots = requested ? [resolve23(ctx.cwd, requested)] : config.searchPaths.map((path) => resolve23(path));
42112
- const plan = planDupeyScans(roots, await listCloudRoots(cloudProbe, roots));
42113
- const scans = await Promise.all(plan.targets.map(({ path, excludeDirs }) => scanner(path, excludeDirs)));
42232
+ const scans = await Promise.all(roots.map((root) => scanner(root)));
42114
42233
  const exactGroups = scans.flatMap(exactGroupsFromScan);
42115
42234
  const extractionErrors = scans.flatMap((scan) => scan.errors);
42116
42235
  if (ctx.json) {
42117
42236
  ctx.stdout(JSON.stringify({
42118
42237
  ok: true,
42119
42238
  roots,
42120
- skippedRoots: plan.skipped,
42121
42239
  exactGroups,
42122
42240
  families: scans.flatMap((scan) => scan.families),
42123
42241
  extractionErrors,
@@ -42126,14 +42244,11 @@ async function runDuplicates(ctx, scanner = (path, excludeDirs) => scanWithDupey
42126
42244
  return 0;
42127
42245
  }
42128
42246
  const lines = [
42129
- `duplicates: scanned ${scans.length} root(s)`,
42247
+ `duplicates: scanned ${roots.length} root(s)`,
42130
42248
  ` exact groups: ${exactGroups.length}`,
42131
42249
  ` other families: ${scans.reduce((count, scan) => count + scan.families.length, 0)}`,
42132
42250
  ` extraction errors: ${extractionErrors.length}`
42133
42251
  ];
42134
- for (const root of plan.skipped) {
42135
- lines.push(` not scanned (inside a cloud or network folder; scanning would download or re-read every file): ${root}`);
42136
- }
42137
42252
  for (const [index, group] of exactGroups.entries()) {
42138
42253
  lines.push(` exact group ${index + 1}:`);
42139
42254
  for (const file of group.files)
@@ -42161,7 +42276,6 @@ function exactGroupsFromScan(scan) {
42161
42276
  }
42162
42277
  var init_duplicates = __esm(() => {
42163
42278
  init_dupey();
42164
- init_cloud_placeholder();
42165
42279
  init_config5();
42166
42280
  init_output();
42167
42281
  });
@@ -42417,7 +42531,7 @@ var init_tui = __esm(() => {
42417
42531
  init_open_target();
42418
42532
  init_output();
42419
42533
  init_update_check();
42420
- init_version3();
42534
+ init_version4();
42421
42535
  init_whatsapp_extension();
42422
42536
  THINKING_LEVELS2 = ["off", "minimal", "low", "medium", "high", "xhigh", "max"];
42423
42537
  });
@@ -42552,7 +42666,7 @@ async function runUpdateCheck(ctx, deps = {}) {
42552
42666
  }
42553
42667
  var init_update_check2 = __esm(() => {
42554
42668
  init_update_check();
42555
- init_version3();
42669
+ init_version4();
42556
42670
  });
42557
42671
 
42558
42672
  // src/cli/commands/attachments.ts
@@ -43675,7 +43789,7 @@ async function runAutomate(ctx) {
43675
43789
 
43676
43790
  // src/cli/index.ts
43677
43791
  init_output();
43678
- init_version3();
43792
+ init_version4();
43679
43793
  var BOOLEAN_FLAGS = new Set([
43680
43794
  "json",
43681
43795
  "debug",