@autorag/librarian 2.13.5 → 2.13.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli/index.js CHANGED
@@ -5032,7 +5032,7 @@ function outdatedMinSyncVersion(binaryPath) {
5032
5032
  }
5033
5033
  return version !== undefined && !isSupportedMinSyncVersion(version) ? version : undefined;
5034
5034
  }
5035
- var MINSYNC_MIN_VERSION = "0.6.0", VERSION_PROBE_TIMEOUT_MS = 5000, VERSION_LINE, probeCache;
5035
+ var MINSYNC_MIN_VERSION = "0.7.1", VERSION_PROBE_TIMEOUT_MS = 5000, VERSION_LINE, probeCache;
5036
5036
  var init_version = __esm(() => {
5037
5037
  VERSION_LINE = /^minsync\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
5038
5038
  probeCache = new Map;
@@ -5276,7 +5276,7 @@ function isGitHubAsset(value) {
5276
5276
  function isRecord2(value) {
5277
5277
  return typeof value === "object" && value !== null;
5278
5278
  }
5279
- var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.6.0", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
5279
+ var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.7.3", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
5280
5280
  var init_installer = __esm(() => {
5281
5281
  init_atomic_write();
5282
5282
  init_process();
@@ -14567,7 +14567,8 @@ class WombatMethod {
14567
14567
  return matchesDatasourceScope(datasourceSourcePath("wombat", this.instanceId, hit.chunkId), options.scope);
14568
14568
  }).map((hit) => ({
14569
14569
  id: `wombat:${this.instanceId}:${hit.chunkId}`,
14570
- content: hit.snippet,
14570
+ content: `${sessionLabel(hit)}
14571
+ ${hit.snippet}`,
14571
14572
  source: datasourceSourcePath("wombat", this.instanceId, hit.chunkId),
14572
14573
  score: hit.score,
14573
14574
  metadata: {
@@ -14590,6 +14591,20 @@ class WombatMethod {
14590
14591
  })).slice(0, topK);
14591
14592
  }
14592
14593
  }
14594
+ function oneLine(value) {
14595
+ return value.replace(/\s+/gu, " ").trim();
14596
+ }
14597
+ function sessionLabel(hit) {
14598
+ const turns = hit.startTurn === undefined ? undefined : hit.endTurn === undefined || hit.endTurn === hit.startTurn ? `turn ${hit.startTurn}` : `turns ${hit.startTurn}-${hit.endTurn}`;
14599
+ const parts = [
14600
+ oneLine(hit.catalogKey),
14601
+ turns,
14602
+ hit.startedAt === undefined ? undefined : hit.startedAt.slice(0, 10),
14603
+ hit.title.length === 0 ? undefined : oneLine(hit.title),
14604
+ hit.cwd === undefined ? undefined : oneLine(hit.cwd)
14605
+ ].filter((part) => part !== undefined && part.length > 0);
14606
+ return `[${parts.join(" | ")}]`;
14607
+ }
14593
14608
  var WOMBAT_MODES, METHOD_TYPE;
14594
14609
  var init_methods7 = __esm(() => {
14595
14610
  init_errors2();
@@ -14702,7 +14717,7 @@ class WombatSkill {
14702
14717
  `);
14703
14718
  return {
14704
14719
  name: "datasource-wombat",
14705
- description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built.",
14720
+ description: "Search past coding-agent conversations (Codex, Claude Code, OpenCode, pi, omp): what was asked, decided, or built. Each hit names its session. Load this skill to open that session's full conversation or its original log with the tool calls that the search does not index.",
14706
14721
  content: [
14707
14722
  "# wombat coding-agent session datasource",
14708
14723
  "",
@@ -14716,10 +14731,10 @@ class WombatSkill {
14716
14731
  scopes,
14717
14732
  "",
14718
14733
  "The selectable retrieval modes are keyword (SPLADE), semantic, and hybrid. Scope arguments can only narrow trusted access. Source identifiers remain opaque and hierarchical.",
14719
- "Each result carries `agent`, `title`, `startedAt`, `startTurn`/`endTurn` and `sessionKey` (`<agent>:<session id>`).",
14734
+ "Every result starts with a label line, `[<sessionKey> | turns A-B | date | title | project]`, followed by the snippet. `sessionKey` is `<agent>:<session id>`.",
14720
14735
  "",
14721
14736
  "## Native CLI",
14722
- "The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the result's `sessionKey`:",
14737
+ "The external `wombat` CLI owns the mirror, the index and the session catalog. To read more than a snippet, call it through `bash` with the `sessionKey` from the result's label:",
14723
14738
  "- `wombat show <sessionKey> [--turn N]`: the full cleaned conversation, or one turn.",
14724
14739
  "- `wombat trace <sessionKey> --turn N --context K`: the original session log around that turn, including the tool calls and outputs that the index leaves out.",
14725
14740
  "- `wombat chunk context <chunk id> --json`: the turns before and after a result.",
@@ -14873,6 +14888,7 @@ function common(config, workspaceRoot) {
14873
14888
  function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
14874
14889
  const skills = [];
14875
14890
  const unknown = [];
14891
+ const retired = [];
14876
14892
  for (const [name, raw] of Object.entries(config ?? {})) {
14877
14893
  if (raw === false)
14878
14894
  continue;
@@ -14884,6 +14900,10 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
14884
14900
  if (entry.enabled === false)
14885
14901
  continue;
14886
14902
  const templateName = entry.type ?? name;
14903
+ if (Object.hasOwn(RETIRED_TEMPLATES, templateName)) {
14904
+ retired.push(name);
14905
+ continue;
14906
+ }
14887
14907
  const builder = Object.hasOwn(BUILDERS, templateName) && datasourceSkillSupported(templateName) ? BUILDERS[templateName] : undefined;
14888
14908
  if (builder === undefined) {
14889
14909
  unknown.push(name);
@@ -14907,9 +14927,9 @@ function buildDatasourceSkills(config, workspaceRoot, embedding = {}) {
14907
14927
  }
14908
14928
  skills.push(typeof entry.description === "string" && entry.description.trim().length > 0 ? new DescribedDatasourceSkill(aliased, entry.description.trim()) : aliased);
14909
14929
  }
14910
- return { skills, unknown };
14930
+ return { skills, unknown, retired };
14911
14931
  }
14912
- var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES;
14932
+ var BUILDERS, PLATFORM_ONLY_SKILLS, BUILTIN_DATASOURCE_SKILL_NAMES, RETIRED_TEMPLATES;
14913
14933
  var init_factory = __esm(() => {
14914
14934
  init_aliased_skill();
14915
14935
  init_config3();
@@ -15065,6 +15085,7 @@ var init_factory = __esm(() => {
15065
15085
  };
15066
15086
  PLATFORM_ONLY_SKILLS = { kakao: "darwin" };
15067
15087
  BUILTIN_DATASOURCE_SKILL_NAMES = builtinDatasourceSkillNames();
15088
+ RETIRED_TEMPLATES = { spotlight: true };
15068
15089
  });
15069
15090
 
15070
15091
  // src/language.ts
@@ -16297,7 +16318,7 @@ function buildAgentOptions(config, agentDir) {
16297
16318
  const embedder = createEmbedderFromConfig({ embedder: config.minSync?.embedder });
16298
16319
  opts.embedder = embedder;
16299
16320
  if (config.datasources !== undefined) {
16300
- const { skills, unknown } = buildDatasourceSkills(config.datasources, config.workspacePath, {
16321
+ const { skills, unknown, retired } = buildDatasourceSkills(config.datasources, config.workspacePath, {
16301
16322
  embedder,
16302
16323
  discrawl: async () => discrawlEmbeddingRuntimeFor(await embedder.describeEndpoint()),
16303
16324
  cliEndpoint: embeddingEndpointSource(embedder),
@@ -16310,16 +16331,27 @@ function buildAgentOptions(config, agentDir) {
16310
16331
  });
16311
16332
  if (skills.length > 0)
16312
16333
  opts.datasourceSkills = skills;
16334
+ const diagnostics = [];
16335
+ if (retired.length > 0) {
16336
+ const names = retired.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
16337
+ diagnostics.push({
16338
+ code: "retired-datasource-skill",
16339
+ severity: "warning",
16340
+ message: `Datasource config ignored because the Spotlight datasource was removed: ${names.join(", ")}. Delete these entries; use fsearch/mirror search (searchPaths) instead.`,
16341
+ source: "datasources"
16342
+ });
16343
+ }
16313
16344
  if (unknown.length > 0) {
16314
- const safeNames = unknown.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
16315
- const diagnostic = {
16345
+ const names = unknown.map((name) => name.replace(/[^A-Za-z0-9._-]/g, "?").slice(0, 80));
16346
+ diagnostics.push({
16316
16347
  code: "unknown-datasource-skill",
16317
16348
  severity: "warning",
16318
- message: `Unknown datasource skill(s) in config were skipped: ${safeNames.join(", ")}`,
16349
+ message: `Unknown datasource skill(s) in config were skipped: ${names.join(", ")}`,
16319
16350
  source: "datasources"
16320
- };
16321
- opts.startupDiagnostics = [diagnostic];
16351
+ });
16322
16352
  }
16353
+ if (diagnostics.length > 0)
16354
+ opts.startupDiagnostics = diagnostics;
16323
16355
  }
16324
16356
  return opts;
16325
16357
  }
@@ -17832,11 +17864,49 @@ var init_matrix = __esm(() => {
17832
17864
  };
17833
17865
  });
17834
17866
 
17867
+ // src/dupey/version.ts
17868
+ function parseDupeyVersion(output) {
17869
+ return VERSION_LINE2.exec(output)?.[1];
17870
+ }
17871
+ function isSupportedDupeyVersion(version) {
17872
+ const have = version.split(".").map((part) => Number.parseInt(part, 10));
17873
+ const need = DUPEY_MIN_VERSION.split(".").map((part) => Number.parseInt(part, 10));
17874
+ for (let index = 0;index < need.length; index += 1) {
17875
+ const left = have[index] ?? 0;
17876
+ const right = need[index] ?? 0;
17877
+ if (left !== right)
17878
+ return left > right;
17879
+ }
17880
+ return true;
17881
+ }
17882
+ var DUPEY_MIN_VERSION = "0.2.0", VERSION_LINE2;
17883
+ var init_version2 = __esm(() => {
17884
+ VERSION_LINE2 = /^dupey\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
17885
+ });
17886
+
17835
17887
  // src/dupey/cli.ts
17836
17888
  import { spawn as spawn14 } from "node:child_process";
17837
17889
  import { resolve as resolve9 } from "node:path";
17838
- function runDupey(args, options) {
17839
- return options.run ? options.run(args) : spawnDupey(options.executable ?? "dupey", args, options);
17890
+ function checkDupeyVersion(executable, options) {
17891
+ const cached = versionChecks.get(executable);
17892
+ if (cached !== undefined)
17893
+ return cached;
17894
+ const check = spawnDupey(executable, ["--version"], options).then((stdout) => {
17895
+ const version = parseDupeyVersion(stdout);
17896
+ if (version !== undefined && !isSupportedDupeyVersion(version)) {
17897
+ throw new DupeyCliError(`dupey ${version} at ${executable} is older than the ${DUPEY_MIN_VERSION} AutoRAG requires; run \`cargo install dupey\` to update it.`);
17898
+ }
17899
+ });
17900
+ check.catch(() => versionChecks.delete(executable));
17901
+ versionChecks.set(executable, check);
17902
+ return check;
17903
+ }
17904
+ async function runDupey(args, options) {
17905
+ if (options.run)
17906
+ return options.run(args);
17907
+ const executable = options.executable ?? "dupey";
17908
+ await checkDupeyVersion(executable, options);
17909
+ return spawnDupey(executable, args, options);
17840
17910
  }
17841
17911
  async function scanWithDupey(dir, options = {}) {
17842
17912
  const absoluteDir = resolve9(dir);
@@ -17970,28 +18040,26 @@ function spawnDupey(executable, args, options) {
17970
18040
  });
17971
18041
  });
17972
18042
  }
17973
- var DupeyCliError, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
18043
+ var DupeyCliError, versionChecks, NEAR_JACCARD = 0.9, CONTAINS_CONTAINMENT = 0.96, CONTAINS_MIN_JACCARD = 0.4;
17974
18044
  var init_cli2 = __esm(() => {
17975
18045
  init_portable_spawn();
18046
+ init_version2();
17976
18047
  DupeyCliError = class DupeyCliError extends Error {
17977
18048
  constructor(message) {
17978
18049
  super(message);
17979
18050
  this.name = "DupeyCliError";
17980
18051
  }
17981
18052
  };
18053
+ versionChecks = new Map;
17982
18054
  });
17983
18055
 
17984
18056
  // src/dupey/filter.ts
17985
18057
  import { stat as stat2 } from "node:fs/promises";
17986
18058
  import { resolve as resolve10 } from "node:path";
17987
- function extractedNoText(file) {
17988
- const signals = file.signals;
17989
- return typeof signals === "object" && signals !== null && "chars" in signals && signals.chars === 0;
17990
- }
17991
18059
  async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerprints) {
17992
18060
  const byHash = new Map;
17993
18061
  for (const file of scan.files) {
17994
- if (typeof file.content_hash !== "string" || file.content_hash.length === 0 || extractedNoText(file))
18062
+ if (typeof file.content_hash !== "string" || file.content_hash.length === 0)
17995
18063
  continue;
17996
18064
  const path = resolve10(root, file.path);
17997
18065
  const group = byHash.get(file.content_hash) ?? [];
@@ -18000,7 +18068,7 @@ async function selectExactDuplicateExclusions(root, scan, isUnavailable, fingerp
18000
18068
  }
18001
18069
  const fingerprintModified = new Map;
18002
18070
  for (const [path, fingerprint] of fingerprints ?? []) {
18003
- if (fingerprint.exact.length === 0 || fingerprint.chars === 0)
18071
+ if (fingerprint.exact.length === 0)
18004
18072
  continue;
18005
18073
  fingerprintModified.set(path, fingerprint.modified);
18006
18074
  const group = byHash.get(fingerprint.exact) ?? [];
@@ -18067,1397 +18135,1368 @@ async function rankExactCandidates(paths, dupeyRanking, fingerprintModified, err
18067
18135
  }
18068
18136
  var init_filter = () => {};
18069
18137
 
18070
- // src/filesystem/cloud-placeholder.ts
18071
- import { execFile as execFile2 } from "node:child_process";
18072
- import { realpathSync as realpathSync3 } from "node:fs";
18073
- import { lstat, readdir } from "node:fs/promises";
18074
- import { homedir as homedir6 } from "node:os";
18075
- import { dirname as dirname15, join as join28 } from "node:path";
18076
- import { promisify } from "node:util";
18077
- function homeCloudRoots(home = homedir6()) {
18078
- const normalizedHome = home.replaceAll("\\", "/").replace(/\/+$/, "");
18079
- return [`${normalizedHome}/Library/CloudStorage`, `${normalizedHome}/Library/Mobile Documents`];
18080
- }
18081
- function datalessFromDarwinFlags(flags) {
18082
- return Number.isFinite(flags) && (flags >>> 0 & SF_DATALESS) !== 0;
18138
+ // src/dupey/index.ts
18139
+ var init_dupey = __esm(() => {
18140
+ init_cli2();
18141
+ init_filter();
18142
+ });
18143
+
18144
+ // src/convert/relate.ts
18145
+ async function convertAndRelate(request, options = {}) {
18146
+ const result = await convertWithFilee(request, options.filee);
18147
+ if (options.dupey === false)
18148
+ return { result, links: [] };
18149
+ const dupeyOptions = options.dupey ?? {};
18150
+ const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
18151
+ const links = await Promise.all(pairs.map(async ({ source, output }) => {
18152
+ try {
18153
+ return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
18154
+ } catch (error) {
18155
+ if (!(error instanceof DupeyCliError))
18156
+ throw error;
18157
+ return { source, output, dupey: { relation: "unavailable", reason: error.message } };
18158
+ }
18159
+ }));
18160
+ return { result, links };
18083
18161
  }
18084
- function datalessFromWindowsAttributes(attributes) {
18085
- return Number.isFinite(attributes) && (attributes & WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) !== 0;
18162
+ var init_relate = __esm(() => {
18163
+ init_dupey();
18164
+ init_cli();
18165
+ });
18166
+
18167
+ // src/convert/index.ts
18168
+ var init_convert = __esm(() => {
18169
+ init_cli();
18170
+ init_installer2();
18171
+ init_matrix();
18172
+ init_relate();
18173
+ });
18174
+
18175
+ // src/datasource/diagnostics.ts
18176
+ function mapDatasourceDiagnostic(ds) {
18177
+ return {
18178
+ code: datasourceCodeToRetrieval(ds.code),
18179
+ severity: ds.severity,
18180
+ message: `[datasource:${ds.code}] ${ds.message}`,
18181
+ source: ds.source ?? (ds.instanceId ? `datasource:${ds.instanceId}` : "datasource")
18182
+ };
18086
18183
  }
18087
- function cloudProviderFromPath(target) {
18088
- const lower = target.toLowerCase();
18089
- if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
18090
- return "google-drive";
18091
- if (lower.includes("onedrive"))
18092
- return "onedrive";
18093
- if (lower.includes("dropbox"))
18094
- return "dropbox";
18095
- if (lower.includes("mobile documents") || lower.includes("clouddocs"))
18096
- return "icloud";
18097
- return "file-provider";
18184
+ function mapDatasourceDiagnostics(diagnostics) {
18185
+ return diagnostics.map(mapDatasourceDiagnostic);
18098
18186
  }
18099
- function cloudProviderFromDomain(domain) {
18100
- const lower = domain.toLowerCase();
18101
- if (lower.includes("clouddocs") || lower.includes("icloud"))
18102
- return "icloud";
18103
- if (lower.includes("google"))
18104
- return "google-drive";
18105
- if (lower.includes("onedrive"))
18106
- return "onedrive";
18107
- if (lower.includes("dropbox"))
18108
- return "dropbox";
18109
- return "file-provider";
18187
+ function datasourceCodeToRetrieval(code) {
18188
+ switch (code) {
18189
+ case "datasource-unavailable":
18190
+ case "datasource-cli-error":
18191
+ case "datasource-empty":
18192
+ case "datasource-rate-limited":
18193
+ case "datasource-auth-error":
18194
+ case "datasource-embedding-egress-rejected":
18195
+ case "datasource-index-failed":
18196
+ case "datasource-index-partial":
18197
+ case "datasource-permission-denied":
18198
+ case "datasource-remote-embedding-rejected":
18199
+ return "retrieval-method-failed";
18200
+ }
18110
18201
  }
18111
- function normalizeCloudPath(value) {
18112
- let normalized = value.replaceAll("\\", "/");
18113
- if (normalized.length > 1)
18114
- normalized = normalized.replace(/\/+$/u, "");
18115
- return normalized;
18202
+
18203
+ // src/datasource/retry.ts
18204
+ import { setTimeout as delay2 } from "node:timers/promises";
18205
+ function datasourceRetryDelayMs(retry, policy) {
18206
+ const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
18207
+ return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
18116
18208
  }
18117
- function isPathInside(target, parent) {
18118
- const normalizedParent = normalizeCloudPath(parent);
18119
- if (normalizedParent === "")
18120
- return false;
18121
- const left = normalizeCloudPath(target);
18122
- const [a, b] = process.platform === "win32" ? [left.toLowerCase(), normalizedParent.toLowerCase()] : [left, normalizedParent];
18123
- return a === b || a.startsWith(`${b}/`);
18209
+ async function indexWithRetry(skill, options = {}, signal) {
18210
+ const policy = {
18211
+ maxRetries: options.maxRetries ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxRetries,
18212
+ baseDelayMs: options.baseDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.baseDelayMs,
18213
+ maxDelayMs: options.maxDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxDelayMs
18214
+ };
18215
+ const sleep = options.sleep ?? ((ms) => delay2(ms, undefined, signal === undefined ? undefined : { signal }));
18216
+ let lastError = "";
18217
+ for (let attempt = 1;; attempt += 1) {
18218
+ const result = await runIndex(skill, signal);
18219
+ signal?.throwIfAborted();
18220
+ if (result.ok) {
18221
+ if (attempt === 1)
18222
+ return result;
18223
+ const recovered = {
18224
+ code: "datasource-index-failed",
18225
+ severity: "info",
18226
+ message: `recovered after ${attempt} attempts; last error: ${lastError}`,
18227
+ instanceId: result.instanceId,
18228
+ source: result.skill
18229
+ };
18230
+ return { ...result, attempts: attempt, diagnostics: [...result.diagnostics, recovered] };
18231
+ }
18232
+ lastError = result.message ?? result.error ?? "datasource-index-failed";
18233
+ if (attempt > policy.maxRetries) {
18234
+ if (attempt === 1)
18235
+ return { ...result, attempts: attempt };
18236
+ const suffix = ` (gave up after ${attempt} attempts)`;
18237
+ const last = result.diagnostics.at(-1);
18238
+ return {
18239
+ ...result,
18240
+ attempts: attempt,
18241
+ message: `${lastError}${suffix}`,
18242
+ diagnostics: last === undefined ? [
18243
+ {
18244
+ code: result.code ?? "datasource-index-failed",
18245
+ severity: "error",
18246
+ message: `${lastError}${suffix}`,
18247
+ instanceId: result.instanceId,
18248
+ source: result.skill
18249
+ }
18250
+ ] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
18251
+ };
18252
+ }
18253
+ await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
18254
+ signal?.throwIfAborted();
18255
+ }
18124
18256
  }
18125
- function canonicalPath(value) {
18257
+ async function runIndex(skill, signal) {
18126
18258
  try {
18127
- return realpathSync3(value);
18128
- } catch {
18129
- return value;
18259
+ return await skill.index(signal === undefined ? undefined : { signal });
18260
+ } catch (error) {
18261
+ signal?.throwIfAborted();
18262
+ const descriptor = skill.describe();
18263
+ const message = error instanceof Error ? error.message : "Datasource indexing failed.";
18264
+ const failed = {
18265
+ ok: false,
18266
+ instanceId: descriptor.instanceId ?? "default",
18267
+ skill: descriptor.name,
18268
+ indexedAt: Date.now(),
18269
+ diagnostics: [
18270
+ {
18271
+ code: "datasource-index-failed",
18272
+ severity: "error",
18273
+ message,
18274
+ source: descriptor.name,
18275
+ instanceId: descriptor.instanceId
18276
+ }
18277
+ ],
18278
+ error: "datasource-index-failed",
18279
+ code: "datasource-index-failed",
18280
+ message
18281
+ };
18282
+ return failed;
18130
18283
  }
18131
18284
  }
18132
- function canonicalPathMapper(searchRoots) {
18133
- const forms = searchRoots.map((root) => ({ raw: normalizeCloudPath(root), canonical: normalizeCloudPath(canonicalPath(root)) })).filter(({ raw, canonical }) => raw !== "" && raw !== canonical).sort((left, right) => right.raw.length - left.raw.length);
18134
- if (forms.length === 0)
18135
- return (path) => path;
18136
- return (path) => {
18137
- const normalized = normalizeCloudPath(path);
18138
- for (const { raw, canonical } of forms) {
18139
- if (isPathInside(normalized, raw))
18140
- return `${canonical}${normalized.slice(raw.length)}`;
18141
- }
18142
- return path;
18285
+ var DEFAULT_DATASOURCE_RETRY_POLICY;
18286
+ var init_retry = __esm(() => {
18287
+ DEFAULT_DATASOURCE_RETRY_POLICY = {
18288
+ maxRetries: 2,
18289
+ baseDelayMs: 1000,
18290
+ maxDelayMs: 30000
18143
18291
  };
18292
+ });
18293
+
18294
+ // src/everything/bundle.ts
18295
+ import { createHash as createHash11 } from "node:crypto";
18296
+ import { existsSync as existsSync18, mkdirSync as mkdirSync15, readFileSync as readFileSync18, rmSync as rmSync7, writeFileSync as writeFileSync10 } from "node:fs";
18297
+ import { dirname as dirname15, join as join28 } from "node:path";
18298
+ import { fileURLToPath as fileURLToPath2 } from "node:url";
18299
+ import JSZip from "jszip";
18300
+ function resolveEverythingBundleDir() {
18301
+ let dir = dirname15(fileURLToPath2(import.meta.url));
18302
+ for (let depth = 0;depth < 6; depth += 1) {
18303
+ const candidate = join28(dir, "vendor", "everything");
18304
+ if (existsSync18(join28(candidate, MANIFEST_FILENAME)))
18305
+ return candidate;
18306
+ const parent = dirname15(dir);
18307
+ if (parent === dir)
18308
+ break;
18309
+ dir = parent;
18310
+ }
18311
+ return;
18144
18312
  }
18145
- function cloudScope(searchRoot, cloudRoots) {
18146
- const canonicalSearchRoot = canonicalPath(searchRoot);
18147
- const roots = cloudRoots.map((root) => ({ path: canonicalPath(root.path), kind: root.kind }));
18148
- return {
18149
- insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
18150
- nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
18151
- };
18313
+ function everythingArch(arch) {
18314
+ if (arch === "x64" || arch === "arm64")
18315
+ return arch;
18316
+ return;
18152
18317
  }
18153
- function parseReleaseOutcome(stdout) {
18154
- let parsed;
18318
+ function everythingBinaryPaths(root, version, arch) {
18319
+ const dir = join28(root, ".autorag", "everything", version, arch);
18320
+ return { everythingPath: join28(dir, "everything.exe"), esPath: join28(dir, "es.exe") };
18321
+ }
18322
+ function loadEverythingBundleManifest(bundleDir) {
18323
+ return JSON.parse(readFileSync18(join28(bundleDir, MANIFEST_FILENAME), "utf8"));
18324
+ }
18325
+ async function ensureEverythingBinaries(options) {
18326
+ const platform = options.platform ?? process.platform;
18327
+ const rawArch = options.arch ?? process.arch;
18328
+ if (platform !== "win32") {
18329
+ return {
18330
+ ok: false,
18331
+ reason: "unsupported-platform",
18332
+ message: `Everything is Windows-only; this host is ${platform}.`
18333
+ };
18334
+ }
18335
+ const arch = everythingArch(rawArch);
18336
+ const bundleDir = options.bundleDir ?? resolveEverythingBundleDir();
18337
+ if (bundleDir === undefined) {
18338
+ return {
18339
+ ok: false,
18340
+ reason: "bundle-missing",
18341
+ message: `Bundled Everything assets (vendor/everything/${MANIFEST_FILENAME}) were not found above ${dirname15(fileURLToPath2(import.meta.url))}.`
18342
+ };
18343
+ }
18344
+ let manifest;
18155
18345
  try {
18156
- parsed = JSON.parse(stdout.trim());
18157
- } catch {
18158
- return;
18346
+ manifest = options.manifest ?? loadEverythingBundleManifest(bundleDir);
18347
+ } catch (error) {
18348
+ return { ok: false, reason: "bundle-missing", message: errorMessage2(error) };
18159
18349
  }
18160
- if (typeof parsed !== "object" || parsed === null || !("ok" in parsed) || typeof parsed.ok !== "boolean") {
18161
- return;
18350
+ const assets = arch === undefined ? undefined : manifest.architectures[arch];
18351
+ if (arch === undefined || assets === undefined) {
18352
+ return {
18353
+ ok: false,
18354
+ reason: "unsupported-platform",
18355
+ message: `No bundled Everything build for Windows architecture ${rawArch}; bundled: ${Object.keys(manifest.architectures).join(", ")}.`
18356
+ };
18162
18357
  }
18163
- return {
18164
- ok: parsed.ok,
18165
- ..."isUploaded" in parsed ? { isUploaded: parsed.isUploaded } : {},
18166
- ..."isUploading" in parsed ? { isUploading: parsed.isUploading } : {},
18167
- ..."downloadingStatus" in parsed ? { downloadingStatus: parsed.downloadingStatus } : {}
18168
- };
18169
- }
18170
- function describeReleaseRefusal(outcome) {
18171
- if (outcome.isUploading === true)
18172
- return "The file is still uploading to the cloud, so it cannot be released yet.";
18173
- if (outcome.isUploaded === false)
18174
- return "The file has not finished uploading to the cloud, so it cannot be released yet.";
18175
- const status = typeof outcome.downloadingStatus === "string" ? outcome.downloadingStatus : "unknown";
18176
- return `The cloud provider declined to release the local copy (download status ${status}).`;
18358
+ const { everythingPath, esPath } = everythingBinaryPaths(options.root, manifest.everythingVersion, arch);
18359
+ const everythingCached = fileHasSha256(everythingPath, assets.everythingSha256);
18360
+ const esCached = fileHasSha256(esPath, assets.esSha256);
18361
+ if (everythingCached && esCached) {
18362
+ return { ok: true, everythingPath, esPath, source: "cached" };
18363
+ }
18364
+ if ((options.mode ?? "install") === "cached") {
18365
+ const state = everythingCached || esCached ? "are incomplete or do not match their pinned SHA-256 digests" : "are not installed";
18366
+ return {
18367
+ ok: false,
18368
+ reason: "not-installed",
18369
+ message: `Everything ${manifest.everythingVersion} (${arch}) binaries ${state} under ${dirname15(everythingPath)}; run \`autorag refresh\` to (re)install and index them.`
18370
+ };
18371
+ }
18372
+ try {
18373
+ mkdirSync15(dirname15(everythingPath), { recursive: true });
18374
+ const everythingBytes = await extractVerified(join28(bundleDir, assets.everythingArchive), assets.everythingArchiveSha256, assets.everythingMember, assets.everythingSha256);
18375
+ const esBytes = await extractVerified(join28(bundleDir, assets.esArchive), assets.esArchiveSha256, assets.esMember, assets.esSha256);
18376
+ writeAtomic(everythingPath, everythingBytes);
18377
+ writeAtomic(esPath, esBytes);
18378
+ } catch (error) {
18379
+ return { ok: false, reason: "install-failed", message: errorMessage2(error) };
18380
+ }
18381
+ return { ok: true, everythingPath, esPath, source: "installed" };
18177
18382
  }
18178
- function describeReleaseHelperFailure(error) {
18179
- const failure = error;
18180
- const parts = [];
18181
- if (failure.killed === true || typeof failure.signal === "string") {
18182
- parts.push(`the helper was killed${typeof failure.signal === "string" ? ` by ${failure.signal}` : ""}`);
18183
- } else if (typeof failure.code === "number") {
18184
- parts.push(`the helper exited with code ${failure.code}`);
18383
+ async function extractVerified(archivePath, archiveSha256, member, memberSha256) {
18384
+ const archive = readFileSync18(archivePath);
18385
+ const actualArchive = sha256(archive);
18386
+ if (actualArchive !== archiveSha256) {
18387
+ throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
18185
18388
  }
18186
- const stderr = typeof failure.stderr === "string" ? failure.stderr.trim() : "";
18187
- if (stderr !== "")
18188
- parts.push(stderr);
18189
- if (parts.length === 0)
18190
- parts.push(error instanceof Error ? error.message : String(error));
18191
- return `osascript could not release the local copy: ${parts.join("; ")}`;
18389
+ const zip = await JSZip.loadAsync(archive);
18390
+ const entry = zip.file(member);
18391
+ if (entry === null)
18392
+ throw new Error(`${archivePath} does not contain ${member}`);
18393
+ const bytes = await entry.async("nodebuffer");
18394
+ const actualMember = sha256(bytes);
18395
+ if (actualMember !== memberSha256) {
18396
+ throw new Error(`${member} from ${archivePath} SHA-256 ${actualMember} does not match pinned ${memberSha256}`);
18397
+ }
18398
+ return bytes;
18192
18399
  }
18193
- async function defaultDarwinReadDir(path) {
18194
- const entries = await readdir(path, { withFileTypes: true });
18195
- return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
18400
+ function fileHasSha256(path, expected) {
18401
+ return existsSync18(path) && sha256(readFileSync18(path)) === expected;
18196
18402
  }
18197
- function parseDarwinMounts(stdout) {
18198
- const mounts = [];
18199
- for (const line of stdout.split(`
18200
- `)) {
18201
- if (line.trim() === "")
18202
- continue;
18203
- const open = line.lastIndexOf(" (");
18204
- if (open <= 0 || !line.endsWith(")"))
18205
- continue;
18206
- const head = line.slice(0, open);
18207
- const on = head.indexOf(" on ");
18208
- if (on <= 0)
18209
- continue;
18210
- const mountPoint = head.slice(on + " on ".length);
18211
- if (mountPoint === "")
18212
- continue;
18213
- const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
18214
- mounts.push({
18215
- device: head.slice(0, on),
18216
- mountPoint,
18217
- fileSystem: fields[0] ?? "",
18218
- options: fields.slice(1)
18219
- });
18403
+ function writeAtomic(path, bytes) {
18404
+ const temp = `${path}.${process.pid}.tmp`;
18405
+ try {
18406
+ writeFileSync10(temp, bytes);
18407
+ renameWithRetry(temp, path);
18408
+ } finally {
18409
+ rmSync7(temp, { force: true });
18220
18410
  }
18221
- return mounts;
18222
18411
  }
18223
- function networkMountPoints(mounts) {
18224
- const points = [];
18225
- const seen = new Set;
18226
- for (const mount of mounts) {
18227
- if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
18228
- continue;
18229
- if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
18230
- continue;
18231
- if (seen.has(mount.mountPoint))
18232
- continue;
18233
- seen.add(mount.mountPoint);
18234
- points.push(mount.mountPoint);
18235
- }
18236
- return points;
18412
+ function sha256(bytes) {
18413
+ return createHash11("sha256").update(bytes).digest("hex");
18237
18414
  }
18238
- function parseXattrValues(stdout, candidates) {
18239
- const values = new Map;
18240
- if (candidates.length === 0)
18241
- return values;
18242
- const ordered = [...candidates].sort((left, right) => right.length - left.length);
18243
- for (const line of stdout.split(`
18244
- `)) {
18245
- if (line === "")
18246
- continue;
18247
- for (const candidate of ordered) {
18248
- if (line.startsWith(`${candidate}: `)) {
18249
- values.set(candidate, line.slice(candidate.length + 2));
18250
- break;
18251
- }
18252
- }
18253
- }
18254
- return values;
18415
+ function errorMessage2(error) {
18416
+ return error instanceof Error ? error.message : String(error);
18417
+ }
18418
+ var MANIFEST_FILENAME = "manifest.json";
18419
+ var init_bundle = __esm(() => {
18420
+ init_atomic_write();
18421
+ });
18422
+
18423
+ // src/filesystem/path-exclusions.ts
18424
+ import { posix as posix3, win32 as win322 } from "node:path";
18425
+ function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
18426
+ const paths = platform === "win32" ? win322 : posix3;
18427
+ const normalize = (path) => {
18428
+ const absolute = paths.resolve(path);
18429
+ return platform === "win32" ? absolute.toLowerCase() : absolute;
18430
+ };
18431
+ const excluded = excludedPaths.map((path) => {
18432
+ const absolute = normalize(path);
18433
+ return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
18434
+ });
18435
+ return (path) => {
18436
+ const absolute = normalize(path);
18437
+ return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
18438
+ };
18255
18439
  }
18440
+ var init_path_exclusions = () => {};
18256
18441
 
18257
- class DarwinCloudProbe {
18258
- exec;
18259
- readDir;
18260
- home;
18261
- constructor(options = {}) {
18262
- this.exec = options.exec ?? defaultCloudExec;
18263
- this.readDir = options.readDir ?? defaultDarwinReadDir;
18264
- this.home = options.home ?? homedir6();
18442
+ // src/filesystem/shared-exclusive-queue.ts
18443
+ class SharedExclusiveQueue {
18444
+ sharedSinceExclusive = [];
18445
+ lastExclusive = Promise.resolve();
18446
+ shared(task) {
18447
+ const run = this.lastExclusive.then(task);
18448
+ const settled = run.then(() => {
18449
+ return;
18450
+ }, () => {
18451
+ return;
18452
+ });
18453
+ this.sharedSinceExclusive.push(settled);
18454
+ return run;
18265
18455
  }
18266
- async cloudRoots(searchRoots = []) {
18267
- const roots = [];
18268
- const seen = new Set;
18269
- const add = (path, kind) => {
18270
- const canonical = canonicalPath(path);
18271
- const key = normalizeCloudPath(canonical);
18272
- if (key === "" || seen.has(key))
18273
- return;
18274
- seen.add(key);
18275
- roots.push({ path: canonical, kind });
18276
- };
18277
- const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
18278
- for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
18279
- if (entry.isDirectory())
18280
- add(join28(cloudStorage, entry.name), "file-provider");
18281
- }
18282
- if (await this.tryReadDir(mobileDocuments) !== undefined)
18283
- add(mobileDocuments, "file-provider");
18284
- for (const mountPoint of await this.networkMountPoints())
18285
- add(mountPoint, "network");
18286
- for (const directory of await this.fileProviderDirectories(searchRoots))
18287
- add(directory, "file-provider");
18288
- return roots;
18289
- }
18290
- async networkMountPoints() {
18291
- try {
18292
- const { stdout } = await this.exec("/sbin/mount", [], { timeoutMs: 1e4, maxBuffer: 4 * 1024 * 1024 });
18293
- return networkMountPoints(parseDarwinMounts(stdout));
18294
- } catch {
18295
- return [];
18296
- }
18297
- }
18298
- async fileProviderDirectories(searchRoots) {
18299
- if (searchRoots.length === 0)
18300
- return [];
18301
- const candidates = new Set;
18302
- for (const searchRoot of searchRoots) {
18303
- const canonical = canonicalPath(searchRoot);
18304
- for (let cursor = canonical;; cursor = dirname15(cursor)) {
18305
- candidates.add(cursor);
18306
- const parent = dirname15(cursor);
18307
- if (parent === cursor)
18308
- break;
18309
- }
18310
- for (const entry of await this.tryReadDir(canonical) ?? []) {
18311
- if (entry.isDirectory())
18312
- candidates.add(join28(canonical, entry.name));
18313
- }
18314
- }
18315
- const paths = [...candidates];
18316
- const managed = [];
18317
- for (let start = 0;start < paths.length; start += XATTR_BATCH) {
18318
- for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
18319
- managed.push(path);
18320
- }
18321
- return managed;
18322
- }
18323
- async xattrValues(paths) {
18324
- try {
18325
- const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
18326
- timeoutMs: 15000,
18327
- maxBuffer: 8 * 1024 * 1024
18328
- });
18329
- return parseXattrValues(stdout, paths);
18330
- } catch (error) {
18331
- const failure = error;
18332
- return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
18333
- }
18334
- }
18335
- async tryReadDir(path) {
18336
- try {
18337
- return await this.readDir(path);
18338
- } catch {
18456
+ exclusive(task) {
18457
+ const earlier = Promise.all([this.lastExclusive, ...this.sharedSinceExclusive]);
18458
+ const run = earlier.then(task);
18459
+ const settled = run.then(() => {
18339
18460
  return;
18340
- }
18341
- }
18342
- async datalessAmong(paths) {
18343
- const dataless = new Set;
18344
- for (let start = 0;start < paths.length; start += STAT_BATCH) {
18345
- const batch = paths.slice(start, start + STAT_BATCH);
18346
- const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
18347
- timeoutMs: 30000,
18348
- maxBuffer: 4 * 1024 * 1024
18349
- });
18350
- const flags = stdout.split(`
18351
- `).filter(Boolean);
18352
- for (const [index, path] of batch.entries()) {
18353
- if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
18354
- dataless.add(path);
18355
- }
18356
- }
18357
- return dataless;
18358
- }
18359
- async providerOf(path) {
18360
- const domain = await readFileProviderDomainUpwards(path);
18361
- return domain === undefined ? undefined : cloudProviderFromDomain(domain);
18362
- }
18363
- canRelease(provider) {
18364
- return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
18365
- }
18366
- async evict(path) {
18367
- let stdout;
18368
- try {
18369
- stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
18370
- timeoutMs: 30000
18371
- })).stdout;
18372
- } catch (error) {
18373
- return { ok: false, message: describeReleaseHelperFailure(error) };
18374
- }
18375
- const outcome = parseReleaseOutcome(stdout);
18376
- if (outcome === undefined) {
18377
- const detail = stdout.trim();
18378
- return {
18379
- ok: false,
18380
- message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
18381
- };
18382
- }
18383
- return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
18384
- }
18385
- }
18386
- function buildWindowsAttributeScript(paths) {
18387
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18388
- ` + `$ErrorActionPreference='SilentlyContinue'
18389
- ` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
18390
- `;
18391
- }
18392
- function buildWindowsSyncRootsScript() {
18393
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18394
- ` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
18395
- ` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
18396
- `;
18397
- }
18398
- function buildWindowsDehydrateScript(path) {
18399
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18400
- ` + `$ErrorActionPreference='Stop'
18401
- ` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
18402
- ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
18403
- ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
18404
- ` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
18405
- ` + `'@
18406
- ` + `$h=[IntPtr]::Zero
18407
- ` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
18408
- ` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
18409
- ` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
18410
- ` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
18411
- ` + `[Autorag.CfApi]::CfCloseHandle($h)
18412
- ` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
18413
- ` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
18414
- `;
18415
- }
18416
- function parseWindowsAttributeOutput(stdout, paths) {
18417
- const lines = stdout.split(/\r?\n/);
18418
- const attributes = new Map;
18419
- for (const [index, path] of paths.entries()) {
18420
- const value = Number.parseInt((lines[index] ?? "").trim(), 10);
18421
- if (Number.isFinite(value))
18422
- attributes.set(path, value);
18423
- }
18424
- return attributes;
18425
- }
18426
- function parseWindowsSyncRoots(stdout) {
18427
- const roots = [];
18428
- for (const line of stdout.split(/\r?\n/)) {
18429
- const tab = line.indexOf("\t");
18430
- if (tab <= 0)
18431
- continue;
18432
- const path = line.slice(0, tab).trim();
18433
- if (path !== "")
18434
- roots.push({ path, label: line.slice(tab + 1).trim() });
18461
+ }, () => {
18462
+ return;
18463
+ });
18464
+ this.sharedSinceExclusive = [];
18465
+ this.lastExclusive = settled;
18466
+ return run;
18435
18467
  }
18436
- return roots;
18437
18468
  }
18438
- function providerFromWindowsSyncRoot(label) {
18439
- const lower = label.toLowerCase();
18440
- if (lower.includes("onedrive") || lower.includes("one drive"))
18441
- return "onedrive";
18442
- if (lower.includes("google"))
18443
- return "google-drive";
18444
- return "file-provider";
18469
+
18470
+ // src/everything/client.ts
18471
+ import { spawn as spawn15 } from "node:child_process";
18472
+ import { createHash as createHash12 } from "node:crypto";
18473
+ import { mkdirSync as mkdirSync16, writeFileSync as writeFileSync11 } from "node:fs";
18474
+ import { join as join29, resolve as resolve11 } from "node:path";
18475
+ import { setTimeout as sleep2 } from "node:timers/promises";
18476
+ function everythingInstanceName(root) {
18477
+ return `autorag-${createHash12("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
18445
18478
  }
18446
- function buildWindowsDiskScript() {
18447
- return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
18448
- ` + "Get-CimInstance -ClassName Win32_LogicalDisk | ForEach-Object {" + `$_.DeviceID + [char]9 + $_.DriveType + [char]9 + $_.FileSystem + [char]9 + $_.VolumeName}
18449
- `;
18479
+ function buildEverythingIni(input) {
18480
+ const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
18481
+ const perFolder = (value) => input.folders.map(() => value).join(",");
18482
+ return [
18483
+ "[Everything]",
18484
+ "app_data=0",
18485
+ "run_as_admin=0",
18486
+ "run_in_background=1",
18487
+ "show_tray_icon=0",
18488
+ "show_in_taskbar=0",
18489
+ "check_for_updates_on_startup=0",
18490
+ "beta_updates=0",
18491
+ "language=1033",
18492
+ "allow_http_server=0",
18493
+ "allow_etp_server=0",
18494
+ "search_history_enabled=0",
18495
+ "run_history_enabled=0",
18496
+ "auto_include_fixed_volumes=0",
18497
+ "auto_include_removable_volumes=0",
18498
+ "auto_include_fixed_refs_volumes=0",
18499
+ "auto_include_removable_refs_volumes=0",
18500
+ "ntfs_volume_paths=",
18501
+ "ntfs_volume_includes=",
18502
+ "refs_volume_paths=",
18503
+ "refs_volume_includes=",
18504
+ "index_size=1",
18505
+ "index_date_modified=1",
18506
+ `folders=${list(input.folders)}`,
18507
+ `folder_monitor_changes=${perFolder("1")}`,
18508
+ `folder_rescan_if_full_list=${perFolder("1")}`,
18509
+ `folder_update_types=${perFolder("0")}`,
18510
+ "exclude_list_enabled=1",
18511
+ `exclude_folders=${list(input.excludeFolders)}`,
18512
+ ""
18513
+ ].join(`\r
18514
+ `);
18450
18515
  }
18451
- function parseWindowsLogicalDisks(stdout) {
18452
- const disks = [];
18453
- for (const line of stdout.split(/\r?\n/)) {
18454
- if (line.trim() === "")
18455
- continue;
18456
- const [deviceId, driveType, fileSystem, ...rest] = line.split("\t");
18457
- const type = Number.parseInt((driveType ?? "").trim(), 10);
18458
- if (!Number.isFinite(type))
18459
- continue;
18460
- disks.push({
18461
- deviceId: (deviceId ?? "").trim(),
18462
- driveType: type,
18463
- fileSystem: (fileSystem ?? "").trim(),
18464
- volumeName: rest.join("\t").trim()
18465
- });
18466
- }
18467
- return disks;
18516
+ function buildEverythingSearchArgs(instance, request, timeoutMs) {
18517
+ const args = [
18518
+ "-argv",
18519
+ "-instance",
18520
+ instance,
18521
+ "-cp",
18522
+ "65001",
18523
+ "-json",
18524
+ "-full-path-and-name",
18525
+ "-size",
18526
+ "-date-modified",
18527
+ "-date-format",
18528
+ "1"
18529
+ ];
18530
+ if (timeoutMs !== undefined)
18531
+ args.push("-timeout", String(timeoutMs));
18532
+ if (request.matchCase)
18533
+ args.push("-case");
18534
+ if (request.matchPath)
18535
+ args.push("-match-path");
18536
+ if (request.wholeWord)
18537
+ args.push("-whole-word");
18538
+ if (request.kind === "files")
18539
+ args.push("/a-d");
18540
+ if (request.kind === "folders")
18541
+ args.push("/ad");
18542
+ if (request.path !== undefined)
18543
+ args.push("-path", request.path);
18544
+ if (request.sort !== undefined)
18545
+ args.push("-sort", request.sort);
18546
+ if (request.offset !== undefined && request.offset > 0)
18547
+ args.push("-offset", String(request.offset));
18548
+ args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
18549
+ if (request.regex)
18550
+ args.push("-regex", "--", request.query);
18551
+ else
18552
+ args.push("--", request.query);
18553
+ return args;
18468
18554
  }
18469
- function windowsRootsFromDisks(disks) {
18470
- const roots = [];
18471
- for (const disk of disks) {
18472
- if (disk.deviceId === "")
18473
- continue;
18474
- const path = `${disk.deviceId}\\`;
18475
- const fileSystem = disk.fileSystem.toLowerCase();
18476
- if (disk.driveType === WINDOWS_DRIVE_TYPE_NETWORK) {
18477
- roots.push({ path, kind: "network" });
18478
- continue;
18479
- }
18480
- if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
18481
- roots.push({ path, kind: "network" });
18482
- continue;
18483
- }
18484
- if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
18485
- roots.push({ path, kind: "cloud-files" });
18486
- }
18555
+ function parseEverythingJson(stdout) {
18556
+ const text = stdout.trim();
18557
+ if (text.length === 0)
18558
+ return [];
18559
+ let parsed;
18560
+ try {
18561
+ parsed = JSON.parse(text);
18562
+ } catch {
18563
+ throw new Error(`ES did not return JSON: ${text.slice(0, 2000)}`);
18487
18564
  }
18488
- return roots;
18489
- }
18490
- function uncShareRoot(target) {
18491
- const normalized = target.replaceAll("/", "\\");
18492
- if (!normalized.startsWith("\\\\") || normalized.startsWith("\\\\?\\") || normalized.startsWith("\\\\.\\"))
18493
- return;
18494
- const rest = normalized.slice(2);
18495
- const separator = rest.indexOf("\\");
18496
- if (separator <= 0)
18497
- return;
18498
- const server = rest.slice(0, separator);
18499
- const share = rest.slice(separator + 1).split("\\")[0] ?? "";
18500
- if (server === "" || share === "")
18501
- return;
18502
- return `\\\\${server}\\${share}`;
18503
- }
18504
- function parseWindowsDehydrateOutput(result) {
18505
- const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
18506
- if (open === null)
18507
- return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
18508
- if (Number.parseInt(open[1], 16) !== 0) {
18565
+ if (!Array.isArray(parsed))
18566
+ throw new Error(`ES did not return a JSON array: ${text.slice(0, 2000)}`);
18567
+ return parsed.map((item) => {
18568
+ const record = item;
18569
+ const filename = String(record.filename ?? "");
18570
+ const isFolder = filename.endsWith("\\") && filename.length > 3;
18509
18571
  return {
18510
- ok: false,
18511
- message: windowsFailureMessage(result, `could not open the file for release (0x${open[1]})`)
18572
+ path: isFolder ? filename.slice(0, -1) : filename,
18573
+ type: filename.endsWith("\\") ? "folder" : "file",
18574
+ size: typeof record.size === "number" ? record.size : undefined,
18575
+ dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
18512
18576
  };
18513
- }
18514
- const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
18515
- if (dehydrate === null) {
18516
- return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
18517
- }
18518
- const hresult = dehydrate[1];
18519
- if (Number.parseInt(hresult, 16) === 0)
18520
- return { ok: true };
18521
- const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
18522
- return {
18523
- ok: false,
18524
- message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
18525
- };
18526
- }
18527
- function windowsFailureMessage(result, fallback) {
18528
- const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
18529
- if (printed !== undefined && printed !== "")
18530
- return printed;
18531
- const stderr = result.stderr.trim();
18532
- return stderr !== "" ? stderr : fallback;
18577
+ });
18533
18578
  }
18534
18579
 
18535
- class WindowsCloudProbe {
18580
+ class EverythingClient {
18581
+ instanceName;
18582
+ options;
18583
+ platform;
18536
18584
  run;
18537
- syncRoots;
18538
- disks;
18539
- constructor(options = {}) {
18540
- this.run = options.run ?? runPowerShell;
18585
+ launch;
18586
+ stateDir;
18587
+ isExcludedPath;
18588
+ binaries;
18589
+ queue = new SharedExclusiveQueue;
18590
+ constructor(options) {
18591
+ this.options = options;
18592
+ this.platform = options.platform ?? process.platform;
18593
+ this.instanceName = everythingInstanceName(options.root);
18594
+ this.run = options.run ?? runProcess;
18595
+ this.launch = options.launch ?? launchDetached;
18596
+ this.stateDir = join29(options.root, ".autorag", "everything");
18597
+ this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
18541
18598
  }
18542
- async cloudRoots(searchRoots = []) {
18543
- const roots = [];
18544
- const seen = new Set;
18545
- const add = (path, kind) => {
18546
- const canonical = canonicalPath(path);
18547
- const key = normalizeCloudPath(canonical).toLowerCase();
18548
- if (key === "" || seen.has(key))
18549
- return;
18550
- seen.add(key);
18551
- roots.push({ path: canonical, kind });
18552
- };
18553
- for (const root of await this.loadSyncRoots())
18554
- add(root.path, "cloud-files");
18555
- for (const root of await this.loadDiskRoots())
18556
- add(root.path, root.kind);
18557
- for (const searchRoot of searchRoots) {
18558
- const share = uncShareRoot(searchRoot);
18559
- if (share !== undefined)
18560
- add(share, "network");
18561
- }
18562
- return roots;
18599
+ isSupported() {
18600
+ return this.platform === "win32";
18563
18601
  }
18564
- async datalessAmong(paths) {
18565
- const dataless = new Set;
18566
- for (let start = 0;start < paths.length; start += WINDOWS_ATTRIBUTE_BATCH) {
18567
- const batch = paths.slice(start, start + WINDOWS_ATTRIBUTE_BATCH);
18568
- const result = await this.run(buildWindowsAttributeScript(batch), 30000);
18569
- if (!result.ok) {
18570
- throw new Error(`Could not read Windows cloud file attributes: ${result.stderr.trim() || "powershell.exe failed"}`);
18602
+ search(request) {
18603
+ return this.queue.shared(async () => {
18604
+ const binaries = await this.resolveBinaries("cached");
18605
+ if (!binaries.ok)
18606
+ return binaries;
18607
+ const ping = await this.ping(binaries.esPath);
18608
+ if (ping.code !== 0) {
18609
+ return {
18610
+ ok: false,
18611
+ reason: "not-running",
18612
+ message: `Everything instance ${this.instanceName} is not running; run \`autorag refresh\` to start and index it.`
18613
+ };
18571
18614
  }
18572
- for (const [path, attributes] of parseWindowsAttributeOutput(result.stdout, batch)) {
18573
- if (datalessFromWindowsAttributes(attributes))
18574
- dataless.add(path);
18615
+ const es = { ok: true, esPath: binaries.esPath };
18616
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
18617
+ if ((this.options.excludeFolders?.length ?? 0) > 0) {
18618
+ return this.searchWithExclusions(es.esPath, request, timeoutMs);
18575
18619
  }
18576
- }
18577
- return dataless;
18578
- }
18579
- async providerOf(path) {
18580
- const root = longestSyncRoot(await this.loadSyncRoots(), path);
18581
- return root === undefined ? cloudProviderFromPath(path) : providerFromWindowsSyncRoot(root.label);
18582
- }
18583
- canRelease(provider) {
18584
- return provider !== undefined;
18620
+ const args = buildEverythingSearchArgs(this.instanceName, request, timeoutMs);
18621
+ const result = await this.run(es.esPath, args, timeoutMs + 5000);
18622
+ if (result.code !== 0)
18623
+ return this.failure("search-failed", es.esPath, args, result);
18624
+ try {
18625
+ return { ok: true, results: parseEverythingJson(result.stdout) };
18626
+ } catch (error) {
18627
+ return { ok: false, reason: "search-failed", message: error.message };
18628
+ }
18629
+ });
18585
18630
  }
18586
- async evict(path) {
18587
- return parseWindowsDehydrateOutput(await this.run(buildWindowsDehydrateScript(path), 60000));
18631
+ async searchWithExclusions(esPath, request, timeoutMs) {
18632
+ const deadline = Date.now() + timeoutMs;
18633
+ const offset = request.offset ?? 0;
18634
+ const maxResults = request.maxResults ?? DEFAULT_MAX_RESULTS;
18635
+ const results = [];
18636
+ let skipped = 0;
18637
+ let nativeOffset = 0;
18638
+ const timedOut = () => ({
18639
+ ok: false,
18640
+ reason: "search-failed",
18641
+ message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
18642
+ });
18643
+ for (;; ) {
18644
+ const remainingMs = deadline - Date.now();
18645
+ if (remainingMs <= 0)
18646
+ return timedOut();
18647
+ const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
18648
+ const response = await this.run(esPath, args, remainingMs + 5000);
18649
+ if (response.code !== 0)
18650
+ return this.failure("search-failed", esPath, args, response);
18651
+ if (Date.now() >= deadline)
18652
+ return timedOut();
18653
+ let entries;
18654
+ try {
18655
+ entries = parseEverythingJson(response.stdout);
18656
+ } catch (error) {
18657
+ return { ok: false, reason: "search-failed", message: error.message };
18658
+ }
18659
+ for (const entry of entries) {
18660
+ if (this.isExcludedPath(entry.path))
18661
+ continue;
18662
+ if (skipped < offset) {
18663
+ skipped += 1;
18664
+ continue;
18665
+ }
18666
+ results.push(entry);
18667
+ if (results.length >= maxResults)
18668
+ return { ok: true, results };
18669
+ }
18670
+ if (entries.length < MAX_FILTER_CANDIDATES)
18671
+ return { ok: true, results };
18672
+ nativeOffset += entries.length;
18673
+ }
18588
18674
  }
18589
- loadSyncRoots() {
18590
- this.syncRoots ??= this.run(buildWindowsSyncRootsScript(), 15000).then((result) => result.ok ? parseWindowsSyncRoots(result.stdout) : [], () => []);
18591
- return this.syncRoots;
18675
+ index() {
18676
+ return this.queue.exclusive(async () => {
18677
+ const es = await this.ensureRunning(true);
18678
+ if (!es.ok)
18679
+ return es;
18680
+ const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
18681
+ const saveArgs = ["-instance", this.instanceName, "-save-db"];
18682
+ const saved = await this.run(es.esPath, saveArgs, timeoutMs);
18683
+ if (saved.code !== 0)
18684
+ return this.failure("index-failed", es.esPath, saveArgs, saved);
18685
+ const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
18686
+ const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
18687
+ if (counted.code !== 0)
18688
+ return this.failure("index-failed", es.esPath, countArgs, counted);
18689
+ return {
18690
+ ok: true,
18691
+ indexedFolders: [...this.options.folders],
18692
+ indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
18693
+ };
18694
+ });
18592
18695
  }
18593
- loadDiskRoots() {
18594
- this.disks ??= this.run(buildWindowsDiskScript(), 30000).then((result) => result.ok ? windowsRootsFromDisks(parseWindowsLogicalDisks(result.stdout)) : [], () => []);
18595
- return this.disks;
18696
+ stop() {
18697
+ return this.queue.exclusive(async () => {
18698
+ const binaries = await this.resolveBinaries("cached");
18699
+ if (!binaries.ok)
18700
+ return;
18701
+ await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
18702
+ });
18596
18703
  }
18597
- }
18598
- function longestSyncRoot(roots, target) {
18599
- const normalized = target.replaceAll("/", "\\").toLowerCase();
18600
- let match;
18601
- let matchLength = -1;
18602
- for (const root of roots) {
18603
- const prefix = root.path.replaceAll("/", "\\").replace(/\\+$/u, "").toLowerCase();
18604
- if (prefix === "" || prefix.length <= matchLength)
18605
- continue;
18606
- if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
18607
- match = root;
18608
- matchLength = prefix.length;
18704
+ async resolveBinaries(mode) {
18705
+ if (!this.isSupported()) {
18706
+ return {
18707
+ ok: false,
18708
+ reason: "unsupported-platform",
18709
+ message: `Everything is Windows-only; this host is ${this.platform}.`
18710
+ };
18609
18711
  }
18712
+ if (this.binaries?.ok)
18713
+ return this.binaries;
18714
+ const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
18715
+ const result = await resolve(mode);
18716
+ if (result.ok)
18717
+ this.binaries = result;
18718
+ return result;
18610
18719
  }
18611
- return match;
18612
- }
18613
- function powerShellLaunch(script) {
18614
- return {
18615
- args: [
18616
- "-NoProfile",
18617
- "-NonInteractive",
18618
- "-Command",
18619
- `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; Invoke-Expression $env:${POWERSHELL_SCRIPT_VARIABLE}`
18620
- ],
18621
- env: { [POWERSHELL_SCRIPT_VARIABLE]: script }
18622
- };
18623
- }
18624
- async function runPowerShell(script, timeoutMs) {
18625
- const launch = powerShellLaunch(script);
18626
- try {
18627
- const { stdout, stderr } = await execFileAsync("powershell.exe", [...launch.args], {
18628
- timeout: timeoutMs,
18629
- maxBuffer: 16 * 1024 * 1024,
18630
- windowsHide: true,
18631
- env: { ...process.env, ...launch.env }
18632
- });
18633
- return { ok: true, stdout, stderr };
18634
- } catch (error) {
18635
- const record = typeof error === "object" && error !== null ? error : {};
18720
+ async ensureRunning(reconfigure) {
18721
+ const binaries = await this.resolveBinaries("install");
18722
+ if (!binaries.ok)
18723
+ return binaries;
18724
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
18725
+ let ping = await this.ping(binaries.esPath);
18726
+ if (ping.code === 0 && !reconfigure)
18727
+ return { ok: true, esPath: binaries.esPath };
18728
+ if (ping.code === 0) {
18729
+ const exitArgs = ["-instance", this.instanceName, "-exit"];
18730
+ const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
18731
+ if (exited.code !== 0)
18732
+ return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
18733
+ }
18734
+ mkdirSync16(this.stateDir, { recursive: true });
18735
+ const iniPath = join29(this.stateDir, "Everything.ini");
18736
+ writeFileSync11(iniPath, buildEverythingIni({
18737
+ folders: this.options.folders,
18738
+ excludeFolders: [
18739
+ ...new Set([
18740
+ join29(this.options.root, ".autorag"),
18741
+ ...this.options.folders.map((folder) => join29(folder, ".autorag")),
18742
+ ...this.options.excludeFolders ?? []
18743
+ ])
18744
+ ]
18745
+ }));
18746
+ const launchArgs = [
18747
+ "-instance",
18748
+ this.instanceName,
18749
+ "-config",
18750
+ iniPath,
18751
+ "-db",
18752
+ join29(this.stateDir, "Everything.db"),
18753
+ "-startup"
18754
+ ];
18755
+ try {
18756
+ this.launch(binaries.everythingPath, launchArgs);
18757
+ } catch (error) {
18758
+ return {
18759
+ ok: false,
18760
+ reason: "startup-failed",
18761
+ message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
18762
+ };
18763
+ }
18764
+ const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
18765
+ const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
18766
+ for (;; ) {
18767
+ ping = await this.ping(binaries.esPath);
18768
+ if (ping.code === 0)
18769
+ return { ok: true, esPath: binaries.esPath };
18770
+ if (Date.now() >= deadline)
18771
+ break;
18772
+ await sleep2(pollMs);
18773
+ }
18636
18774
  return {
18637
18775
  ok: false,
18638
- stdout: typeof record.stdout === "string" ? record.stdout : "",
18639
- stderr: typeof record.stderr === "string" ? record.stderr : String(error)
18776
+ reason: "startup-failed",
18777
+ message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
18640
18778
  };
18641
18779
  }
18642
- }
18643
- function defaultCloudProbe() {
18644
- if (process.platform === "darwin")
18645
- return new DarwinCloudProbe;
18646
- if (process.platform === "win32")
18647
- return new WindowsCloudProbe;
18648
- return;
18649
- }
18650
- async function readFileProviderDomain(target) {
18651
- try {
18652
- await lstat(target);
18653
- const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
18654
- timeout: 2000
18655
- });
18656
- const value = stdout.trim();
18657
- return value.length > 0 ? value : undefined;
18658
- } catch {
18659
- return;
18780
+ async ping(esPath) {
18781
+ const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
18782
+ return result.code === 0 && !VERSION_PATTERN.test(result.stdout.trim()) ? { ...result, code: -1 } : result;
18783
+ }
18784
+ failure(reason, esPath, args, result) {
18785
+ return {
18786
+ ok: false,
18787
+ reason,
18788
+ message: `${esPath} ${args.join(" ")} exit ${result.code}: ${(result.stderr || result.stdout).trim()}`
18789
+ };
18660
18790
  }
18661
18791
  }
18662
- async function readFileProviderDomainUpwards(target) {
18663
- let cursor = target;
18664
- for (let depth = 0;depth < 12; depth += 1) {
18665
- const domain = await readFileProviderDomain(cursor);
18666
- if (domain !== undefined)
18667
- return domain;
18668
- const parent = dirname15(cursor);
18669
- if (parent === cursor)
18792
+ function runProcess(command, args, timeoutMs) {
18793
+ let resolvePromise;
18794
+ const promise = new Promise((resolve) => {
18795
+ resolvePromise = resolve;
18796
+ });
18797
+ const child = spawn15(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
18798
+ const stdout = [];
18799
+ const stderr = [];
18800
+ let bytes = 0;
18801
+ let overflow = false;
18802
+ const collect = (target) => (chunk) => {
18803
+ bytes += chunk.length;
18804
+ if (bytes > MAX_OUTPUT_BYTES) {
18805
+ overflow = true;
18806
+ child.kill();
18670
18807
  return;
18671
- cursor = parent;
18672
- }
18673
- return;
18808
+ }
18809
+ target.push(chunk);
18810
+ };
18811
+ child.stdout.on("data", collect(stdout));
18812
+ child.stderr.on("data", collect(stderr));
18813
+ const timer = setTimeout(() => child.kill(), timeoutMs);
18814
+ child.on("error", (error) => {
18815
+ clearTimeout(timer);
18816
+ resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
18817
+ });
18818
+ child.on("close", (code, signal) => {
18819
+ clearTimeout(timer);
18820
+ const err = Buffer.concat(stderr).toString("utf8");
18821
+ resolvePromise({
18822
+ code,
18823
+ stdout: Buffer.concat(stdout).toString("utf8"),
18824
+ stderr: overflow ? `${err}
18825
+ output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
18826
+ killed by ${signal} after ${timeoutMs}ms` : err
18827
+ });
18828
+ });
18829
+ return promise;
18674
18830
  }
18675
- function quotePowerShell(value) {
18676
- return `'${value.replace(/['\u2018\u2019\u201A\u201B]/gu, (quote) => quote + quote)}'`;
18831
+ function launchDetached(command, args) {
18832
+ const child = spawn15(command, args, { detached: true, stdio: "ignore", windowsHide: true });
18833
+ child.on("error", () => {});
18834
+ child.unref();
18677
18835
  }
18678
- var execFileAsync, SF_DATALESS = 1073741824, FILE_PROVIDER_XATTR = "com.apple.file-provider-domain-id", WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 4194304, STAT_BATCH = 200, defaultCloudExec = (file, args, options) => execFileAsync(file, [...args], {
18679
- timeout: options?.timeoutMs ?? 30000,
18680
- ...options?.maxBuffer === undefined ? {} : { maxBuffer: options.maxBuffer }
18681
- }), JXA_RELEASE_PROGRAM = `ObjC.import("Foundation");
18682
- function run(argv) {
18683
- var url = $.NSURL.fileURLWithPath(argv[0]);
18684
- var errorRef = Ref();
18685
- var released = $.NSFileManager.defaultManager.evictUbiquitousItemAtURLError(url, errorRef);
18686
- if (released) return JSON.stringify({ ok: true });
18687
- function resourceValue(key) {
18688
- var ref = Ref();
18689
- if (!url.getResourceValueForKeyError(ref, key, null)) return null;
18690
- var value = ref[0];
18691
- return value === undefined || value === null ? null : ObjC.unwrap(value);
18692
- }
18693
- return JSON.stringify({
18694
- ok: false,
18695
- isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
18696
- isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
18697
- downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
18698
- });
18699
- }`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
18700
- var init_cloud_placeholder = __esm(() => {
18701
- execFileAsync = promisify(execFile2);
18702
- RELEASABLE_DARWIN_PROVIDERS = {
18703
- icloud: true,
18704
- "google-drive": true,
18705
- onedrive: true,
18706
- dropbox: true
18707
- };
18708
- DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
18709
- WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
18710
- WINDOWS_DEHYDRATE_REFUSALS = {
18711
- "80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
18712
- "80070188": "The file is pinned to stay on this device, so it is not released."
18713
- };
18836
+ var DEFAULT_TIMEOUT_MS8 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
18837
+ var init_client9 = __esm(() => {
18838
+ init_path_exclusions();
18839
+ init_bundle();
18840
+ DEFAULT_INDEX_TIMEOUT_MS3 = 10 * 60000;
18841
+ MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
18842
+ VERSION_PATTERN = /^\d+\.\d+\.\d+\.\d+$/;
18714
18843
  });
18715
18844
 
18716
- // src/dupey/scan-plan.ts
18717
- import { basename as basename7 } from "node:path";
18718
- async function listCloudRoots(probe, roots) {
18719
- if (probe === undefined)
18720
- return [];
18845
+ // src/everything/index.ts
18846
+ var init_everything = __esm(() => {
18847
+ init_bundle();
18848
+ init_client9();
18849
+ });
18850
+
18851
+ // src/filesystem/cloud-placeholder.ts
18852
+ import { execFile as execFile2 } from "node:child_process";
18853
+ import { realpathSync as realpathSync3 } from "node:fs";
18854
+ import { lstat, readdir } from "node:fs/promises";
18855
+ import { homedir as homedir6 } from "node:os";
18856
+ import { dirname as dirname16, join as join30 } from "node:path";
18857
+ import { promisify } from "node:util";
18858
+ function homeCloudRoots(home = homedir6()) {
18859
+ const normalizedHome = home.replaceAll("\\", "/").replace(/\/+$/, "");
18860
+ return [`${normalizedHome}/Library/CloudStorage`, `${normalizedHome}/Library/Mobile Documents`];
18861
+ }
18862
+ function datalessFromDarwinFlags(flags) {
18863
+ return Number.isFinite(flags) && (flags >>> 0 & SF_DATALESS) !== 0;
18864
+ }
18865
+ function datalessFromWindowsAttributes(attributes) {
18866
+ return Number.isFinite(attributes) && (attributes & WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) !== 0;
18867
+ }
18868
+ function cloudProviderFromPath(target) {
18869
+ const lower = target.toLowerCase();
18870
+ if (lower.includes("googledrive") || lower.includes("google drive") || lower.includes("drivefs"))
18871
+ return "google-drive";
18872
+ if (lower.includes("onedrive"))
18873
+ return "onedrive";
18874
+ if (lower.includes("dropbox"))
18875
+ return "dropbox";
18876
+ if (lower.includes("mobile documents") || lower.includes("clouddocs"))
18877
+ return "icloud";
18878
+ return "file-provider";
18879
+ }
18880
+ function cloudProviderFromDomain(domain) {
18881
+ const lower = domain.toLowerCase();
18882
+ if (lower.includes("clouddocs") || lower.includes("icloud"))
18883
+ return "icloud";
18884
+ if (lower.includes("google"))
18885
+ return "google-drive";
18886
+ if (lower.includes("onedrive"))
18887
+ return "onedrive";
18888
+ if (lower.includes("dropbox"))
18889
+ return "dropbox";
18890
+ return "file-provider";
18891
+ }
18892
+ function normalizeCloudPath(value) {
18893
+ let normalized = value.replaceAll("\\", "/");
18894
+ if (normalized.length > 1)
18895
+ normalized = normalized.replace(/\/+$/u, "");
18896
+ return normalized;
18897
+ }
18898
+ function isPathInside(target, parent) {
18899
+ const normalizedParent = normalizeCloudPath(parent);
18900
+ if (normalizedParent === "")
18901
+ return false;
18902
+ const left = normalizeCloudPath(target);
18903
+ const [a, b] = process.platform === "win32" ? [left.toLowerCase(), normalizedParent.toLowerCase()] : [left, normalizedParent];
18904
+ return a === b || a.startsWith(`${b}/`);
18905
+ }
18906
+ function canonicalPath(value) {
18721
18907
  try {
18722
- return await probe.cloudRoots(roots);
18908
+ return realpathSync3(value);
18723
18909
  } catch {
18724
- return [];
18910
+ return value;
18725
18911
  }
18726
18912
  }
18727
- function planDupeyScans(roots, cloudRoots) {
18728
- const targets = [];
18729
- const skipped = [];
18730
- for (const path of roots) {
18731
- const scope = cloudScope(path, cloudRoots);
18732
- if (scope.insideCloudRoot) {
18733
- skipped.push(path);
18734
- continue;
18913
+ function canonicalPathMapper(searchRoots) {
18914
+ const forms = searchRoots.map((root) => ({ raw: normalizeCloudPath(root), canonical: normalizeCloudPath(canonicalPath(root)) })).filter(({ raw, canonical }) => raw !== "" && raw !== canonical).sort((left, right) => right.raw.length - left.raw.length);
18915
+ if (forms.length === 0)
18916
+ return (path) => path;
18917
+ return (path) => {
18918
+ const normalized = normalizeCloudPath(path);
18919
+ for (const { raw, canonical } of forms) {
18920
+ if (isPathInside(normalized, raw))
18921
+ return `${canonical}${normalized.slice(raw.length)}`;
18735
18922
  }
18736
- targets.push({ path, excludeDirs: scope.nestedCloudRoots.map((root) => basename7(root.path)) });
18737
- }
18738
- return { targets, skipped };
18923
+ return path;
18924
+ };
18739
18925
  }
18740
- var init_scan_plan = __esm(() => {
18741
- init_cloud_placeholder();
18742
- });
18743
-
18744
- // src/dupey/index.ts
18745
- var init_dupey = __esm(() => {
18746
- init_cli2();
18747
- init_filter();
18748
- init_scan_plan();
18749
- });
18750
-
18751
- // src/convert/relate.ts
18752
- async function convertAndRelate(request, options = {}) {
18753
- const result = await convertWithFilee(request, options.filee);
18754
- if (options.dupey === false)
18755
- return { result, links: [] };
18756
- const dupeyOptions = options.dupey ?? {};
18757
- const pairs = result.files.filter((file) => file.state === "done").flatMap((file) => file.outputs.map((output) => ({ source: file.source, output })));
18758
- const links = await Promise.all(pairs.map(async ({ source, output }) => {
18759
- try {
18760
- return { source, output, dupey: await compareWithDupey(source, output, dupeyOptions) };
18761
- } catch (error) {
18762
- if (!(error instanceof DupeyCliError))
18763
- throw error;
18764
- return { source, output, dupey: { relation: "unavailable", reason: error.message } };
18765
- }
18766
- }));
18767
- return { result, links };
18926
+ function cloudScope(searchRoot, cloudRoots) {
18927
+ const canonicalSearchRoot = canonicalPath(searchRoot);
18928
+ const roots = cloudRoots.map((root) => ({ path: canonicalPath(root.path), kind: root.kind }));
18929
+ return {
18930
+ insideCloudRoot: roots.some((root) => isPathInside(canonicalSearchRoot, root.path)),
18931
+ nestedCloudRoots: roots.filter((root) => isPathInside(root.path, canonicalSearchRoot) && !isPathInside(canonicalSearchRoot, root.path))
18932
+ };
18768
18933
  }
18769
- var init_relate = __esm(() => {
18770
- init_dupey();
18771
- init_cli();
18772
- });
18773
-
18774
- // src/convert/index.ts
18775
- var init_convert = __esm(() => {
18776
- init_cli();
18777
- init_installer2();
18778
- init_matrix();
18779
- init_relate();
18780
- });
18781
-
18782
- // src/datasource/diagnostics.ts
18783
- function mapDatasourceDiagnostic(ds) {
18934
+ function parseReleaseOutcome(stdout) {
18935
+ let parsed;
18936
+ try {
18937
+ parsed = JSON.parse(stdout.trim());
18938
+ } catch {
18939
+ return;
18940
+ }
18941
+ if (typeof parsed !== "object" || parsed === null || !("ok" in parsed) || typeof parsed.ok !== "boolean") {
18942
+ return;
18943
+ }
18784
18944
  return {
18785
- code: datasourceCodeToRetrieval(ds.code),
18786
- severity: ds.severity,
18787
- message: `[datasource:${ds.code}] ${ds.message}`,
18788
- source: ds.source ?? (ds.instanceId ? `datasource:${ds.instanceId}` : "datasource")
18945
+ ok: parsed.ok,
18946
+ ..."isUploaded" in parsed ? { isUploaded: parsed.isUploaded } : {},
18947
+ ..."isUploading" in parsed ? { isUploading: parsed.isUploading } : {},
18948
+ ..."downloadingStatus" in parsed ? { downloadingStatus: parsed.downloadingStatus } : {}
18789
18949
  };
18790
18950
  }
18791
- function mapDatasourceDiagnostics(diagnostics) {
18792
- return diagnostics.map(mapDatasourceDiagnostic);
18951
+ function describeReleaseRefusal(outcome) {
18952
+ if (outcome.isUploading === true)
18953
+ return "The file is still uploading to the cloud, so it cannot be released yet.";
18954
+ if (outcome.isUploaded === false)
18955
+ return "The file has not finished uploading to the cloud, so it cannot be released yet.";
18956
+ const status = typeof outcome.downloadingStatus === "string" ? outcome.downloadingStatus : "unknown";
18957
+ return `The cloud provider declined to release the local copy (download status ${status}).`;
18793
18958
  }
18794
- function datasourceCodeToRetrieval(code) {
18795
- switch (code) {
18796
- case "datasource-unavailable":
18797
- case "datasource-cli-error":
18798
- case "datasource-empty":
18799
- case "datasource-rate-limited":
18800
- case "datasource-auth-error":
18801
- case "datasource-embedding-egress-rejected":
18802
- case "datasource-index-failed":
18803
- case "datasource-index-partial":
18804
- case "datasource-permission-denied":
18805
- case "datasource-remote-embedding-rejected":
18806
- return "retrieval-method-failed";
18959
+ function describeReleaseHelperFailure(error) {
18960
+ const failure = error;
18961
+ const parts = [];
18962
+ if (failure.killed === true || typeof failure.signal === "string") {
18963
+ parts.push(`the helper was killed${typeof failure.signal === "string" ? ` by ${failure.signal}` : ""}`);
18964
+ } else if (typeof failure.code === "number") {
18965
+ parts.push(`the helper exited with code ${failure.code}`);
18807
18966
  }
18967
+ const stderr = typeof failure.stderr === "string" ? failure.stderr.trim() : "";
18968
+ if (stderr !== "")
18969
+ parts.push(stderr);
18970
+ if (parts.length === 0)
18971
+ parts.push(error instanceof Error ? error.message : String(error));
18972
+ return `osascript could not release the local copy: ${parts.join("; ")}`;
18808
18973
  }
18809
-
18810
- // src/datasource/retry.ts
18811
- import { setTimeout as delay2 } from "node:timers/promises";
18812
- function datasourceRetryDelayMs(retry, policy) {
18813
- const step = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (retry - 1));
18814
- return Math.round(step * (0.5 + 0.5 * (policy.random ?? Math.random)()));
18974
+ async function defaultDarwinReadDir(path) {
18975
+ const entries = await readdir(path, { withFileTypes: true });
18976
+ return entries.map((entry) => ({ name: entry.name, isDirectory: () => entry.isDirectory() }));
18815
18977
  }
18816
- async function indexWithRetry(skill, options = {}, signal) {
18817
- const policy = {
18818
- maxRetries: options.maxRetries ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxRetries,
18819
- baseDelayMs: options.baseDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.baseDelayMs,
18820
- maxDelayMs: options.maxDelayMs ?? DEFAULT_DATASOURCE_RETRY_POLICY.maxDelayMs
18821
- };
18822
- const sleep = options.sleep ?? ((ms) => delay2(ms, undefined, signal === undefined ? undefined : { signal }));
18823
- let lastError = "";
18824
- for (let attempt = 1;; attempt += 1) {
18825
- const result = await runIndex(skill, signal);
18826
- signal?.throwIfAborted();
18827
- if (result.ok) {
18828
- if (attempt === 1)
18829
- return result;
18830
- const recovered = {
18831
- code: "datasource-index-failed",
18832
- severity: "info",
18833
- message: `recovered after ${attempt} attempts; last error: ${lastError}`,
18834
- instanceId: result.instanceId,
18835
- source: result.skill
18836
- };
18837
- return { ...result, attempts: attempt, diagnostics: [...result.diagnostics, recovered] };
18838
- }
18839
- lastError = result.message ?? result.error ?? "datasource-index-failed";
18840
- if (attempt > policy.maxRetries) {
18841
- if (attempt === 1)
18842
- return { ...result, attempts: attempt };
18843
- const suffix = ` (gave up after ${attempt} attempts)`;
18844
- const last = result.diagnostics.at(-1);
18845
- return {
18846
- ...result,
18847
- attempts: attempt,
18848
- message: `${lastError}${suffix}`,
18849
- diagnostics: last === undefined ? [
18850
- {
18851
- code: result.code ?? "datasource-index-failed",
18852
- severity: "error",
18853
- message: `${lastError}${suffix}`,
18854
- instanceId: result.instanceId,
18855
- source: result.skill
18856
- }
18857
- ] : [...result.diagnostics.slice(0, -1), { ...last, message: `${last.message}${suffix}` }]
18858
- };
18859
- }
18860
- await sleep(datasourceRetryDelayMs(attempt, { ...policy, ...options.random ? { random: options.random } : {} }));
18861
- signal?.throwIfAborted();
18978
+ function parseDarwinMounts(stdout) {
18979
+ const mounts = [];
18980
+ for (const line of stdout.split(`
18981
+ `)) {
18982
+ if (line.trim() === "")
18983
+ continue;
18984
+ const open = line.lastIndexOf(" (");
18985
+ if (open <= 0 || !line.endsWith(")"))
18986
+ continue;
18987
+ const head = line.slice(0, open);
18988
+ const on = head.indexOf(" on ");
18989
+ if (on <= 0)
18990
+ continue;
18991
+ const mountPoint = head.slice(on + " on ".length);
18992
+ if (mountPoint === "")
18993
+ continue;
18994
+ const fields = line.slice(open + 2, -1).split(",").map((field) => field.trim()).filter((field) => field !== "");
18995
+ mounts.push({
18996
+ device: head.slice(0, on),
18997
+ mountPoint,
18998
+ fileSystem: fields[0] ?? "",
18999
+ options: fields.slice(1)
19000
+ });
18862
19001
  }
19002
+ return mounts;
18863
19003
  }
18864
- async function runIndex(skill, signal) {
18865
- try {
18866
- return await skill.index(signal === undefined ? undefined : { signal });
18867
- } catch (error) {
18868
- signal?.throwIfAborted();
18869
- const descriptor = skill.describe();
18870
- const message = error instanceof Error ? error.message : "Datasource indexing failed.";
18871
- const failed = {
18872
- ok: false,
18873
- instanceId: descriptor.instanceId ?? "default",
18874
- skill: descriptor.name,
18875
- indexedAt: Date.now(),
18876
- diagnostics: [
18877
- {
18878
- code: "datasource-index-failed",
18879
- severity: "error",
18880
- message,
18881
- source: descriptor.name,
18882
- instanceId: descriptor.instanceId
18883
- }
18884
- ],
18885
- error: "datasource-index-failed",
18886
- code: "datasource-index-failed",
18887
- message
18888
- };
18889
- return failed;
19004
+ function networkMountPoints(mounts) {
19005
+ const points = [];
19006
+ const seen = new Set;
19007
+ for (const mount of mounts) {
19008
+ if (DARWIN_NON_STORAGE_FILE_SYSTEMS[mount.fileSystem] === true)
19009
+ continue;
19010
+ if (mount.options.includes(DARWIN_LOCAL_MOUNT_OPTION))
19011
+ continue;
19012
+ if (seen.has(mount.mountPoint))
19013
+ continue;
19014
+ seen.add(mount.mountPoint);
19015
+ points.push(mount.mountPoint);
18890
19016
  }
19017
+ return points;
18891
19018
  }
18892
- var DEFAULT_DATASOURCE_RETRY_POLICY;
18893
- var init_retry = __esm(() => {
18894
- DEFAULT_DATASOURCE_RETRY_POLICY = {
18895
- maxRetries: 2,
18896
- baseDelayMs: 1000,
18897
- maxDelayMs: 30000
18898
- };
18899
- });
18900
-
18901
- // src/everything/bundle.ts
18902
- import { createHash as createHash11 } from "node:crypto";
18903
- import { existsSync as existsSync18, mkdirSync as mkdirSync15, readFileSync as readFileSync18, rmSync as rmSync7, writeFileSync as writeFileSync10 } from "node:fs";
18904
- import { dirname as dirname16, join as join29 } from "node:path";
18905
- import { fileURLToPath as fileURLToPath2 } from "node:url";
18906
- import JSZip from "jszip";
18907
- function resolveEverythingBundleDir() {
18908
- let dir = dirname16(fileURLToPath2(import.meta.url));
18909
- for (let depth = 0;depth < 6; depth += 1) {
18910
- const candidate = join29(dir, "vendor", "everything");
18911
- if (existsSync18(join29(candidate, MANIFEST_FILENAME)))
18912
- return candidate;
18913
- const parent = dirname16(dir);
18914
- if (parent === dir)
18915
- break;
18916
- dir = parent;
19019
+ function parseXattrValues(stdout, candidates) {
19020
+ const values = new Map;
19021
+ if (candidates.length === 0)
19022
+ return values;
19023
+ const ordered = [...candidates].sort((left, right) => right.length - left.length);
19024
+ for (const line of stdout.split(`
19025
+ `)) {
19026
+ if (line === "")
19027
+ continue;
19028
+ for (const candidate of ordered) {
19029
+ if (line.startsWith(`${candidate}: `)) {
19030
+ values.set(candidate, line.slice(candidate.length + 2));
19031
+ break;
19032
+ }
19033
+ }
18917
19034
  }
18918
- return;
18919
- }
18920
- function everythingArch(arch) {
18921
- if (arch === "x64" || arch === "arm64")
18922
- return arch;
18923
- return;
18924
- }
18925
- function everythingBinaryPaths(root, version, arch) {
18926
- const dir = join29(root, ".autorag", "everything", version, arch);
18927
- return { everythingPath: join29(dir, "everything.exe"), esPath: join29(dir, "es.exe") };
18928
- }
18929
- function loadEverythingBundleManifest(bundleDir) {
18930
- return JSON.parse(readFileSync18(join29(bundleDir, MANIFEST_FILENAME), "utf8"));
19035
+ return values;
18931
19036
  }
18932
- async function ensureEverythingBinaries(options) {
18933
- const platform = options.platform ?? process.platform;
18934
- const rawArch = options.arch ?? process.arch;
18935
- if (platform !== "win32") {
18936
- return {
18937
- ok: false,
18938
- reason: "unsupported-platform",
18939
- message: `Everything is Windows-only; this host is ${platform}.`
18940
- };
19037
+
19038
+ class DarwinCloudProbe {
19039
+ exec;
19040
+ readDir;
19041
+ home;
19042
+ constructor(options = {}) {
19043
+ this.exec = options.exec ?? defaultCloudExec;
19044
+ this.readDir = options.readDir ?? defaultDarwinReadDir;
19045
+ this.home = options.home ?? homedir6();
18941
19046
  }
18942
- const arch = everythingArch(rawArch);
18943
- const bundleDir = options.bundleDir ?? resolveEverythingBundleDir();
18944
- if (bundleDir === undefined) {
18945
- return {
18946
- ok: false,
18947
- reason: "bundle-missing",
18948
- message: `Bundled Everything assets (vendor/everything/${MANIFEST_FILENAME}) were not found above ${dirname16(fileURLToPath2(import.meta.url))}.`
19047
+ async cloudRoots(searchRoots = []) {
19048
+ const roots = [];
19049
+ const seen = new Set;
19050
+ const add = (path, kind) => {
19051
+ const canonical = canonicalPath(path);
19052
+ const key = normalizeCloudPath(canonical);
19053
+ if (key === "" || seen.has(key))
19054
+ return;
19055
+ seen.add(key);
19056
+ roots.push({ path: canonical, kind });
18949
19057
  };
19058
+ const [cloudStorage, mobileDocuments] = homeCloudRoots(this.home);
19059
+ for (const entry of await this.tryReadDir(cloudStorage) ?? []) {
19060
+ if (entry.isDirectory())
19061
+ add(join30(cloudStorage, entry.name), "file-provider");
19062
+ }
19063
+ if (await this.tryReadDir(mobileDocuments) !== undefined)
19064
+ add(mobileDocuments, "file-provider");
19065
+ for (const mountPoint of await this.networkMountPoints())
19066
+ add(mountPoint, "network");
19067
+ for (const directory of await this.fileProviderDirectories(searchRoots))
19068
+ add(directory, "file-provider");
19069
+ return roots;
18950
19070
  }
18951
- let manifest;
18952
- try {
18953
- manifest = options.manifest ?? loadEverythingBundleManifest(bundleDir);
18954
- } catch (error) {
18955
- return { ok: false, reason: "bundle-missing", message: errorMessage2(error) };
19071
+ async networkMountPoints() {
19072
+ try {
19073
+ const { stdout } = await this.exec("/sbin/mount", [], { timeoutMs: 1e4, maxBuffer: 4 * 1024 * 1024 });
19074
+ return networkMountPoints(parseDarwinMounts(stdout));
19075
+ } catch {
19076
+ return [];
19077
+ }
18956
19078
  }
18957
- const assets = arch === undefined ? undefined : manifest.architectures[arch];
18958
- if (arch === undefined || assets === undefined) {
18959
- return {
18960
- ok: false,
18961
- reason: "unsupported-platform",
18962
- message: `No bundled Everything build for Windows architecture ${rawArch}; bundled: ${Object.keys(manifest.architectures).join(", ")}.`
18963
- };
19079
+ async fileProviderDirectories(searchRoots) {
19080
+ if (searchRoots.length === 0)
19081
+ return [];
19082
+ const candidates = new Set;
19083
+ for (const searchRoot of searchRoots) {
19084
+ const canonical = canonicalPath(searchRoot);
19085
+ for (let cursor = canonical;; cursor = dirname16(cursor)) {
19086
+ candidates.add(cursor);
19087
+ const parent = dirname16(cursor);
19088
+ if (parent === cursor)
19089
+ break;
19090
+ }
19091
+ for (const entry of await this.tryReadDir(canonical) ?? []) {
19092
+ if (entry.isDirectory())
19093
+ candidates.add(join30(canonical, entry.name));
19094
+ }
19095
+ }
19096
+ const paths = [...candidates];
19097
+ const managed = [];
19098
+ for (let start = 0;start < paths.length; start += XATTR_BATCH) {
19099
+ for (const path of (await this.xattrValues(paths.slice(start, start + XATTR_BATCH))).keys())
19100
+ managed.push(path);
19101
+ }
19102
+ return managed;
18964
19103
  }
18965
- const { everythingPath, esPath } = everythingBinaryPaths(options.root, manifest.everythingVersion, arch);
18966
- const everythingCached = fileHasSha256(everythingPath, assets.everythingSha256);
18967
- const esCached = fileHasSha256(esPath, assets.esSha256);
18968
- if (everythingCached && esCached) {
18969
- return { ok: true, everythingPath, esPath, source: "cached" };
19104
+ async xattrValues(paths) {
19105
+ try {
19106
+ const { stdout } = await this.exec("/usr/bin/xattr", ["-v", "-p", FILE_PROVIDER_XATTR, ...paths], {
19107
+ timeoutMs: 15000,
19108
+ maxBuffer: 8 * 1024 * 1024
19109
+ });
19110
+ return parseXattrValues(stdout, paths);
19111
+ } catch (error) {
19112
+ const failure = error;
19113
+ return typeof failure.stdout === "string" ? parseXattrValues(failure.stdout, paths) : new Map;
19114
+ }
18970
19115
  }
18971
- if ((options.mode ?? "install") === "cached") {
18972
- const state = everythingCached || esCached ? "are incomplete or do not match their pinned SHA-256 digests" : "are not installed";
18973
- return {
18974
- ok: false,
18975
- reason: "not-installed",
18976
- message: `Everything ${manifest.everythingVersion} (${arch}) binaries ${state} under ${dirname16(everythingPath)}; run \`autorag refresh\` to (re)install and index them.`
18977
- };
19116
+ async tryReadDir(path) {
19117
+ try {
19118
+ return await this.readDir(path);
19119
+ } catch {
19120
+ return;
19121
+ }
18978
19122
  }
18979
- try {
18980
- mkdirSync15(dirname16(everythingPath), { recursive: true });
18981
- const everythingBytes = await extractVerified(join29(bundleDir, assets.everythingArchive), assets.everythingArchiveSha256, assets.everythingMember, assets.everythingSha256);
18982
- const esBytes = await extractVerified(join29(bundleDir, assets.esArchive), assets.esArchiveSha256, assets.esMember, assets.esSha256);
18983
- writeAtomic(everythingPath, everythingBytes);
18984
- writeAtomic(esPath, esBytes);
18985
- } catch (error) {
18986
- return { ok: false, reason: "install-failed", message: errorMessage2(error) };
19123
+ async datalessAmong(paths) {
19124
+ const dataless = new Set;
19125
+ for (let start = 0;start < paths.length; start += STAT_BATCH) {
19126
+ const batch = paths.slice(start, start + STAT_BATCH);
19127
+ const { stdout } = await this.exec("/usr/bin/stat", ["-f", "%f", ...batch], {
19128
+ timeoutMs: 30000,
19129
+ maxBuffer: 4 * 1024 * 1024
19130
+ });
19131
+ const flags = stdout.split(`
19132
+ `).filter(Boolean);
19133
+ for (const [index, path] of batch.entries()) {
19134
+ if (datalessFromDarwinFlags(Number.parseInt(flags[index] ?? "", 10)))
19135
+ dataless.add(path);
19136
+ }
19137
+ }
19138
+ return dataless;
18987
19139
  }
18988
- return { ok: true, everythingPath, esPath, source: "installed" };
18989
- }
18990
- async function extractVerified(archivePath, archiveSha256, member, memberSha256) {
18991
- const archive = readFileSync18(archivePath);
18992
- const actualArchive = sha256(archive);
18993
- if (actualArchive !== archiveSha256) {
18994
- throw new Error(`${archivePath} SHA-256 ${actualArchive} does not match pinned ${archiveSha256}`);
19140
+ async providerOf(path) {
19141
+ const domain = await readFileProviderDomainUpwards(path);
19142
+ return domain === undefined ? undefined : cloudProviderFromDomain(domain);
18995
19143
  }
18996
- const zip = await JSZip.loadAsync(archive);
18997
- const entry = zip.file(member);
18998
- if (entry === null)
18999
- throw new Error(`${archivePath} does not contain ${member}`);
19000
- const bytes = await entry.async("nodebuffer");
19001
- const actualMember = sha256(bytes);
19002
- if (actualMember !== memberSha256) {
19003
- throw new Error(`${member} from ${archivePath} SHA-256 ${actualMember} does not match pinned ${memberSha256}`);
19144
+ canRelease(provider) {
19145
+ return provider !== undefined && RELEASABLE_DARWIN_PROVIDERS[provider] === true;
19004
19146
  }
19005
- return bytes;
19006
- }
19007
- function fileHasSha256(path, expected) {
19008
- return existsSync18(path) && sha256(readFileSync18(path)) === expected;
19009
- }
19010
- function writeAtomic(path, bytes) {
19011
- const temp = `${path}.${process.pid}.tmp`;
19012
- try {
19013
- writeFileSync10(temp, bytes);
19014
- renameWithRetry(temp, path);
19015
- } finally {
19016
- rmSync7(temp, { force: true });
19147
+ async evict(path) {
19148
+ let stdout;
19149
+ try {
19150
+ stdout = (await this.exec("/usr/bin/osascript", ["-l", "JavaScript", "-e", JXA_RELEASE_PROGRAM, path], {
19151
+ timeoutMs: 30000
19152
+ })).stdout;
19153
+ } catch (error) {
19154
+ return { ok: false, message: describeReleaseHelperFailure(error) };
19155
+ }
19156
+ const outcome = parseReleaseOutcome(stdout);
19157
+ if (outcome === undefined) {
19158
+ const detail = stdout.trim();
19159
+ return {
19160
+ ok: false,
19161
+ message: `osascript returned no release result${detail === "" ? "" : `: ${detail}`}`
19162
+ };
19163
+ }
19164
+ return outcome.ok ? { ok: true } : { ok: false, message: describeReleaseRefusal(outcome) };
19017
19165
  }
19018
19166
  }
19019
- function sha256(bytes) {
19020
- return createHash11("sha256").update(bytes).digest("hex");
19167
+ function buildWindowsAttributeScript(paths) {
19168
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19169
+ ` + `$ErrorActionPreference='SilentlyContinue'
19170
+ ` + `foreach($p in @(${paths.map(quotePowerShell).join(",")})){` + "$i=Get-Item -LiteralPath $p -Force -ErrorAction SilentlyContinue;" + `if($null -eq $i){'ERR'}else{$i.Attributes.value__}}
19171
+ `;
19021
19172
  }
19022
- function errorMessage2(error) {
19023
- return error instanceof Error ? error.message : String(error);
19173
+ function buildWindowsSyncRootsScript() {
19174
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19175
+ ` + `$k='HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager'
19176
+ ` + "if(Test-Path $k){Get-ChildItem $k|ForEach-Object{" + "$u=Join-Path $_.PSPath 'UserSyncRoots';" + "if(Test-Path $u){$p=Get-ItemProperty -LiteralPath $u -ErrorAction SilentlyContinue;" + "$p.PSObject.Properties|Where-Object{$_.Name -notlike 'PS*'}|ForEach-Object{" + `"$($_.Name)$([char]9)$($_.Value)"}}}}
19177
+ `;
19024
19178
  }
19025
- var MANIFEST_FILENAME = "manifest.json";
19026
- var init_bundle = __esm(() => {
19027
- init_atomic_write();
19028
- });
19029
-
19030
- // src/filesystem/path-exclusions.ts
19031
- import { posix as posix3, win32 as win322 } from "node:path";
19032
- function createPathExclusionMatcher(excludedPaths, platform = process.platform) {
19033
- const paths = platform === "win32" ? win322 : posix3;
19034
- const normalize = (path) => {
19035
- const absolute = paths.resolve(path);
19036
- return platform === "win32" ? absolute.toLowerCase() : absolute;
19037
- };
19038
- const excluded = excludedPaths.map((path) => {
19039
- const absolute = normalize(path);
19040
- return { absolute, prefix: absolute.endsWith(paths.sep) ? absolute : `${absolute}${paths.sep}` };
19041
- });
19042
- return (path) => {
19043
- const absolute = normalize(path);
19044
- return excluded.some((entry) => absolute === entry.absolute || absolute.startsWith(entry.prefix));
19045
- };
19179
+ function buildWindowsDehydrateScript(path) {
19180
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19181
+ ` + `$ErrorActionPreference='Stop'
19182
+ ` + `Add-Type -Namespace Autorag -Name CfApi -MemberDefinition @'
19183
+ ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfOpenFileWithOplock(string FilePath, uint Flags, out System.IntPtr ProtectedHandle);
19184
+ ` + `[DllImport("cldapi.dll", CharSet=CharSet.Unicode, SetLastError=false)] public static extern int CfDehydratePlaceholder(System.IntPtr FileHandle, long StartingOffset, long Length, uint DehydrateFlags, System.IntPtr Overlapped);
19185
+ ` + `[DllImport("cldapi.dll", SetLastError=false)] public static extern void CfCloseHandle(System.IntPtr FileHandle);
19186
+ ` + `'@
19187
+ ` + `$h=[IntPtr]::Zero
19188
+ ` + `$open=[Autorag.CfApi]::CfOpenFileWithOplock(${quotePowerShell(path)},0x1,[ref]$h)
19189
+ ` + `'OPEN={0:X8}' -f ($open -band 0xFFFFFFFF)
19190
+ ` + "if($open -ne 0){if($h -ne [IntPtr]::Zero){[Autorag.CfApi]::CfCloseHandle($h)};" + `$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($open)).Message;"MSG=$m";exit 0}
19191
+ ` + `$dry=[Autorag.CfApi]::CfDehydratePlaceholder($h,0,-1,0,[IntPtr]::Zero)
19192
+ ` + `[Autorag.CfApi]::CfCloseHandle($h)
19193
+ ` + `'DEHYDRATE={0:X8}' -f ($dry -band 0xFFFFFFFF)
19194
+ ` + `if($dry -ne 0){$m=([System.Runtime.InteropServices.Marshal]::GetExceptionForHR($dry)).Message;"MSG=$m"}
19195
+ `;
19046
19196
  }
19047
- var init_path_exclusions = () => {};
19048
-
19049
- // src/filesystem/shared-exclusive-queue.ts
19050
- class SharedExclusiveQueue {
19051
- sharedSinceExclusive = [];
19052
- lastExclusive = Promise.resolve();
19053
- shared(task) {
19054
- const run = this.lastExclusive.then(task);
19055
- const settled = run.then(() => {
19056
- return;
19057
- }, () => {
19058
- return;
19059
- });
19060
- this.sharedSinceExclusive.push(settled);
19061
- return run;
19197
+ function parseWindowsAttributeOutput(stdout, paths) {
19198
+ const lines = stdout.split(/\r?\n/);
19199
+ const attributes = new Map;
19200
+ for (const [index, path] of paths.entries()) {
19201
+ const value = Number.parseInt((lines[index] ?? "").trim(), 10);
19202
+ if (Number.isFinite(value))
19203
+ attributes.set(path, value);
19062
19204
  }
19063
- exclusive(task) {
19064
- const earlier = Promise.all([this.lastExclusive, ...this.sharedSinceExclusive]);
19065
- const run = earlier.then(task);
19066
- const settled = run.then(() => {
19067
- return;
19068
- }, () => {
19069
- return;
19070
- });
19071
- this.sharedSinceExclusive = [];
19072
- this.lastExclusive = settled;
19073
- return run;
19205
+ return attributes;
19206
+ }
19207
+ function parseWindowsSyncRoots(stdout) {
19208
+ const roots = [];
19209
+ for (const line of stdout.split(/\r?\n/)) {
19210
+ const tab = line.indexOf("\t");
19211
+ if (tab <= 0)
19212
+ continue;
19213
+ const path = line.slice(0, tab).trim();
19214
+ if (path !== "")
19215
+ roots.push({ path, label: line.slice(tab + 1).trim() });
19074
19216
  }
19217
+ return roots;
19075
19218
  }
19076
-
19077
- // src/everything/client.ts
19078
- import { spawn as spawn15 } from "node:child_process";
19079
- import { createHash as createHash12 } from "node:crypto";
19080
- import { mkdirSync as mkdirSync16, writeFileSync as writeFileSync11 } from "node:fs";
19081
- import { join as join30, resolve as resolve11 } from "node:path";
19082
- import { setTimeout as sleep2 } from "node:timers/promises";
19083
- function everythingInstanceName(root) {
19084
- return `autorag-${createHash12("sha256").update(resolve11(root).toLowerCase()).digest("hex").slice(0, 12)}`;
19219
+ function providerFromWindowsSyncRoot(label) {
19220
+ const lower = label.toLowerCase();
19221
+ if (lower.includes("onedrive") || lower.includes("one drive"))
19222
+ return "onedrive";
19223
+ if (lower.includes("google"))
19224
+ return "google-drive";
19225
+ return "file-provider";
19085
19226
  }
19086
- function buildEverythingIni(input) {
19087
- const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
19088
- const perFolder = (value) => input.folders.map(() => value).join(",");
19089
- return [
19090
- "[Everything]",
19091
- "app_data=0",
19092
- "run_as_admin=0",
19093
- "run_in_background=1",
19094
- "show_tray_icon=0",
19095
- "show_in_taskbar=0",
19096
- "check_for_updates_on_startup=0",
19097
- "beta_updates=0",
19098
- "language=1033",
19099
- "allow_http_server=0",
19100
- "allow_etp_server=0",
19101
- "search_history_enabled=0",
19102
- "run_history_enabled=0",
19103
- "auto_include_fixed_volumes=0",
19104
- "auto_include_removable_volumes=0",
19105
- "auto_include_fixed_refs_volumes=0",
19106
- "auto_include_removable_refs_volumes=0",
19107
- "ntfs_volume_paths=",
19108
- "ntfs_volume_includes=",
19109
- "refs_volume_paths=",
19110
- "refs_volume_includes=",
19111
- "index_size=1",
19112
- "index_date_modified=1",
19113
- `folders=${list(input.folders)}`,
19114
- `folder_monitor_changes=${perFolder("1")}`,
19115
- `folder_rescan_if_full_list=${perFolder("1")}`,
19116
- `folder_update_types=${perFolder("0")}`,
19117
- "exclude_list_enabled=1",
19118
- `exclude_folders=${list(input.excludeFolders)}`,
19119
- ""
19120
- ].join(`\r
19121
- `);
19227
+ function buildWindowsDiskScript() {
19228
+ return `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8
19229
+ ` + "Get-CimInstance -ClassName Win32_LogicalDisk | ForEach-Object {" + `$_.DeviceID + [char]9 + $_.DriveType + [char]9 + $_.FileSystem + [char]9 + $_.VolumeName}
19230
+ `;
19122
19231
  }
19123
- function buildEverythingSearchArgs(instance, request, timeoutMs) {
19124
- const args = [
19125
- "-argv",
19126
- "-instance",
19127
- instance,
19128
- "-cp",
19129
- "65001",
19130
- "-json",
19131
- "-full-path-and-name",
19132
- "-size",
19133
- "-date-modified",
19134
- "-date-format",
19135
- "1"
19136
- ];
19137
- if (timeoutMs !== undefined)
19138
- args.push("-timeout", String(timeoutMs));
19139
- if (request.matchCase)
19140
- args.push("-case");
19141
- if (request.matchPath)
19142
- args.push("-match-path");
19143
- if (request.wholeWord)
19144
- args.push("-whole-word");
19145
- if (request.kind === "files")
19146
- args.push("/a-d");
19147
- if (request.kind === "folders")
19148
- args.push("/ad");
19149
- if (request.path !== undefined)
19150
- args.push("-path", request.path);
19151
- if (request.sort !== undefined)
19152
- args.push("-sort", request.sort);
19153
- if (request.offset !== undefined && request.offset > 0)
19154
- args.push("-offset", String(request.offset));
19155
- args.push("-n", String(request.maxResults ?? DEFAULT_MAX_RESULTS));
19156
- if (request.regex)
19157
- args.push("-regex", "--", request.query);
19158
- else
19159
- args.push("--", request.query);
19160
- return args;
19232
+ function parseWindowsLogicalDisks(stdout) {
19233
+ const disks = [];
19234
+ for (const line of stdout.split(/\r?\n/)) {
19235
+ if (line.trim() === "")
19236
+ continue;
19237
+ const [deviceId, driveType, fileSystem, ...rest] = line.split("\t");
19238
+ const type = Number.parseInt((driveType ?? "").trim(), 10);
19239
+ if (!Number.isFinite(type))
19240
+ continue;
19241
+ disks.push({
19242
+ deviceId: (deviceId ?? "").trim(),
19243
+ driveType: type,
19244
+ fileSystem: (fileSystem ?? "").trim(),
19245
+ volumeName: rest.join("\t").trim()
19246
+ });
19247
+ }
19248
+ return disks;
19161
19249
  }
19162
- function parseEverythingJson(stdout) {
19163
- const text = stdout.trim();
19164
- if (text.length === 0)
19165
- return [];
19166
- let parsed;
19167
- try {
19168
- parsed = JSON.parse(text);
19169
- } catch {
19170
- throw new Error(`ES did not return JSON: ${text.slice(0, 2000)}`);
19250
+ function windowsRootsFromDisks(disks) {
19251
+ const roots = [];
19252
+ for (const disk of disks) {
19253
+ if (disk.deviceId === "")
19254
+ continue;
19255
+ const path = `${disk.deviceId}\\`;
19256
+ const fileSystem = disk.fileSystem.toLowerCase();
19257
+ if (disk.driveType === WINDOWS_DRIVE_TYPE_NETWORK) {
19258
+ roots.push({ path, kind: "network" });
19259
+ continue;
19260
+ }
19261
+ if (WINDOWS_FUSE_FILE_SYSTEM_PREFIXES.some((prefix) => fileSystem.startsWith(prefix))) {
19262
+ roots.push({ path, kind: "network" });
19263
+ continue;
19264
+ }
19265
+ if (fileSystem.startsWith("googledrivefs") || disk.volumeName.toLowerCase().includes("google drive")) {
19266
+ roots.push({ path, kind: "cloud-files" });
19267
+ }
19171
19268
  }
19172
- if (!Array.isArray(parsed))
19173
- throw new Error(`ES did not return a JSON array: ${text.slice(0, 2000)}`);
19174
- return parsed.map((item) => {
19175
- const record = item;
19176
- const filename = String(record.filename ?? "");
19177
- const isFolder = filename.endsWith("\\") && filename.length > 3;
19269
+ return roots;
19270
+ }
19271
+ function uncShareRoot(target) {
19272
+ const normalized = target.replaceAll("/", "\\");
19273
+ if (!normalized.startsWith("\\\\") || normalized.startsWith("\\\\?\\") || normalized.startsWith("\\\\.\\"))
19274
+ return;
19275
+ const rest = normalized.slice(2);
19276
+ const separator = rest.indexOf("\\");
19277
+ if (separator <= 0)
19278
+ return;
19279
+ const server = rest.slice(0, separator);
19280
+ const share = rest.slice(separator + 1).split("\\")[0] ?? "";
19281
+ if (server === "" || share === "")
19282
+ return;
19283
+ return `\\\\${server}\\${share}`;
19284
+ }
19285
+ function parseWindowsDehydrateOutput(result) {
19286
+ const open = /^OPEN=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
19287
+ if (open === null)
19288
+ return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
19289
+ if (Number.parseInt(open[1], 16) !== 0) {
19178
19290
  return {
19179
- path: isFolder ? filename.slice(0, -1) : filename,
19180
- type: filename.endsWith("\\") ? "folder" : "file",
19181
- size: typeof record.size === "number" ? record.size : undefined,
19182
- dateModified: typeof record.date_modified === "string" || typeof record.date_modified === "number" ? String(record.date_modified) : undefined
19291
+ ok: false,
19292
+ message: windowsFailureMessage(result, `could not open the file for release (0x${open[1]})`)
19183
19293
  };
19184
- });
19294
+ }
19295
+ const dehydrate = /^DEHYDRATE=(?:0x)?([0-9A-Fa-f]{8})$/mu.exec(result.stdout);
19296
+ if (dehydrate === null) {
19297
+ return { ok: false, message: windowsFailureMessage(result, "the release command produced no result") };
19298
+ }
19299
+ const hresult = dehydrate[1];
19300
+ if (Number.parseInt(hresult, 16) === 0)
19301
+ return { ok: true };
19302
+ const refusal = WINDOWS_DEHYDRATE_REFUSALS[hresult.toUpperCase()];
19303
+ return {
19304
+ ok: false,
19305
+ message: refusal ?? windowsFailureMessage(result, `the provider refused to dehydrate the file (0x${hresult})`)
19306
+ };
19307
+ }
19308
+ function windowsFailureMessage(result, fallback) {
19309
+ const printed = /^MSG=(.*)$/mu.exec(result.stdout)?.[1]?.trim();
19310
+ if (printed !== undefined && printed !== "")
19311
+ return printed;
19312
+ const stderr = result.stderr.trim();
19313
+ return stderr !== "" ? stderr : fallback;
19185
19314
  }
19186
19315
 
19187
- class EverythingClient {
19188
- instanceName;
19189
- options;
19190
- platform;
19316
+ class WindowsCloudProbe {
19191
19317
  run;
19192
- launch;
19193
- stateDir;
19194
- isExcludedPath;
19195
- binaries;
19196
- queue = new SharedExclusiveQueue;
19197
- constructor(options) {
19198
- this.options = options;
19199
- this.platform = options.platform ?? process.platform;
19200
- this.instanceName = everythingInstanceName(options.root);
19201
- this.run = options.run ?? runProcess;
19202
- this.launch = options.launch ?? launchDetached;
19203
- this.stateDir = join30(options.root, ".autorag", "everything");
19204
- this.isExcludedPath = createPathExclusionMatcher(options.excludeFolders ?? [], "win32");
19205
- }
19206
- isSupported() {
19207
- return this.platform === "win32";
19318
+ syncRoots;
19319
+ disks;
19320
+ constructor(options = {}) {
19321
+ this.run = options.run ?? runPowerShell;
19208
19322
  }
19209
- search(request) {
19210
- return this.queue.shared(async () => {
19211
- const binaries = await this.resolveBinaries("cached");
19212
- if (!binaries.ok)
19213
- return binaries;
19214
- const ping = await this.ping(binaries.esPath);
19215
- if (ping.code !== 0) {
19216
- return {
19217
- ok: false,
19218
- reason: "not-running",
19219
- message: `Everything instance ${this.instanceName} is not running; run \`autorag refresh\` to start and index it.`
19220
- };
19221
- }
19222
- const es = { ok: true, esPath: binaries.esPath };
19223
- const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
19224
- if ((this.options.excludeFolders?.length ?? 0) > 0) {
19225
- return this.searchWithExclusions(es.esPath, request, timeoutMs);
19226
- }
19227
- const args = buildEverythingSearchArgs(this.instanceName, request, timeoutMs);
19228
- const result = await this.run(es.esPath, args, timeoutMs + 5000);
19229
- if (result.code !== 0)
19230
- return this.failure("search-failed", es.esPath, args, result);
19231
- try {
19232
- return { ok: true, results: parseEverythingJson(result.stdout) };
19233
- } catch (error) {
19234
- return { ok: false, reason: "search-failed", message: error.message };
19235
- }
19236
- });
19323
+ async cloudRoots(searchRoots = []) {
19324
+ const roots = [];
19325
+ const seen = new Set;
19326
+ const add = (path, kind) => {
19327
+ const canonical = canonicalPath(path);
19328
+ const key = normalizeCloudPath(canonical).toLowerCase();
19329
+ if (key === "" || seen.has(key))
19330
+ return;
19331
+ seen.add(key);
19332
+ roots.push({ path: canonical, kind });
19333
+ };
19334
+ for (const root of await this.loadSyncRoots())
19335
+ add(root.path, "cloud-files");
19336
+ for (const root of await this.loadDiskRoots())
19337
+ add(root.path, root.kind);
19338
+ for (const searchRoot of searchRoots) {
19339
+ const share = uncShareRoot(searchRoot);
19340
+ if (share !== undefined)
19341
+ add(share, "network");
19342
+ }
19343
+ return roots;
19237
19344
  }
19238
- async searchWithExclusions(esPath, request, timeoutMs) {
19239
- const deadline = Date.now() + timeoutMs;
19240
- const offset = request.offset ?? 0;
19241
- const maxResults = request.maxResults ?? DEFAULT_MAX_RESULTS;
19242
- const results = [];
19243
- let skipped = 0;
19244
- let nativeOffset = 0;
19245
- const timedOut = () => ({
19246
- ok: false,
19247
- reason: "search-failed",
19248
- message: `Everything search exceeded ${timeoutMs}ms while filtering excluded paths; narrow the query or increase the Everything timeout.`
19249
- });
19250
- for (;; ) {
19251
- const remainingMs = deadline - Date.now();
19252
- if (remainingMs <= 0)
19253
- return timedOut();
19254
- const args = buildEverythingSearchArgs(this.instanceName, { ...request, offset: nativeOffset, maxResults: MAX_FILTER_CANDIDATES }, remainingMs);
19255
- const response = await this.run(esPath, args, remainingMs + 5000);
19256
- if (response.code !== 0)
19257
- return this.failure("search-failed", esPath, args, response);
19258
- if (Date.now() >= deadline)
19259
- return timedOut();
19260
- let entries;
19261
- try {
19262
- entries = parseEverythingJson(response.stdout);
19263
- } catch (error) {
19264
- return { ok: false, reason: "search-failed", message: error.message };
19345
+ async datalessAmong(paths) {
19346
+ const dataless = new Set;
19347
+ for (let start = 0;start < paths.length; start += WINDOWS_ATTRIBUTE_BATCH) {
19348
+ const batch = paths.slice(start, start + WINDOWS_ATTRIBUTE_BATCH);
19349
+ const result = await this.run(buildWindowsAttributeScript(batch), 30000);
19350
+ if (!result.ok) {
19351
+ throw new Error(`Could not read Windows cloud file attributes: ${result.stderr.trim() || "powershell.exe failed"}`);
19265
19352
  }
19266
- for (const entry of entries) {
19267
- if (this.isExcludedPath(entry.path))
19268
- continue;
19269
- if (skipped < offset) {
19270
- skipped += 1;
19271
- continue;
19272
- }
19273
- results.push(entry);
19274
- if (results.length >= maxResults)
19275
- return { ok: true, results };
19353
+ for (const [path, attributes] of parseWindowsAttributeOutput(result.stdout, batch)) {
19354
+ if (datalessFromWindowsAttributes(attributes))
19355
+ dataless.add(path);
19276
19356
  }
19277
- if (entries.length < MAX_FILTER_CANDIDATES)
19278
- return { ok: true, results };
19279
- nativeOffset += entries.length;
19280
19357
  }
19358
+ return dataless;
19281
19359
  }
19282
- index() {
19283
- return this.queue.exclusive(async () => {
19284
- const es = await this.ensureRunning(true);
19285
- if (!es.ok)
19286
- return es;
19287
- const timeoutMs = this.options.indexTimeoutMs ?? DEFAULT_INDEX_TIMEOUT_MS3;
19288
- const saveArgs = ["-instance", this.instanceName, "-save-db"];
19289
- const saved = await this.run(es.esPath, saveArgs, timeoutMs);
19290
- if (saved.code !== 0)
19291
- return this.failure("index-failed", es.esPath, saveArgs, saved);
19292
- const countArgs = ["-instance", this.instanceName, "-get-result-count", "-timeout", String(timeoutMs), "*"];
19293
- const counted = await this.run(es.esPath, countArgs, timeoutMs + 5000);
19294
- if (counted.code !== 0)
19295
- return this.failure("index-failed", es.esPath, countArgs, counted);
19296
- return {
19297
- ok: true,
19298
- indexedFolders: [...this.options.folders],
19299
- indexedItems: Number.parseInt(counted.stdout.trim(), 10) || 0
19300
- };
19301
- });
19360
+ async providerOf(path) {
19361
+ const root = longestSyncRoot(await this.loadSyncRoots(), path);
19362
+ return root === undefined ? cloudProviderFromPath(path) : providerFromWindowsSyncRoot(root.label);
19302
19363
  }
19303
- stop() {
19304
- return this.queue.exclusive(async () => {
19305
- const binaries = await this.resolveBinaries("cached");
19306
- if (!binaries.ok)
19307
- return;
19308
- await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
19309
- });
19364
+ canRelease(provider) {
19365
+ return provider !== undefined;
19310
19366
  }
19311
- async resolveBinaries(mode) {
19312
- if (!this.isSupported()) {
19313
- return {
19314
- ok: false,
19315
- reason: "unsupported-platform",
19316
- message: `Everything is Windows-only; this host is ${this.platform}.`
19317
- };
19318
- }
19319
- if (this.binaries?.ok)
19320
- return this.binaries;
19321
- const resolve = this.options.resolveBinaries ?? ((resolveMode) => ensureEverythingBinaries({ root: this.options.root, mode: resolveMode }));
19322
- const result = await resolve(mode);
19323
- if (result.ok)
19324
- this.binaries = result;
19325
- return result;
19367
+ async evict(path) {
19368
+ return parseWindowsDehydrateOutput(await this.run(buildWindowsDehydrateScript(path), 60000));
19326
19369
  }
19327
- async ensureRunning(reconfigure) {
19328
- const binaries = await this.resolveBinaries("install");
19329
- if (!binaries.ok)
19330
- return binaries;
19331
- const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
19332
- let ping = await this.ping(binaries.esPath);
19333
- if (ping.code === 0 && !reconfigure)
19334
- return { ok: true, esPath: binaries.esPath };
19335
- if (ping.code === 0) {
19336
- const exitArgs = ["-instance", this.instanceName, "-exit"];
19337
- const exited = await this.run(binaries.esPath, exitArgs, timeoutMs);
19338
- if (exited.code !== 0)
19339
- return this.failure("startup-failed", binaries.esPath, exitArgs, exited);
19340
- }
19341
- mkdirSync16(this.stateDir, { recursive: true });
19342
- const iniPath = join30(this.stateDir, "Everything.ini");
19343
- writeFileSync11(iniPath, buildEverythingIni({
19344
- folders: this.options.folders,
19345
- excludeFolders: [
19346
- ...new Set([
19347
- join30(this.options.root, ".autorag"),
19348
- ...this.options.folders.map((folder) => join30(folder, ".autorag")),
19349
- ...this.options.excludeFolders ?? []
19350
- ])
19351
- ]
19352
- }));
19353
- const launchArgs = [
19354
- "-instance",
19355
- this.instanceName,
19356
- "-config",
19357
- iniPath,
19358
- "-db",
19359
- join30(this.stateDir, "Everything.db"),
19360
- "-startup"
19361
- ];
19362
- try {
19363
- this.launch(binaries.everythingPath, launchArgs);
19364
- } catch (error) {
19365
- return {
19366
- ok: false,
19367
- reason: "startup-failed",
19368
- message: `${binaries.everythingPath} ${launchArgs.join(" ")} failed to start: ${error.message}`
19369
- };
19370
- }
19371
- const deadline = Date.now() + (this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS);
19372
- const pollMs = this.options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
19373
- for (;; ) {
19374
- ping = await this.ping(binaries.esPath);
19375
- if (ping.code === 0)
19376
- return { ok: true, esPath: binaries.esPath };
19377
- if (Date.now() >= deadline)
19378
- break;
19379
- await sleep2(pollMs);
19380
- }
19381
- return {
19382
- ok: false,
19383
- reason: "startup-failed",
19384
- message: `Everything instance ${this.instanceName} (${binaries.everythingPath}) did not answer ES IPC within ${this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS}ms; last ES exit ${ping.code}: ${(ping.stderr || ping.stdout).trim()}`
19385
- };
19370
+ loadSyncRoots() {
19371
+ this.syncRoots ??= this.run(buildWindowsSyncRootsScript(), 15000).then((result) => result.ok ? parseWindowsSyncRoots(result.stdout) : [], () => []);
19372
+ return this.syncRoots;
19386
19373
  }
19387
- async ping(esPath) {
19388
- const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
19389
- return result.code === 0 && !VERSION_PATTERN.test(result.stdout.trim()) ? { ...result, code: -1 } : result;
19374
+ loadDiskRoots() {
19375
+ this.disks ??= this.run(buildWindowsDiskScript(), 30000).then((result) => result.ok ? windowsRootsFromDisks(parseWindowsLogicalDisks(result.stdout)) : [], () => []);
19376
+ return this.disks;
19390
19377
  }
19391
- failure(reason, esPath, args, result) {
19378
+ }
19379
+ function longestSyncRoot(roots, target) {
19380
+ const normalized = target.replaceAll("/", "\\").toLowerCase();
19381
+ let match;
19382
+ let matchLength = -1;
19383
+ for (const root of roots) {
19384
+ const prefix = root.path.replaceAll("/", "\\").replace(/\\+$/u, "").toLowerCase();
19385
+ if (prefix === "" || prefix.length <= matchLength)
19386
+ continue;
19387
+ if (normalized === prefix || normalized.startsWith(`${prefix}\\`)) {
19388
+ match = root;
19389
+ matchLength = prefix.length;
19390
+ }
19391
+ }
19392
+ return match;
19393
+ }
19394
+ function powerShellLaunch(script) {
19395
+ return {
19396
+ args: [
19397
+ "-NoProfile",
19398
+ "-NonInteractive",
19399
+ "-Command",
19400
+ `[Console]::OutputEncoding=[System.Text.Encoding]::UTF8; Invoke-Expression $env:${POWERSHELL_SCRIPT_VARIABLE}`
19401
+ ],
19402
+ env: { [POWERSHELL_SCRIPT_VARIABLE]: script }
19403
+ };
19404
+ }
19405
+ async function runPowerShell(script, timeoutMs) {
19406
+ const launch = powerShellLaunch(script);
19407
+ try {
19408
+ const { stdout, stderr } = await execFileAsync("powershell.exe", [...launch.args], {
19409
+ timeout: timeoutMs,
19410
+ maxBuffer: 16 * 1024 * 1024,
19411
+ windowsHide: true,
19412
+ env: { ...process.env, ...launch.env }
19413
+ });
19414
+ return { ok: true, stdout, stderr };
19415
+ } catch (error) {
19416
+ const record = typeof error === "object" && error !== null ? error : {};
19392
19417
  return {
19393
19418
  ok: false,
19394
- reason,
19395
- message: `${esPath} ${args.join(" ")} exit ${result.code}: ${(result.stderr || result.stdout).trim()}`
19419
+ stdout: typeof record.stdout === "string" ? record.stdout : "",
19420
+ stderr: typeof record.stderr === "string" ? record.stderr : String(error)
19396
19421
  };
19397
19422
  }
19398
19423
  }
19399
- function runProcess(command, args, timeoutMs) {
19400
- let resolvePromise;
19401
- const promise = new Promise((resolve) => {
19402
- resolvePromise = resolve;
19403
- });
19404
- const child = spawn15(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
19405
- const stdout = [];
19406
- const stderr = [];
19407
- let bytes = 0;
19408
- let overflow = false;
19409
- const collect = (target) => (chunk) => {
19410
- bytes += chunk.length;
19411
- if (bytes > MAX_OUTPUT_BYTES) {
19412
- overflow = true;
19413
- child.kill();
19414
- return;
19415
- }
19416
- target.push(chunk);
19417
- };
19418
- child.stdout.on("data", collect(stdout));
19419
- child.stderr.on("data", collect(stderr));
19420
- const timer = setTimeout(() => child.kill(), timeoutMs);
19421
- child.on("error", (error) => {
19422
- clearTimeout(timer);
19423
- resolvePromise({ code: null, stdout: "", stderr: `${command}: ${error.message}` });
19424
- });
19425
- child.on("close", (code, signal) => {
19426
- clearTimeout(timer);
19427
- const err = Buffer.concat(stderr).toString("utf8");
19428
- resolvePromise({
19429
- code,
19430
- stdout: Buffer.concat(stdout).toString("utf8"),
19431
- stderr: overflow ? `${err}
19432
- output exceeded ${MAX_OUTPUT_BYTES} bytes` : signal !== null ? `${err}
19433
- killed by ${signal} after ${timeoutMs}ms` : err
19424
+ function defaultCloudProbe() {
19425
+ if (process.platform === "darwin")
19426
+ return new DarwinCloudProbe;
19427
+ if (process.platform === "win32")
19428
+ return new WindowsCloudProbe;
19429
+ return;
19430
+ }
19431
+ async function readFileProviderDomain(target) {
19432
+ try {
19433
+ await lstat(target);
19434
+ const { stdout } = await execFileAsync("/usr/bin/xattr", ["-p", FILE_PROVIDER_XATTR, target], {
19435
+ timeout: 2000
19434
19436
  });
19435
- });
19436
- return promise;
19437
+ const value = stdout.trim();
19438
+ return value.length > 0 ? value : undefined;
19439
+ } catch {
19440
+ return;
19441
+ }
19437
19442
  }
19438
- function launchDetached(command, args) {
19439
- const child = spawn15(command, args, { detached: true, stdio: "ignore", windowsHide: true });
19440
- child.on("error", () => {});
19441
- child.unref();
19443
+ async function readFileProviderDomainUpwards(target) {
19444
+ let cursor = target;
19445
+ for (let depth = 0;depth < 12; depth += 1) {
19446
+ const domain = await readFileProviderDomain(cursor);
19447
+ if (domain !== undefined)
19448
+ return domain;
19449
+ const parent = dirname16(cursor);
19450
+ if (parent === cursor)
19451
+ return;
19452
+ cursor = parent;
19453
+ }
19454
+ return;
19442
19455
  }
19443
- var DEFAULT_TIMEOUT_MS8 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
19444
- var init_client9 = __esm(() => {
19445
- init_path_exclusions();
19446
- init_bundle();
19447
- DEFAULT_INDEX_TIMEOUT_MS3 = 10 * 60000;
19448
- MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
19449
- VERSION_PATTERN = /^\d+\.\d+\.\d+\.\d+$/;
19450
- });
19451
-
19452
- // src/everything/index.ts
19453
- var init_everything = __esm(() => {
19454
- init_bundle();
19455
- init_client9();
19456
+ function quotePowerShell(value) {
19457
+ return `'${value.replace(/['\u2018\u2019\u201A\u201B]/gu, (quote) => quote + quote)}'`;
19458
+ }
19459
+ var execFileAsync, SF_DATALESS = 1073741824, FILE_PROVIDER_XATTR = "com.apple.file-provider-domain-id", WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 4194304, STAT_BATCH = 200, defaultCloudExec = (file, args, options) => execFileAsync(file, [...args], {
19460
+ timeout: options?.timeoutMs ?? 30000,
19461
+ ...options?.maxBuffer === undefined ? {} : { maxBuffer: options.maxBuffer }
19462
+ }), JXA_RELEASE_PROGRAM = `ObjC.import("Foundation");
19463
+ function run(argv) {
19464
+ var url = $.NSURL.fileURLWithPath(argv[0]);
19465
+ var errorRef = Ref();
19466
+ var released = $.NSFileManager.defaultManager.evictUbiquitousItemAtURLError(url, errorRef);
19467
+ if (released) return JSON.stringify({ ok: true });
19468
+ function resourceValue(key) {
19469
+ var ref = Ref();
19470
+ if (!url.getResourceValueForKeyError(ref, key, null)) return null;
19471
+ var value = ref[0];
19472
+ return value === undefined || value === null ? null : ObjC.unwrap(value);
19473
+ }
19474
+ return JSON.stringify({
19475
+ ok: false,
19476
+ isUploaded: resourceValue($.NSURLUbiquitousItemIsUploadedKey),
19477
+ isUploading: resourceValue($.NSURLUbiquitousItemIsUploadingKey),
19478
+ downloadingStatus: resourceValue($.NSURLUbiquitousItemDownloadingStatusKey)
19479
+ });
19480
+ }`, RELEASABLE_DARWIN_PROVIDERS, XATTR_BATCH = 200, DARWIN_NON_STORAGE_FILE_SYSTEMS, DARWIN_LOCAL_MOUNT_OPTION = "local", WINDOWS_ATTRIBUTE_BATCH = 100, WINDOWS_DRIVE_TYPE_NETWORK = 4, WINDOWS_FUSE_FILE_SYSTEM_PREFIXES, WINDOWS_DEHYDRATE_REFUSALS, POWERSHELL_SCRIPT_VARIABLE = "AUTORAG_POWERSHELL_SCRIPT";
19481
+ var init_cloud_placeholder = __esm(() => {
19482
+ execFileAsync = promisify(execFile2);
19483
+ RELEASABLE_DARWIN_PROVIDERS = {
19484
+ icloud: true,
19485
+ "google-drive": true,
19486
+ onedrive: true,
19487
+ dropbox: true
19488
+ };
19489
+ DARWIN_NON_STORAGE_FILE_SYSTEMS = { autofs: true };
19490
+ WINDOWS_FUSE_FILE_SYSTEM_PREFIXES = ["fuse", "winfsp", "dokan"];
19491
+ WINDOWS_DEHYDRATE_REFUSALS = {
19492
+ "80070179": "The file has changes the cloud provider has not synced yet, so it cannot be released.",
19493
+ "80070188": "The file is pinned to stay on this device, so it is not released."
19494
+ };
19456
19495
  });
19457
19496
 
19458
19497
  // src/fsearch/walk.ts
19459
19498
  import { readdir as readdir2, stat as stat3 } from "node:fs/promises";
19460
- import { basename as basename8, join as join31, sep as sep5 } from "node:path";
19499
+ import { basename as basename7, join as join31, sep as sep5 } from "node:path";
19461
19500
  function compare(a, b) {
19462
19501
  return a < b ? -1 : a > b ? 1 : 0;
19463
19502
  }
@@ -19468,11 +19507,11 @@ async function walkFileSearch(folders, request, options = {}) {
19468
19507
  const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_DEADLINE_MS);
19469
19508
  const matches = request.regex ? (() => {
19470
19509
  const pattern = new RegExp(request.query, request.matchCase ? "" : "i");
19471
- return (path) => pattern.test(request.matchPath ? path : basename8(path));
19510
+ return (path) => pattern.test(request.matchPath ? path : basename7(path));
19472
19511
  })() : (() => {
19473
19512
  const needle = request.matchCase ? request.query : request.query.toLowerCase();
19474
19513
  return (path) => {
19475
- const haystack = request.matchPath ? path : basename8(path);
19514
+ const haystack = request.matchPath ? path : basename7(path);
19476
19515
  return (request.matchCase ? haystack : haystack.toLowerCase()).includes(needle);
19477
19516
  };
19478
19517
  })();
@@ -19527,7 +19566,7 @@ async function walkFileSearch(folders, request, options = {}) {
19527
19566
  continue;
19528
19567
  collected.push({
19529
19568
  path,
19530
- name: basename8(path),
19569
+ name: basename7(path),
19531
19570
  type: info.isDirectory() ? "folder" : "file",
19532
19571
  size: info.isDirectory() ? undefined : info.size,
19533
19572
  dateModified: info.mtime.toISOString()
@@ -19552,7 +19591,7 @@ async function walkFileSearch(folders, request, options = {}) {
19552
19591
  continue;
19553
19592
  collected.push({
19554
19593
  path: folder,
19555
- name: basename8(folder),
19594
+ name: basename7(folder),
19556
19595
  type: "folder",
19557
19596
  size: undefined,
19558
19597
  dateModified: info.mtime.toISOString()
@@ -19585,7 +19624,7 @@ var init_walk = __esm(() => {
19585
19624
  import { spawn as spawn16 } from "node:child_process";
19586
19625
  import { createHash as createHash13 } from "node:crypto";
19587
19626
  import { existsSync as existsSync19, mkdirSync as mkdirSync17, readFileSync as readFileSync19, realpathSync as realpathSync4, rmSync as rmSync8, writeFileSync as writeFileSync12 } from "node:fs";
19588
- import { basename as basename9, join as join32, resolve as resolve12, sep as sep6 } from "node:path";
19627
+ import { basename as basename8, join as join32, resolve as resolve12, sep as sep6 } from "node:path";
19589
19628
  import { setTimeout as sleep3 } from "node:timers/promises";
19590
19629
  function fsearchSocketPath(dbPath) {
19591
19630
  let canonical;
@@ -19649,7 +19688,7 @@ function parseFSearchJsonLines(stdout) {
19649
19688
  const path = String(record.path ?? "");
19650
19689
  entries.push({
19651
19690
  path,
19652
- name: basename9(path),
19691
+ name: basename8(path),
19653
19692
  type: record.type === "folder" ? "folder" : "file",
19654
19693
  size: typeof record.size === "number" ? record.size : undefined,
19655
19694
  dateModified: typeof record.mtime === "number" ? new Date(record.mtime * 1000).toISOString() : undefined
@@ -20067,7 +20106,7 @@ var init_diagnostics = __esm(() => {
20067
20106
  });
20068
20107
 
20069
20108
  // src/jikji/retry-command.ts
20070
- import { basename as basename10 } from "node:path";
20109
+ import { basename as basename9 } from "node:path";
20071
20110
  function shellWords(command) {
20072
20111
  const words = [];
20073
20112
  let current = "";
@@ -20134,7 +20173,7 @@ function parseJikjiRetryCommand(command, envelopeProof) {
20134
20173
  const words = shellWords(command);
20135
20174
  if (words === undefined || words.length < 4)
20136
20175
  return;
20137
- const program = basename10(words[0]).replace(/\.exe$/iu, "");
20176
+ const program = basename9(words[0]).replace(/\.exe$/iu, "");
20138
20177
  if (program !== "jikji" || words[1] !== "discover")
20139
20178
  return;
20140
20179
  const root = words[2];
@@ -20542,7 +20581,7 @@ class SerialQueue {
20542
20581
 
20543
20582
  // src/jikji/version.ts
20544
20583
  function parseJikjiVersion(output) {
20545
- return VERSION_LINE2.exec(output)?.[1];
20584
+ return VERSION_LINE3.exec(output)?.[1];
20546
20585
  }
20547
20586
  function parts(version) {
20548
20587
  return version.split(".").map((part) => Number.parseInt(part, 10));
@@ -20558,9 +20597,9 @@ function isSupportedJikjiVersion(version) {
20558
20597
  }
20559
20598
  return true;
20560
20599
  }
20561
- var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1", VERSION_LINE2;
20562
- var init_version2 = __esm(() => {
20563
- VERSION_LINE2 = /^jikji(?:-cli)?\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
20600
+ var JIKJI_MIN_VERSION = "0.2.0", JIKJI_INSTALL_VERSION = "0.2.1", VERSION_LINE3;
20601
+ var init_version3 = __esm(() => {
20602
+ VERSION_LINE3 = /^jikji(?:-cli)?\s+(\d+\.\d+\.\d+)(?:[-+][^\s]*)?\s*$/mu;
20564
20603
  });
20565
20604
 
20566
20605
  // src/jikji/client.ts
@@ -20928,7 +20967,7 @@ var init_client11 = __esm(() => {
20928
20967
  init_answer_pack();
20929
20968
  init_installer3();
20930
20969
  init_prepare_budget();
20931
- init_version2();
20970
+ init_version3();
20932
20971
  ABORTED_BEFORE_START = { ok: false, reason: "aborted", stdout: "", stderr: "", code: null };
20933
20972
  });
20934
20973
 
@@ -22013,7 +22052,7 @@ import {
22013
22052
  rmSync as rmSync10,
22014
22053
  writeFileSync as writeFileSync14
22015
22054
  } from "node:fs";
22016
- import { basename as basename11, dirname as dirname19, isAbsolute as isAbsolute9, join as join37, relative as relative7, sep as sep8 } from "node:path";
22055
+ import { basename as basename10, dirname as dirname19, isAbsolute as isAbsolute9, join as join37, relative as relative7, sep as sep8 } from "node:path";
22017
22056
  function syncMinSyncWorkspace(root, options = {}) {
22018
22057
  const workspacePath = options.workspacePath ?? minSyncWorkspaceRoot(root);
22019
22058
  const filesRoot = join37(workspacePath, "files");
@@ -22054,7 +22093,7 @@ function syncMinSyncWorkspace(root, options = {}) {
22054
22093
  return { workspacePath, entries, excluded };
22055
22094
  }
22056
22095
  function stagingStatePath(workspacePath) {
22057
- return join37(dirname19(workspacePath), `.${basename11(workspacePath)}-autorag-staging.json`);
22096
+ return join37(dirname19(workspacePath), `.${basename10(workspacePath)}-autorag-staging.json`);
22058
22097
  }
22059
22098
  function loadStagingState2(workspacePath) {
22060
22099
  const path = stagingStatePath(workspacePath);
@@ -22105,7 +22144,7 @@ function ensureManagedDirectory(filesRoot, directory) {
22105
22144
  }
22106
22145
  }
22107
22146
  function copyFileAtomically(source, destination) {
22108
- const temporaryPath = join37(dirname19(destination), `.${basename11(destination)}.${randomUUID6()}.tmp`);
22147
+ const temporaryPath = join37(dirname19(destination), `.${basename10(destination)}.${randomUUID6()}.tmp`);
22109
22148
  try {
22110
22149
  copyFileSync4(source, temporaryPath);
22111
22150
  renameWithRetry(temporaryPath, destination);
@@ -22159,7 +22198,7 @@ var init_workspace = __esm(() => {
22159
22198
 
22160
22199
  // src/minsync/method.ts
22161
22200
  import { existsSync as existsSync24, readFileSync as readFileSync22, realpathSync as realpathSync7, statSync as statSync8 } from "node:fs";
22162
- import { basename as basename12, join as join38, normalize } from "node:path";
22201
+ import { basename as basename11, join as join38, normalize } from "node:path";
22163
22202
 
22164
22203
  class MinSyncVectorMethod {
22165
22204
  root;
@@ -22260,7 +22299,7 @@ class MinSyncVectorMethod {
22260
22299
  const entry = byPath.get(hit.path);
22261
22300
  if (!entry || !matchesVirtualPathScope(entry.virtualPath, options.scope))
22262
22301
  continue;
22263
- const chunkDiscriminator = hit.docId ?? basename12(hit.path);
22302
+ const chunkDiscriminator = hit.docId ?? basename11(hit.path);
22264
22303
  const chunkId = `minsync:${entry.virtualPath}:${chunkDiscriminator}`;
22265
22304
  const location = locateChunkInMirror(mirrorTexts, entry, hit.text);
22266
22305
  results.push({
@@ -22360,18 +22399,24 @@ var init_minsync = __esm(() => {
22360
22399
 
22361
22400
  // src/mirror/cloud-fingerprints.ts
22362
22401
  import { existsSync as existsSync25, readFileSync as readFileSync23 } from "node:fs";
22363
- import { join as join39 } from "node:path";
22402
+ import { extname as extname5, join as join39 } from "node:path";
22364
22403
  function cloudFingerprintStorePath(root) {
22365
22404
  return join39(root, ".autorag", CLOUD_FINGERPRINTS_FILE);
22366
22405
  }
22367
22406
  function emptyCloudFingerprints() {
22368
- return { version: 1, entries: {} };
22407
+ return { version: 2, entries: {} };
22369
22408
  }
22370
- function isCloudFingerprint(value) {
22409
+ function readCloudFingerprint(sourcePath, value, storeVersion) {
22371
22410
  if (typeof value !== "object" || value === null)
22372
- return false;
22411
+ return;
22373
22412
  const record = value;
22374
- return typeof record.sourceVersion === "string" && typeof record.exact === "string" && typeof record.chars === "number" && (record.modified === null || typeof record.modified === "string") && (record.revision === null || typeof record.revision === "string");
22413
+ const { sourceVersion, exact, modified, revision } = record;
22414
+ if (typeof sourceVersion !== "string" || typeof exact !== "string" || modified !== null && typeof modified !== "string" || revision !== null && typeof revision !== "string") {
22415
+ return;
22416
+ }
22417
+ if (storeVersion !== 2 && (record.chars === 0 || extname5(sourcePath).toLowerCase() === ".hwp"))
22418
+ return;
22419
+ return { sourceVersion, exact, modified, revision };
22375
22420
  }
22376
22421
  function loadCloudFingerprints(root) {
22377
22422
  const path = cloudFingerprintStorePath(root);
@@ -22384,24 +22429,26 @@ function loadCloudFingerprints(root) {
22384
22429
  const { entries } = parsed;
22385
22430
  if (typeof entries !== "object" || entries === null)
22386
22431
  return emptyCloudFingerprints();
22432
+ const storeVersion = "version" in parsed ? parsed.version : undefined;
22387
22433
  const valid = {};
22388
- for (const [sourcePath, fingerprint] of Object.entries(entries)) {
22389
- if (isCloudFingerprint(fingerprint))
22434
+ for (const [sourcePath, value] of Object.entries(entries)) {
22435
+ const fingerprint = readCloudFingerprint(sourcePath, value, storeVersion);
22436
+ if (fingerprint !== undefined)
22390
22437
  valid[sourcePath] = fingerprint;
22391
22438
  }
22392
- return { version: 1, entries: valid };
22439
+ return { version: 2, entries: valid };
22393
22440
  } catch {
22394
22441
  return emptyCloudFingerprints();
22395
22442
  }
22396
22443
  }
22397
22444
  function saveCloudFingerprints(root, entries) {
22398
- writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version: 1, entries }, null, 2)}
22445
+ writeFileAtomic(cloudFingerprintStorePath(root), `${JSON.stringify({ version: 2, entries }, null, 2)}
22399
22446
  `);
22400
22447
  }
22401
22448
  function cloudFingerprintExclusions(store) {
22402
22449
  const map = new Map;
22403
22450
  for (const [sourcePath, fingerprint] of Object.entries(store.entries)) {
22404
- map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified, chars: fingerprint.chars });
22451
+ map.set(sourcePath, { exact: fingerprint.exact, modified: fingerprint.modified });
22405
22452
  }
22406
22453
  return map;
22407
22454
  }
@@ -22793,7 +22840,7 @@ class Parser {
22793
22840
  }
22794
22841
 
22795
22842
  // src/parser/csv.ts
22796
- import { basename as basename13, extname as extname5 } from "node:path";
22843
+ import { basename as basename12, extname as extname6 } from "node:path";
22797
22844
  function toCell(text) {
22798
22845
  return text.trim() === "" ? undefined : { text, kind: classifyText(text) };
22799
22846
  }
@@ -22924,7 +22971,7 @@ var init_csv = __esm(() => {
22924
22971
  this.#maxRows = options.maxRows ?? DEFAULT_MAX_ROWS;
22925
22972
  }
22926
22973
  async parse(input) {
22927
- const extension = extname5(input.virtualPath);
22974
+ const extension = extname6(input.virtualPath);
22928
22975
  const format = extension.slice(1).toLowerCase();
22929
22976
  let text = decodeText(input.bytes).replace(/^\uFEFF/, "");
22930
22977
  const directive = SEPARATOR_DIRECTIVE.exec(text);
@@ -22936,10 +22983,10 @@ var init_csv = __esm(() => {
22936
22983
  {
22937
22984
  code: "parser-warning",
22938
22985
  severity: "warning",
22939
- message: `"${basename13(input.virtualPath)}": indexed the first ${this.#maxRows} of ${total} rows`
22986
+ message: `"${basename12(input.virtualPath)}": indexed the first ${this.#maxRows} of ${total} rows`
22940
22987
  }
22941
22988
  ] : [];
22942
- const grid = { name: basename13(input.virtualPath, extension), rows: rows.map((row) => row.map(toCell)) };
22989
+ const grid = { name: basename12(input.virtualPath, extension), rows: rows.map((row) => row.map(toCell)) };
22943
22990
  return {
22944
22991
  markdown: normalizeMarkdown(renderWorkbook([grid])),
22945
22992
  metadata: { parser: this.name, format },
@@ -23478,7 +23525,7 @@ var init_plain_text = __esm(() => {
23478
23525
  });
23479
23526
 
23480
23527
  // src/parser/registry.ts
23481
- import { extname as extname6 } from "node:path";
23528
+ import { extname as extname7 } from "node:path";
23482
23529
 
23483
23530
  class ParserRegistry {
23484
23531
  parsersByExtension = new Map;
@@ -23497,7 +23544,7 @@ class ParserRegistry {
23497
23544
  }
23498
23545
  }
23499
23546
  getForVirtualPath(virtualPath) {
23500
- return this.parsersByExtension.get(normalizeExtension(extname6(virtualPath)));
23547
+ return this.parsersByExtension.get(normalizeExtension(extname7(virtualPath)));
23501
23548
  }
23502
23549
  list() {
23503
23550
  return Array.from(new Set(this.parsersByExtension.values()));
@@ -23510,7 +23557,7 @@ function normalizeExtension(extension) {
23510
23557
  var init_registry = () => {};
23511
23558
 
23512
23559
  // src/parser/spreadsheet.ts
23513
- import { extname as extname7 } from "node:path";
23560
+ import { extname as extname8 } from "node:path";
23514
23561
  import * as XLSX from "xlsx";
23515
23562
  function assertWorkbookSignature(format, bytes) {
23516
23563
  if (startsWith(bytes, ZIP_SIGNATURE) || startsWith(bytes, OLE_SIGNATURE))
@@ -23609,7 +23656,7 @@ var init_spreadsheet = __esm(() => {
23609
23656
  this.#maxRowsPerSheet = options.maxRowsPerSheet ?? DEFAULT_MAX_ROWS_PER_SHEET;
23610
23657
  }
23611
23658
  async parse(input) {
23612
- const format = extname7(input.virtualPath).slice(1).toLowerCase();
23659
+ const format = extname8(input.virtualPath).slice(1).toLowerCase();
23613
23660
  let workbook;
23614
23661
  try {
23615
23662
  assertWorkbookSignature(format, input.bytes);
@@ -23883,6 +23930,10 @@ class CloudGate {
23883
23930
  isPlaceholder(path) {
23884
23931
  return this.placeholders.has(path);
23885
23932
  }
23933
+ startBatch() {
23934
+ this.downloadedFiles = 0;
23935
+ this.downloadedBytes = 0;
23936
+ }
23886
23937
  async decide(path, sizeBytes, directory) {
23887
23938
  if (this.probe === undefined || !this.placeholders.has(path))
23888
23939
  return LOCAL;
@@ -23892,7 +23943,7 @@ class CloudGate {
23892
23943
  const overFiles = this.downloadedFiles >= this.maxFiles;
23893
23944
  const overBytes = this.downloadedFiles > 0 && this.downloadedBytes + sizeBytes > this.maxBytes;
23894
23945
  if (overFiles || overBytes)
23895
- return { kind: "defer" };
23946
+ return { kind: "next-batch" };
23896
23947
  this.downloadedFiles += 1;
23897
23948
  this.downloadedBytes += sizeBytes;
23898
23949
  return { kind: "download" };
@@ -23937,7 +23988,7 @@ var init_cloud = __esm(() => {
23937
23988
  // src/mirror/sync.ts
23938
23989
  import { createHash as createHash17 } from "node:crypto";
23939
23990
  import { opendir as opendir2, readFile, stat as stat4 } from "node:fs/promises";
23940
- import { basename as basename14, dirname as dirname22, extname as extname8, join as join43, resolve as resolve15, sep as sep9 } from "node:path";
23991
+ import { basename as basename13, dirname as dirname22, extname as extname9, join as join43, resolve as resolve15, sep as sep9 } from "node:path";
23941
23992
  import { setTimeout as sleep4 } from "node:timers/promises";
23942
23993
  function errorCode(error) {
23943
23994
  if (!(error instanceof Error))
@@ -24141,195 +24192,221 @@ async function syncWithRegistry(options, registry, isolated) {
24141
24192
  });
24142
24193
  const pending = new Set(plan.decisions.filter((decision) => decision.action === "process" || decision.action === "retry").map((decision) => decision.key));
24143
24194
  const deferred = new Map(plan.decisions.filter((decision) => decision.deferred === true).map((decision) => [decision.key, decision]));
24144
- for (const decision of plan.decisions) {
24145
- if (decision.action !== "delete")
24146
- continue;
24147
- const path = pathsByKey.get(decision.key);
24148
- await session.delete(decision.key);
24149
- if (path === undefined)
24150
- continue;
24151
- if (previous.entries[path])
24152
- deleted += 1;
24153
- if (!currentPaths.has(path))
24154
- diagnostics.push({
24155
- code: "deleted-mirror",
24156
- severity: "info",
24157
- message: "A previously indexed document is gone; its parsed mirror was removed.",
24158
- source: path
24159
- });
24160
- }
24161
- await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
24162
- for (const [position, entry] of current.entries()) {
24163
- options.onProgress?.({
24164
- processed: position,
24165
- total: current.length,
24166
- written,
24167
- deleted,
24168
- skipped,
24169
- currentVirtualPath: entry.virtualPath
24170
- });
24171
- if (options.signal?.aborted)
24172
- throw abortReason(options.signal);
24173
- const omission = omitted.get(entry.virtualPath);
24174
- if (omission !== undefined) {
24175
- if (pending.has(mirrorStageKey(entry.virtualPath))) {
24176
- if (previous.entries[entry.virtualPath])
24177
- deleted += 1;
24178
- await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
24179
- }
24180
- skipped += 1;
24181
- if (omission.diagnostic)
24182
- diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
24183
- continue;
24184
- }
24185
- const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
24186
- if (deferredFailure !== undefined) {
24187
- skipped += 1;
24188
- const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
24189
- diagnostics.push({
24190
- code,
24191
- severity: "warning",
24192
- source: entry.virtualPath,
24193
- message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
24194
- });
24195
- continue;
24196
- }
24197
- const parser = registry.getForVirtualPath(entry.virtualPath);
24198
- const outputPath = parsedOutputPath(options.root, entry.virtualPath);
24199
- const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname22(entry.sourcePath)) : { kind: "local" };
24200
- if (cloudDecision.kind === "skip") {
24201
- skipped += 1;
24202
- diagnostics.push({
24203
- code: "cloud-placeholder-skipped",
24204
- severity: "info",
24205
- message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
24206
- source: entry.virtualPath,
24207
- ...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
24208
- });
24209
- continue;
24210
- }
24211
- if (cloudDecision.kind === "defer") {
24212
- skipped += 1;
24213
- diagnostics.push({
24214
- code: "cloud-download-deferred",
24215
- severity: "info",
24216
- message: "The download budget for this refresh is used up; the file stays queued for the next refresh.",
24217
- source: entry.virtualPath
24218
- });
24219
- continue;
24220
- }
24221
- if (pending.has(mirrorStageKey(entry.virtualPath))) {
24222
- let bytes;
24223
- try {
24224
- bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
24225
- } catch (error) {
24226
- const failure = sourceReadFailure(error);
24227
- await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
24228
- continue;
24229
- }
24230
- const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
24231
- if (tooSmallImage !== undefined) {
24232
- await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
24195
+ const applyDeleteDecisions = async () => {
24196
+ for (const decision of plan.decisions) {
24197
+ if (decision.action !== "delete")
24233
24198
  continue;
24234
- }
24235
- let parsed;
24236
- try {
24237
- const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
24238
- parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
24239
- } catch (error) {
24240
- if (options.signal?.aborted)
24241
- throw abortReason(options.signal);
24242
- if (error instanceof ParseError)
24243
- await skipFile(entry, {
24244
- code: "parser-failed",
24245
- severity: "warning",
24246
- message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
24247
- }, "parser-failed");
24248
- else
24249
- await skipFile(entry, {
24250
- code: "parser-crashed",
24251
- severity: "warning",
24252
- message: PARSER_CRASH_MESSAGE,
24253
- ...error instanceof Error ? { reason: error.name } : {}
24254
- });
24199
+ const path = pathsByKey.get(decision.key);
24200
+ await session.delete(decision.key);
24201
+ if (path === undefined)
24255
24202
  continue;
24256
- }
24257
- let info;
24258
- try {
24259
- info = await stat4(entry.sourcePath, { bigint: true });
24260
- } catch (error) {
24261
- await skipFile(entry, sourceReadFailure(error));
24203
+ if (previous.entries[path])
24204
+ deleted += 1;
24205
+ if (!currentPaths.has(path))
24206
+ diagnostics.push({
24207
+ code: "deleted-mirror",
24208
+ severity: "info",
24209
+ message: "A previously indexed document is gone; its parsed mirror was removed.",
24210
+ source: path
24211
+ });
24212
+ }
24213
+ };
24214
+ const deferDeletions = options.onBatch !== undefined;
24215
+ const batchSize = Math.max(1, Math.floor(options.batchSize ?? DEFAULT_MIRROR_BATCH_SIZE));
24216
+ let batchIndex = 0;
24217
+ let batchStopped = false;
24218
+ const runBatch = async (virtualPaths, final) => {
24219
+ if (options.onBatch === undefined)
24220
+ return;
24221
+ batchIndex += 1;
24222
+ const outcome = await options.onBatch({ index: batchIndex, virtualPaths, final });
24223
+ if (outcome?.stop === true)
24224
+ batchStopped = true;
24225
+ };
24226
+ if (!deferDeletions)
24227
+ await applyDeleteDecisions();
24228
+ await gate.classify(current.filter((entry) => pending.has(mirrorStageKey(entry.virtualPath))).map((entry) => entry.sourcePath));
24229
+ let batchStart = 0;
24230
+ for (;; ) {
24231
+ const batchLimit = Math.min(batchStart + batchSize, current.length);
24232
+ let batchEnd = batchLimit;
24233
+ const changedBefore = written + deleted;
24234
+ gate.startBatch();
24235
+ for (let position = batchStart;position < batchLimit; position += 1) {
24236
+ const entry = current[position];
24237
+ options.onProgress?.({
24238
+ processed: position,
24239
+ total: current.length,
24240
+ written,
24241
+ deleted,
24242
+ skipped,
24243
+ currentVirtualPath: entry.virtualPath
24244
+ });
24245
+ if (options.signal?.aborted)
24246
+ throw abortReason(options.signal);
24247
+ const omission = omitted.get(entry.virtualPath);
24248
+ if (omission !== undefined) {
24249
+ if (pending.has(mirrorStageKey(entry.virtualPath))) {
24250
+ if (previous.entries[entry.virtualPath])
24251
+ deleted += 1;
24252
+ await session.skip(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), omission.reason);
24253
+ }
24254
+ skipped += 1;
24255
+ if (omission.diagnostic)
24256
+ diagnostics.push({ ...omission.diagnostic, source: entry.virtualPath });
24262
24257
  continue;
24263
24258
  }
24264
- const unchanged = cloudDecision.kind === "download" ? contentIdentity(info) === entry.contentIdentity : statIdentity(info) === entry.statIdentity;
24265
- if (!unchanged) {
24259
+ const deferredFailure = deferred.get(mirrorStageKey(entry.virtualPath));
24260
+ if (deferredFailure !== undefined) {
24266
24261
  skipped += 1;
24262
+ const code = deferredFailure.error === "source-unreadable" ? "source-unreadable" : "parser-failed";
24267
24263
  diagnostics.push({
24268
- code: "source-changed",
24264
+ code,
24269
24265
  severity: "warning",
24270
- message: "The source changed while parsing; its last good mirror was kept and it will be retried on the next refresh.",
24271
- source: entry.virtualPath
24266
+ source: entry.virtualPath,
24267
+ message: code === "parser-failed" ? PARSER_FAILURE_MESSAGE : "The source could not be read; retry is deferred until the source or settings change, or an explicit retry is requested."
24272
24268
  });
24273
- await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
24274
24269
  continue;
24275
24270
  }
24276
- const markdown = normalizeMarkdown(parsed.markdown);
24277
- try {
24278
- await withTransientRetry(() => writeFileAtomic(outputPath, markdown), transientRetries, transientBackoffMs);
24279
- } catch (error) {
24280
- const code = errorCode(error);
24281
- if (code !== undefined && Object.hasOwn(WORKSPACE_ERROR_CODES, code))
24282
- throw new MirrorWorkspaceError(options.root, error);
24283
- await skipFile(entry, {
24284
- code: "mirror-write-failed",
24285
- severity: "warning",
24286
- message: error instanceof Error ? error.message : String(error),
24287
- ...code !== undefined ? { reason: code } : {}
24271
+ const parser = registry.getForVirtualPath(entry.virtualPath);
24272
+ const outputPath = parsedOutputPath(options.root, entry.virtualPath);
24273
+ const cloudDecision = pending.has(mirrorStageKey(entry.virtualPath)) ? await gate.decide(entry.sourcePath, entry.sizeBytes, dirname22(entry.sourcePath)) : { kind: "local" };
24274
+ if (cloudDecision.kind === "skip") {
24275
+ skipped += 1;
24276
+ diagnostics.push({
24277
+ code: "cloud-placeholder-skipped",
24278
+ severity: "info",
24279
+ message: `The file is not downloaded and its provider has no verified release path, so it was not read; its last indexed copy was kept. Make it available offline to index it.`,
24280
+ source: entry.virtualPath,
24281
+ ...cloudDecision.provider === undefined ? {} : { reason: cloudDecision.provider }
24288
24282
  });
24289
24283
  continue;
24290
24284
  }
24291
- await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
24292
- path: basename14(outputPath),
24293
- size: Buffer.byteLength(markdown)
24294
- });
24295
- for (const diagnostic of parsed.diagnostics ?? [])
24296
- diagnostics.push({ ...diagnostic, source: entry.virtualPath });
24297
- written += 1;
24298
- if (fingerprintCloud && cloudFilePaths.has(entry.sourcePath)) {
24299
- const existing = cloudFingerprintEntries[entry.sourcePath];
24300
- if (existing === undefined || existing.sourceVersion !== entry.sourceVersion) {
24301
- try {
24302
- const fingerprintResult = await fingerprintWithDupey(entry.sourcePath, options.cloudFingerprints?.dupey ?? {});
24303
- cloudFingerprintEntries[entry.sourcePath] = {
24304
- sourceVersion: entry.sourceVersion,
24305
- exact: fingerprintResult.exact,
24306
- chars: fingerprintResult.chars,
24307
- modified: fingerprintResult.modified,
24308
- revision: fingerprintResult.revision
24309
- };
24310
- cloudFingerprintsDirty = true;
24311
- } catch (error) {
24312
- diagnostics.push({
24313
- code: "cloud-fingerprint-failed",
24285
+ if (cloudDecision.kind === "next-batch") {
24286
+ batchEnd = position;
24287
+ break;
24288
+ }
24289
+ if (pending.has(mirrorStageKey(entry.virtualPath))) {
24290
+ let bytes;
24291
+ try {
24292
+ bytes = await withTransientRetry(() => readFile(entry.sourcePath), transientRetries, transientBackoffMs);
24293
+ } catch (error) {
24294
+ const failure = sourceReadFailure(error);
24295
+ await skipFile(entry, failure, failure.code === "source-unreadable" ? "source-unreadable" : undefined);
24296
+ continue;
24297
+ }
24298
+ const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
24299
+ if (tooSmallImage !== undefined) {
24300
+ await skipFile(entry, { code: "parser-skipped", severity: "warning", message: tooSmallImage }, "parser-skipped");
24301
+ continue;
24302
+ }
24303
+ let parsed;
24304
+ try {
24305
+ const input = { virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes };
24306
+ parsed = isolated !== undefined && ISOLATED_PARSER_NAMES.has(parser.name) ? await isolated.parse(parser.name, input, options.signal) : await parser.parse(input);
24307
+ } catch (error) {
24308
+ if (options.signal?.aborted)
24309
+ throw abortReason(options.signal);
24310
+ if (error instanceof ParseError)
24311
+ await skipFile(entry, {
24312
+ code: "parser-failed",
24314
24313
  severity: "warning",
24315
- message: `The cloud or network file was indexed but its duplicate fingerprint could not be read, so it will not take part in exact-duplicate exclusion: ${error instanceof Error ? error.message : String(error)}`,
24316
- source: entry.virtualPath
24314
+ message: error.cause instanceof ParseIsolationError ? `${PARSER_FAILURE_MESSAGE} ${error.cause.message}.` : PARSER_FAILURE_MESSAGE
24315
+ }, "parser-failed");
24316
+ else
24317
+ await skipFile(entry, {
24318
+ code: "parser-crashed",
24319
+ severity: "warning",
24320
+ message: PARSER_CRASH_MESSAGE,
24321
+ ...error instanceof Error ? { reason: error.name } : {}
24317
24322
  });
24318
- }
24323
+ continue;
24319
24324
  }
24320
- }
24321
- if (cloudDecision.kind === "download" && gate.evictAfterParse) {
24322
- const released = await gate.release(entry.sourcePath);
24323
- if (!released.ok) {
24325
+ let info;
24326
+ try {
24327
+ info = await stat4(entry.sourcePath, { bigint: true });
24328
+ } catch (error) {
24329
+ await skipFile(entry, sourceReadFailure(error));
24330
+ continue;
24331
+ }
24332
+ const unchanged = cloudDecision.kind === "download" ? contentIdentity(info) === entry.contentIdentity : statIdentity(info) === entry.statIdentity;
24333
+ if (!unchanged) {
24334
+ skipped += 1;
24324
24335
  diagnostics.push({
24325
- code: "cloud-evict-failed",
24336
+ code: "source-changed",
24326
24337
  severity: "warning",
24327
- message: `The file was indexed but its local copy could not be released: ${released.message}`,
24338
+ message: "The source changed while parsing; its last good mirror was kept and it will be retried on the next refresh.",
24328
24339
  source: entry.virtualPath
24329
24340
  });
24341
+ await session.fail(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), "source-changed");
24342
+ continue;
24343
+ }
24344
+ const markdown = normalizeMarkdown(parsed.markdown);
24345
+ try {
24346
+ await withTransientRetry(() => writeFileAtomic(outputPath, markdown), transientRetries, transientBackoffMs);
24347
+ } catch (error) {
24348
+ const code = errorCode(error);
24349
+ if (code !== undefined && Object.hasOwn(WORKSPACE_ERROR_CODES, code))
24350
+ throw new MirrorWorkspaceError(options.root, error);
24351
+ await skipFile(entry, {
24352
+ code: "mirror-write-failed",
24353
+ severity: "warning",
24354
+ message: error instanceof Error ? error.message : String(error),
24355
+ ...code !== undefined ? { reason: code } : {}
24356
+ });
24357
+ continue;
24358
+ }
24359
+ await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
24360
+ path: basename13(outputPath),
24361
+ size: Buffer.byteLength(markdown)
24362
+ });
24363
+ for (const diagnostic of parsed.diagnostics ?? [])
24364
+ diagnostics.push({ ...diagnostic, source: entry.virtualPath });
24365
+ written += 1;
24366
+ if (fingerprintCloud && cloudFilePaths.has(entry.sourcePath)) {
24367
+ const existing = cloudFingerprintEntries[entry.sourcePath];
24368
+ if (existing === undefined || existing.sourceVersion !== entry.sourceVersion) {
24369
+ try {
24370
+ const fingerprintResult = await fingerprintWithDupey(entry.sourcePath, options.cloudFingerprints?.dupey ?? {});
24371
+ cloudFingerprintEntries[entry.sourcePath] = {
24372
+ sourceVersion: entry.sourceVersion,
24373
+ exact: fingerprintResult.exact,
24374
+ modified: fingerprintResult.modified,
24375
+ revision: fingerprintResult.revision
24376
+ };
24377
+ cloudFingerprintsDirty = true;
24378
+ } catch (error) {
24379
+ diagnostics.push({
24380
+ code: "cloud-fingerprint-failed",
24381
+ severity: "warning",
24382
+ message: `The cloud or network file was indexed but its duplicate fingerprint could not be read, so it will not take part in exact-duplicate exclusion: ${error instanceof Error ? error.message : String(error)}`,
24383
+ source: entry.virtualPath
24384
+ });
24385
+ }
24386
+ }
24387
+ }
24388
+ if (cloudDecision.kind === "download" && gate.evictAfterParse) {
24389
+ const released = await gate.release(entry.sourcePath);
24390
+ if (!released.ok) {
24391
+ diagnostics.push({
24392
+ code: "cloud-evict-failed",
24393
+ severity: "warning",
24394
+ message: `The file was indexed but its local copy could not be released: ${released.message}`,
24395
+ source: entry.virtualPath
24396
+ });
24397
+ }
24330
24398
  }
24331
24399
  }
24332
24400
  }
24401
+ const finalBatch = batchEnd >= current.length;
24402
+ if (finalBatch && deferDeletions)
24403
+ await applyDeleteDecisions();
24404
+ if (finalBatch || written + deleted !== changedBefore) {
24405
+ await runBatch(current.slice(batchStart, batchEnd).map((item) => item.virtualPath), finalBatch);
24406
+ }
24407
+ if (finalBatch || batchStopped)
24408
+ break;
24409
+ batchStart = batchEnd;
24333
24410
  }
24334
24411
  await session.finish();
24335
24412
  if (fingerprintCloud) {
@@ -24419,7 +24496,7 @@ async function detectMirrorStaleness(options) {
24419
24496
  }
24420
24497
  }
24421
24498
  function standaloneImageTooSmallMessage(virtualPath, bytes) {
24422
- if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname8(virtualPath))))
24499
+ if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname9(virtualPath))))
24423
24500
  return;
24424
24501
  return ocrImageTooSmallMessage(bytes);
24425
24502
  }
@@ -24587,7 +24664,7 @@ async function collectFiles(state, directory, depth) {
24587
24664
  }
24588
24665
  if (!entry.isFile())
24589
24666
  continue;
24590
- const extension = normalizeExtension2(extname8(entry.name));
24667
+ const extension = normalizeExtension2(extname9(entry.name));
24591
24668
  if (!state.supportedExtensions.has(extension))
24592
24669
  continue;
24593
24670
  let fileStat;
@@ -24621,7 +24698,7 @@ async function collectFiles(state, directory, depth) {
24621
24698
  warnDirectoryUnreadable(state, directory, code);
24622
24699
  }
24623
24700
  }
24624
- var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
24701
+ var DEFAULT_MAX_SOURCE_BYTES, DEFAULT_MAX_DELETE_RATIO = 0.3, MASS_DELETE_FLOOR = 100, DEFAULT_MAX_SCANNED_ENTRIES = 2000000, DEFAULT_MAX_WALK_DEPTH = 64, DEFAULT_MIRROR_BATCH_SIZE = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", PARSER_CRASH_MESSAGE = "The registered parser crashed unexpectedly on this file; it was skipped during indexing.", DEFAULT_TRANSIENT_RETRIES = 3, DEFAULT_TRANSIENT_BACKOFF_MS = 50, TRANSIENT_ERROR_CODES, WORKSPACE_ERROR_CODES, MirrorWorkspaceError, STANDALONE_OCR_IMAGE_EXTENSIONS, SEARCH_ROOT_UNAVAILABLE_MESSAGES;
24625
24702
  var init_sync = __esm(() => {
24626
24703
  init_cli2();
24627
24704
  init_cloud_placeholder();
@@ -29632,7 +29709,7 @@ var init_automation_tool = __esm(() => {
29632
29709
  });
29633
29710
 
29634
29711
  // src/agent/citation-links.ts
29635
- import { basename as basename15, isAbsolute as isAbsolute12 } from "node:path";
29712
+ import { basename as basename14, isAbsolute as isAbsolute12 } from "node:path";
29636
29713
  import { pathToFileURL as pathToFileURL3 } from "node:url";
29637
29714
  function openableTarget(source) {
29638
29715
  if (/^https?:\/\//iu.test(source)) {
@@ -29650,7 +29727,7 @@ function linkCitations(markdown, sourceOf, isFile) {
29650
29727
  for (const marker of answerMarkers(markdown, isFile)) {
29651
29728
  if (marker.kind === "number")
29652
29729
  continue;
29653
- const labels = marker.kind === "file" ? [{ label: basename15(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
29730
+ const labels = marker.kind === "file" ? [{ label: basename14(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
29654
29731
  let linked = false;
29655
29732
  const parts = labels.map(({ label, source }) => {
29656
29733
  const target = source === undefined ? undefined : openableTarget(source);
@@ -31776,10 +31853,7 @@ function renderScanReport(details) {
31776
31853
  const lines = [
31777
31854
  `dupey scanned ${details.scans.length} configured root(s).`,
31778
31855
  `families=${details.familyCount} exactDuplicates=${details.exactDuplicateCount} extractionErrors=${errors}`,
31779
- "Review exact families before cleanup; near/contains families are not safe deletion evidence.",
31780
- ...details.skippedRoots.length === 0 ? [] : [
31781
- `Not scanned (inside a cloud or network folder; scanning would download or re-read every file): ${details.skippedRoots.join(", ")}. Duplicates there are not reported, and an exact duplicate is only found against files an earlier refresh fingerprinted.`
31782
- ]
31856
+ "Review exact families before cleanup; near/contains families are not safe deletion evidence."
31783
31857
  ];
31784
31858
  let rendered = 0;
31785
31859
  let omitted = 0;
@@ -32757,6 +32831,56 @@ var init_packaged_skill = __esm(() => {
32757
32831
  PACKAGED_SKILL_ROOTS = ["skills", "internal-skills"];
32758
32832
  });
32759
32833
 
32834
+ // src/agent/pipelined-sync.ts
32835
+ class PipelinedSync {
32836
+ #sync;
32837
+ #covered = [];
32838
+ #dirty = false;
32839
+ #running;
32840
+ #failure;
32841
+ #stopped = false;
32842
+ #runs = 0;
32843
+ constructor(options) {
32844
+ this.#sync = options.sync;
32845
+ }
32846
+ async batch(batch) {
32847
+ this.#raise();
32848
+ this.#covered.push(...batch.virtualPaths);
32849
+ this.#dirty = true;
32850
+ if (!this.#stopped)
32851
+ this.#running ??= this.#loop();
32852
+ if (batch.final)
32853
+ await this.#running;
32854
+ this.#raise();
32855
+ return this.#stopped ? { stop: true } : undefined;
32856
+ }
32857
+ async settle() {
32858
+ this.#stopped = true;
32859
+ await this.#running;
32860
+ }
32861
+ #raise() {
32862
+ if (this.#failure !== undefined)
32863
+ throw this.#failure.error;
32864
+ }
32865
+ async#loop() {
32866
+ try {
32867
+ while (this.#dirty && !this.#stopped) {
32868
+ this.#dirty = false;
32869
+ const virtualPaths = this.#covered;
32870
+ this.#covered = [];
32871
+ this.#runs += 1;
32872
+ const outcome = await this.#sync({ index: this.#runs, virtualPaths });
32873
+ if (outcome?.stop === true)
32874
+ this.#stopped = true;
32875
+ }
32876
+ } catch (error) {
32877
+ this.#failure = { error };
32878
+ } finally {
32879
+ this.#running = undefined;
32880
+ }
32881
+ }
32882
+ }
32883
+
32760
32884
  // src/agent/query-decomposition.ts
32761
32885
  import { completeSimple as completeSimple2 } from "@earendil-works/pi-ai/compat";
32762
32886
  function buildDecompositionPrompt(query) {
@@ -32996,7 +33120,7 @@ var init_query_routing = __esm(() => {
32996
33120
  false: "Every part of the question is answered with a concrete, specific, consistent fact (a name, date, number, or definite statement)."
32997
33121
  });
32998
33122
  BASE_ROUTE_OPTIONS = {
32999
- local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. When unsure whether intrinsic knowledge suffices, choose local.",
33123
+ local: "Answering needs additional information beyond the assistant's intrinsic knowledge: private information only the user can access (files on their computer, their chats such as Discord, KakaoTalk, Slack, their email, notes, calendar, or history), documents, or any fact that may be recent, specific, niche, or changing (current events, recent releases, prices, versions). Prefer local whenever the question refers to the user's own life, situation, plans, or records — questions phrased with words like 'my', 'I', 'we', 'our', a named friend, family member, or colleague, or 'my case/hearing/appointment/routine' — even when a generic answer would also be possible. A request to find, locate, open, or look up a document, file, slide deck, presentation, paper, note, message, or other material is local even when it names no owner ('find the slides about X', 'where is the Y presentation', 'Z 관련 자료 찾아줘'): such material lives in the user's own files and chats, and a well-known topic is not a reason to answer from general knowledge or the web. When unsure whether intrinsic knowledge suffices, choose local.",
33000
33124
  extra: "None of the other options fits and no search of the user's own records is needed first. Either the request is answerable without looking anything up — stable common knowledge, a definition, simple reasoning, or small talk — or it is a task for the assistant's tools that is not file organization and not AutoRAG configuration: committing code, pushing a branch, opening a pull request, running tests or a build, installing a package, running a script, writing or editing code. Programming work is extra even when the code names the user's files or folders ('write a script that lowercases every PNG', 'add this to my parser'): the assistant is asked to produce or change code, not to act on those files now. So is a request to run something (tests, a build, a script, a command) and report the outcome, even when what it runs lives in the user's own repository: running it is a task for the tool, not a search of the user's records. Never extra when the request refers to the user's own life, files, records, chats, calendar, or history (e.g. 'my', 'I', 'our', 'my girlfriend', 'my case', 'my routine'), and never extra when the answer depends on facts that may be recent, specific, or changing (current events, recent releases, prices, versions): search those locally instead. If any other option fits, do not choose extra.",
33001
33125
  organize: "The user asks the assistant to DO something to files or folders instead of finding information: move or copy files into folders, tidy or sort a folder, bulk or pattern rename (for example by date or number), rename files according to what their content says, convert files to another format, create folder structures, group files by type or topic, archive, or move to the trash ('tidy this up', 'move these into folders', 'bulk rename these files', 'rename the files to match their content', 'convert these to PDF', 'sort my Downloads folder'). It is about acting on files and folders, never about looking up what a document says or where something is. A question that only asks to find or read a file's content is local, not organize."
33002
33126
  };
@@ -34234,7 +34358,7 @@ var init_page_loader = __esm(() => {
34234
34358
  });
34235
34359
 
34236
34360
  // src/web/fetch/url-target.ts
34237
- import { extname as extname9 } from "node:path";
34361
+ import { extname as extname10 } from "node:path";
34238
34362
  function repairCollapsedScheme(value) {
34239
34363
  const m = value.match(/^(https?):\/(?!\/)/i);
34240
34364
  return m ? `${m[1]}://${value.slice(m[0].length)}` : value;
@@ -34253,7 +34377,7 @@ function getFilenameExtensionHint(filename) {
34253
34377
  const lower = filename.toLowerCase();
34254
34378
  if (lower.endsWith(".tar.gz"))
34255
34379
  return ".tar.gz";
34256
- return extname9(filename).toLowerCase();
34380
+ return extname10(filename).toLowerCase();
34257
34381
  }
34258
34382
  function getExtensionHint(url, contentDisposition) {
34259
34383
  if (contentDisposition) {
@@ -34430,7 +34554,7 @@ var init_convert2 = __esm(() => {
34430
34554
  });
34431
34555
 
34432
34556
  // src/web/fetch/feeds.ts
34433
- import { extname as extname10 } from "node:path";
34557
+ import { extname as extname11 } from "node:path";
34434
34558
  import { XMLParser as XMLParser3 } from "fast-xml-parser";
34435
34559
  function getHtmlAttribute(tag, attribute) {
34436
34560
  const pattern = new RegExp(`\\b${attribute}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'=<>]+))`, "i");
@@ -34493,7 +34617,7 @@ function extractDocumentLinks(html, baseUrl) {
34493
34617
  const href = getHtmlAttribute(tag, "href");
34494
34618
  if (!href)
34495
34619
  continue;
34496
- const ext = extname10(href).toLowerCase();
34620
+ const ext = extname11(href).toLowerCase();
34497
34621
  if (!CONVERTIBLE_EXTENSIONS2.has(ext))
34498
34622
  continue;
34499
34623
  const resolved = href.startsWith("http") ? href : new URL(href, baseUrl).href;
@@ -36161,7 +36285,7 @@ var init_web_search_tool = __esm(() => {
36161
36285
  // src/agent/agent.ts
36162
36286
  import { randomUUID as randomUUID7 } from "node:crypto";
36163
36287
  import { existsSync as existsSync32, watch as fsWatch, mkdirSync as mkdirSync23, realpathSync as realpathSync8, statSync as statSync11, writeFileSync as writeFileSync17 } from "node:fs";
36164
- import { basename as basename16, dirname as dirname27, join as join53, relative as relative8, resolve as resolve17, sep as sep10 } from "node:path";
36288
+ import { basename as basename15, dirname as dirname27, join as join53, relative as relative8, resolve as resolve17, sep as sep10 } from "node:path";
36165
36289
  import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
36166
36290
  import { getAgentDir as getAgentDir5, keyText } from "@earendil-works/pi-coding-agent";
36167
36291
  function describeDecisionModel(status) {
@@ -36596,10 +36720,20 @@ class AutoRAGAgent {
36596
36720
  }
36597
36721
  async scanDuplicateDocuments() {
36598
36722
  if (this.dupeyOptions === false) {
36599
- return { scans: [], skippedRoots: [], familyCount: 0, exactDuplicateCount: 0 };
36723
+ return { scans: [], familyCount: 0, exactDuplicateCount: 0 };
36600
36724
  }
36601
- const { targets, skipped } = planDupeyScans(this.searchPaths, await this.cloudRoots());
36602
- const scans = await Promise.all(targets.map(({ path, excludeDirs }) => scanWithDupey(path, {
36725
+ const cloudRoots = await this.cloudRoots();
36726
+ const scanRoots = [];
36727
+ for (const searchPath of this.searchPaths) {
36728
+ const scope = cloudScope(searchPath, cloudRoots);
36729
+ if (scope.insideCloudRoot)
36730
+ continue;
36731
+ scanRoots.push({
36732
+ path: searchPath,
36733
+ excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path))
36734
+ });
36735
+ }
36736
+ const scans = await Promise.all(scanRoots.map(({ path, excludeDirs }) => scanWithDupey(path, {
36603
36737
  ...this.dupeyOptions || {},
36604
36738
  ...excludeDirs.length === 0 ? {} : { excludeDirs }
36605
36739
  })));
@@ -36613,7 +36747,7 @@ class AutoRAGAgent {
36613
36747
  }
36614
36748
  return count + [...hashes.values()].reduce((sum, size) => sum + Math.max(0, size - 1), 0);
36615
36749
  }, 0);
36616
- return { scans, skippedRoots: skipped, familyCount, exactDuplicateCount };
36750
+ return { scans, familyCount, exactDuplicateCount };
36617
36751
  }
36618
36752
  loadMemoryContextFor(query) {
36619
36753
  if (this.memoryContextCache?.query === query)
@@ -38266,13 +38400,67 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38266
38400
  inFlight: true,
38267
38401
  lastStartedAt: startedAt
38268
38402
  };
38403
+ const syncAbort = new AbortController;
38404
+ const forwardAbort = () => syncAbort.abort(signal?.reason);
38405
+ signal?.addEventListener("abort", forwardAbort, { once: true });
38406
+ let minSyncPipeline;
38269
38407
  try {
38270
38408
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38409
+ const batchedMinSync = needsParsed && wants("minsync");
38410
+ const minSyncBatches = [];
38411
+ const minSyncBatchFailures = [];
38412
+ let minSyncSyncedBase = 0;
38413
+ let minSyncTotalBase = 0;
38414
+ minSyncPipeline = batchedMinSync ? new PipelinedSync({
38415
+ sync: async ({ index, virtualPaths }) => {
38416
+ syncAbort.signal.throwIfAborted();
38417
+ let syncTotal = minSyncTotalBase;
38418
+ const result = await this.syncMinSync(force && index === 1, syncAbort.signal, (minsyncProgress) => {
38419
+ syncTotal = minSyncTotalBase + minsyncProgress.total;
38420
+ progress = updateRefreshProgress(progress, {
38421
+ minsync: { synced: minSyncSyncedBase + minsyncProgress.processed, total: syncTotal }
38422
+ });
38423
+ writeRefreshProgress(this.workspaceProjectRoot, progress);
38424
+ emitProgress({
38425
+ phase: "minsync",
38426
+ processed: minSyncSyncedBase + minsyncProgress.processed,
38427
+ total: syncTotal
38428
+ });
38429
+ });
38430
+ minSyncBatches.push(result);
38431
+ if (result.ok)
38432
+ minSyncBatchFailures.length = 0;
38433
+ else if (result.reason !== "lock-held") {
38434
+ for (const virtualPath of virtualPaths) {
38435
+ minSyncBatchFailures.push({
38436
+ code: "minsync-sync-failed",
38437
+ severity: "warning",
38438
+ message: `This file stayed at its previous index entry: the sync that covered it could not index it and the next sync retries it. ${sanitizeDiagnosticMessage(result.diagnostic?.message ?? result.reason ?? "MinSync sync failed.")}`,
38439
+ source: virtualPath
38440
+ });
38441
+ }
38442
+ }
38443
+ minSyncSyncedBase = syncTotal;
38444
+ minSyncTotalBase = syncTotal;
38445
+ return !result.ok && result.reason === "lock-held" ? { stop: true } : undefined;
38446
+ }
38447
+ }) : undefined;
38448
+ const pipeline = minSyncPipeline;
38271
38449
  const summary = needsParsed ? await this.syncParsedMirrors(force, {
38272
38450
  ...opts?.retrySkipped === true ? { retrySkipped: true } : {},
38273
38451
  ...signal === undefined ? {} : { signal },
38452
+ ...pipeline === undefined ? {} : {
38453
+ onBatch: (batch) => {
38454
+ if (batch.final) {
38455
+ progress = updateRefreshProgress(progress, { phase: "minsync" });
38456
+ writeRefreshProgress(this.workspaceProjectRoot, progress);
38457
+ }
38458
+ return pipeline.batch(batch);
38459
+ }
38460
+ },
38274
38461
  onProgress: (parsedProgress) => {
38275
38462
  progress = updateRefreshProgress(progress, {
38463
+ phase: "parsed",
38276
38464
  sourceFiles: { total: parsedProgress.total },
38277
38465
  parsedCounts: {
38278
38466
  scanned: parsedProgress.processed,
@@ -38294,7 +38482,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38294
38482
  }) : await this.scanMirrorStaleness();
38295
38483
  signal?.throwIfAborted();
38296
38484
  progress = updateRefreshProgress(progress, {
38297
- phase: wants("minsync") ? "minsync" : wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
38485
+ phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
38298
38486
  sourceFiles: { total: summary.scanned },
38299
38487
  parsedCounts: {
38300
38488
  scanned: summary.scanned,
@@ -38304,24 +38492,12 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38304
38492
  }
38305
38493
  });
38306
38494
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38307
- const minsync = wants("minsync") ? await this.syncMinSync(force, signal, (minsyncProgress) => {
38308
- progress = updateRefreshProgress(progress, {
38309
- minsync: { synced: minsyncProgress.processed, total: minsyncProgress.total }
38310
- });
38311
- writeRefreshProgress(this.workspaceProjectRoot, progress);
38312
- emitProgress({
38313
- phase: "minsync",
38314
- processed: minsyncProgress.processed,
38315
- total: minsyncProgress.total
38316
- });
38317
- }) : undefined;
38318
- signal?.throwIfAborted();
38495
+ const minsync = batchedMinSync ? aggregateMinSyncBatches(minSyncBatches) : undefined;
38319
38496
  if (minsync !== undefined) {
38320
38497
  progress = updateRefreshProgress(progress, {
38321
- phase: wants("datasources") ? "datasources" : wants("jikji") ? "jikji" : "finalizing",
38322
38498
  minsync: {
38323
- ...progress.minsync,
38324
38499
  synced: minsync.synced,
38500
+ total: Math.max(minSyncTotalBase, minsync.synced),
38325
38501
  ...minsync.filesFailed !== undefined && minsync.filesFailed.length > 0 ? { filesSkipped: minsync.filesFailed.length } : {}
38326
38502
  }
38327
38503
  });
@@ -38383,7 +38559,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38383
38559
  finishedAt: new Date().toISOString()
38384
38560
  });
38385
38561
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38386
- const minsyncDiagnostics = minSyncRefreshDiagnostics(minsync);
38562
+ const minsyncDiagnostics = [...minSyncRefreshDiagnostics(minsync), ...minSyncBatchFailures];
38387
38563
  const publicMinsync = minsync ? {
38388
38564
  ok: minsync.ok,
38389
38565
  synced: minsync.synced,
@@ -38417,6 +38593,7 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38417
38593
  ...fsearch !== undefined ? { fsearch } : {}
38418
38594
  };
38419
38595
  } catch (error) {
38596
+ syncAbort.abort(error);
38420
38597
  this.refreshState = {
38421
38598
  ...this.refreshState,
38422
38599
  lastOutcome: "failed",
@@ -38430,6 +38607,8 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38430
38607
  writeRefreshProgress(this.workspaceProjectRoot, progress);
38431
38608
  throw error;
38432
38609
  } finally {
38610
+ signal?.removeEventListener("abort", forwardAbort);
38611
+ await minSyncPipeline?.settle();
38433
38612
  this.refreshState = {
38434
38613
  ...this.refreshState,
38435
38614
  inFlight: false,
@@ -38651,15 +38830,15 @@ ${this.discoveryHint((tools) => `When exploring local files and folders, activel
38651
38830
  const cloudRoots = await this.cloudRoots();
38652
38831
  const fingerprints = cloudFingerprintExclusions(loadCloudFingerprints(this.workspaceProjectRoot));
38653
38832
  for (const searchPath of this.searchPaths) {
38654
- const target = planDupeyScans([searchPath], cloudRoots).targets[0];
38833
+ const scope = cloudScope(searchPath, cloudRoots);
38655
38834
  let scan;
38656
- if (target === undefined) {
38835
+ if (scope.insideCloudRoot) {
38657
38836
  scan = { dir: searchPath, files: [], families: [], errors: [] };
38658
38837
  } else {
38659
38838
  try {
38660
38839
  scan = await scanWithDupey(searchPath, {
38661
38840
  ...this.dupeyOptions || {},
38662
- ...target.excludeDirs.length === 0 ? {} : { excludeDirs: target.excludeDirs }
38841
+ ...scope.nestedCloudRoots.length === 0 ? {} : { excludeDirs: scope.nestedCloudRoots.map((root) => basename15(root.path)) }
38663
38842
  });
38664
38843
  } catch (error) {
38665
38844
  if (!(error instanceof DupeyCliError))
@@ -39188,6 +39367,12 @@ function fsearchRefreshDiagnostics(fsearch) {
39188
39367
  }
39189
39368
  ];
39190
39369
  }
39370
+ function aggregateMinSyncBatches(batches) {
39371
+ const last = batches.at(-1);
39372
+ if (last === undefined)
39373
+ return;
39374
+ return { ...last, synced: batches.reduce((total, batch) => total + batch.synced, 0) };
39375
+ }
39191
39376
  function minSyncRefreshDiagnostics(minsync) {
39192
39377
  if (!minsync)
39193
39378
  return [];
@@ -39218,7 +39403,7 @@ function toMinSyncReasonDiagnostic(reason) {
39218
39403
  function pinExcludedPath(path) {
39219
39404
  const resolvedPath = resolve17(path);
39220
39405
  try {
39221
- return join53(realpathSync8(dirname27(resolvedPath)), basename16(resolvedPath));
39406
+ return join53(realpathSync8(dirname27(resolvedPath)), basename15(resolvedPath));
39222
39407
  } catch {
39223
39408
  return resolvedPath;
39224
39409
  }
@@ -40207,7 +40392,7 @@ function readPackageVersion() {
40207
40392
  return "0.0.0";
40208
40393
  }
40209
40394
  }
40210
- var init_version3 = () => {};
40395
+ var init_version4 = () => {};
40211
40396
 
40212
40397
  // src/cli/commands/init.ts
40213
40398
  import { existsSync as existsSync35, readFileSync as readFileSync30 } from "node:fs";
@@ -42039,20 +42224,18 @@ var init_health = __esm(() => {
42039
42224
 
42040
42225
  // src/cli/commands/duplicates.ts
42041
42226
  import { resolve as resolve23 } from "node:path";
42042
- async function runDuplicates(ctx, scanner = (path, excludeDirs) => scanWithDupey(path, { excludeDirs }), cloudProbe = defaultCloudProbe()) {
42227
+ async function runDuplicates(ctx, scanner = (path) => scanWithDupey(path)) {
42043
42228
  try {
42044
42229
  const config = resolveConfig({ flags: ctx.flags, cwd: ctx.cwd });
42045
42230
  const requested = ctx.positionals[0];
42046
42231
  const roots = requested ? [resolve23(ctx.cwd, requested)] : config.searchPaths.map((path) => resolve23(path));
42047
- const plan = planDupeyScans(roots, await listCloudRoots(cloudProbe, roots));
42048
- const scans = await Promise.all(plan.targets.map(({ path, excludeDirs }) => scanner(path, excludeDirs)));
42232
+ const scans = await Promise.all(roots.map((root) => scanner(root)));
42049
42233
  const exactGroups = scans.flatMap(exactGroupsFromScan);
42050
42234
  const extractionErrors = scans.flatMap((scan) => scan.errors);
42051
42235
  if (ctx.json) {
42052
42236
  ctx.stdout(JSON.stringify({
42053
42237
  ok: true,
42054
42238
  roots,
42055
- skippedRoots: plan.skipped,
42056
42239
  exactGroups,
42057
42240
  families: scans.flatMap((scan) => scan.families),
42058
42241
  extractionErrors,
@@ -42061,14 +42244,11 @@ async function runDuplicates(ctx, scanner = (path, excludeDirs) => scanWithDupey
42061
42244
  return 0;
42062
42245
  }
42063
42246
  const lines = [
42064
- `duplicates: scanned ${scans.length} root(s)`,
42247
+ `duplicates: scanned ${roots.length} root(s)`,
42065
42248
  ` exact groups: ${exactGroups.length}`,
42066
42249
  ` other families: ${scans.reduce((count, scan) => count + scan.families.length, 0)}`,
42067
42250
  ` extraction errors: ${extractionErrors.length}`
42068
42251
  ];
42069
- for (const root of plan.skipped) {
42070
- lines.push(` not scanned (inside a cloud or network folder; scanning would download or re-read every file): ${root}`);
42071
- }
42072
42252
  for (const [index, group] of exactGroups.entries()) {
42073
42253
  lines.push(` exact group ${index + 1}:`);
42074
42254
  for (const file of group.files)
@@ -42096,7 +42276,6 @@ function exactGroupsFromScan(scan) {
42096
42276
  }
42097
42277
  var init_duplicates = __esm(() => {
42098
42278
  init_dupey();
42099
- init_cloud_placeholder();
42100
42279
  init_config5();
42101
42280
  init_output();
42102
42281
  });
@@ -42352,7 +42531,7 @@ var init_tui = __esm(() => {
42352
42531
  init_open_target();
42353
42532
  init_output();
42354
42533
  init_update_check();
42355
- init_version3();
42534
+ init_version4();
42356
42535
  init_whatsapp_extension();
42357
42536
  THINKING_LEVELS2 = ["off", "minimal", "low", "medium", "high", "xhigh", "max"];
42358
42537
  });
@@ -42487,7 +42666,7 @@ async function runUpdateCheck(ctx, deps = {}) {
42487
42666
  }
42488
42667
  var init_update_check2 = __esm(() => {
42489
42668
  init_update_check();
42490
- init_version3();
42669
+ init_version4();
42491
42670
  });
42492
42671
 
42493
42672
  // src/cli/commands/attachments.ts
@@ -43610,7 +43789,7 @@ async function runAutomate(ctx) {
43610
43789
 
43611
43790
  // src/cli/index.ts
43612
43791
  init_output();
43613
- init_version3();
43792
+ init_version4();
43614
43793
  var BOOLEAN_FLAGS = new Set([
43615
43794
  "json",
43616
43795
  "debug",