@autorag/librarian 2.10.0 → 2.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/LICENSE +195 -21
  2. package/NOTICE +17 -21
  3. package/README.md +335 -161
  4. package/dist/agent/agent.d.ts +67 -2
  5. package/dist/agent/context-token-guard.d.ts +23 -0
  6. package/dist/agent/jev-extension.d.ts +5 -4
  7. package/dist/agent/jikji-find-tool.d.ts +1 -0
  8. package/dist/agent/pi-session.d.ts +35 -2
  9. package/dist/agent/request-limits.d.ts +20 -0
  10. package/dist/agent/search-all-tool.d.ts +1 -0
  11. package/dist/agent/search-single-datasource-tool.d.ts +1 -0
  12. package/dist/cli/config.d.ts +13 -11
  13. package/dist/cli/index.js +2573 -800
  14. package/dist/cloud/config.d.ts +18 -0
  15. package/dist/cloud/jev-backend.d.ts +90 -0
  16. package/dist/core.js +2185 -418
  17. package/dist/datasource/crawler-skill.d.ts +17 -0
  18. package/dist/datasource/sidecar/config.d.ts +26 -0
  19. package/dist/datasource/sidecar/descriptors.d.ts +67 -0
  20. package/dist/datasource/sidecar/fusion.d.ts +25 -0
  21. package/dist/datasource/sidecar/index.d.ts +131 -0
  22. package/dist/datasource/sidecar/methods.d.ts +27 -0
  23. package/dist/datasource/types.d.ts +3 -1
  24. package/dist/embedding-runtime/index.d.ts +6 -6
  25. package/dist/embedding-runtime/manifest.d.ts +0 -12
  26. package/dist/embedding-runtime/types.d.ts +1 -1
  27. package/dist/index.js +2210 -439
  28. package/dist/mcp/index.js +2205 -438
  29. package/dist/minsync/client.d.ts +3 -3
  30. package/dist/minsync/installer.d.ts +1 -1
  31. package/dist/minsync/process.d.ts +1 -0
  32. package/dist/minsync/types.d.ts +1 -1
  33. package/dist/parser/defaults.d.ts +6 -4
  34. package/dist/parser/ocr-engines.d.ts +6 -0
  35. package/dist/retrieval/index.d.ts +1 -1
  36. package/dist/retrieval/rerank.d.ts +20 -3
  37. package/licenses/embedding-assets.json +0 -35
  38. package/package.json +24 -27
  39. package/skills/autorag/SKILL.md +1 -1
  40. package/skills/autorag-doctor/SKILL.md +1 -1
  41. package/skills/autorag-lite-setup/SKILL.md +2 -2
  42. package/skills/autorag-setup/SKILL.md +2 -2
  43. package/licenses/gemma-notice.txt +0 -24
  44. package/licenses/gemma-section-3.2-flow-down.txt +0 -9
  45. package/licenses/gemma-terms-of-use.html +0 -4750
  46. package/licenses/gemma-terms-of-use.txt +0 -158
package/dist/mcp/index.js CHANGED
@@ -62,6 +62,7 @@ function datasourceCodeToRetrieval(code) {
62
62
  case "datasource-auth-error":
63
63
  case "datasource-embedding-egress-rejected":
64
64
  case "datasource-index-failed":
65
+ case "datasource-index-partial":
65
66
  case "datasource-permission-denied":
66
67
  case "datasource-remote-embedding-rejected":
67
68
  return "retrieval-method-failed";
@@ -1113,19 +1114,6 @@ var init_manifest = __esm(() => {
1113
1114
  artifactSha256: "06507c7b42688469c4e7298b0a1e16deff06caf291cf0a5b278c308249c3e439",
1114
1115
  backend: "auto",
1115
1116
  protocolCapabilities
1116
- },
1117
- "embeddinggemma-300m": {
1118
- profileId: "embeddinggemma-300m",
1119
- provider: "google",
1120
- model: "embeddinggemma-300M-Q8_0.gguf",
1121
- dimension: 768,
1122
- queryPrefix: "task: search result | query: ",
1123
- passagePrefix: "title: none | text: ",
1124
- runtimeBuild: RUNTIME_BUILD,
1125
- modelRevision: "0f741b5a6585bd53aeb15cd1372c56f2a0f65e12",
1126
- artifactSha256: "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63",
1127
- backend: "auto",
1128
- protocolCapabilities
1129
1117
  }
1130
1118
  };
1131
1119
  runtimeAssets = {
@@ -1171,17 +1159,6 @@ var init_manifest = __esm(() => {
1171
1159
  unsuitableUse: "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256.",
1172
1160
  upstreamCardUrl: "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B",
1173
1161
  modelCard: "docs/model-cards/qwen3-embedding-0.6b.md"
1174
- },
1175
- embeddinggemma: {
1176
- url: "https://huggingface.co/ggml-org/embeddinggemma-300M-GGUF/resolve/0f741b5a6585bd53aeb15cd1372c56f2a0f65e12/embeddinggemma-300M-Q8_0.gguf",
1177
- revision: "0f741b5a6585bd53aeb15cd1372c56f2a0f65e12",
1178
- sha256: "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63",
1179
- license: "Gemma Terms of Use",
1180
- noticeReference: "Google EmbeddingGemma model card and Gemma Terms of Use",
1181
- intendedUse: "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.",
1182
- unsuitableUse: "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256, uses restricted by the Gemma Prohibited Use Policy.",
1183
- upstreamCardUrl: "https://ai.google.dev/gemma/docs/embeddinggemma/model_card",
1184
- modelCard: "docs/model-cards/embeddinggemma-300m.md"
1185
1162
  }
1186
1163
  };
1187
1164
  });
@@ -1619,7 +1596,7 @@ var init_supervisor = __esm(() => {
1619
1596
  // src/embedding-runtime/index.ts
1620
1597
  import { basename as basename4 } from "node:path";
1621
1598
  function modelAsset(profile) {
1622
- return profile.profileId === "qwen3-embedding-0.6b" ? { id: profile.profileId, filename: basename4(profile.model), ...MODEL_ASSETS.qwen3 } : { id: profile.profileId, filename: basename4(profile.model), ...MODEL_ASSETS.embeddinggemma };
1599
+ return { id: profile.profileId, filename: basename4(profile.model), ...MODEL_ASSETS.qwen3 };
1623
1600
  }
1624
1601
  function runtimeAsset(platform, backend) {
1625
1602
  const selected = platform ?? "darwin-arm64-metal";
@@ -5021,7 +4998,12 @@ var init_local_embedder = __esm(() => {
5021
4998
  // src/minsync/process.ts
5022
4999
  import { spawn as spawn7 } from "node:child_process";
5023
5000
  function spawnProcess(command, args, cwd, options = {}) {
5024
- return new Promise((resolve) => {
5001
+ return new Promise((resolve, reject) => {
5002
+ const { signal } = options;
5003
+ if (signal?.aborted) {
5004
+ reject(abortError(signal));
5005
+ return;
5006
+ }
5025
5007
  const portable = portableSpawnCommand(command, args);
5026
5008
  const child = spawn7(portable.command, [...portable.args], { cwd, stdio: ["ignore", "pipe", "pipe"] });
5027
5009
  let stdout = "";
@@ -5039,15 +5021,26 @@ function spawnProcess(command, args, cwd, options = {}) {
5039
5021
  timedOut = true;
5040
5022
  child.kill("SIGTERM");
5041
5023
  }, options.timeoutMs) : undefined;
5024
+ const onAbort = () => {
5025
+ child.kill("SIGTERM");
5026
+ };
5027
+ signal?.addEventListener("abort", onAbort, { once: true });
5028
+ const settle = () => {
5029
+ clearTimeout(timer);
5030
+ signal?.removeEventListener("abort", onAbort);
5031
+ };
5042
5032
  child.on("error", (error) => {
5043
- if (timer)
5044
- clearTimeout(timer);
5045
- resolve({ ok: false, stdout, stderr: error.message, code: null });
5033
+ settle();
5034
+ if (signal?.aborted)
5035
+ reject(abortError(signal));
5036
+ else
5037
+ resolve({ ok: false, stdout, stderr: error.message, code: null });
5046
5038
  });
5047
5039
  child.on("close", (code) => {
5048
- if (timer)
5049
- clearTimeout(timer);
5050
- if (timedOut) {
5040
+ settle();
5041
+ if (signal?.aborted) {
5042
+ reject(abortError(signal));
5043
+ } else if (timedOut) {
5051
5044
  resolve({ ok: false, stdout, stderr: "process timed out", code: null, timedOut: true });
5052
5045
  } else {
5053
5046
  resolve({ ok: code === 0, stdout, stderr, code });
@@ -5055,6 +5048,9 @@ function spawnProcess(command, args, cwd, options = {}) {
5055
5048
  });
5056
5049
  });
5057
5050
  }
5051
+ function abortError(signal) {
5052
+ return signal.reason instanceof Error ? signal.reason : new DOMException("The operation was aborted", "AbortError");
5053
+ }
5058
5054
  var init_process = __esm(() => {
5059
5055
  init_portable_spawn();
5060
5056
  });
@@ -5287,10 +5283,12 @@ class MinSyncClient {
5287
5283
  } : {}
5288
5284
  };
5289
5285
  }
5290
- async query(text, topK, mode = "vector") {
5286
+ async query(text, topK, mode = "vector", signal) {
5291
5287
  if (!existsSync11(this.binaryPath))
5292
5288
  throw minSyncBinaryMissingError();
5293
- const result = await this.spawn(["query", "--format", "json", "--mode", mode, "-k", String(topK), text]);
5289
+ const result = await this.spawn(["query", "--format", "json", "--mode", mode, "-k", String(topK), text], {
5290
+ ...signal === undefined ? {} : { signal }
5291
+ });
5294
5292
  if (!result.ok)
5295
5293
  throw new MinSyncQueryError(result.code, result.stderr);
5296
5294
  return parseQueryHits(result.stdout);
@@ -5363,7 +5361,7 @@ function isIdentity(value) {
5363
5361
  function isRecord4(value) {
5364
5362
  return typeof value === "object" && value !== null;
5365
5363
  }
5366
- var MINSYNC_OLLAMA_MIGRATION_MESSAGE = "This workspace uses the legacy 768-dimensional Ollama/TEI embedding path. Reindex explicitly, or pin an explicit profile config before using the new default runtime.", API_KEY_ENV_PATTERN, MinSyncQueryError;
5364
+ var MINSYNC_OLLAMA_MIGRATION_MESSAGE = "This workspace uses the legacy 768-dimensional Ollama/TEI embedding path. Reindex explicitly, or pin an explicit Ollama/TEI embedder config before using the new default runtime.", API_KEY_ENV_PATTERN, MinSyncQueryError;
5367
5365
  var init_client4 = __esm(() => {
5368
5366
  init_embedder_config();
5369
5367
  init_errors();
@@ -5607,7 +5605,7 @@ function isGitHubAsset(value) {
5607
5605
  function isRecord5(value) {
5608
5606
  return typeof value === "object" && value !== null;
5609
5607
  }
5610
- var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.4.6", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
5608
+ var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.4.7", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
5611
5609
  var init_installer2 = __esm(() => {
5612
5610
  init_process();
5613
5611
  SHA256_HEX_PATTERN = /^[a-f0-9]{64}$/;
@@ -5916,11 +5914,11 @@ class MinSyncVectorMethod {
5916
5914
  });
5917
5915
  let hits;
5918
5916
  try {
5919
- hits = await client.query(query, queryK, this.mode);
5917
+ hits = await client.query(query, queryK, this.mode, options.signal);
5920
5918
  } catch (error) {
5921
- if (this.mode !== "hybrid")
5919
+ if (this.mode !== "hybrid" || options.signal?.aborted)
5922
5920
  throw error;
5923
- hits = await client.query(query, queryK, "bm25");
5921
+ hits = await client.query(query, queryK, "bm25", options.signal);
5924
5922
  }
5925
5923
  const results = [];
5926
5924
  for (const hit of hits) {
@@ -6243,10 +6241,16 @@ async function runTesseract(recognition, signal, cleanupResolve, cleanupReject)
6243
6241
  };
6244
6242
  signal.addEventListener("abort", abort, { once: true });
6245
6243
  try {
6244
+ const tooSmall = ocrImageTooSmallMessage(recognition.bytes);
6245
+ if (tooSmall !== undefined)
6246
+ throw new Error(tooSmall);
6246
6247
  const { cachePath } = recognition;
6247
6248
  if (cachePath !== undefined)
6248
6249
  await mkdir3(cachePath, { recursive: true });
6249
- worker = await createWorker(recognition.language, undefined, cachePath === undefined ? {} : { cachePath });
6250
+ worker = await createWorker(recognition.language, undefined, {
6251
+ ...cachePath === undefined ? {} : { cachePath },
6252
+ errorHandler: containTesseractWorkerError
6253
+ });
6250
6254
  if (signal.aborted)
6251
6255
  throw new Error("OCR aborted before worker was ready");
6252
6256
  const result = await worker.recognize(Buffer.from(recognition.bytes));
@@ -6256,6 +6260,129 @@ async function runTesseract(recognition, signal, cleanupResolve, cleanupReject)
6256
6260
  await terminate().then(cleanupResolve, cleanupReject);
6257
6261
  }
6258
6262
  }
6263
+ function containTesseractWorkerError(_payload) {}
6264
+ function ocrImageTooSmallMessage(bytes) {
6265
+ const size = readImagePixelSize(bytes);
6266
+ if (size === undefined || size.width >= MIN_LEPTONICA_SIDE_PIXELS && size.height >= MIN_LEPTONICA_SIDE_PIXELS) {
6267
+ return;
6268
+ }
6269
+ return `Image is too small to scale (${size.width}x${size.height} vs minimum of ${MIN_LEPTONICA_SIDE_PIXELS} pixels).`;
6270
+ }
6271
+ function readImagePixelSize(bytes) {
6272
+ if (bytes.length >= 8 && bytes[0] === 137 && bytes[1] === 80)
6273
+ return readPngSize(bytes);
6274
+ if (bytes.length >= 2 && bytes[0] === 255 && bytes[1] === 216)
6275
+ return readJpegSize(bytes);
6276
+ if (bytes.length >= 12 && bytes[0] === 82 && bytes[1] === 73)
6277
+ return readWebpSize(bytes);
6278
+ return;
6279
+ }
6280
+ function readPngSize(bytes) {
6281
+ if (bytes.length < 24)
6282
+ return;
6283
+ if (bytes[0] !== 137 || bytes[1] !== 80 || bytes[2] !== 78 || bytes[3] !== 71 || bytes[4] !== 13 || bytes[5] !== 10 || bytes[6] !== 26 || bytes[7] !== 10) {
6284
+ return;
6285
+ }
6286
+ if (bytes[12] !== 73 || bytes[13] !== 72 || bytes[14] !== 68 || bytes[15] !== 82)
6287
+ return;
6288
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
6289
+ return { width: view.getUint32(16), height: view.getUint32(20) };
6290
+ }
6291
+ function readJpegSize(bytes) {
6292
+ if (bytes.length < 4 || bytes[0] !== 255 || bytes[1] !== 216)
6293
+ return;
6294
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
6295
+ let offset = 2;
6296
+ while (offset + 1 < bytes.length) {
6297
+ if (bytes[offset] !== 255)
6298
+ return;
6299
+ offset += 1;
6300
+ while (offset < bytes.length && bytes[offset] === 255)
6301
+ offset += 1;
6302
+ if (offset >= bytes.length)
6303
+ return;
6304
+ const marker = bytes[offset] ?? 0;
6305
+ offset += 1;
6306
+ if (marker === 0)
6307
+ return;
6308
+ if (marker === 1 || marker === 217 || marker >= 208 && marker <= 216)
6309
+ continue;
6310
+ if (offset + 2 > bytes.length)
6311
+ return;
6312
+ const segmentLength = view.getUint16(offset);
6313
+ if (segmentLength < 2)
6314
+ return;
6315
+ if (isJpegStartOfFrame(marker)) {
6316
+ if (segmentLength < 7 || offset + 7 > bytes.length)
6317
+ return;
6318
+ return { width: view.getUint16(offset + 5), height: view.getUint16(offset + 3) };
6319
+ }
6320
+ if (marker === 218)
6321
+ return;
6322
+ const next = offset + segmentLength;
6323
+ if (next <= offset || next > bytes.length)
6324
+ return;
6325
+ offset = next;
6326
+ }
6327
+ return;
6328
+ }
6329
+ function isJpegStartOfFrame(marker) {
6330
+ return marker >= 192 && marker <= 207 && marker !== 196 && marker !== 200 && marker !== 204;
6331
+ }
6332
+ function readWebpSize(bytes) {
6333
+ if (bytes.length < 20 || !asciiAt(bytes, 0, "RIFF") || !asciiAt(bytes, 8, "WEBP"))
6334
+ return;
6335
+ const chunkSize = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(16, true);
6336
+ if (asciiAt(bytes, 12, "VP8X"))
6337
+ return readVp8xSize(bytes, chunkSize);
6338
+ if (asciiAt(bytes, 12, "VP8L"))
6339
+ return readVp8lSize(bytes, chunkSize);
6340
+ if (asciiAt(bytes, 12, "VP8 "))
6341
+ return readVp8Size(bytes, chunkSize);
6342
+ return;
6343
+ }
6344
+ function readVp8xSize(bytes, chunkSize) {
6345
+ if (!webpPayloadFits(bytes, chunkSize, 10))
6346
+ return;
6347
+ const payload = 20;
6348
+ const width = 1 + readU24LE(bytes, payload + 4);
6349
+ const height = 1 + readU24LE(bytes, payload + 7);
6350
+ return { width, height };
6351
+ }
6352
+ function readVp8lSize(bytes, chunkSize) {
6353
+ if (!webpPayloadFits(bytes, chunkSize, 5) || bytes[20] !== 47)
6354
+ return;
6355
+ const b1 = bytes[21] ?? 0;
6356
+ const b2 = bytes[22] ?? 0;
6357
+ const b3 = bytes[23] ?? 0;
6358
+ const b4 = bytes[24] ?? 0;
6359
+ const width = 1 + (b1 | (b2 & 63) << 8);
6360
+ const height = 1 + (b2 >> 6 | b3 << 2 | (b4 & 15) << 10);
6361
+ return { width, height };
6362
+ }
6363
+ function readVp8Size(bytes, chunkSize) {
6364
+ if (!webpPayloadFits(bytes, chunkSize, 10) || (bytes[20] & 1) !== 0)
6365
+ return;
6366
+ if (bytes[23] !== 157 || bytes[24] !== 1 || bytes[25] !== 42)
6367
+ return;
6368
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
6369
+ return { width: view.getUint16(26, true) & 16383, height: view.getUint16(28, true) & 16383 };
6370
+ }
6371
+ function webpPayloadFits(bytes, chunkSize, needed) {
6372
+ return chunkSize >= needed && bytes.length >= 20 + needed;
6373
+ }
6374
+ function readU24LE(bytes, offset) {
6375
+ return (bytes[offset] ?? 0) | (bytes[offset + 1] ?? 0) << 8 | (bytes[offset + 2] ?? 0) << 16;
6376
+ }
6377
+ function asciiAt(bytes, offset, text) {
6378
+ if (offset + text.length > bytes.length)
6379
+ return false;
6380
+ for (let index = 0;index < text.length; index += 1) {
6381
+ if (bytes[offset + index] !== text.charCodeAt(index))
6382
+ return false;
6383
+ }
6384
+ return true;
6385
+ }
6259
6386
  function withTimeout(operation, timeoutMs, onTimeout) {
6260
6387
  return new Promise((resolve, reject) => {
6261
6388
  const timeout = setTimeout(() => {
@@ -6271,7 +6398,7 @@ function withTimeout(operation, timeoutMs, onTimeout) {
6271
6398
  });
6272
6399
  });
6273
6400
  }
6274
- var TESSERACT_LANGUAGES, BUILTIN_OCR_LANGUAGES, DEFAULT_TESSERACT_TIMEOUT_MS = 30000;
6401
+ var TESSERACT_LANGUAGES, BUILTIN_OCR_LANGUAGES, DEFAULT_TESSERACT_TIMEOUT_MS = 30000, MIN_LEPTONICA_SIDE_PIXELS = 3;
6275
6402
  var init_ocr_engines = __esm(() => {
6276
6403
  TESSERACT_LANGUAGES = {
6277
6404
  ko: "kor",
@@ -6480,8 +6607,8 @@ function createDefaultParserRegistry(options = {}) {
6480
6607
  const kordocOptions = {
6481
6608
  ...options.kordoc,
6482
6609
  images: ocrEnabled,
6483
- ...options.kordoc?.ocr === undefined && ocrEnabled ? {
6484
- ocr: builtinCovers ? true : createTesseractOcrProvider({
6610
+ ...options.kordoc?.ocr === undefined ? {
6611
+ ocr: !ocrEnabled ? false : builtinCovers ? true : createTesseractOcrProvider({
6485
6612
  languages,
6486
6613
  timeoutMs: options.ocr?.timeoutMs,
6487
6614
  cachePath: options.ocr?.cachePath
@@ -6619,6 +6746,23 @@ async function syncParsedMirrors(options) {
6619
6746
  checkpoint();
6620
6747
  continue;
6621
6748
  }
6749
+ const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
6750
+ if (tooSmallImage !== undefined) {
6751
+ deleted += removePrevious(options.root, previous, entry.virtualPath);
6752
+ handledPrevious.add(entry.virtualPath);
6753
+ skipped += 1;
6754
+ recordSkip(entry, "parser-skipped");
6755
+ diagnostics.push({
6756
+ code: "parser-skipped",
6757
+ severity: "warning",
6758
+ message: tooSmallImage,
6759
+ source: entry.virtualPath
6760
+ });
6761
+ sinceCheckpoint += 1;
6762
+ if (sinceCheckpoint >= MIRROR_CHECKPOINT_EVERY)
6763
+ checkpoint();
6764
+ continue;
6765
+ }
6622
6766
  let parsed;
6623
6767
  try {
6624
6768
  parsed = await parser.parse({ virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes });
@@ -6720,6 +6864,11 @@ async function detectMirrorStaleness(options) {
6720
6864
  }
6721
6865
  return diagnostics;
6722
6866
  }
6867
+ function standaloneImageTooSmallMessage(virtualPath, bytes) {
6868
+ if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname4(virtualPath))))
6869
+ return;
6870
+ return ocrImageTooSmallMessage(bytes);
6871
+ }
6723
6872
  function sourceReadFailure(error) {
6724
6873
  if (!(error instanceof Error))
6725
6874
  return;
@@ -6815,11 +6964,13 @@ function removeFile(path) {
6815
6964
  if (existsSync15(path))
6816
6965
  rmSync6(path, { force: true });
6817
6966
  }
6818
- var DEFAULT_MAX_SOURCE_BYTES, MIRROR_CHECKPOINT_EVERY = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.";
6967
+ var DEFAULT_MAX_SOURCE_BYTES, MIRROR_CHECKPOINT_EVERY = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", STANDALONE_OCR_IMAGE_EXTENSIONS;
6819
6968
  var init_sync = __esm(() => {
6820
6969
  init_source_paths();
6821
6970
  init_defaults();
6822
6971
  init_errors2();
6972
+ init_kordoc();
6973
+ init_ocr_engines();
6823
6974
  init_text();
6824
6975
  init_index_store();
6825
6976
  init_paths();
@@ -6847,6 +6998,7 @@ var init_sync = __esm(() => {
6847
6998
  "GrShaderCache",
6848
6999
  "ShaderCache"
6849
7000
  ]);
7001
+ STANDALONE_OCR_IMAGE_EXTENSIONS = new Set(KORDOC_IMAGE_EXTENSIONS);
6850
7002
  });
6851
7003
 
6852
7004
  // src/observability/run-log.ts
@@ -7712,8 +7864,36 @@ class RetrievalMethodRegistry {
7712
7864
  }
7713
7865
  }
7714
7866
 
7867
+ // src/cloud/config.ts
7868
+ function normalizeBaseUrl(value) {
7869
+ const trimmed = value.trim();
7870
+ let end = trimmed.length;
7871
+ while (end > 0 && trimmed.charCodeAt(end - 1) === 47)
7872
+ end--;
7873
+ return trimmed.slice(0, end);
7874
+ }
7875
+ function resolveBaseUrl(env) {
7876
+ const configured = env[AUTORAG_BASE_URL_ENV];
7877
+ const value = typeof configured === "string" && configured.trim() !== "" ? configured : DEFAULT_BASE_URL;
7878
+ return normalizeBaseUrl(value);
7879
+ }
7880
+ var PROVIDER_ID = "autorag", PROVIDER_NAME = "AutoRAG", DEFAULT_BASE_URL = "https://api.dazziapp.com", AUTORAG_BASE_URL_ENV = "AUTORAG_BASE_URL", AUTORAG_API_KEY_ENV = "AUTORAG_API_KEY", API_PATH = "/v1", MODELS_PATH = "/v1/models", AUTHORIZE_PATH = "/desktop/authorize", TOKEN_PATH = "/api/desktop/token", AUTORAG_INPUT_LIMITS, OAUTH_CLIENT_ID = "agent", API_KEY_CONFIG_VALUE;
7881
+ var init_config = __esm(() => {
7882
+ AUTORAG_INPUT_LIMITS = {
7883
+ maxRequestBytes: 3500000,
7884
+ images: { resize: { maxBytes: 1e6 } }
7885
+ };
7886
+ API_KEY_CONFIG_VALUE = `$${AUTORAG_API_KEY_ENV}`;
7887
+ });
7888
+
7715
7889
  // src/retrieval/rerank.ts
7716
7890
  import { OpenRouter } from "@openrouter/sdk";
7891
+ function defaultRerankModel(provider) {
7892
+ return provider === PROVIDER_ID ? DEFAULT_AUTORAG_RERANK_MODEL : DEFAULT_RERANK_MODEL;
7893
+ }
7894
+ function defaultRerankApiKeyEnv(provider) {
7895
+ return provider === PROVIDER_ID ? DEFAULT_AUTORAG_RERANK_API_KEY_ENV : DEFAULT_RERANK_API_KEY_ENV;
7896
+ }
7717
7897
  function createReranker(config) {
7718
7898
  if (config === false || config === undefined)
7719
7899
  return;
@@ -7721,30 +7901,36 @@ function createReranker(config) {
7721
7901
  if (!SUPPORTED_RERANK_PROVIDERS.includes(provider)) {
7722
7902
  throw new Error(`Unsupported rerank provider "${provider}"; supported providers: ${SUPPORTED_RERANK_PROVIDERS.join(", ")}`);
7723
7903
  }
7724
- return new OpenRouterReranker(config);
7904
+ return new OpenRouterReranker({ ...config, provider });
7905
+ }
7906
+ function autoragApiBaseUrl(env) {
7907
+ const root = resolveBaseUrl(env);
7908
+ return root.endsWith(API_PATH) ? root : `${root}${API_PATH}`;
7725
7909
  }
7726
7910
 
7727
7911
  class OpenRouterReranker {
7728
7912
  model;
7729
- provider = DEFAULT_RERANK_PROVIDER;
7913
+ provider;
7730
7914
  client;
7731
7915
  unavailableReason;
7732
7916
  constructor(options = {}) {
7733
- this.model = options.model ?? DEFAULT_RERANK_MODEL;
7917
+ this.provider = options.provider ?? DEFAULT_RERANK_PROVIDER;
7918
+ this.model = options.model ?? defaultRerankModel(this.provider);
7734
7919
  if (options.client !== undefined) {
7735
7920
  this.client = options.client;
7736
7921
  return;
7737
7922
  }
7738
- const apiKeyEnv = options.apiKeyEnv ?? DEFAULT_RERANK_API_KEY_ENV;
7739
7923
  const env = options.env ?? process.env;
7924
+ const apiKeyEnv = options.apiKeyEnv ?? defaultRerankApiKeyEnv(this.provider);
7740
7925
  const apiKey = options.apiKey ?? env[apiKeyEnv];
7741
7926
  if (apiKey === undefined || apiKey.length === 0) {
7742
7927
  this.unavailableReason = `no API key: set ${apiKeyEnv} or configure rerank.apiKeyEnv`;
7743
7928
  return;
7744
7929
  }
7930
+ const baseUrl = options.baseUrl ?? (this.provider === PROVIDER_ID ? autoragApiBaseUrl(env) : undefined);
7745
7931
  const sdkOptions = { apiKey };
7746
- if (options.baseUrl !== undefined)
7747
- sdkOptions.serverURL = options.baseUrl;
7932
+ if (baseUrl !== undefined)
7933
+ sdkOptions.serverURL = baseUrl;
7748
7934
  if (options.timeoutMs !== undefined)
7749
7935
  sdkOptions.timeoutMs = options.timeoutMs;
7750
7936
  const sdk = new OpenRouter(sdkOptions);
@@ -7809,9 +7995,10 @@ class OpenRouterReranker {
7809
7995
  return reranked;
7810
7996
  }
7811
7997
  }
7812
- var DEFAULT_RERANK_MODEL = "voyageai/rerank-3-lite", DEFAULT_RERANK_PROVIDER = "openrouter", DEFAULT_RERANK_API_KEY_ENV = "OPENROUTER_API_KEY", DEFAULT_RERANK_TOP_N = 25, SUPPORTED_RERANK_PROVIDERS;
7998
+ var DEFAULT_RERANK_MODEL = "voyageai/rerank-3-lite", DEFAULT_RERANK_PROVIDER = "openrouter", DEFAULT_RERANK_API_KEY_ENV = "OPENROUTER_API_KEY", DEFAULT_AUTORAG_RERANK_MODEL = "qwen/qwen3-reranker-8b", DEFAULT_AUTORAG_RERANK_API_KEY_ENV = "AUTORAG_API_KEY", DEFAULT_RERANK_TOP_N = 25, SUPPORTED_RERANK_PROVIDERS;
7813
7999
  var init_rerank = __esm(() => {
7814
- SUPPORTED_RERANK_PROVIDERS = [DEFAULT_RERANK_PROVIDER];
8000
+ init_config();
8001
+ SUPPORTED_RERANK_PROVIDERS = [DEFAULT_RERANK_PROVIDER, PROVIDER_ID];
7815
8002
  });
7816
8003
 
7817
8004
  // src/retrieval/selection.ts
@@ -9030,22 +9217,219 @@ var init_fsearch_search_tool = __esm(() => {
9030
9217
  });
9031
9218
  });
9032
9219
 
9033
- // src/cloud/config.ts
9034
- function normalizeBaseUrl(value) {
9035
- const trimmed = value.trim();
9036
- let end = trimmed.length;
9037
- while (end > 0 && trimmed.charCodeAt(end - 1) === 47)
9038
- end--;
9039
- return trimmed.slice(0, end);
9220
+ // src/cloud/jev-backend.ts
9221
+ import {
9222
+ BackendError
9223
+ } from "jev-use";
9224
+
9225
+ class AutoragJevBackend {
9226
+ name = "autorag";
9227
+ apiKey;
9228
+ baseUrl;
9229
+ defaultModel;
9230
+ timeoutMs;
9231
+ maxRetries;
9232
+ fetchImpl;
9233
+ constructor(options = {}) {
9234
+ const env = options.env ?? process.env;
9235
+ const apiKey = options.apiKey ?? env[AUTORAG_API_KEY_ENV];
9236
+ if (apiKey === undefined || apiKey.trim() === "") {
9237
+ throw new Error(`jev.backend "autorag" needs ${AUTORAG_API_KEY_ENV}.`);
9238
+ }
9239
+ this.apiKey = apiKey;
9240
+ this.baseUrl = options.baseUrl ?? resolveBaseUrl(env);
9241
+ this.defaultModel = options.defaultModel ?? AUTORAG_DEFAULT_JEV_MODEL;
9242
+ this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS6;
9243
+ this.maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
9244
+ this.fetchImpl = options.fetchImpl;
9245
+ }
9246
+ async judge(request) {
9247
+ const model = request.model ?? this.defaultModel;
9248
+ const body = toNativeBody(request.state, request.questions, model);
9249
+ const started = Date.now();
9250
+ const response = await this.postJson(`${this.baseUrl}${JEV_PATH}`, body);
9251
+ return {
9252
+ answers: parseNativeAnswers(this.name, response.answers, request.questions),
9253
+ model: response.model ?? model,
9254
+ latencyMs: Date.now() - started,
9255
+ usage: {
9256
+ inputTokens: response.usage?.input_tokens,
9257
+ outputTokens: response.usage?.output_tokens
9258
+ }
9259
+ };
9260
+ }
9261
+ async postJson(url, body) {
9262
+ const doFetch = this.fetchImpl ?? fetch;
9263
+ let lastError;
9264
+ for (let attempt = 0;attempt <= this.maxRetries; attempt += 1) {
9265
+ if (attempt > 0)
9266
+ await sleep3(retryDelay(attempt, lastError));
9267
+ const controller = new AbortController;
9268
+ const timer = setTimeout(() => controller.abort(), this.timeoutMs);
9269
+ try {
9270
+ const response = await doFetch(url, {
9271
+ method: "POST",
9272
+ headers: {
9273
+ "Content-Type": "application/json",
9274
+ Accept: "application/json",
9275
+ Authorization: `Bearer ${this.apiKey}`
9276
+ },
9277
+ body: JSON.stringify(body),
9278
+ signal: controller.signal
9279
+ });
9280
+ if (response.ok)
9281
+ return await response.json();
9282
+ const message = await extractErrorMessage(response);
9283
+ lastError = new BackendError(this.name, `HTTP ${response.status}: ${message}`, response.status);
9284
+ lastError.retryAfterMs = parseRetryAfter(response);
9285
+ if (!isRetryable(response.status) || attempt === this.maxRetries)
9286
+ throw lastError;
9287
+ } catch (error) {
9288
+ if (error instanceof BackendError) {
9289
+ if (attempt === this.maxRetries || !isRetryable(error.status))
9290
+ throw error;
9291
+ continue;
9292
+ }
9293
+ const aborted = error?.name === "AbortError";
9294
+ lastError = new BackendError(this.name, aborted ? `request timed out after ${this.timeoutMs}ms` : `network error: ${String(error)}`, undefined, error);
9295
+ if (attempt === this.maxRetries)
9296
+ throw lastError;
9297
+ } finally {
9298
+ clearTimeout(timer);
9299
+ }
9300
+ }
9301
+ throw lastError ?? new BackendError(this.name, "unreachable");
9302
+ }
9040
9303
  }
9041
- function resolveBaseUrl(env) {
9042
- const configured = env[AUTORAG_BASE_URL_ENV];
9043
- const value = typeof configured === "string" && configured.trim() !== "" ? configured : DEFAULT_BASE_URL;
9044
- return normalizeBaseUrl(value);
9304
+ function serializeState(state) {
9305
+ return typeof state === "string" ? state : JSON.stringify(state);
9045
9306
  }
9046
- var PROVIDER_ID = "autorag", PROVIDER_NAME = "AutoRAG", DEFAULT_BASE_URL = "https://api.dazziapp.com", AUTORAG_BASE_URL_ENV = "AUTORAG_BASE_URL", AUTORAG_API_KEY_ENV = "AUTORAG_API_KEY", API_PATH = "/v1", MODELS_PATH = "/v1/models", AUTHORIZE_PATH = "/desktop/authorize", TOKEN_PATH = "/api/desktop/token", OAUTH_CLIENT_ID = "agent", API_KEY_CONFIG_VALUE;
9047
- var init_config = __esm(() => {
9048
- API_KEY_CONFIG_VALUE = `$${AUTORAG_API_KEY_ENV}`;
9307
+ function toNativeQuestion(question) {
9308
+ switch (question.type) {
9309
+ case "noul":
9310
+ return {
9311
+ type: "noul",
9312
+ instructions: question.question,
9313
+ ...question.criteria ? { criteria: question.criteria } : {}
9314
+ };
9315
+ case "choice":
9316
+ return {
9317
+ type: "choice",
9318
+ instructions: question.question,
9319
+ criteria: Object.fromEntries(optionEntries(question.options ?? []).map(([label, meaning]) => [label, meaning || label]))
9320
+ };
9321
+ case "score":
9322
+ return {
9323
+ type: "score",
9324
+ instructions: question.question,
9325
+ criteria: question.levels ?? []
9326
+ };
9327
+ }
9328
+ }
9329
+ function toNativeBody(state, questions, model) {
9330
+ return {
9331
+ model,
9332
+ state: serializeState(state),
9333
+ questions: Object.fromEntries(questions.map((question) => [question.id, toNativeQuestion(question)]))
9334
+ };
9335
+ }
9336
+ function parseNativeAnswers(backend, answers, questions) {
9337
+ if (!answers || typeof answers !== "object") {
9338
+ throw new BackendError(backend, "response has no answers object");
9339
+ }
9340
+ const byId = answers;
9341
+ return questions.map((question) => parseNativeAnswer(backend, question, byId[question.id]));
9342
+ }
9343
+ function parseNativeAnswer(backend, question, answer) {
9344
+ if (!answer) {
9345
+ throw new BackendError(backend, `response missing answer for question "${question.id}"`);
9346
+ }
9347
+ switch (question.type) {
9348
+ case "noul":
9349
+ return {
9350
+ answer: numberOrThrow(backend, question.id, answer.noul),
9351
+ confidence: answer.confidence
9352
+ };
9353
+ case "choice":
9354
+ return {
9355
+ answer: stringOrThrow(backend, question.id, answer.choice),
9356
+ confidence: answer.confidence,
9357
+ distribution: answer.probabilities
9358
+ };
9359
+ case "score":
9360
+ return {
9361
+ answer: numberOrThrow(backend, question.id, answer.score),
9362
+ confidence: answer.confidence,
9363
+ distribution: answer.probabilities,
9364
+ legend: answer.legend
9365
+ };
9366
+ }
9367
+ }
9368
+ function optionEntries(options) {
9369
+ return Array.isArray(options) ? options.map((label) => [label, ""]) : Object.entries(options);
9370
+ }
9371
+ function numberOrThrow(backend, id, value) {
9372
+ if (typeof value !== "number" || !Number.isFinite(value)) {
9373
+ throw new BackendError(backend, `answer "${id}" has no numeric value`);
9374
+ }
9375
+ return value;
9376
+ }
9377
+ function stringOrThrow(backend, id, value) {
9378
+ if (typeof value !== "string" || !value) {
9379
+ throw new BackendError(backend, `answer "${id}" has no choice value`);
9380
+ }
9381
+ return value;
9382
+ }
9383
+ function isRetryable(status) {
9384
+ if (status === undefined)
9385
+ return true;
9386
+ return status === 408 || status === 429 || status >= 500;
9387
+ }
9388
+ function retryDelay(attempt, lastError) {
9389
+ if (lastError?.retryAfterMs !== undefined)
9390
+ return Math.min(lastError.retryAfterMs, 60000);
9391
+ const base = Math.min(RETRY_BASE_MS * 2 ** (attempt - 1), RETRY_MAX_MS);
9392
+ return base * (1 + Math.random() * 0.25);
9393
+ }
9394
+ function parseRetryAfter(response) {
9395
+ const milliseconds = response.headers.get("retry-after-ms");
9396
+ if (milliseconds && Number.isFinite(Number(milliseconds)))
9397
+ return Number(milliseconds);
9398
+ const seconds = response.headers.get("retry-after");
9399
+ if (seconds && Number.isFinite(Number(seconds)))
9400
+ return Number(seconds) * 1000;
9401
+ return;
9402
+ }
9403
+ async function extractErrorMessage(response) {
9404
+ try {
9405
+ const data = await response.json();
9406
+ for (const key of ["error", "message", "detail"]) {
9407
+ const value = data[key];
9408
+ if (typeof value === "string")
9409
+ return value;
9410
+ if (value && typeof value === "object") {
9411
+ const message = value.message;
9412
+ if (typeof message === "string")
9413
+ return message;
9414
+ }
9415
+ if (Array.isArray(value) && value.length > 0) {
9416
+ const first = value[0];
9417
+ if (typeof first?.msg === "string")
9418
+ return first.msg;
9419
+ }
9420
+ }
9421
+ return JSON.stringify(data).slice(0, 300);
9422
+ } catch {
9423
+ return response.statusText || "request failed";
9424
+ }
9425
+ }
9426
+ function sleep3(ms) {
9427
+ return new Promise((resolve) => setTimeout(resolve, ms));
9428
+ }
9429
+ var JEV_PATH, AUTORAG_DEFAULT_JEV_MODEL = "jev-1.13", DEFAULT_TIMEOUT_MS6 = 1e4, DEFAULT_MAX_RETRIES = 2, RETRY_BASE_MS = 500, RETRY_MAX_MS = 5000;
9430
+ var init_jev_backend = __esm(() => {
9431
+ init_config();
9432
+ JEV_PATH = `${API_PATH}/jev`;
9049
9433
  });
9050
9434
 
9051
9435
  // src/cloud/errors.ts
@@ -9176,7 +9560,8 @@ function mapServerModel(entry) {
9176
9560
  cost: readPricing(entry.pricing),
9177
9561
  reasoning: entry.reasoning === true,
9178
9562
  contextWindow: readPositiveNumber(entry.context_window) ?? DEFAULT_CONTEXT_WINDOW,
9179
- maxTokens: readPositiveNumber(entry.max_output_tokens) ?? DEFAULT_MAX_TOKENS
9563
+ maxTokens: readPositiveNumber(entry.max_output_tokens) ?? DEFAULT_MAX_TOKENS,
9564
+ inputLimits: AUTORAG_INPUT_LIMITS
9180
9565
  };
9181
9566
  }
9182
9567
  function buildStoreEntry(models, providerId, baseUrl) {
@@ -9215,7 +9600,8 @@ function modelsFromStore(stored) {
9215
9600
  cost: entry.cost,
9216
9601
  reasoning: entry.reasoning,
9217
9602
  contextWindow: entry.contextWindow,
9218
- maxTokens: entry.maxTokens
9603
+ maxTokens: entry.maxTokens,
9604
+ inputLimits: AUTORAG_INPUT_LIMITS
9219
9605
  });
9220
9606
  }
9221
9607
  return models;
@@ -9623,69 +10009,456 @@ var init_provider = __esm(() => {
9623
10009
  init_oauth_login();
9624
10010
  });
9625
10011
 
9626
- // src/agent/pi-session.ts
9627
- import { join as join21 } from "node:path";
9628
- import { streamSimple as compatStreamSimple } from "@earendil-works/pi-ai/compat";
9629
- import {
9630
- createAgentSession,
9631
- createAgentSessionFromServices,
9632
- createAgentSessionRuntime,
9633
- createAgentSessionServices,
9634
- DefaultResourceLoader,
9635
- getAgentDir,
9636
- ModelRuntime,
9637
- SessionManager,
9638
- SettingsManager
9639
- } from "@earendil-works/pi-coding-agent";
9640
- function resolveInteractiveTuiMode(settingsManager) {
9641
- const saved = settingsManager.getProjectSettings().tuiMode ?? settingsManager.getGlobalSettings().tuiMode;
9642
- return saved ?? "regular";
9643
- }
9644
- function toToolDefinition(tool) {
9645
- return {
9646
- name: tool.name,
9647
- label: tool.label,
9648
- description: tool.description,
9649
- parameters: tool.parameters,
9650
- ...tool.prepareArguments === undefined ? {} : { prepareArguments: tool.prepareArguments },
9651
- execute: async (toolCallId, params, signal, onUpdate) => {
9652
- return tool.execute(toolCallId, params, signal, onUpdate);
9653
- }
10012
+ // src/agent/context-token-guard.ts
10013
+ import { calculateContextTokens, estimateTokens } from "@earendil-works/pi-coding-agent";
10014
+ function createContextTokenGuardedTransform(model, base, fixedOverheadTokens = () => 0) {
10015
+ return async (messages) => {
10016
+ const transformed = base === undefined ? messages : await base(messages);
10017
+ const currentModel = typeof model === "function" ? model() : model;
10018
+ return guardAgentContextMessages(currentModel, transformed, transformed === messages, fixedOverheadTokens());
9654
10019
  };
9655
10020
  }
9656
- function modelConfig(model, configuredApiKey) {
9657
- return {
9658
- name: model.name,
9659
- baseUrl: model.baseUrl,
9660
- api: model.api,
9661
- ...configuredApiKey === undefined ? {} : { apiKey: configuredApiKey },
9662
- authHeader: false,
9663
- streamSimple: (requestModel, context, options) => compatStreamSimple(requestModel, context, {
9664
- ...options,
9665
- ...configuredApiKey === EMBEDDED_AUTH_MARKER ? { apiKey: undefined } : {}
9666
- }),
9667
- models: [
9668
- {
9669
- id: model.id,
9670
- name: model.name,
9671
- reasoning: model.reasoning,
9672
- input: [...model.input],
9673
- cost: model.cost,
9674
- contextWindow: model.contextWindow,
9675
- maxTokens: model.maxTokens,
9676
- ...model.thinkingLevelMap === undefined ? {} : { thinkingLevelMap: model.thinkingLevelMap }
10021
+ function guardAgentContextMessages(model, messages, useReportedUsage = true, fixedOverheadTokens = 0) {
10022
+ if (model === undefined || !Number.isFinite(model.contextWindow) || model.contextWindow <= 0 || messages.length === 0) {
10023
+ return messages;
10024
+ }
10025
+ const usableContext = Math.floor(model.contextWindow / CONTEXT_HEADROOM);
10026
+ const outputReserve = Math.min(usableContext, Number.isFinite(model.maxTokens) ? Math.max(0, model.maxTokens) : 0);
10027
+ const inputBudget = Math.max(0, usableContext - outputReserve);
10028
+ const budget = Math.max(0, inputBudget - fixedOverheadTokens);
10029
+ const reported = useReportedUsage && findLastAssistantUsage(messages) !== undefined;
10030
+ if (estimateContextMessageTokens(messages, useReportedUsage) <= (reported ? inputBudget : budget)) {
10031
+ return messages;
10032
+ }
10033
+ const candidates = [];
10034
+ const candidateIndexes = [];
10035
+ const base = messages.slice();
10036
+ for (const [index, message] of base.entries()) {
10037
+ const text = textContent(message);
10038
+ if (text === undefined)
10039
+ continue;
10040
+ const section = parseCandidateSection(message, text);
10041
+ if (section === undefined)
10042
+ continue;
10043
+ candidates.push(section);
10044
+ candidateIndexes.push(index);
10045
+ base[index] = replaceTextContent(message, `${section.prefix}${section.suffix}`);
10046
+ }
10047
+ let guarded = messages;
10048
+ if (candidates.length > 0) {
10049
+ const fixedTokens = estimateContextMessageTokens(base, false);
10050
+ const maximumOmitted = candidates.reduce((total, section) => total + section.blocks.length, 0);
10051
+ const omissionMarker = (count) => `
10052
+
10053
+ ${GUARD_MARKER} Retrieval candidates omitted: ${count}.
10054
+ `;
10055
+ let remaining = Math.max(0, budget - fixedTokens - Math.ceil(omissionMarker(maximumOmitted).length / 4));
10056
+ const kept = candidates.map(() => []);
10057
+ let omitted = 0;
10058
+ let firstOmitted = -1;
10059
+ for (const [sectionIndex, section] of candidates.entries()) {
10060
+ for (const block of section.blocks) {
10061
+ const tokens = Math.max(1, Math.ceil(block.length / 4));
10062
+ if (tokens <= remaining) {
10063
+ kept[sectionIndex]?.push(block);
10064
+ remaining -= tokens;
10065
+ continue;
10066
+ }
10067
+ omitted += section.blocks.length - (kept[sectionIndex]?.length ?? 0);
10068
+ for (const later of candidates.slice(sectionIndex + 1))
10069
+ omitted += later.blocks.length;
10070
+ firstOmitted = sectionIndex;
10071
+ break;
9677
10072
  }
9678
- ]
9679
- };
10073
+ if (firstOmitted !== -1)
10074
+ break;
10075
+ }
10076
+ if (omitted > 0) {
10077
+ const candidateByMessage = new Map;
10078
+ for (const [sectionIndex, messageIndex] of candidateIndexes.entries())
10079
+ candidateByMessage.set(messageIndex, sectionIndex);
10080
+ guarded = base.map((message, index) => {
10081
+ const sectionIndex = candidateByMessage.get(index);
10082
+ if (sectionIndex === undefined)
10083
+ return message;
10084
+ const section = candidates[sectionIndex];
10085
+ const marker = sectionIndex === firstOmitted ? omissionMarker(omitted) : "";
10086
+ return replaceTextContent(message, `${section.prefix}${kept[sectionIndex]?.join("") ?? ""}${marker}${section.suffix}`);
10087
+ });
10088
+ }
10089
+ }
10090
+ if (estimateContextMessageTokens(guarded, false) <= budget && guarded !== messages)
10091
+ return guarded;
10092
+ return trimFixedContext(guarded, budget);
9680
10093
  }
9681
- async function configureModelRuntime(modelRuntime, model, apiKey, providerApiKeys) {
9682
- if (model !== undefined) {
9683
- const configuredKey = apiKey ?? providerApiKeys?.[model.provider];
9684
- if (modelRuntime.getModel(model.provider, model.id) === undefined) {
9685
- modelRuntime.registerProvider(model.provider, modelConfig(model, configuredKey ?? EMBEDDED_AUTH_MARKER));
10094
+ function estimateContextMessageTokens(messages, useReportedUsage = false) {
10095
+ if (useReportedUsage) {
10096
+ const usage = findLastAssistantUsage(messages);
10097
+ if (usage !== undefined) {
10098
+ return calculateContextTokens(usage.usage) + messages.slice(usage.index + 1).reduce((total, message) => total + estimateMessageTokens(message), 0);
9686
10099
  }
9687
10100
  }
9688
- const keys = new Map;
10101
+ return messages.reduce((total, message) => total + estimateMessageTokens(message), 0);
10102
+ }
10103
+ function trimFixedContext(messages, inputBudget) {
10104
+ const trimmed = messages.slice();
10105
+ const lastIndex = trimmed.length - 1;
10106
+ const shrinkable = trimmed.map((message, index) => ({ message, index })).filter(({ index, message }) => index !== lastIndex && message.role !== "system");
10107
+ for (const { message, index } of shrinkable) {
10108
+ const replacement = replaceMessageWithMarker(message);
10109
+ if (replacement === message || estimateMessageTokens(replacement) >= estimateMessageTokens(message))
10110
+ continue;
10111
+ trimmed[index] = replacement;
10112
+ if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
10113
+ return trimmed;
10114
+ }
10115
+ const lastMessage = trimmed[lastIndex];
10116
+ if (lastMessage !== undefined && lastMessage.role !== "system") {
10117
+ const replacement = replaceMessageWithMarker(lastMessage);
10118
+ if (replacement !== lastMessage) {
10119
+ trimmed[lastIndex] = replacement;
10120
+ if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
10121
+ return trimmed;
10122
+ }
10123
+ const fixedWithoutLast = trimmed.reduce((total, message, index) => index === lastIndex ? total : total + estimateMessageTokens(message), 0);
10124
+ if (fixedWithoutLast < inputBudget) {
10125
+ const fitted = fitMessageToTokens(lastMessage, inputBudget - fixedWithoutLast);
10126
+ if (fitted !== lastMessage) {
10127
+ trimmed[lastIndex] = fitted;
10128
+ if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
10129
+ return trimmed;
10130
+ }
10131
+ }
10132
+ }
10133
+ for (const [index, message] of trimmed.entries()) {
10134
+ if (message.role !== "system")
10135
+ continue;
10136
+ const replacement = replaceMessageWithMarker(message);
10137
+ if (replacement === message || estimateMessageTokens(replacement) >= estimateMessageTokens(message))
10138
+ continue;
10139
+ trimmed[index] = replacement;
10140
+ if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
10141
+ return trimmed;
10142
+ }
10143
+ return trimmed;
10144
+ }
10145
+ function fitMessageToTokens(message, maxTokens) {
10146
+ const original = textContent(message);
10147
+ if (original === undefined)
10148
+ return message;
10149
+ const marker = `
10150
+
10151
+ ${GUARD_MARKER}
10152
+ `;
10153
+ let low = 0;
10154
+ let high = original.length;
10155
+ let best = replaceTextContent(message, marker);
10156
+ while (low <= high) {
10157
+ const middle = Math.floor((low + high) / 2);
10158
+ const candidate = replaceTextContent(message, excerpt(original, middle, marker));
10159
+ if (estimateMessageTokens(candidate) <= maxTokens) {
10160
+ best = candidate;
10161
+ low = middle + 1;
10162
+ } else {
10163
+ high = middle - 1;
10164
+ }
10165
+ }
10166
+ return best;
10167
+ }
10168
+ function excerpt(text, maxChars, marker) {
10169
+ if (text.length <= maxChars)
10170
+ return text;
10171
+ const available = Math.max(0, maxChars - marker.length);
10172
+ const head = Math.ceil(available * 0.7);
10173
+ const tail = Math.max(0, available - head);
10174
+ return `${text.slice(0, head)}${marker}${tail > 0 ? text.slice(-tail) : ""}`;
10175
+ }
10176
+ function replaceMessageWithMarker(message) {
10177
+ return replaceTextContent(message, `
10178
+
10179
+ ${GUARD_MARKER}
10180
+ `);
10181
+ }
10182
+ function estimateMessageTokens(message) {
10183
+ if (message.role === "system")
10184
+ return estimateSystemMessageTokens(message);
10185
+ const estimate = estimateTokens(message);
10186
+ return message.role === "toolResult" ? estimate + Math.ceil(message.toolName.length / 4) : estimate;
10187
+ }
10188
+ function estimateSystemMessageTokens(message) {
10189
+ let chars = typeof message.content === "string" ? message.content.length : 0;
10190
+ if (Array.isArray(message.content)) {
10191
+ for (const block of message.content) {
10192
+ if (block.type === "text")
10193
+ chars += block.text.length;
10194
+ else if (block.type === "image")
10195
+ chars += 4800;
10196
+ }
10197
+ }
10198
+ chars += JSON.stringify(message.sections ?? {}).length;
10199
+ chars += JSON.stringify(message.toolsAdded ?? []).length;
10200
+ chars += JSON.stringify(message.toolsRemoved ?? []).length;
10201
+ return Math.ceil(chars / 4);
10202
+ }
10203
+ function findLastAssistantUsage(messages) {
10204
+ for (let index = messages.length - 1;index >= 0; index--) {
10205
+ const message = messages[index];
10206
+ if (message?.role !== "assistant" || message.stopReason === "aborted" || message.stopReason === "error")
10207
+ continue;
10208
+ if (calculateContextTokens(message.usage) > 0)
10209
+ return { index, usage: message.usage };
10210
+ }
10211
+ return;
10212
+ }
10213
+ function textContent(message) {
10214
+ if (message.role === "system") {
10215
+ if (typeof message.content === "string")
10216
+ return message.content;
10217
+ return message.content.filter((block) => block.type === "text").map((block) => block.text).join("");
10218
+ }
10219
+ if (message.role === "user" || message.role === "toolResult") {
10220
+ if (typeof message.content === "string")
10221
+ return message.content;
10222
+ let text = "";
10223
+ for (const block of message.content) {
10224
+ if (block.type !== "text")
10225
+ return;
10226
+ text += block.text;
10227
+ }
10228
+ return text;
10229
+ }
10230
+ if (message.role === "assistant") {
10231
+ let text = "";
10232
+ for (const block of message.content)
10233
+ if (block.type === "text")
10234
+ text += block.text;
10235
+ return text.length > 0 ? text : undefined;
10236
+ }
10237
+ return;
10238
+ }
10239
+ function replaceTextContent(message, text) {
10240
+ if (message.role === "system")
10241
+ return { ...message, content: text };
10242
+ if (message.role === "user") {
10243
+ if (typeof message.content === "string")
10244
+ return { ...message, content: text };
10245
+ return { ...message, content: [{ type: "text", text }] };
10246
+ }
10247
+ if (message.role === "toolResult")
10248
+ return { ...message, content: [{ type: "text", text }] };
10249
+ if (message.role === "assistant") {
10250
+ const content = [];
10251
+ let replaced = false;
10252
+ for (const block of message.content) {
10253
+ if (block.type !== "text") {
10254
+ content.push(block);
10255
+ continue;
10256
+ }
10257
+ if (replaced)
10258
+ continue;
10259
+ replaced = true;
10260
+ content.push({ ...block, text });
10261
+ }
10262
+ return replaced ? { ...message, content } : message;
10263
+ }
10264
+ return message;
10265
+ }
10266
+ function parseCandidateSection(message, text) {
10267
+ if (message.role === "user") {
10268
+ const markerMatch = /(?:Baseline retrieval (?:evidence \(already gathered for you\)|context):|initial candidates:)/u.exec(text);
10269
+ if (markerMatch === null)
10270
+ return;
10271
+ const sectionStart = (markerMatch.index ?? 0) + markerMatch[0].length;
10272
+ const sectionEnd = findBaselineSectionEnd(text, sectionStart);
10273
+ if (!/^\[e?\d+\] /mu.test(text.slice(sectionStart, sectionEnd)))
10274
+ return;
10275
+ return splitCandidates(text, sectionStart, sectionEnd);
10276
+ }
10277
+ if (message.role !== "toolResult")
10278
+ return;
10279
+ const isRetrieval = RETRIEVAL_TOOL_NAMES[message.toolName] === true || message.toolName.startsWith(RETRIEVAL_TOOL_PREFIX);
10280
+ if (!isRetrieval)
10281
+ return;
10282
+ if (message.toolName === "jikji_find") {
10283
+ const directiveStart = text.indexOf(`
10284
+
10285
+ directive:`);
10286
+ return splitCandidates(text, 0, directiveStart >= 0 ? directiveStart : text.length, /^- [^\n]*(?:\n|$)/gmu);
10287
+ }
10288
+ const diagnosticsStart = text.indexOf(`
10289
+
10290
+ Diagnostics:`);
10291
+ return splitCandidates(text, 0, diagnosticsStart >= 0 ? diagnosticsStart : text.length);
10292
+ }
10293
+ function findBaselineSectionEnd(text, startIndex) {
10294
+ const endMarkers = [
10295
+ `
10296
+
10297
+ Produce the best`,
10298
+ `
10299
+
10300
+ Treat candidates as unverified evidence`,
10301
+ `
10302
+
10303
+ Formatting and content rules`
10304
+ ];
10305
+ const positions = endMarkers.map((marker) => text.indexOf(marker, startIndex)).filter((index) => index >= 0);
10306
+ return positions.length > 0 ? Math.min(...positions) : text.length;
10307
+ }
10308
+ function splitCandidates(text, sectionStart, sectionEnd, pattern = /^\[e?\d+\] [^\n]*(?:\n|$)/gmu) {
10309
+ const region = text.slice(sectionStart, sectionEnd);
10310
+ const matches = [...region.matchAll(pattern)];
10311
+ const firstStart = matches[0]?.index;
10312
+ if (firstStart === undefined)
10313
+ return;
10314
+ const blocks = [];
10315
+ for (const [index, match] of matches.entries()) {
10316
+ const start = match.index ?? 0;
10317
+ const end = index + 1 < matches.length ? matches[index + 1]?.index ?? region.length : region.length;
10318
+ blocks.push(region.slice(start, end));
10319
+ }
10320
+ return {
10321
+ prefix: text.slice(0, sectionStart) + region.slice(0, firstStart),
10322
+ suffix: text.slice(sectionEnd),
10323
+ blocks
10324
+ };
10325
+ }
10326
+ var CONTEXT_HEADROOM = 1.2, GUARD_MARKER = "[AutoRAG context guard: omitted context to stay within the model context budget.]", RETRIEVAL_TOOL_PREFIX = "search_datasource_", RETRIEVAL_TOOL_NAMES;
10327
+ var init_context_token_guard = __esm(() => {
10328
+ RETRIEVAL_TOOL_NAMES = {
10329
+ jikji_find: true,
10330
+ search_all_documents: true,
10331
+ semantic_search_local_docs: true,
10332
+ search_single_datasource_documents: true,
10333
+ everything_search: true,
10334
+ fsearch_search: true
10335
+ };
10336
+ });
10337
+
10338
+ // src/agent/request-limits.ts
10339
+ function fitRequestLimits(messages, limits) {
10340
+ const maxBytes = limits?.maxRequestBytes;
10341
+ const maxImages = limits?.images?.maxPerRequest;
10342
+ if (maxBytes === undefined && maxImages === undefined)
10343
+ return;
10344
+ const images = imageLocations(messages);
10345
+ if (images.length === 0)
10346
+ return;
10347
+ let keptImages = images.length;
10348
+ let bytes = maxBytes === undefined ? 0 : Buffer.byteLength(JSON.stringify(messages));
10349
+ const dropped = [];
10350
+ for (const image of images) {
10351
+ const overCount = maxImages !== undefined && keptImages > maxImages;
10352
+ const overBytes = maxBytes !== undefined && bytes > maxBytes;
10353
+ if (!overCount && !overBytes)
10354
+ break;
10355
+ dropped.push(image);
10356
+ keptImages--;
10357
+ bytes -= image.bytes - NOTE_BYTES;
10358
+ }
10359
+ if (dropped.length === 0)
10360
+ return;
10361
+ const byMessage = new Map;
10362
+ for (const { message, block } of dropped) {
10363
+ const blocks = byMessage.get(message) ?? new Set;
10364
+ blocks.add(block);
10365
+ byMessage.set(message, blocks);
10366
+ }
10367
+ return messages.map((message, index) => {
10368
+ const blocks = byMessage.get(index);
10369
+ if (blocks === undefined)
10370
+ return message;
10371
+ const content = message.content.map((part, block) => blocks.has(block) ? { type: "text", text: IMAGE_OMITTED_NOTE } : part);
10372
+ return { ...message, content };
10373
+ });
10374
+ }
10375
+ function imageLocations(messages) {
10376
+ const locations = [];
10377
+ messages.forEach((message, index) => {
10378
+ const content = message.content;
10379
+ if (!Array.isArray(content))
10380
+ return;
10381
+ content.forEach((part, block) => {
10382
+ if (!isImage(part))
10383
+ return;
10384
+ locations.push({ message: index, block, bytes: Buffer.byteLength(JSON.stringify(part)) });
10385
+ });
10386
+ });
10387
+ return locations;
10388
+ }
10389
+ function isImage(part) {
10390
+ return typeof part === "object" && part !== null && part.type === "image";
10391
+ }
10392
+ var IMAGE_OMITTED_NOTE = "[Image omitted from this request to stay within the model provider's request limits. Read the file again if you still need to see it.]", NOTE_BYTES;
10393
+ var init_request_limits = __esm(() => {
10394
+ NOTE_BYTES = Buffer.byteLength(JSON.stringify({ type: "text", text: IMAGE_OMITTED_NOTE }));
10395
+ });
10396
+
10397
+ // src/agent/pi-session.ts
10398
+ import { join as join21 } from "node:path";
10399
+ import { streamSimple as compatStreamSimple } from "@earendil-works/pi-ai/compat";
10400
+ import {
10401
+ createAgentSession,
10402
+ createAgentSessionFromServices,
10403
+ createAgentSessionRuntime,
10404
+ createAgentSessionServices,
10405
+ DefaultResourceLoader,
10406
+ getAgentDir,
10407
+ ModelRuntime,
10408
+ SessionManager,
10409
+ SettingsManager
10410
+ } from "@earendil-works/pi-coding-agent";
10411
+ import { getKeybindings, matchesKey } from "@earendil-works/pi-tui";
10412
+ function resolveInteractiveTuiMode(settingsManager) {
10413
+ const saved = settingsManager.getProjectSettings().tuiMode ?? settingsManager.getGlobalSettings().tuiMode;
10414
+ return saved ?? "regular";
10415
+ }
10416
+ function toToolDefinition(tool) {
10417
+ return {
10418
+ name: tool.name,
10419
+ label: tool.label,
10420
+ description: tool.description,
10421
+ parameters: tool.parameters,
10422
+ ...tool.prepareArguments === undefined ? {} : { prepareArguments: tool.prepareArguments },
10423
+ execute: async (toolCallId, params, signal, onUpdate) => {
10424
+ return tool.execute(toolCallId, params, signal, onUpdate);
10425
+ }
10426
+ };
10427
+ }
10428
+ function modelConfig(model, configuredApiKey) {
10429
+ return {
10430
+ name: model.name,
10431
+ baseUrl: model.baseUrl,
10432
+ api: model.api,
10433
+ ...configuredApiKey === undefined ? {} : { apiKey: configuredApiKey },
10434
+ authHeader: false,
10435
+ streamSimple: (requestModel, context, options) => compatStreamSimple(requestModel, context, {
10436
+ ...options,
10437
+ ...configuredApiKey === EMBEDDED_AUTH_MARKER ? { apiKey: undefined } : {}
10438
+ }),
10439
+ models: [
10440
+ {
10441
+ id: model.id,
10442
+ name: model.name,
10443
+ reasoning: model.reasoning,
10444
+ input: [...model.input],
10445
+ cost: model.cost,
10446
+ contextWindow: model.contextWindow,
10447
+ maxTokens: model.maxTokens,
10448
+ ...model.inputLimits === undefined ? {} : { inputLimits: model.inputLimits },
10449
+ ...model.thinkingLevelMap === undefined ? {} : { thinkingLevelMap: model.thinkingLevelMap }
10450
+ }
10451
+ ]
10452
+ };
10453
+ }
10454
+ async function configureModelRuntime(modelRuntime, model, apiKey, providerApiKeys) {
10455
+ if (model !== undefined) {
10456
+ const configuredKey = apiKey ?? providerApiKeys?.[model.provider];
10457
+ if (modelRuntime.getModel(model.provider, model.id) === undefined) {
10458
+ modelRuntime.registerProvider(model.provider, modelConfig(model, configuredKey ?? EMBEDDED_AUTH_MARKER));
10459
+ }
10460
+ }
10461
+ const keys = new Map;
9689
10462
  if (model !== undefined && apiKey !== undefined)
9690
10463
  keys.set(model.provider, apiKey);
9691
10464
  for (const [provider, key] of Object.entries(providerApiKeys ?? {}))
@@ -9693,20 +10466,47 @@ async function configureModelRuntime(modelRuntime, model, apiKey, providerApiKey
9693
10466
  for (const [provider, key] of keys)
9694
10467
  await modelRuntime.setRuntimeApiKey(provider, key);
9695
10468
  }
10469
+ function enforceRequestLimits(pi) {
10470
+ pi.on("context", (event, ctx) => {
10471
+ const messages = fitRequestLimits(event.messages, ctx.model?.inputLimits);
10472
+ return messages === undefined ? undefined : { messages };
10473
+ });
10474
+ }
10475
+ function guardContext(pi, contextTransform) {
10476
+ pi.on("context", async (event, ctx) => {
10477
+ const activeTools = new Set(pi.getActiveTools());
10478
+ const toolDeclarations = pi.getAllTools().filter((tool) => activeTools.has(tool.name)).map(({ name, description, parameters }) => ({ name, description, parameters }));
10479
+ const fixedOverheadTokens = Math.ceil((ctx.getSystemPrompt().length + JSON.stringify(toolDeclarations).length) / 4);
10480
+ return {
10481
+ messages: await createContextTokenGuardedTransform(ctx.model, contextTransform, () => fixedOverheadTokens)(event.messages)
10482
+ };
10483
+ });
10484
+ }
9696
10485
  function createAutoRAGExtension(getSystemPrompt, contextTransform) {
9697
10486
  return (pi) => {
9698
10487
  pi.on("before_agent_start", () => ({ systemPrompt: getSystemPrompt() }));
9699
- if (contextTransform !== undefined) {
9700
- pi.on("context", async (event) => ({ messages: await contextTransform(event.messages) }));
9701
- }
10488
+ guardContext(pi, contextTransform);
10489
+ enforceRequestLimits(pi);
10490
+ };
10491
+ }
10492
+ function interactiveKeyOf(data) {
10493
+ const bindings = getKeybindings();
10494
+ const keysOf = (action, fallback) => {
10495
+ const keys = bindings.getKeys(action);
10496
+ return keys.length > 0 ? keys : [fallback];
9702
10497
  };
10498
+ const windows = process.platform === "win32" || process.platform === "linux" && Boolean(process.env.WSL_DISTRO_NAME || process.env.WSL_INTEROP);
10499
+ if (keysOf("app.interrupt", "escape").some((key) => matchesKey(data, key)))
10500
+ return "interrupt";
10501
+ if (keysOf("app.message.followUp", windows ? "ctrl+q" : "alt+enter").some((key) => matchesKey(data, key)))
10502
+ return "followUp";
10503
+ return;
9703
10504
  }
9704
- function createAutoRAGInteractiveExtension(getSystemPrompt, contextTransform, onQuery, updateNotice) {
10505
+ function createAutoRAGInteractiveExtension(getSystemPrompt, contextTransform, options, updateNotice) {
9705
10506
  return (pi) => {
9706
10507
  pi.on("before_agent_start", () => ({ systemPrompt: getSystemPrompt() }));
9707
- if (contextTransform !== undefined) {
9708
- pi.on("context", async (event) => ({ messages: await contextTransform(event.messages) }));
9709
- }
10508
+ guardContext(pi, contextTransform);
10509
+ enforceRequestLimits(pi);
9710
10510
  if (updateNotice !== undefined) {
9711
10511
  pi.on("session_start", () => {
9712
10512
  updateNotice().then((text) => {
@@ -9722,20 +10522,37 @@ function createAutoRAGInteractiveExtension(getSystemPrompt, contextTransform, on
9722
10522
  });
9723
10523
  });
9724
10524
  }
9725
- pi.on("input", async (event) => {
9726
- const query = event.text.trim();
9727
- if (event.source !== "interactive" || query.length === 0 || query.startsWith("/"))
10525
+ const { onInput, isSearchRunning, onKey } = options;
10526
+ if (onKey !== undefined) {
10527
+ pi.on("session_start", (_event, ctx) => {
10528
+ ctx.ui.onTerminalInput((data) => {
10529
+ const key = interactiveKeyOf(data);
10530
+ return key !== undefined && onKey(key) ? { consume: true } : undefined;
10531
+ });
10532
+ });
10533
+ }
10534
+ pi.on("input", (event) => {
10535
+ const text = event.text.trim();
10536
+ if (event.source !== "interactive" || text.length === 0)
10537
+ return;
10538
+ if (text.startsWith("/") && isSearchRunning?.() !== true)
9728
10539
  return;
10540
+ let disposition;
9729
10541
  try {
9730
- await onQuery(query, pi);
10542
+ disposition = onInput({ text, images: event.images, streamingBehavior: event.streamingBehavior }, pi);
9731
10543
  } catch (error) {
9732
10544
  pi.sendMessage({
9733
10545
  customType: "autorag.error",
9734
10546
  content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
9735
10547
  display: true
9736
- });
9737
- }
9738
- return { action: "handled" };
10548
+ }, { triggerTurn: false });
10549
+ return { action: "handled" };
10550
+ }
10551
+ return disposition.kind === "steer" ? {
10552
+ action: "transform",
10553
+ text: disposition.text,
10554
+ ...event.images === undefined ? {} : { images: event.images }
10555
+ } : { action: "handled" };
9739
10556
  });
9740
10557
  };
9741
10558
  }
@@ -9815,7 +10632,7 @@ async function createAutoRAGPiInteractiveRuntime(options) {
9815
10632
  settingsManager,
9816
10633
  resourceLoaderOptions: {
9817
10634
  extensionFactories: [
9818
- createAutoRAGInteractiveExtension(options.getSystemPrompt, options.contextTransform, options.onQuery, options.updateNotice),
10635
+ createAutoRAGInteractiveExtension(options.getSystemPrompt, options.contextTransform, { onInput: options.onInput, isSearchRunning: options.isSearchRunning, onKey: options.onKey }, options.updateNotice),
9819
10636
  ...options.extensionFactories ?? []
9820
10637
  ],
9821
10638
  systemPrompt: options.getSystemPrompt(),
@@ -9850,6 +10667,8 @@ async function createAutoRAGPiInteractiveRuntime(options) {
9850
10667
  var PI_BUILTIN_TOOL_NAMES, EMBEDDED_AUTH_MARKER = "<autorag-embedded-model>";
9851
10668
  var init_pi_session = __esm(() => {
9852
10669
  init_provider();
10670
+ init_context_token_guard();
10671
+ init_request_limits();
9853
10672
  PI_BUILTIN_TOOL_NAMES = ["read", "bash", "edit", "write", "grep", "find", "ls"];
9854
10673
  });
9855
10674
 
@@ -9859,12 +10678,26 @@ import { Type as Type6 } from "typebox";
9859
10678
  function createJevJudge(options = {}) {
9860
10679
  let client;
9861
10680
  return async (state, questions, callOptions = {}) => {
9862
- client ??= new Jev({
9863
- ...options.backend !== undefined ? { backend: options.backend } : {},
9864
- ...options.env !== undefined ? { env: options.env } : {},
9865
- ...options.model !== undefined ? { model: options.model } : {},
9866
- ...options.confidenceThreshold !== undefined ? { confidenceThreshold: options.confidenceThreshold } : {}
9867
- });
10681
+ if (client === undefined) {
10682
+ const env = options.env ?? process.env;
10683
+ const explicit = options.backend;
10684
+ let backend;
10685
+ if (typeof explicit === "string") {
10686
+ backend = explicit === "autorag" ? new AutoragJevBackend({ env }) : explicit;
10687
+ } else if (explicit !== undefined) {
10688
+ backend = explicit;
10689
+ } else if (env.JEV_BACKEND?.toLowerCase() === "autorag") {
10690
+ backend = new AutoragJevBackend({ env });
10691
+ } else {
10692
+ backend = undefined;
10693
+ }
10694
+ client = new Jev({
10695
+ ...backend !== undefined ? { backend } : {},
10696
+ ...options.env !== undefined ? { env: options.env } : {},
10697
+ ...options.model !== undefined ? { model: options.model } : {},
10698
+ ...options.confidenceThreshold !== undefined ? { confidenceThreshold: options.confidenceThreshold } : {}
10699
+ });
10700
+ }
9868
10701
  const model = options.model ?? BACKEND_DEFAULT_MODEL[client.backend.name];
9869
10702
  return client.judge(state, questions, {
9870
10703
  ...model !== undefined ? { model } : {},
@@ -9906,8 +10739,12 @@ function createJevExtension(judgeOrOptions = {}) {
9906
10739
  }
9907
10740
  var JEV_TOOL_NAME = "jev", OPENROUTER_DEFAULT_MODEL = "typesafe/jev-1.13", BACKEND_DEFAULT_MODEL, jevQuestionSchema, jevParameters;
9908
10741
  var init_jev_extension = __esm(() => {
10742
+ init_jev_backend();
9909
10743
  init_pi_session();
9910
- BACKEND_DEFAULT_MODEL = { openrouter: OPENROUTER_DEFAULT_MODEL };
10744
+ BACKEND_DEFAULT_MODEL = {
10745
+ openrouter: OPENROUTER_DEFAULT_MODEL,
10746
+ autorag: AUTORAG_DEFAULT_JEV_MODEL
10747
+ };
9911
10748
  jevQuestionSchema = Type6.Object({
9912
10749
  id: Type6.Optional(Type6.String({ description: "Caller-assigned id, echoed back in the verdict" })),
9913
10750
  type: Type6.Union([Type6.Literal("noul"), Type6.Literal("choice"), Type6.Literal("score")], {
@@ -9949,7 +10786,7 @@ function createJikjiFindTool(provider, ledger = new EvidenceLedger) {
9949
10786
  label: "Jikji Find",
9950
10787
  description: "Discover local files via Jikji. Returns answer_paths with per-candidate next_read hints and a tool-call policy directive. Call this FIRST for local file discovery when Jikji is configured.",
9951
10788
  parameters: jikjiFindSchema,
9952
- async execute(_toolCallId, params) {
10789
+ async execute(_toolCallId, params, signal) {
9953
10790
  const query = String(params.query ?? "").trim();
9954
10791
  if (query.length === 0) {
9955
10792
  return {
@@ -9972,7 +10809,8 @@ function createJikjiFindTool(provider, ledger = new EvidenceLedger) {
9972
10809
  }
9973
10810
  const { answerPack, policy, diagnostics, perRoot } = await provider.findJikji(query, {
9974
10811
  topK: params.topK,
9975
- first: params.first
10812
+ first: params.first,
10813
+ ...signal === undefined ? {} : { signal }
9976
10814
  });
9977
10815
  if (answerPack === undefined) {
9978
10816
  const diagMsg = diagnostics.length > 0 ? diagnostics.map((d) => d.message).join(" ") : "Jikji is unavailable or all roots failed.";
@@ -12286,7 +13124,7 @@ function createSearchAllDocumentsTool(provider, ledger = new EvidenceLedger) {
12286
13124
  label: "Search All Documents",
12287
13125
  description: "Search across all configured retrieval methods (posix, MinSync, datasources) and return every distinct chunk they found, with only pure duplicates removed. This is the exhaustive retrieval surface: one call can return many candidates, including several passages of the same document, so judge the evidence yourself rather than assuming it was pre-filtered. Authority is server-configured; tool arguments can only provide query, topK, and an optional narrowing scope.",
12288
13126
  parameters: searchAllSchema,
12289
- async execute(_toolCallId, params) {
13127
+ async execute(_toolCallId, params, signal) {
12290
13128
  const query = params.query.trim();
12291
13129
  if (query.length === 0) {
12292
13130
  return {
@@ -12296,7 +13134,8 @@ function createSearchAllDocumentsTool(provider, ledger = new EvidenceLedger) {
12296
13134
  }
12297
13135
  const { results, diagnostics, perMethodCounts } = await provider.searchAllDocuments(query, {
12298
13136
  topK: params.topK,
12299
- scope: params.scope
13137
+ scope: params.scope,
13138
+ ...signal === undefined ? {} : { signal }
12300
13139
  });
12301
13140
  const evidenceIds = results.map((result) => ledger.registerResult(SEARCH_ALL_DOCUMENTS_TOOL_NAME, result));
12302
13141
  return {
@@ -12320,7 +13159,7 @@ Diagnostics: ${diagnostics.map((d) => `${d.source ?? "all"}:${d.code}`).join(",
12320
13159
  if (results.length === 0)
12321
13160
  return `No results.${diagnosticSummary}`;
12322
13161
  const rows = results.map((result, index) => {
12323
- const line = result.content.replace(/\s+/gu, " ").slice(0, 500);
13162
+ const line = result.content.replace(/\s+/gu, " ");
12324
13163
  return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
12325
13164
  ${line}`;
12326
13165
  });
@@ -12351,7 +13190,7 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
12351
13190
  label: "Search MinSync Documents",
12352
13191
  description: "Search parsed document mirrors with MinSync semantic vector retrieval. Use for conceptual and meaning-based search.",
12353
13192
  parameters: searchMinSyncSchema,
12354
- async execute(_toolCallId, params) {
13193
+ async execute(_toolCallId, params, signal) {
12355
13194
  if (params.query.trim().length === 0) {
12356
13195
  return {
12357
13196
  content: [{ type: "text", text: "MinSync query was empty; no documents searched." }],
@@ -12361,7 +13200,8 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
12361
13200
  const scope = resolveScope(params.scope);
12362
13201
  const results = await getMethod().retrieve(params.query, {
12363
13202
  topK: params.topK,
12364
- scope
13203
+ scope,
13204
+ ...signal === undefined ? {} : { signal }
12365
13205
  });
12366
13206
  const evidenceIds = results.map((result) => ledger.registerResult(SEARCH_MINSYNC_DOCUMENTS_TOOL_NAME, result));
12367
13207
  return {
@@ -12380,7 +13220,7 @@ function formatResults2(results, evidenceIds) {
12380
13220
  if (results.length === 0)
12381
13221
  return "No MinSync results.";
12382
13222
  const rows = results.map((result, index) => {
12383
- const line = result.content.replace(/\s+/gu, " ").slice(0, 500);
13223
+ const line = result.content.replace(/\s+/gu, " ");
12384
13224
  return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
12385
13225
  ${line}`;
12386
13226
  });
@@ -12428,7 +13268,7 @@ function createTool(provider, spec, ledger) {
12428
13268
  label: `Search ${spec.datasourceId}`,
12429
13269
  description: `Search only the "${spec.datasourceId}" datasource connection: ${spec.description} ` + `This tool spawns only this connection's CLIs and returns only this connection's hits.${scopeLine} ` + `Authority is server-configured; tool arguments can only provide query, topK, and an optional narrowing scope.`,
12430
13270
  parameters: searchSingleDatasourceSchema,
12431
- async execute(_toolCallId, params) {
13271
+ async execute(_toolCallId, params, signal) {
12432
13272
  const query = params.query.trim();
12433
13273
  if (query.length === 0) {
12434
13274
  return {
@@ -12444,7 +13284,8 @@ function createTool(provider, spec, ledger) {
12444
13284
  }
12445
13285
  const { results, diagnostics } = await provider.searchSingleDatasourceDocuments(spec.datasourceId, query, {
12446
13286
  topK: params.topK,
12447
- scope: params.scope
13287
+ scope: params.scope,
13288
+ ...signal === undefined ? {} : { signal }
12448
13289
  });
12449
13290
  const evidenceIds = results.map((result) => ledger.registerResult(`search_datasource_${spec.datasourceId}`, result));
12450
13291
  return {
@@ -12468,7 +13309,7 @@ Diagnostics: ${diagnostics.map((d) => `${d.source ?? "datasource"}:${d.code}`).j
12468
13309
  if (results.length === 0)
12469
13310
  return `No datasource results.${diagnosticSummary}`;
12470
13311
  const rows = results.map((result, index) => {
12471
- const line = result.content.replace(/\s+/gu, " ").slice(0, 500);
13312
+ const line = result.content.replace(/\s+/gu, " ");
12472
13313
  return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
12473
13314
  ${line}`;
12474
13315
  });
@@ -18031,7 +18872,7 @@ function mergeSources(merged, sources) {
18031
18872
  existing.source.ageSeconds ??= source.ageSeconds;
18032
18873
  }
18033
18874
  }
18034
- function sleep3(ms, signal) {
18875
+ function sleep4(ms, signal) {
18035
18876
  if (ms <= 0)
18036
18877
  return Promise.resolve();
18037
18878
  let resolveFn;
@@ -18081,9 +18922,9 @@ async function searchPublicWeb(params, deadlines = {}) {
18081
18922
  failures.push({ provider: { id, label: id }, error });
18082
18923
  }
18083
18924
  }));
18084
- await Promise.race([all, sleep3(softMs, signal)]);
18925
+ await Promise.race([all, sleep4(softMs, signal)]);
18085
18926
  if (!responses.some(isUseful) && failures.length < engineIds.length) {
18086
- await Promise.race([all, firstSuccess, sleep3(Math.max(0, hardMs - softMs), signal)]);
18927
+ await Promise.race([all, firstSuccess, sleep4(Math.max(0, hardMs - softMs), signal)]);
18087
18928
  }
18088
18929
  straggler.abort();
18089
18930
  const merged = new Map;
@@ -18499,6 +19340,20 @@ class AutoRAGAgent {
18499
19340
  lastQuery;
18500
19341
  lastSessionId;
18501
19342
  activeRun = false;
19343
+ interactive = {
19344
+ run: undefined,
19345
+ followUps: [],
19346
+ steers: [],
19347
+ queuedSteers: [],
19348
+ preSessionSteers: [],
19349
+ acceptingSteers: false,
19350
+ harnessAbort: false,
19351
+ userAborted: false,
19352
+ steerInterrupt: new AbortController,
19353
+ nextSubmitIsFollowUp: false,
19354
+ steerTexts: new Set
19355
+ };
19356
+ activeRunAbort;
18502
19357
  modelNativeSearchAuth;
18503
19358
  retrievalTrace = [];
18504
19359
  evidenceLedger = new EvidenceLedger;
@@ -18975,9 +19830,9 @@ ${context.text}
18975
19830
  apiKey: resolved.apiKey,
18976
19831
  providerApiKeys: resolved.providerApiKeys,
18977
19832
  getSystemPrompt: () => systemPrompt,
18978
- customTools: this.tools.filter((tool) => !PI_BUILTIN_TOOL_NAMES.includes(tool.name)),
19833
+ customTools: this.steerableTools(),
18979
19834
  remoteSession: this.remoteSession,
18980
- contextTransform: (messages) => this.withMemoryContext(messages),
19835
+ contextTransform: (messages) => this.withMemoryContext(frameSteers(messages, this.interactive.steerTexts)),
18981
19836
  ...this.jevExtension !== undefined ? { extensionFactories: [this.jevExtension], extensionToolNames: [JEV_TOOL_NAME] } : {}
18982
19837
  });
18983
19838
  const agent = piSession.session.agent;
@@ -18993,6 +19848,8 @@ ${context.text}
18993
19848
  const unsubscribers = [...this.listeners].map((listener) => session.agent.subscribe(listener));
18994
19849
  unsubscribers.push(session.agent.subscribe((event) => {
18995
19850
  this.recordSearchToolEvent(event);
19851
+ if (event.type === "message_start" && event.message.role === "user")
19852
+ this.steersReachedModel();
18996
19853
  }));
18997
19854
  return unsubscribers;
18998
19855
  }
@@ -19009,6 +19866,7 @@ ${context.text}
19009
19866
  return;
19010
19867
  this.searchToolCallCount += 1;
19011
19868
  if (this.searchToolCallCount >= this.maxSearchToolCalls) {
19869
+ this.interactive.harnessAbort = true;
19012
19870
  this.activeSession?.abort();
19013
19871
  }
19014
19872
  const details = event.result.details;
@@ -19055,9 +19913,11 @@ ${context.text}
19055
19913
  getSystemPrompt: () => buildSystemPrompt(this.currentSystemPromptConfig({
19056
19914
  modelId: this.boundPiRuntime?.session.model?.id ?? resolved?.model.id
19057
19915
  })),
19058
- contextTransform: (messages) => this.withMemoryContext(messages),
19059
- customTools: this.tools.filter((tool) => !PI_BUILTIN_TOOL_NAMES.includes(tool.name)),
19060
- onQuery: (query, pi) => this.runInteractivePiQuery(query, pi),
19916
+ contextTransform: async (messages) => this.withMemoryContext(frameSteers(messages, this.interactive.steerTexts)),
19917
+ customTools: this.steerableTools(),
19918
+ onInput: (input, pi) => this.routeInteractiveInput(input, pi),
19919
+ isSearchRunning: () => this.interactive.run !== undefined,
19920
+ onKey: (key) => this.handleInteractiveKey(key),
19061
19921
  ...this.jevExtension !== undefined ? { extensionFactories: [this.jevExtension], extensionToolNames: [JEV_TOOL_NAME] } : {},
19062
19922
  ...this.updateNotice === undefined ? {} : { updateNotice: this.updateNotice }
19063
19923
  });
@@ -19072,18 +19932,154 @@ ${context.text}
19072
19932
  }
19073
19933
  };
19074
19934
  }
19935
+ steerableTools() {
19936
+ return this.tools.filter((tool) => !PI_BUILTIN_TOOL_NAMES.includes(tool.name)).map((tool) => this.searchToolNames.has(tool.name) || tool.name === WEB_SEARCH_TOOL_NAME || tool.name === WEB_FETCH_TOOL_NAME ? this.interruptibleBySteer(tool) : tool);
19937
+ }
19938
+ interruptibleBySteer(tool) {
19939
+ return {
19940
+ ...tool,
19941
+ execute: async (toolCallId, params, signal, onUpdate) => {
19942
+ const steer = this.interactive.steerInterrupt.signal;
19943
+ if (steer.aborted)
19944
+ return skippedBySteer(tool.name);
19945
+ const combined = signal === undefined ? steer : AbortSignal.any([signal, steer]);
19946
+ let onSteer = () => {};
19947
+ const steered = new Promise((resolve) => {
19948
+ onSteer = () => resolve("steered");
19949
+ steer.addEventListener("abort", onSteer, { once: true });
19950
+ });
19951
+ try {
19952
+ const outcome = await Promise.race([tool.execute(toolCallId, params, combined, onUpdate), steered]);
19953
+ return outcome === "steered" ? skippedBySteer(tool.name) : outcome;
19954
+ } catch (error) {
19955
+ if (steer.aborted && signal?.aborted !== true)
19956
+ return skippedBySteer(tool.name);
19957
+ throw error;
19958
+ } finally {
19959
+ steer.removeEventListener("abort", onSteer);
19960
+ }
19961
+ }
19962
+ };
19963
+ }
19075
19964
  deriveAnswer(text) {
19076
19965
  const derived = deriveResultsFromAnswer(text, this.evidenceLedger, { allowLocalFiles: !this.remoteSession });
19077
19966
  this.routingDiagnostics.push(...derived.diagnostics);
19078
19967
  return derived.details;
19079
19968
  }
19969
+ routeInteractiveInput(input, pi) {
19970
+ const state = this.interactive;
19971
+ const followUp = input.streamingBehavior === "followUp" || state.nextSubmitIsFollowUp;
19972
+ state.nextSubmitIsFollowUp = false;
19973
+ if (state.run === undefined) {
19974
+ this.startInteractiveQuery(input.text, pi);
19975
+ return { kind: "handled" };
19976
+ }
19977
+ if (followUp) {
19978
+ state.followUps.push(input.text);
19979
+ return { kind: "handled" };
19980
+ }
19981
+ if (input.streamingBehavior === "steer") {
19982
+ state.steers.push(input.text);
19983
+ state.steerTexts.add(input.text);
19984
+ state.steerInterrupt.abort();
19985
+ return { kind: "steer", text: input.text };
19986
+ }
19987
+ if (!this.steer(input.text, input.images))
19988
+ state.followUps.push(input.text);
19989
+ return { kind: "handled" };
19990
+ }
19991
+ handleInteractiveKey(key) {
19992
+ const state = this.interactive;
19993
+ if (state.run === undefined || this.boundPiRuntime?.session.isStreaming === true)
19994
+ return false;
19995
+ if (key === "followUp") {
19996
+ state.nextSubmitIsFollowUp = true;
19997
+ return false;
19998
+ }
19999
+ state.userAborted = true;
20000
+ state.steerInterrupt.abort();
20001
+ return true;
20002
+ }
20003
+ startInteractiveQuery(query, pi) {
20004
+ const state = this.interactive;
20005
+ const run = (async () => {
20006
+ let next = query;
20007
+ while (next !== undefined) {
20008
+ try {
20009
+ await this.runInteractivePiQuery(next, pi);
20010
+ } catch (error) {
20011
+ pi.sendMessage({
20012
+ customType: "autorag.error",
20013
+ content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
20014
+ display: true
20015
+ }, { triggerTurn: false });
20016
+ }
20017
+ next = state.followUps.shift();
20018
+ }
20019
+ })().finally(() => {
20020
+ state.run = undefined;
20021
+ });
20022
+ state.run = run;
20023
+ }
19080
20024
  async runInteractivePiQuery(query, pi) {
19081
20025
  pi.setSessionName(query.slice(0, 80));
19082
- for await (const _event of this.searchDocumentsStream(query)) {}
20026
+ const state = this.interactive;
20027
+ const session = this.boundPiRuntime?.session;
20028
+ const unsubscribe = session?.subscribe((event) => {
20029
+ if (event.type === "queue_update" && state.queuedSteers.length > 0 && !session.agent.hasQueuedMessages()) {
20030
+ state.queuedSteers.length = 0;
20031
+ }
20032
+ if (event.type === "agent_settled" && event.aborted && !state.harnessAbort)
20033
+ state.userAborted = true;
20034
+ });
20035
+ try {
20036
+ for await (const _event of this.searchDocumentsStream(query)) {}
20037
+ } finally {
20038
+ unsubscribe?.();
20039
+ if (state.queuedSteers.length > 0) {
20040
+ session?.agent.clearSteeringQueue();
20041
+ if (!state.userAborted)
20042
+ state.followUps.unshift(...state.queuedSteers);
20043
+ state.queuedSteers.length = 0;
20044
+ }
20045
+ }
20046
+ }
20047
+ async waitForIdle() {
20048
+ while (this.interactive.run !== undefined)
20049
+ await this.interactive.run;
20050
+ }
20051
+ steer(text, images = []) {
20052
+ const state = this.interactive;
20053
+ if (!this.activeRun || !state.acceptingSteers || state.userAborted || text.trim() === "")
20054
+ return false;
20055
+ state.steers.push(text);
20056
+ state.steerTexts.add(text);
20057
+ state.steerInterrupt.abort();
20058
+ const content = [{ type: "text", text }, ...images];
20059
+ const message = { role: "user", content, timestamp: Date.now() };
20060
+ const agent = this.activeSession?.agent;
20061
+ if (agent === undefined)
20062
+ state.preSessionSteers.push(message);
20063
+ else
20064
+ agent.steer(message);
20065
+ state.queuedSteers.push(text);
20066
+ return true;
19083
20067
  }
19084
20068
  abort() {
20069
+ if (!this.activeRun)
20070
+ return;
20071
+ const state = this.interactive;
20072
+ state.userAborted = true;
20073
+ state.steerInterrupt.abort();
20074
+ this.activeRunAbort?.abort();
19085
20075
  this.activeSession?.abort();
19086
20076
  }
20077
+ steersReachedModel() {
20078
+ const state = this.interactive;
20079
+ state.queuedSteers.length = 0;
20080
+ if (state.steerInterrupt.signal.aborted && !state.userAborted)
20081
+ state.steerInterrupt = new AbortController;
20082
+ }
19087
20083
  startAutoRefresh(intervalMs, options = {}) {
19088
20084
  this.stopAutoRefresh();
19089
20085
  const tick = () => {
@@ -19143,6 +20139,15 @@ ${context.text}
19143
20139
  const publishOnCapture = this.jevJudge === undefined;
19144
20140
  let published = false;
19145
20141
  const planAbort = new AbortController;
20142
+ this.activeRunAbort = planAbort;
20143
+ const steering = this.interactive;
20144
+ steering.steers.length = 0;
20145
+ steering.queuedSteers.length = 0;
20146
+ steering.preSessionSteers.length = 0;
20147
+ steering.harnessAbort = false;
20148
+ steering.userAborted = false;
20149
+ steering.steerInterrupt = new AbortController;
20150
+ steering.acceptingSteers = true;
19146
20151
  const publishPreliminary = (details) => {
19147
20152
  if (planAbort.signal.aborted || published)
19148
20153
  return;
@@ -19178,6 +20183,9 @@ ${context.text}
19178
20183
  session = await this.createSearchSession(resolved, buildSystemPrompt(this.currentSystemPromptConfig({ modelId: resolved.model.id })));
19179
20184
  this.activeSession = session;
19180
20185
  unsubscribers = this.configureSearchSession(session);
20186
+ for (const message of steering.preSessionSteers)
20187
+ session.agent.steer(message);
20188
+ steering.preSessionSteers.length = 0;
19181
20189
  let timeout;
19182
20190
  let timedOutAfterFastAnswer = false;
19183
20191
  try {
@@ -19238,25 +20246,42 @@ Warning: the edited config did not validate, so the previous config was restored
19238
20246
  }
19239
20247
  return;
19240
20248
  }
19241
- if (plan.route === "direct") {
20249
+ const steered = () => this.interactive.steers.length > 0;
20250
+ const stopped = () => this.interactive.userAborted;
20251
+ if (stopped())
20252
+ return;
20253
+ let directText;
20254
+ if (plan.route === "direct" && !steered()) {
19242
20255
  setThinking(this.fastThinkingLevel);
19243
20256
  await activeSession.prompt(this.buildDirectAnswerPrompt(trimmedQuery));
19244
- const answer = finalAnswerText(transcript());
19245
- if (answer !== undefined)
19246
- captured = this.deriveAnswer(answer);
20257
+ if (stopped())
20258
+ return;
20259
+ directText = finalAnswerText(transcript());
20260
+ if (!steered()) {
20261
+ if (directText !== undefined)
20262
+ captured = this.deriveAnswer(directText);
20263
+ return;
20264
+ }
20265
+ }
20266
+ const baseline = plan.route === "direct" ? "" : await this.prefetchInitialRetrievalContext(trimmedQuery, plan.queries, options, plan.datasources);
20267
+ if (stopped())
19247
20268
  return;
20269
+ let fastText = directText;
20270
+ let preliminary;
20271
+ if (plan.route !== "direct") {
20272
+ setThinking(this.fastThinkingLevel);
20273
+ await activeSession.prompt(this.buildFastAnswerPrompt(trimmedQuery, options, baseline));
20274
+ if (stopped())
20275
+ return;
20276
+ fastText = finalAnswerText(transcript());
20277
+ preliminary = fastText === undefined ? undefined : this.deriveAnswer(fastText);
19248
20278
  }
19249
- const baseline = await this.prefetchInitialRetrievalContext(trimmedQuery, plan.queries, options, plan.datasources);
19250
- setThinking(this.fastThinkingLevel);
19251
- await activeSession.prompt(this.buildFastAnswerPrompt(trimmedQuery, options, baseline));
19252
- const fastText = finalAnswerText(transcript());
19253
- const preliminary = fastText === undefined ? undefined : this.deriveAnswer(fastText);
19254
20279
  if (preliminary !== undefined) {
19255
20280
  emitPreliminary(preliminary);
19256
- const followUp = await this.shouldFollowUp(trimmedQuery, preliminary);
19257
- if (planAbort.signal.aborted)
20281
+ const followUp = steered() || await this.shouldFollowUp(trimmedQuery, preliminary);
20282
+ if (planAbort.signal.aborted || stopped())
19258
20283
  return;
19259
- if (!followUp) {
20284
+ if (!followUp && !steered()) {
19260
20285
  captured = preliminary;
19261
20286
  return;
19262
20287
  }
@@ -19264,14 +20289,23 @@ Warning: the edited config did not validate, so the previous config was restored
19264
20289
  }
19265
20290
  setThinking(this.finalThinkingLevel);
19266
20291
  const fastAnswerDelivered = preliminary !== undefined && this.preliminaryCallback !== undefined;
19267
- await activeSession.prompt(this.buildRefinementPrompt(trimmedQuery, options, fastText, fastAnswerDelivered));
20292
+ await activeSession.prompt(this.buildRefinementPrompt(trimmedQuery, options, fastText, fastAnswerDelivered, this.interactive.steers));
20293
+ while (!stopped() && !planAbort.signal.aborted && this.interactive.queuedSteers.length > 0) {
20294
+ await activeSession.prompt(buildSteerContinuationPrompt(this.interactive.steers));
20295
+ }
20296
+ steering.acceptingSteers = false;
20297
+ if (stopped())
20298
+ return;
19268
20299
  const finalText = finalAnswerText(transcript());
19269
20300
  if (finalText !== undefined)
19270
20301
  captured = this.deriveAnswer(finalText);
19271
- })(),
20302
+ })().finally(() => {
20303
+ steering.acceptingSteers = false;
20304
+ }),
19272
20305
  new Promise((resolve, reject) => {
19273
20306
  timeout = setTimeout(() => {
19274
20307
  planAbort.abort();
20308
+ this.interactive.harnessAbort = true;
19275
20309
  Promise.resolve(session?.abort());
19276
20310
  if (fastCaptured !== undefined && !this.remoteSession) {
19277
20311
  timedOutAfterFastAnswer = true;
@@ -19718,12 +20752,13 @@ Warning: the edited config did not validate, so the previous config was restored
19718
20752
  return selection.selected;
19719
20753
  }
19720
20754
  async prefetchInitialRetrievalContext(query, searchQueries, options, datasources = []) {
19721
- const retrieveOptions = { topK: this.limits.prefetch.minSyncTopK, scope: options.scope };
20755
+ const signal = this.interactive.steerInterrupt.signal;
20756
+ const retrieveOptions = { topK: this.limits.prefetch.minSyncTopK, scope: options.scope, signal };
19722
20757
  const formatSearchQueries = (list) => `Search queries (decomposed from the original question):
19723
20758
  ${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
19724
20759
  `)}`;
19725
20760
  const perQuery = await Promise.all(searchQueries.map((searchQuery) => Promise.all([
19726
- this.jikjiClient === undefined ? Promise.resolve(undefined) : this.findJikji(searchQuery, { topK: this.limits.prefetch.jikjiTopK }).catch(() => {
20761
+ this.jikjiClient === undefined ? Promise.resolve(undefined) : this.findJikji(searchQuery, { topK: this.limits.prefetch.jikjiTopK, signal }).catch(() => {
19727
20762
  return;
19728
20763
  }),
19729
20764
  this.minSyncMethod.retrieve(searchQuery, retrieveOptions).catch((error) => {
@@ -19733,7 +20768,8 @@ ${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
19733
20768
  }),
19734
20769
  Promise.all(datasources.map((datasourceId) => this.searchSingleDatasourceDocuments(datasourceId, searchQuery, {
19735
20770
  topK: this.limits.singleDatasourceTopK,
19736
- scope: options.scope
20771
+ scope: options.scope,
20772
+ signal
19737
20773
  }).then((outcome) => outcome.results).catch(() => [])))
19738
20774
  ])));
19739
20775
  for (const [index, [jikji, vector, perDatasource]] of perQuery.entries()) {
@@ -19783,7 +20819,7 @@ ${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
19783
20819
  options.observedSources?.add(result.source);
19784
20820
  for (const result of datasourceResults)
19785
20821
  options.observedSources?.add(result.source);
19786
- const reranked = await this.rerankPrefetchPool(query, jikjiPaths, minSyncResults, datasourceResults);
20822
+ const reranked = await this.rerankPrefetchPool(query, jikjiPaths, minSyncResults, datasourceResults, signal);
19787
20823
  if (reranked !== undefined) {
19788
20824
  const baseline = formatRerankedBaseline(reranked, reranked.map((result) => this.evidenceLedger.registerResult("baseline", result)));
19789
20825
  return searchQueries.length > 1 ? `${formatSearchQueries(searchQueries)}
@@ -19815,7 +20851,7 @@ ${result.content.replace(/\s+/gu, " ")}`).join(`
19815
20851
 
19816
20852
  `);
19817
20853
  }
19818
- async rerankPrefetchPool(query, answerPaths, minSyncResults, datasourceResults) {
20854
+ async rerankPrefetchPool(query, answerPaths, minSyncResults, datasourceResults, signal) {
19819
20855
  const reranker = this.reranker;
19820
20856
  if (reranker === undefined)
19821
20857
  return;
@@ -19833,7 +20869,7 @@ ${result.content.replace(/\s+/gu, " ")}`).join(`
19833
20869
  if (!reranker.describe().available)
19834
20870
  return;
19835
20871
  try {
19836
- const reranked = await reranker.rerank(query, candidates, { topN: this.rerankTopN });
20872
+ const reranked = await reranker.rerank(query, candidates, { topN: this.rerankTopN, signal });
19837
20873
  return reranked.length > 0 ? reranked : undefined;
19838
20874
  } catch {
19839
20875
  return;
@@ -19876,7 +20912,7 @@ ${baseline}
19876
20912
  const fetch2 = names.has(WEB_FETCH_TOOL_NAME) ? ` (and ${WEB_FETCH_TOOL_NAME} to read a promising page)` : "";
19877
20913
  return `When local evidence is missing, thin, or possibly outdated — or the question concerns current events, recent releases, prices, or other public facts — use ${WEB_SEARCH_TOOL_NAME}${fetch2} to confirm or correct claims and fill gaps; use URLs as result sources for web-derived evidence. `;
19878
20914
  }
19879
- buildRefinementPrompt(query, options, fastAnswer, fastAnswerDelivered) {
20915
+ buildRefinementPrompt(query, options, fastAnswer, fastAnswerDelivered, steers = []) {
19880
20916
  const limit = typeof options.topK === "number" ? ` Cite at most ${options.topK} sources.` : "";
19881
20917
  const scope = options.scope ? ` Restrict search to virtual path scope ${options.scope}.` : "";
19882
20918
  const firstAnswer = formatFirstAnswerContext(fastAnswer, fastAnswerDelivered);
@@ -19899,7 +20935,7 @@ ${baseline}
19899
20935
 
19900
20936
  ` + `${firstAnswer}
19901
20937
 
19902
- ` + `Now verify it rigorously. ${this.discoveryHint((tools) => `Actively use ${tools} when discovering or exploring local files and folders. `)}Check important claims against source files with bash when needed, correct anything wrong or unsupported, fill gaps with retrieval tools, and resolve conflicts and freshness. ${this.webFallbackHint()}
20938
+ ` + formatSteerContext(steers) + `Now verify it rigorously. ${this.discoveryHint((tools) => `Actively use ${tools} when discovering or exploring local files and folders. `)}Check important claims against source files with bash when needed, correct anything wrong or unsupported, fill gaps with retrieval tools, and resolve conflicts and freshness. ${this.webFallbackHint()}
19903
20939
 
19904
20940
  ` + `${answerRules}
19905
20941
 
@@ -20302,7 +21338,8 @@ ${initialRetrievalContext ?? "Pending; continue only with a brief progress state
20302
21338
  const sourceRoots = planJikjiSourceRoots(this.searchPaths);
20303
21339
  const findOpts = {
20304
21340
  topK: opts?.topK,
20305
- first: opts?.first
21341
+ first: opts?.first,
21342
+ ...opts?.signal === undefined ? {} : { signal: opts.signal }
20306
21343
  };
20307
21344
  const diagnostics = [];
20308
21345
  const okPacks = [];
@@ -20453,13 +21490,18 @@ ${initialRetrievalContext ?? "Pending; continue only with a brief progress state
20453
21490
  return { results, diagnostics };
20454
21491
  }
20455
21492
  async searchAllDocuments(query, options = {}) {
20456
- return this.retrieveWithDiagnostics(query, { topK: options.topK, scope: options.scope });
21493
+ return this.retrieveWithDiagnostics(query, {
21494
+ topK: options.topK,
21495
+ scope: options.scope,
21496
+ ...options.signal === undefined ? {} : { signal: options.signal }
21497
+ });
20457
21498
  }
20458
21499
  async searchSingleDatasourceDocuments(datasourceId, query, options = {}) {
20459
21500
  const retrievalOptions = {
20460
21501
  ...this.activeRetrievalOptions,
20461
21502
  topK: options.topK,
20462
- scope: options.scope
21503
+ scope: options.scope,
21504
+ ...options.signal === undefined ? {} : { signal: options.signal }
20463
21505
  };
20464
21506
  const methods = this.methodRegistry.list().filter((method) => {
20465
21507
  const descriptor = method.describe();
@@ -20562,6 +21604,49 @@ ${result.content.replace(/\s+/gu, " ")}`);
20562
21604
  ${lines.join(`
20563
21605
  `)}`;
20564
21606
  }
21607
+ function skippedBySteer(toolName) {
21608
+ return {
21609
+ content: [
21610
+ {
21611
+ type: "text",
21612
+ text: `Skipped due to the user's steering message. ${toolName} was stopped before it returned; do not count it as completed work. Apply the steering message, then retry the search if it is still needed.`
21613
+ }
21614
+ ],
21615
+ details: { method: toolName, skipped: "steer" }
21616
+ };
21617
+ }
21618
+ function buildSteerMessage(text) {
21619
+ return `The user interrupted the running search with this correction:
21620
+
21621
+ ${text}
21622
+
21623
+ ` + `Treat it as direction for the search in progress, not as a new question: adjust which sources, queries, or claims you pursue, then continue. ` + `When finished, reply with the final answer to the original question, updated for this correction, and no tool call.`;
21624
+ }
21625
+ function frameSteers(messages, steerTexts) {
21626
+ if (steerTexts.size === 0)
21627
+ return messages;
21628
+ return messages.map((message) => {
21629
+ if (message.role !== "user" || typeof message.content === "string")
21630
+ return message;
21631
+ const [first, ...rest] = message.content;
21632
+ if (first?.type !== "text" || !steerTexts.has(first.text))
21633
+ return message;
21634
+ return { ...message, content: [{ ...first, text: buildSteerMessage(first.text) }, ...rest] };
21635
+ });
21636
+ }
21637
+ function formatSteerContext(steers) {
21638
+ if (steers.length === 0)
21639
+ return "";
21640
+ return `While the search ran, the user corrected it:
21641
+ ${steers.map((steer) => `- ${steer}`).join(`
21642
+ `)}
21643
+ ` + `The first answer predates these corrections; follow them over it.
21644
+
21645
+ `;
21646
+ }
21647
+ function buildSteerContinuationPrompt(steers) {
21648
+ return `${formatSteerContext(steers)}Apply the latest correction: search or verify further if it needs that, ` + `then reply with the complete final answer to the original question and no tool call.`;
21649
+ }
20565
21650
  function formatFirstAnswerContext(fastAnswer, delivered) {
20566
21651
  if (fastAnswer === undefined) {
20567
21652
  return "(the fast phase produced no answer — write the complete answer)";
@@ -20970,23 +22055,175 @@ ${this.description}`
20970
22055
  }
20971
22056
  }
20972
22057
 
20973
- // src/datasource/skills/clawgallery/client.ts
20974
- import { spawn as spawn9 } from "node:child_process";
20975
- import { existsSync as existsSync23 } from "node:fs";
20976
-
20977
- class ClawGalleryClient {
20978
- options;
20979
- constructor(options = {}) {
20980
- this.options = options;
20981
- }
20982
- async bootstrap(signal) {
20983
- if (this.options.path !== undefined && !existsSync23(this.options.path)) {
20984
- return {
20985
- ok: false,
20986
- reason: "nonzero-exit",
20987
- stdout: "",
20988
- stderr: "configured ClawGallery folder does not exist",
20989
- code: null
22058
+ // src/datasource/connector.ts
22059
+ import { createHash as createHash12 } from "node:crypto";
22060
+ function connectorFailureToDiagnosticCode(reason) {
22061
+ switch (reason) {
22062
+ case "auth":
22063
+ return "datasource-auth-error";
22064
+ case "permission":
22065
+ return "datasource-permission-denied";
22066
+ case "rate-limited":
22067
+ return "datasource-rate-limited";
22068
+ case "unavailable":
22069
+ case "not-configured":
22070
+ return "datasource-unavailable";
22071
+ case "empty":
22072
+ return "datasource-empty";
22073
+ case "api-error":
22074
+ case "invalid-data":
22075
+ return "datasource-index-failed";
22076
+ }
22077
+ }
22078
+ function boundDiagnosticText(value) {
22079
+ const trimmed = value.trim();
22080
+ return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
22081
+ }
22082
+ function sanitizeIdSegment(value) {
22083
+ const normalized = value.replace(/[^\p{L}\p{N}._-]+/gu, "-");
22084
+ let start = 0;
22085
+ while (start < normalized.length && (normalized[start] === "-" || normalized[start] === "."))
22086
+ start += 1;
22087
+ let end = normalized.length;
22088
+ while (end > start && (normalized[end - 1] === "-" || normalized[end - 1] === "."))
22089
+ end -= 1;
22090
+ const cleaned = normalized.slice(start, end);
22091
+ if (cleaned === value && cleaned.length > 0 && cleaned.length <= 80)
22092
+ return cleaned;
22093
+ const hash = createHash12("sha256").update(value).digest("hex").slice(0, 8);
22094
+ const stem = cleaned.slice(0, 60);
22095
+ return stem.length > 0 ? `${stem}-${hash}` : hash;
22096
+ }
22097
+ var init_connector = () => {};
22098
+
22099
+ // src/datasource/sidecar/config.ts
22100
+ import { readFileSync as readFileSync19 } from "node:fs";
22101
+ import { homedir as homedir3 } from "node:os";
22102
+ import { dirname as dirname17, isAbsolute as isAbsolute8, join as join30, resolve as resolve14, sep as sep7 } from "node:path";
22103
+ import { parse as parseToml2 } from "smol-toml";
22104
+ function resolveSidecarOptions(input) {
22105
+ const { connector, workspaceRoot } = input;
22106
+ const setting = connector?.sidecar;
22107
+ if (setting === false || workspaceRoot === undefined)
22108
+ return;
22109
+ const object = typeof setting === "object" && setting !== null && !Array.isArray(setting) ? setting : {};
22110
+ if (object.enabled === false)
22111
+ return;
22112
+ const configured = [object.crawlerDbPath, connector?.databasePath].find((value) => typeof value === "string" && value.length > 0);
22113
+ const crawlerDbPath = expandHome(configured ?? archivePathFromCrawlerConfig(input) ?? input.defaultCrawlerDb);
22114
+ const workspaceId = typeof connector?.workspace === "string" && connector.workspace.length > 0 ? connector.workspace : undefined;
22115
+ return {
22116
+ descriptor: input.descriptor,
22117
+ crawlerDbPath,
22118
+ sidecarDbPath: join30(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(input.registrationName), sanitizeIdSegment(input.instanceId), "sidecar.sqlite"),
22119
+ ...workspaceId === undefined ? {} : { workspaceId },
22120
+ ...object.semantic === false ? {} : { embedder: createGatewayEmbedder() }
22121
+ };
22122
+ }
22123
+ function archivePathFromCrawlerConfig(input) {
22124
+ const configPath = input.connector?.configPath;
22125
+ if (input.descriptor.archivePathInConfig !== true || typeof configPath !== "string" || configPath.length === 0) {
22126
+ return;
22127
+ }
22128
+ const file = expandHome(configPath);
22129
+ try {
22130
+ const dbPath = parseToml2(readFileSync19(file, "utf8")).db_path;
22131
+ if (typeof dbPath !== "string" || dbPath.length === 0)
22132
+ return;
22133
+ const expanded = expandHome(dbPath);
22134
+ return isAbsolute8(expanded) ? expanded : resolve14(dirname17(file), expanded);
22135
+ } catch {
22136
+ return;
22137
+ }
22138
+ }
22139
+ function expandHome(path) {
22140
+ if (path === "~")
22141
+ return homedir3();
22142
+ const prefixed = path.startsWith("~/") || sep7 === "\\" && path.startsWith("~\\");
22143
+ return prefixed ? join30(homedir3(), path.slice(2)) : path;
22144
+ }
22145
+ var init_config2 = __esm(() => {
22146
+ init_connector();
22147
+ });
22148
+
22149
+ // src/datasource/sidecar/descriptors.ts
22150
+ function chatCrawler(id) {
22151
+ return {
22152
+ id,
22153
+ keyExpression: "m.event_id",
22154
+ chunkId: (key) => key,
22155
+ fromClause: "messages m",
22156
+ liveWhere: "(m.deleted_at is null or m.deleted_at = 0)",
22157
+ textExpression: "coalesce(m.text, '')",
22158
+ titleExpression: "m.chat_name",
22159
+ epochSecondsExpression: "m.ts",
22160
+ fts: { table: "messages_fts", joinOn: "messages_fts.rowid = m.rowid" },
22161
+ metadata: [
22162
+ { sql: "m.chat_jid", key: "chatId" },
22163
+ { sql: "m.chat_name", key: "chatName" },
22164
+ { sql: "m.sender_name", key: "senderName" },
22165
+ { sql: "m.ts", key: "timestamp" }
22166
+ ],
22167
+ channelIdExpression: "m.chat_jid",
22168
+ channelNameExpression: "m.chat_name",
22169
+ hierarchy: (metadata) => ["chats", str(metadata.chatName) ?? str(metadata.chatId) ?? "unknown"]
22170
+ };
22171
+ }
22172
+ var str = (value) => typeof value === "string" && value.length > 0 ? value : undefined, slack, CRAWLER_DESCRIPTORS;
22173
+ var init_descriptors = __esm(() => {
22174
+ slack = {
22175
+ id: "slack",
22176
+ archivePathInConfig: true,
22177
+ keyExpression: "m.channel_id || '|' || m.ts",
22178
+ chunkId: (_key, metadata) => `${metadata.channelId ?? ""}-${(metadata.timestamp ?? "").replaceAll(".", "-")}`,
22179
+ fromClause: "messages m left join channels c on c.id = m.channel_id and c.workspace_id = m.workspace_id " + "left join users u on u.id = m.user_id",
22180
+ liveWhere: "(m.deleted_ts is null or m.deleted_ts = '')",
22181
+ scopeExpression: "m.workspace_id",
22182
+ textExpression: "coalesce(nullif(m.normalized_text, ''), m.text)",
22183
+ titleExpression: "c.name",
22184
+ epochSecondsExpression: "cast(m.ts as real)",
22185
+ fts: { table: "message_fts", joinOn: "message_fts.message_key = m.channel_id || '|' || m.ts" },
22186
+ metadata: [
22187
+ { sql: "m.channel_id", key: "channelId" },
22188
+ { sql: "c.name", key: "channelName" },
22189
+ { sql: "m.workspace_id", key: "workspaceId" },
22190
+ { sql: "m.user_id", key: "userId" },
22191
+ { sql: "coalesce(nullif(u.real_name, ''), u.name)", key: "senderName" },
22192
+ { sql: "m.ts", key: "timestamp" }
22193
+ ],
22194
+ channelIdExpression: "m.channel_id",
22195
+ channelNameExpression: "c.name",
22196
+ hierarchy: (metadata) => [
22197
+ "workspaces",
22198
+ str(metadata.workspaceId) ?? "unknown",
22199
+ "channels",
22200
+ str(metadata.channelName) ?? str(metadata.channelId) ?? "unknown"
22201
+ ]
22202
+ };
22203
+ CRAWLER_DESCRIPTORS = {
22204
+ slack,
22205
+ telegram: chatCrawler("telegram"),
22206
+ whatsapp: chatCrawler("whatsapp")
22207
+ };
22208
+ });
22209
+
22210
+ // src/datasource/skills/clawgallery/client.ts
22211
+ import { spawn as spawn9 } from "node:child_process";
22212
+ import { existsSync as existsSync23 } from "node:fs";
22213
+
22214
+ class ClawGalleryClient {
22215
+ options;
22216
+ constructor(options = {}) {
22217
+ this.options = options;
22218
+ }
22219
+ async bootstrap(signal) {
22220
+ if (this.options.path !== undefined && !existsSync23(this.options.path)) {
22221
+ return {
22222
+ ok: false,
22223
+ reason: "nonzero-exit",
22224
+ stdout: "",
22225
+ stderr: "configured ClawGallery folder does not exist",
22226
+ code: null
20990
22227
  };
20991
22228
  }
20992
22229
  const args = ["bootstrap"];
@@ -21085,7 +22322,7 @@ function spawnCli(binary, args, env, options, signal) {
21085
22322
  const timeout = setTimeout(() => {
21086
22323
  reason = "timeout";
21087
22324
  terminate2(child);
21088
- }, options.timeoutMs ?? DEFAULT_TIMEOUT_MS6);
22325
+ }, options.timeoutMs ?? DEFAULT_TIMEOUT_MS7);
21089
22326
  const abort = () => {
21090
22327
  reason = "aborted";
21091
22328
  terminate2(child);
@@ -21191,16 +22428,16 @@ function failure2(result, reason) {
21191
22428
  ok: false,
21192
22429
  reason: reason ?? result.reason ?? "nonzero-exit",
21193
22430
  stdout: result.stdout,
21194
- stderr: boundDiagnosticText(result.stderr),
22431
+ stderr: boundDiagnosticText2(result.stderr),
21195
22432
  code: result.code
21196
22433
  };
21197
22434
  }
21198
- function boundDiagnosticText(value) {
22435
+ function boundDiagnosticText2(value) {
21199
22436
  if (value.length === 0)
21200
22437
  return "";
21201
22438
  return value.trim().slice(0, 4000);
21202
22439
  }
21203
- var DEFAULT_BINARY2 = "clawgallery", DEFAULT_TIMEOUT_MS6 = 60000, DEFAULT_MAX_BUFFER_BYTES2, SAFE_ENV;
22440
+ var DEFAULT_BINARY2 = "clawgallery", DEFAULT_TIMEOUT_MS7 = 60000, DEFAULT_MAX_BUFFER_BYTES2, SAFE_ENV;
21204
22441
  var init_client5 = __esm(() => {
21205
22442
  init_portable_spawn();
21206
22443
  DEFAULT_MAX_BUFFER_BYTES2 = 4 * 1024 * 1024;
@@ -21494,7 +22731,7 @@ function statusToReason(status) {
21494
22731
  }
21495
22732
  async function requestText(url, options) {
21496
22733
  const fetchImpl = options.fetchImpl ?? globalThis.fetch;
21497
- const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS7;
22734
+ const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
21498
22735
  let response;
21499
22736
  try {
21500
22737
  response = await fetchImpl(url, {
@@ -21558,13 +22795,13 @@ function resolveToken(token, tokenEnv) {
21558
22795
  }
21559
22796
  return;
21560
22797
  }
21561
- var DEFAULT_TIMEOUT_MS7 = 15000;
22798
+ var DEFAULT_TIMEOUT_MS8 = 15000;
21562
22799
 
21563
22800
  // src/datasource/skills/cloud-drive/rclone-connector.ts
21564
22801
  import { spawn as spawn10 } from "node:child_process";
21565
- import { mkdirSync as mkdirSync18, readFileSync as readFileSync19, renameSync as renameSync10, rmSync as rmSync9, writeFileSync as writeFileSync17 } from "node:fs";
22802
+ import { mkdirSync as mkdirSync18, readFileSync as readFileSync20, renameSync as renameSync10, rmSync as rmSync9, writeFileSync as writeFileSync17 } from "node:fs";
21566
22803
  import { readFile as readFile4 } from "node:fs/promises";
21567
- import { dirname as dirname17, join as join30, normalize as normalize4 } from "node:path";
22804
+ import { dirname as dirname18, join as join31, normalize as normalize4 } from "node:path";
21568
22805
 
21569
22806
  class RcloneConnector {
21570
22807
  options;
@@ -21580,7 +22817,7 @@ class RcloneConnector {
21580
22817
  if (remote === undefined || remote.length === 0) {
21581
22818
  return { ok: false, reason: "not-configured", message: "rclone remote not configured" };
21582
22819
  }
21583
- const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
22820
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS9;
21584
22821
  const exportFormats = this.options.exportFormats ?? DEFAULT_EXPORT_FORMATS;
21585
22822
  let listResult;
21586
22823
  try {
@@ -21651,7 +22888,7 @@ class RcloneConnector {
21651
22888
  return;
21652
22889
  const destination = mirrorFilePath(paths.mirrorRoot, entry.path);
21653
22890
  const temp = `${destination}.autorag-tmp`;
21654
- mkdirSync18(dirname17(temp), { recursive: true });
22891
+ mkdirSync18(dirname18(temp), { recursive: true });
21655
22892
  let copied;
21656
22893
  try {
21657
22894
  copied = await this.runner(copyArgs(remote, entry.path, temp, exportFormats, this.options.concurrency, this.options.bandwidthLimit), timeoutMs);
@@ -21756,12 +22993,12 @@ function sameEntry(previous, current) {
21756
22993
  return previous?.path === current.path && previous.size === current.size && previous.modTime === current.modTime && previous.remoteId === current.remoteId && JSON.stringify(previous.hashes) === JSON.stringify(current.hashes);
21757
22994
  }
21758
22995
  function mirrorPaths(workspaceRoot, skillName, instanceId) {
21759
- const base = join30(workspaceRoot, ".autorag", "datasources", skillName, instanceId);
21760
- return { mirrorRoot: join30(base, "mirror"), manifestPath: join30(base, "manifest.json") };
22996
+ const base = join31(workspaceRoot, ".autorag", "datasources", skillName, instanceId);
22997
+ return { mirrorRoot: join31(base, "mirror"), manifestPath: join31(base, "manifest.json") };
21761
22998
  }
21762
22999
  function mirrorFilePath(root, path) {
21763
23000
  const safe = normalize4(path).replace(/^(\.\.(?:[/\\]|$))+|^[/\\]+/gu, "");
21764
- return join30(root, safe);
23001
+ return join31(root, safe);
21765
23002
  }
21766
23003
  function remoteTarget(remote, path) {
21767
23004
  return remote.endsWith(":") || remote.endsWith("/") ? `${remote}${path}` : `${remote}/${path}`;
@@ -21780,14 +23017,14 @@ function copyArgs(remote, path, destination, exportFormats, concurrency, bandwid
21780
23017
  }
21781
23018
  function loadManifest2(path) {
21782
23019
  try {
21783
- const parsed = JSON.parse(readFileSync19(path, "utf8"));
23020
+ const parsed = JSON.parse(readFileSync20(path, "utf8"));
21784
23021
  return parsed.version === 1 && Array.isArray(parsed.entries) ? parsed : undefined;
21785
23022
  } catch {
21786
23023
  return;
21787
23024
  }
21788
23025
  }
21789
23026
  function saveManifest(path, manifest) {
21790
- mkdirSync18(dirname17(path), { recursive: true });
23027
+ mkdirSync18(dirname18(path), { recursive: true });
21791
23028
  const temp = `${path}.autorag-tmp`;
21792
23029
  writeFileSync17(temp, `${JSON.stringify(manifest)}
21793
23030
  `, "utf8");
@@ -21889,7 +23126,7 @@ function runBinary(binary, args, timeoutMs, env, cwd) {
21889
23126
  });
21890
23127
  });
21891
23128
  }
21892
- var DEFAULT_BINARY3 = "rclone", DEFAULT_EXTENSIONS, DEFAULT_EXPORT_FORMATS = "txt,csv", DEFAULT_MAX_DOCUMENTS = 200, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_TIMEOUT_MS8 = 60000, MAX_CONTENT_CHARS2 = 1e5;
23129
+ var DEFAULT_BINARY3 = "rclone", DEFAULT_EXTENSIONS, DEFAULT_EXPORT_FORMATS = "txt,csv", DEFAULT_MAX_DOCUMENTS = 200, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_TIMEOUT_MS9 = 60000, MAX_CONTENT_CHARS2 = 1e5;
21893
23130
  var init_rclone_connector = __esm(() => {
21894
23131
  init_defaults();
21895
23132
  init_portable_spawn();
@@ -21897,52 +23134,11 @@ var init_rclone_connector = __esm(() => {
21897
23134
  DEFAULT_MAX_BYTES_PER_FILE = 2 * 1024 * 1024;
21898
23135
  });
21899
23136
 
21900
- // src/datasource/connector.ts
21901
- import { createHash as createHash12 } from "node:crypto";
21902
- function connectorFailureToDiagnosticCode(reason) {
21903
- switch (reason) {
21904
- case "auth":
21905
- return "datasource-auth-error";
21906
- case "permission":
21907
- return "datasource-permission-denied";
21908
- case "rate-limited":
21909
- return "datasource-rate-limited";
21910
- case "unavailable":
21911
- case "not-configured":
21912
- return "datasource-unavailable";
21913
- case "empty":
21914
- return "datasource-empty";
21915
- case "api-error":
21916
- case "invalid-data":
21917
- return "datasource-index-failed";
21918
- }
21919
- }
21920
- function boundDiagnosticText2(value) {
21921
- const trimmed = value.trim();
21922
- return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
21923
- }
21924
- function sanitizeIdSegment(value) {
21925
- const normalized = value.replace(/[^\p{L}\p{N}._-]+/gu, "-");
21926
- let start = 0;
21927
- while (start < normalized.length && (normalized[start] === "-" || normalized[start] === "."))
21928
- start += 1;
21929
- let end = normalized.length;
21930
- while (end > start && (normalized[end - 1] === "-" || normalized[end - 1] === "."))
21931
- end -= 1;
21932
- const cleaned = normalized.slice(start, end);
21933
- if (cleaned === value && cleaned.length > 0 && cleaned.length <= 80)
21934
- return cleaned;
21935
- const hash = createHash12("sha256").update(value).digest("hex").slice(0, 8);
21936
- const stem = cleaned.slice(0, 60);
21937
- return stem.length > 0 ? `${stem}-${hash}` : hash;
21938
- }
21939
- var init_connector = () => {};
21940
-
21941
23137
  // src/datasource/chunk-store.ts
21942
- import { existsSync as existsSync24, mkdirSync as mkdirSync19, readFileSync as readFileSync20, writeFileSync as writeFileSync18 } from "node:fs";
21943
- import { dirname as dirname18, join as join31 } from "node:path";
23138
+ import { existsSync as existsSync24, mkdirSync as mkdirSync19, readFileSync as readFileSync21, writeFileSync as writeFileSync18 } from "node:fs";
23139
+ import { dirname as dirname19, join as join32 } from "node:path";
21944
23140
  function datasourceChunkStorePath(workspaceRoot, skillName, instanceId) {
21945
- return join31(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId), "chunks.json");
23141
+ return join32(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId), "chunks.json");
21946
23142
  }
21947
23143
 
21948
23144
  class DatasourceChunkStore {
@@ -21981,7 +23177,7 @@ class DatasourceChunkStore {
21981
23177
  try {
21982
23178
  if (!existsSync24(this.storePath))
21983
23179
  return false;
21984
- const parsed = JSON.parse(readFileSync20(this.storePath, "utf8"));
23180
+ const parsed = JSON.parse(readFileSync21(this.storePath, "utf8"));
21985
23181
  if (parsed.version !== STORE_VERSION || !Array.isArray(parsed.chunks))
21986
23182
  return false;
21987
23183
  this.chunkList = parsed.chunks.filter((chunk) => typeof chunk?.chunkId === "string" && typeof chunk?.content === "string");
@@ -22027,7 +23223,7 @@ class DatasourceChunkStore {
22027
23223
  if (this.storePath === undefined)
22028
23224
  return;
22029
23225
  try {
22030
- mkdirSync19(dirname18(this.storePath), { recursive: true });
23226
+ mkdirSync19(dirname19(this.storePath), { recursive: true });
22031
23227
  const payload = {
22032
23228
  version: STORE_VERSION,
22033
23229
  skill: this.skillName,
@@ -22169,7 +23365,7 @@ class ConnectorDatasourceSkill {
22169
23365
  }
22170
23366
  if (!fetched.ok) {
22171
23367
  const code = connectorFailureToDiagnosticCode(fetched.reason);
22172
- const message = boundDiagnosticText2(fetched.message ?? fetched.reason);
23368
+ const message = boundDiagnosticText(fetched.message ?? fetched.reason);
22173
23369
  return this.fail(code, `${fetched.reason}: ${message}`);
22174
23370
  }
22175
23371
  const documents = this.applyDedupeWindow(fetched.documents);
@@ -22181,7 +23377,7 @@ class ConnectorDatasourceSkill {
22181
23377
  const diagnostics = (fetched.warnings ?? []).map((warning) => ({
22182
23378
  code: "datasource-index-failed",
22183
23379
  severity: "warning",
22184
- message: boundDiagnosticText2(warning),
23380
+ message: boundDiagnosticText(warning),
22185
23381
  instanceId: this.instanceId,
22186
23382
  source: this.definition.skillName
22187
23383
  }));
@@ -22298,7 +23494,7 @@ class ConnectorDatasourceSkill {
22298
23494
  return out;
22299
23495
  }
22300
23496
  fail(code, message) {
22301
- const sanitized = boundDiagnosticText2(message);
23497
+ const sanitized = boundDiagnosticText(message);
22302
23498
  this.lastError = sanitized;
22303
23499
  const diagnostic = {
22304
23500
  code,
@@ -22490,8 +23686,8 @@ var init_types4 = __esm(() => {
22490
23686
 
22491
23687
  // src/datasource/skills/discrawl/client.ts
22492
23688
  import { spawn as spawn12 } from "node:child_process";
22493
- import { mkdirSync as mkdirSync20, readFileSync as readFileSync21, writeFileSync as writeFileSync19 } from "node:fs";
22494
- import { dirname as dirname19 } from "node:path";
23689
+ import { mkdirSync as mkdirSync20, readFileSync as readFileSync22, writeFileSync as writeFileSync19 } from "node:fs";
23690
+ import { dirname as dirname20 } from "node:path";
22495
23691
 
22496
23692
  class DiscrawlClient {
22497
23693
  options;
@@ -22507,7 +23703,7 @@ class DiscrawlClient {
22507
23703
  const path = `${workspace}/.autorag/datasources/discrawl/config.toml`;
22508
23704
  let existing = "";
22509
23705
  try {
22510
- existing = readFileSync21(path, "utf8");
23706
+ existing = readFileSync22(path, "utf8");
22511
23707
  } catch {}
22512
23708
  const managed = existing.startsWith(`# AutoRAG managed discrawl embeddings v1
22513
23709
  `);
@@ -22515,7 +23711,7 @@ class DiscrawlClient {
22515
23711
  return { configured: false, rebuildRequired: false };
22516
23712
  const previous = managed ? parseManagedEmbedding(existing) : undefined;
22517
23713
  const config = managedDiscrawlConfig({ ...runtime, baseUrl: nativeGatewayBaseUrl(runtime.baseUrl) });
22518
- mkdirSync20(dirname19(path), { recursive: true });
23714
+ mkdirSync20(dirname20(path), { recursive: true });
22519
23715
  if (existing !== config)
22520
23716
  writeFileSync19(path, config, "utf8");
22521
23717
  return {
@@ -23540,8 +24736,8 @@ var init_github = __esm(() => {
23540
24736
 
23541
24737
  // src/datasource/skills/github-gist/connector.ts
23542
24738
  import { execFile as execFile2 } from "node:child_process";
23543
- import { existsSync as existsSync25, mkdirSync as mkdirSync21, readFileSync as readFileSync22, writeFileSync as writeFileSync20 } from "node:fs";
23544
- import { dirname as dirname20 } from "node:path";
24739
+ import { existsSync as existsSync25, mkdirSync as mkdirSync21, readFileSync as readFileSync23, writeFileSync as writeFileSync20 } from "node:fs";
24740
+ import { dirname as dirname21 } from "node:path";
23545
24741
  async function defaultGhTokenRunner() {
23546
24742
  return new Promise((resolve) => {
23547
24743
  execFile2("gh", ["auth", "token"], { timeout: GH_TOKEN_TIMEOUT_MS }, (error, stdout) => {
@@ -23663,7 +24859,7 @@ class GitHubGistConnector {
23663
24859
  if (statePath === undefined || !existsSync25(statePath))
23664
24860
  return { version: STATE_VERSION, gists: {} };
23665
24861
  try {
23666
- const parsed = JSON.parse(readFileSync22(statePath, "utf8"));
24862
+ const parsed = JSON.parse(readFileSync23(statePath, "utf8"));
23667
24863
  if (parsed.version !== STATE_VERSION || typeof parsed.gists !== "object" || parsed.gists === null) {
23668
24864
  return { version: STATE_VERSION, gists: {} };
23669
24865
  }
@@ -23677,7 +24873,7 @@ class GitHubGistConnector {
23677
24873
  if (statePath === undefined)
23678
24874
  return;
23679
24875
  try {
23680
- mkdirSync21(dirname20(statePath), { recursive: true });
24876
+ mkdirSync21(dirname21(statePath), { recursive: true });
23681
24877
  writeFileSync20(statePath, `${JSON.stringify(state)}
23682
24878
  `, "utf8");
23683
24879
  } catch {}
@@ -23723,8 +24919,8 @@ var init_connector3 = () => {};
23723
24919
 
23724
24920
  // src/datasource/skills/github-gist/semantic.ts
23725
24921
  import { createHash as createHash13 } from "node:crypto";
23726
- import { existsSync as existsSync26, mkdirSync as mkdirSync22, readFileSync as readFileSync23, writeFileSync as writeFileSync21 } from "node:fs";
23727
- import { dirname as dirname21 } from "node:path";
24922
+ import { existsSync as existsSync26, mkdirSync as mkdirSync22, readFileSync as readFileSync24, writeFileSync as writeFileSync21 } from "node:fs";
24923
+ import { dirname as dirname22 } from "node:path";
23728
24924
  function contentHash(chunk) {
23729
24925
  return createHash13("sha256").update(`${chunk.title ?? ""}
23730
24926
  ${chunk.content}`).digest("hex");
@@ -23817,7 +25013,7 @@ ${chunk.content}`));
23817
25013
  if (this.statePath === undefined || !existsSync26(this.statePath))
23818
25014
  return;
23819
25015
  try {
23820
- const parsed = JSON.parse(readFileSync23(this.statePath, "utf8"));
25016
+ const parsed = JSON.parse(readFileSync24(this.statePath, "utf8"));
23821
25017
  if (parsed.version !== VECTORS_VERSION || typeof parsed.entries !== "object" || parsed.entries === null)
23822
25018
  return;
23823
25019
  if (typeof parsed.identity?.dimension !== "number")
@@ -23832,7 +25028,7 @@ ${chunk.content}`));
23832
25028
  if (this.identity === undefined)
23833
25029
  return;
23834
25030
  try {
23835
- mkdirSync22(dirname21(this.statePath), { recursive: true });
25031
+ mkdirSync22(dirname22(this.statePath), { recursive: true });
23836
25032
  const payload = {
23837
25033
  version: VECTORS_VERSION,
23838
25034
  identity: this.identity,
@@ -23915,9 +25111,9 @@ var init_semantic = __esm(() => {
23915
25111
  });
23916
25112
 
23917
25113
  // src/datasource/skills/github-gist/skill.ts
23918
- import { join as join32 } from "node:path";
25114
+ import { join as join33 } from "node:path";
23919
25115
  function gistDatasourceDir(workspaceRoot, skillName, instanceId) {
23920
- return join32(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId));
25116
+ return join33(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId));
23921
25117
  }
23922
25118
  var GITHUB_GIST_SKILL_DEFINITION, DEFAULT_INSTANCE_ID3 = "default", GitHubGistSkill;
23923
25119
  var init_skill5 = __esm(() => {
@@ -23945,7 +25141,7 @@ var init_skill5 = __esm(() => {
23945
25141
  constructor(options = {}) {
23946
25142
  const { connector, connectorOptions, semantic, ...rest } = options;
23947
25143
  const instanceId = options.instanceId ?? DEFAULT_INSTANCE_ID3;
23948
- const statePath = connectorOptions?.statePath ?? (options.workspaceRoot === undefined ? undefined : join32(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "state.json"));
25144
+ const statePath = connectorOptions?.statePath ?? (options.workspaceRoot === undefined ? undefined : join33(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "state.json"));
23949
25145
  super(GITHUB_GIST_SKILL_DEFINITION, {
23950
25146
  ...rest,
23951
25147
  connector: connector ?? new GitHubGistConnector({
@@ -23955,7 +25151,7 @@ var init_skill5 = __esm(() => {
23955
25151
  });
23956
25152
  if (semantic?.enabled !== false) {
23957
25153
  this.semanticIndex = new GistSemanticIndex({
23958
- statePath: semantic?.statePath ?? (options.workspaceRoot === undefined ? undefined : join32(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "vectors.json"))
25154
+ statePath: semantic?.statePath ?? (options.workspaceRoot === undefined ? undefined : join33(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "vectors.json"))
23959
25155
  });
23960
25156
  this.semanticEmbedder = semantic?.embedder ?? createGatewayEmbedder(semantic?.embedderOptions ?? {});
23961
25157
  this.semanticMethod = new GitHubGistSemanticMethod({
@@ -23983,7 +25179,7 @@ var init_skill5 = __esm(() => {
23983
25179
  {
23984
25180
  code: "datasource-index-failed",
23985
25181
  severity: "warning",
23986
- message: boundDiagnosticText2(`semantic-unavailable: ${sync.message}`),
25182
+ message: boundDiagnosticText(`semantic-unavailable: ${sync.message}`),
23987
25183
  instanceId: this.instanceId,
23988
25184
  source: this.describe().name
23989
25185
  }
@@ -24678,7 +25874,7 @@ var init_types5 = __esm(() => {
24678
25874
  // src/datasource/skills/lazykatok/client.ts
24679
25875
  import { spawn as spawn14 } from "node:child_process";
24680
25876
  import { existsSync as existsSync27 } from "node:fs";
24681
- import { delimiter as delimiter4, join as join33 } from "node:path";
25877
+ import { delimiter as delimiter4, join as join34 } from "node:path";
24682
25878
 
24683
25879
  class LazykatokClient {
24684
25880
  options;
@@ -24866,7 +26062,7 @@ function lookupInPath2(executable, env) {
24866
26062
  for (const dir of pathEnv.split(delimiter4)) {
24867
26063
  if (dir.length === 0)
24868
26064
  continue;
24869
- const candidate = join33(dir, executable);
26065
+ const candidate = join34(dir, executable);
24870
26066
  if (existsSync27(candidate))
24871
26067
  return candidate;
24872
26068
  }
@@ -25356,7 +26552,7 @@ var init_lazykatok = __esm(() => {
25356
26552
 
25357
26553
  // src/datasource/skills/mail-export/connector.ts
25358
26554
  import { readdir as readdir3, readFile as readFile5, stat as stat6 } from "node:fs/promises";
25359
- import { basename as basename12, dirname as dirname22, extname as extname7, join as join34 } from "node:path";
26555
+ import { basename as basename12, dirname as dirname23, extname as extname7, join as join35 } from "node:path";
25360
26556
  import { simpleParser as simpleParser2 } from "mailparser";
25361
26557
 
25362
26558
  class MailExportConnector {
@@ -25382,7 +26578,7 @@ class MailExportConnector {
25382
26578
  continue;
25383
26579
  const extension = extname7(entry.name).toLowerCase();
25384
26580
  if (extension === ".eml" || extension === ".mbox") {
25385
- files.push(join34(entry.parentPath, entry.name));
26581
+ files.push(join35(entry.parentPath, entry.name));
25386
26582
  }
25387
26583
  }
25388
26584
  } else if (info.isFile()) {
@@ -25417,7 +26613,7 @@ class MailExportConnector {
25417
26613
  continue;
25418
26614
  }
25419
26615
  const stem = basename12(file, extname7(file));
25420
- const mailboxDir = basename12(dirname22(file));
26616
+ const mailboxDir = basename12(dirname23(file));
25421
26617
  const hierarchy = ["mailboxes", mailboxDir, stem];
25422
26618
  const extension = extname7(file).toLowerCase();
25423
26619
  const rawMessages = extension === ".mbox" ? splitMbox(bytes.toString("utf8")) : [bytes.toString("utf8")];
@@ -25538,7 +26734,7 @@ function ensurePrivateMailcrawlDataDir(path) {
25538
26734
  if (process.platform !== "win32")
25539
26735
  chmodSync(path, 448);
25540
26736
  }
25541
- var init_config2 = () => {};
26737
+ var init_config3 = () => {};
25542
26738
 
25543
26739
  // src/datasource/skills/mailcrawl/client.ts
25544
26740
  import { spawn as spawn15 } from "node:child_process";
@@ -25864,7 +27060,7 @@ function truncateUtf8(value, maxBytes) {
25864
27060
  var DEFAULT_BINARY4 = "mailcrawl", DEFAULT_SEARCH_TIMEOUT_MS3 = 60000, DEFAULT_INDEX_TIMEOUT_MS3 = 1800000, DEFAULT_MAX_BUFFER_BYTES4 = 1048576, LOOPBACK_HOSTS2, SAFE_ENV_KEYS2, SAFE_MAILCRAWL_ENV_KEYS, REMOTE_EMBEDDING_ENV_KEYS;
25865
27061
  var init_client9 = __esm(() => {
25866
27062
  init_portable_spawn();
25867
- init_config2();
27063
+ init_config3();
25868
27064
  LOOPBACK_HOSTS2 = new Set(["127.0.0.1", "localhost", "[::1]", "::1"]);
25869
27065
  SAFE_ENV_KEYS2 = new Set(["HOME", "LANG", "LC_ALL", "PATH", "TMPDIR", "TMP", "TEMP", "NO_COLOR"]);
25870
27066
  SAFE_MAILCRAWL_ENV_KEYS = new Set([
@@ -26111,7 +27307,7 @@ var TAGS;
26111
27307
  var init_skill9 = __esm(() => {
26112
27308
  init_scope2();
26113
27309
  init_client9();
26114
- init_config2();
27310
+ init_config3();
26115
27311
  init_methods4();
26116
27312
  TAGS = ["mailcrawl", "email", "pii"];
26117
27313
  });
@@ -26123,6 +27319,504 @@ var init_mailcrawl = __esm(() => {
26123
27319
  init_skill9();
26124
27320
  });
26125
27321
 
27322
+ // src/datasource/sidecar/fusion.ts
27323
+ function encodeVector(vector) {
27324
+ const floats = Float32Array.from(vector);
27325
+ return new Uint8Array(floats.buffer, floats.byteOffset, floats.byteLength);
27326
+ }
27327
+ function decodeVector(blob) {
27328
+ if (blob.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0)
27329
+ return;
27330
+ return new Float32Array(new Uint8Array(blob).buffer);
27331
+ }
27332
+ function cosine3(a, b) {
27333
+ if (a.length === 0 || a.length !== b.length)
27334
+ return 0;
27335
+ let dot = 0;
27336
+ let normA = 0;
27337
+ let normB = 0;
27338
+ for (let index = 0;index < a.length; index += 1) {
27339
+ const x = a[index] ?? 0;
27340
+ const y = b[index] ?? 0;
27341
+ dot += x * y;
27342
+ normA += x * x;
27343
+ normB += y * y;
27344
+ }
27345
+ if (normA === 0 || normB === 0)
27346
+ return 0;
27347
+ return dot / Math.sqrt(normA * normB);
27348
+ }
27349
+ function topKByCosine(query, rows, k) {
27350
+ if (k <= 0)
27351
+ return [];
27352
+ const scored = [];
27353
+ for (const row of rows) {
27354
+ const score = cosine3(query, row.vector);
27355
+ if (score > 0)
27356
+ scored.push({ key: row.key, score });
27357
+ }
27358
+ scored.sort((a, b) => b.score - a.score || a.key.localeCompare(b.key));
27359
+ return scored.slice(0, k);
27360
+ }
27361
+ function reciprocalRankFusion(lists) {
27362
+ const scores = new Map;
27363
+ for (const list of lists) {
27364
+ const seen = new Set;
27365
+ for (const [index, key] of list.entries()) {
27366
+ if (seen.has(key))
27367
+ continue;
27368
+ seen.add(key);
27369
+ scores.set(key, (scores.get(key) ?? 0) + 1 / (RRF_K2 + index + 1));
27370
+ }
27371
+ }
27372
+ return [...scores.entries()].map(([key, score]) => ({ key, score })).sort((a, b) => b.score - a.score);
27373
+ }
27374
+ var RRF_K2 = 60;
27375
+
27376
+ // src/datasource/sidecar/index.ts
27377
+ import { createHash as createHash14 } from "node:crypto";
27378
+ import { mkdirSync as mkdirSync24, statSync as statSync3 } from "node:fs";
27379
+ import { dirname as dirname24 } from "node:path";
27380
+ import { DatabaseSync as DatabaseSync2 } from "node:sqlite";
27381
+ import { pathToFileURL } from "node:url";
27382
+
27383
+ class CrawlerSidecar {
27384
+ descriptor;
27385
+ options;
27386
+ select;
27387
+ where;
27388
+ opened;
27389
+ attached;
27390
+ lastSemantic;
27391
+ constructor(options) {
27392
+ this.options = options;
27393
+ const { descriptor } = options;
27394
+ this.descriptor = descriptor;
27395
+ const metadataColumns = descriptor.metadata.map((column, index) => `${column.sql} as m${index}`);
27396
+ this.select = [
27397
+ `${descriptor.keyExpression} as k`,
27398
+ `${descriptor.textExpression} as content`,
27399
+ `${descriptor.titleExpression ?? "null"} as title`,
27400
+ `${descriptor.epochSecondsExpression ?? "null"} as epoch`,
27401
+ ...metadataColumns
27402
+ ].join(", ");
27403
+ const scope = descriptor.scopeExpression !== undefined && options.workspaceId !== undefined;
27404
+ const allowlist = [
27405
+ ...(options.channelIds?.length ?? 0) > 0 ? [`${descriptor.channelIdExpression} in (${options.channelIds?.map(() => "?").join(", ")})`] : [],
27406
+ ...(options.channelNames?.length ?? 0) > 0 ? [`${descriptor.channelNameExpression} in (${options.channelNames?.map(() => "?").join(", ")})`] : []
27407
+ ];
27408
+ this.where = [
27409
+ descriptor.liveWhere,
27410
+ ...scope ? [`${descriptor.scopeExpression} = ?`] : [],
27411
+ ...allowlist.length > 0 ? [`(${allowlist.join(" or ")})`] : []
27412
+ ].join(" and ");
27413
+ }
27414
+ get db() {
27415
+ if (this.opened !== undefined)
27416
+ return this.opened;
27417
+ mkdirSync24(dirname24(this.options.sidecarDbPath), { recursive: true });
27418
+ const db = new DatabaseSync2(this.options.sidecarDbPath);
27419
+ db.exec(`pragma busy_timeout = ${BUSY_TIMEOUT_MS}`);
27420
+ db.exec("pragma journal_mode = wal");
27421
+ db.exec(`
27422
+ create table if not exists sidecar_meta (k text primary key, v text not null);
27423
+ create table if not exists sidecar_vec (message_key text primary key, hash text not null, vec blob not null);
27424
+ `);
27425
+ this.opened = db;
27426
+ return db;
27427
+ }
27428
+ attachCrawler() {
27429
+ let file;
27430
+ try {
27431
+ const { dev, ino } = statSync3(this.options.crawlerDbPath);
27432
+ file = { dev, ino };
27433
+ } catch {
27434
+ this.detachCrawler();
27435
+ return false;
27436
+ }
27437
+ if (this.attached !== undefined) {
27438
+ if (this.attached.dev === file.dev && this.attached.ino === file.ino)
27439
+ return true;
27440
+ this.detachCrawler();
27441
+ }
27442
+ const crawlerUri = `${pathToFileURL(this.options.crawlerDbPath).href}?mode=ro`.replaceAll("'", "''");
27443
+ try {
27444
+ this.db.exec(`attach database '${crawlerUri}' as crawler`);
27445
+ } catch {
27446
+ return false;
27447
+ }
27448
+ const { descriptor } = this.options;
27449
+ try {
27450
+ this.db.prepare(`select ${this.select} from ${descriptor.fromClause} where ${this.where} limit 0`).all(...this.whereParams());
27451
+ this.db.prepare(`select 1 from ${descriptor.fts.table} limit 0`).all();
27452
+ } catch {
27453
+ this.db.exec("detach database crawler");
27454
+ return false;
27455
+ }
27456
+ this.attached = file;
27457
+ this.lastSemantic = undefined;
27458
+ return true;
27459
+ }
27460
+ detachCrawler() {
27461
+ if (this.attached === undefined)
27462
+ return;
27463
+ this.attached = undefined;
27464
+ this.lastSemantic = undefined;
27465
+ try {
27466
+ this.opened?.exec("detach database crawler");
27467
+ } catch {}
27468
+ }
27469
+ get semanticEnabled() {
27470
+ return this.options.embedder !== undefined;
27471
+ }
27472
+ close() {
27473
+ this.opened?.close();
27474
+ this.opened = undefined;
27475
+ this.attached = undefined;
27476
+ this.lastSemantic = undefined;
27477
+ }
27478
+ whereParams() {
27479
+ const { descriptor, workspaceId, channelIds = [], channelNames = [] } = this.options;
27480
+ return [
27481
+ ...descriptor.scopeExpression !== undefined && workspaceId !== undefined ? [workspaceId] : [],
27482
+ ...channelIds,
27483
+ ...channelNames
27484
+ ];
27485
+ }
27486
+ searchLexical(query, topK) {
27487
+ const match = toMatchQuery(query);
27488
+ if (match === undefined || topK <= 0 || !this.attachCrawler())
27489
+ return [];
27490
+ const { descriptor } = this.options;
27491
+ const rows = this.db.prepare(`select ${this.select}, bm25(${descriptor.fts.table}) as rank
27492
+ from ${descriptor.fromClause} join ${descriptor.fts.table} on ${descriptor.fts.joinOn}
27493
+ where ${descriptor.fts.table} match ? and ${this.where}
27494
+ order by rank limit ?`).all(match, ...this.whereParams(), topK);
27495
+ return rows.map((row) => this.toHit(this.toRow(row), -Number(row.rank)));
27496
+ }
27497
+ async searchSemantic(query, topK) {
27498
+ const trimmed = query.trim();
27499
+ if (this.options.embedder === undefined || trimmed.length === 0 || topK <= 0 || !this.attachCrawler())
27500
+ return [];
27501
+ const ranked = await this.rankSemantic(trimmed, topK * POOL_FACTOR);
27502
+ if (ranked.length === 0)
27503
+ return [];
27504
+ const live = this.hydrate(ranked.map((entry) => entry.key));
27505
+ const hits = [];
27506
+ for (const entry of ranked) {
27507
+ const row = live.get(entry.key);
27508
+ if (row !== undefined)
27509
+ hits.push(this.toHit(row, entry.score));
27510
+ if (hits.length >= topK)
27511
+ break;
27512
+ }
27513
+ return hits;
27514
+ }
27515
+ async rankSemantic(query, pool) {
27516
+ const now = Date.now();
27517
+ let cached = this.lastSemantic;
27518
+ if (cached === undefined || cached.query !== query || now - cached.at > SEMANTIC_REUSE_MS) {
27519
+ const embedder = this.options.embedder;
27520
+ if (embedder === undefined)
27521
+ return [];
27522
+ cached = {
27523
+ query,
27524
+ at: now,
27525
+ vector: embedder.embed([query]).then((vectors) => vectors[0], () => {
27526
+ return;
27527
+ }),
27528
+ ranked: undefined
27529
+ };
27530
+ this.lastSemantic = cached;
27531
+ }
27532
+ const vector = await cached.vector;
27533
+ if (vector === undefined) {
27534
+ if (this.lastSemantic === cached)
27535
+ this.lastSemantic = undefined;
27536
+ return [];
27537
+ }
27538
+ if (cached.ranked === undefined || cached.ranked.pool < pool) {
27539
+ const wide = pool * POOL_FACTOR;
27540
+ cached.ranked = { pool: wide, keys: topKByCosine(vector, this.storedVectors(), wide) };
27541
+ }
27542
+ return cached.ranked.keys.slice(0, pool);
27543
+ }
27544
+ async searchHybrid(query, topK) {
27545
+ if (topK <= 0)
27546
+ return [];
27547
+ const pool = topK * POOL_FACTOR;
27548
+ const lexical = this.searchLexical(query, pool);
27549
+ const semantic = await this.searchSemantic(query, pool);
27550
+ if (semantic.length === 0)
27551
+ return lexical.slice(0, topK);
27552
+ const byKey = new Map;
27553
+ for (const hit of [...semantic, ...lexical])
27554
+ byKey.set(hit.key, hit);
27555
+ const fused = [];
27556
+ for (const entry of reciprocalRankFusion([lexical.map((hit) => hit.key), semantic.map((hit) => hit.key)])) {
27557
+ const hit = byKey.get(entry.key);
27558
+ if (hit !== undefined)
27559
+ fused.push({ ...hit, score: entry.score });
27560
+ if (fused.length >= topK)
27561
+ break;
27562
+ }
27563
+ return fused;
27564
+ }
27565
+ async sync() {
27566
+ const { embedder } = this.options;
27567
+ if (embedder === undefined)
27568
+ return { ok: false, message: "no embedder configured" };
27569
+ if (!this.attachCrawler())
27570
+ return { ok: false, message: "crawler archive not available yet" };
27571
+ let identity;
27572
+ try {
27573
+ identity = await embedder.identity();
27574
+ } catch (error) {
27575
+ return { ok: false, message: errorMessage2(error) };
27576
+ }
27577
+ this.adoptIdentity(identity);
27578
+ const pruned = this.prune();
27579
+ const pending = this.pendingKeys();
27580
+ const batchSize = Math.max(1, this.options.batchSize ?? DEFAULT_BATCH_SIZE);
27581
+ const budget = Math.max(1, this.options.maxEmbedPerSync ?? DEFAULT_MAX_EMBED_PER_SYNC);
27582
+ const target = Math.min(pending.length, budget);
27583
+ const embed = async (rows) => {
27584
+ const vectors = await embedder.embed(rows.map((row) => row.input));
27585
+ if (vectors.length !== rows.length) {
27586
+ throw new Error(`embedder returned ${vectors.length} vectors for ${rows.length} messages`);
27587
+ }
27588
+ const wrong = vectors.find((vector) => vector.length !== identity.dimension);
27589
+ if (wrong !== undefined) {
27590
+ throw new Error(`embedder returned a ${wrong.length}-dimension vector, expected ${identity.dimension}`);
27591
+ }
27592
+ return vectors;
27593
+ };
27594
+ let embedded = 0;
27595
+ let skipped = 0;
27596
+ for (let offset = 0;offset < target; offset += batchSize) {
27597
+ const rows = [...this.hydrate(pending.slice(offset, Math.min(offset + batchSize, target))).values()];
27598
+ if (rows.length === 0)
27599
+ continue;
27600
+ try {
27601
+ this.store(rows, await embed(rows));
27602
+ embedded += rows.length;
27603
+ } catch (batchError) {
27604
+ const rejected = [];
27605
+ let lastError = batchError;
27606
+ for (const row of rows) {
27607
+ try {
27608
+ this.store([row], await embed([row]));
27609
+ embedded += 1;
27610
+ } catch (error) {
27611
+ rejected.push(row);
27612
+ lastError = error;
27613
+ }
27614
+ }
27615
+ if (rejected.length === 0)
27616
+ continue;
27617
+ if (embedded === 0)
27618
+ return { ok: false, message: errorMessage2(lastError) };
27619
+ this.store(rejected, undefined);
27620
+ skipped += rejected.length;
27621
+ }
27622
+ }
27623
+ return { ok: true, embedded, skipped, pruned, remaining: pending.length - embedded - skipped };
27624
+ }
27625
+ adoptIdentity(identity) {
27626
+ const serialized = JSON.stringify(identity);
27627
+ const current = this.db.prepare("select v from sidecar_meta where k = ?").get(IDENTITY_KEY);
27628
+ if (current?.v === serialized)
27629
+ return;
27630
+ this.transaction(() => {
27631
+ this.db.exec("delete from sidecar_vec");
27632
+ this.db.prepare("insert or replace into sidecar_meta (k, v) values (?, ?)").run(IDENTITY_KEY, serialized);
27633
+ });
27634
+ this.lastSemantic = undefined;
27635
+ }
27636
+ prune() {
27637
+ const { descriptor } = this.options;
27638
+ const result = this.db.prepare(`delete from sidecar_vec where message_key not in (
27639
+ select ${descriptor.keyExpression} from ${descriptor.fromClause} where ${this.where}
27640
+ )`).run(...this.whereParams());
27641
+ if (Number(result.changes) > 0)
27642
+ this.lastSemantic = undefined;
27643
+ return Number(result.changes);
27644
+ }
27645
+ pendingKeys() {
27646
+ const { descriptor } = this.options;
27647
+ const rows = this.db.prepare(`select ${this.select}, v.hash as stored_hash
27648
+ from ${descriptor.fromClause}
27649
+ left join sidecar_vec v on v.message_key = ${descriptor.keyExpression}
27650
+ where ${this.where}`).iterate(...this.whereParams());
27651
+ const pending = [];
27652
+ for (const row of rows) {
27653
+ if (row.stored_hash !== this.toRow(row).hash)
27654
+ pending.push(String(row.k));
27655
+ }
27656
+ return pending;
27657
+ }
27658
+ hydrate(keys) {
27659
+ const { descriptor } = this.options;
27660
+ const out = new Map;
27661
+ if (keys.length === 0)
27662
+ return out;
27663
+ const rows = this.db.prepare(`select ${this.select} from ${descriptor.fromClause}
27664
+ where ${descriptor.keyExpression} in (${keys.map(() => "?").join(", ")}) and ${this.where}`).all(...keys, ...this.whereParams());
27665
+ const byKey = new Map(rows.map((row) => [String(row.k), this.toRow(row)]));
27666
+ for (const key of keys) {
27667
+ const row = byKey.get(key);
27668
+ if (row !== undefined)
27669
+ out.set(key, row);
27670
+ }
27671
+ return out;
27672
+ }
27673
+ store(rows, vectors) {
27674
+ const upsert = this.db.prepare("insert or replace into sidecar_vec (message_key, hash, vec) values (?, ?, ?)");
27675
+ this.transaction(() => {
27676
+ for (const [index, row] of rows.entries()) {
27677
+ upsert.run(row.key, row.hash, encodeVector(vectors?.[index] ?? []));
27678
+ }
27679
+ });
27680
+ this.lastSemantic = undefined;
27681
+ }
27682
+ *storedVectors() {
27683
+ const rows = this.db.prepare("select message_key, vec from sidecar_vec where length(vec) > 0").iterate();
27684
+ for (const row of rows) {
27685
+ const vector = row.vec instanceof Uint8Array ? decodeVector(row.vec) : undefined;
27686
+ if (vector !== undefined)
27687
+ yield { key: String(row.message_key), vector };
27688
+ }
27689
+ }
27690
+ transaction(work) {
27691
+ this.db.exec("begin immediate");
27692
+ try {
27693
+ work();
27694
+ this.db.exec("commit");
27695
+ } catch (error) {
27696
+ this.db.exec("rollback");
27697
+ throw error;
27698
+ }
27699
+ }
27700
+ toRow(row) {
27701
+ const metadata = {};
27702
+ for (const [index, column] of this.options.descriptor.metadata.entries()) {
27703
+ const value = row[`m${index}`];
27704
+ if (typeof value === "string" && value.length > 0 || typeof value === "number")
27705
+ metadata[column.key] = String(value);
27706
+ }
27707
+ const title = typeof row.title === "string" && row.title.length > 0 ? row.title : undefined;
27708
+ const epoch = typeof row.epoch === "number" && Number.isFinite(row.epoch) ? row.epoch : undefined;
27709
+ const content = String(row.content ?? "");
27710
+ const input = `${title ?? ""}
27711
+ ${content}`.slice(0, MAX_EMBED_INPUT_CHARS);
27712
+ return {
27713
+ key: String(row.k),
27714
+ content,
27715
+ title,
27716
+ epoch,
27717
+ metadata,
27718
+ input,
27719
+ hash: createHash14("sha1").update(input).digest("hex")
27720
+ };
27721
+ }
27722
+ toHit(row, score) {
27723
+ return {
27724
+ key: row.key,
27725
+ content: row.content,
27726
+ score,
27727
+ ...row.title !== undefined ? { title: row.title } : {},
27728
+ ...row.epoch !== undefined ? { publishedAt: Math.round(row.epoch * 1000) } : {},
27729
+ metadata: row.metadata
27730
+ };
27731
+ }
27732
+ }
27733
+ function toMatchQuery(text) {
27734
+ const terms = [...new Set(text.match(/[\p{L}\p{N}_]+/gu) ?? [])];
27735
+ return terms.length === 0 ? undefined : terms.map((term) => `"${term}"`).join(" OR ");
27736
+ }
27737
+ function errorMessage2(error) {
27738
+ return error instanceof Error ? error.message : String(error);
27739
+ }
27740
+ var DEFAULT_BATCH_SIZE = 256, DEFAULT_MAX_EMBED_PER_SYNC = 2048, MAX_EMBED_INPUT_CHARS = 8192, BUSY_TIMEOUT_MS = 5000, POOL_FACTOR = 3, SEMANTIC_REUSE_MS = 1e4, IDENTITY_KEY = "identity";
27741
+ var init_sidecar = () => {};
27742
+
27743
+ // src/datasource/sidecar/methods.ts
27744
+ function sidecarMethodName(datasourceId, mode) {
27745
+ return mode === "bm25" ? `${datasourceId}-sidecar-bm25` : `${datasourceId}-${mode}`;
27746
+ }
27747
+
27748
+ class SidecarRetrievalMethod {
27749
+ options;
27750
+ constructor(options) {
27751
+ this.options = options;
27752
+ }
27753
+ describe() {
27754
+ const { datasourceId, skillType, mode, tags } = this.options;
27755
+ const { type, text, capabilities } = DESCRIPTIONS[mode];
27756
+ return {
27757
+ name: sidecarMethodName(datasourceId, mode),
27758
+ type,
27759
+ description: `${text} over ${skillType} via a read-only sidecar index`,
27760
+ status: "active",
27761
+ capabilities: [...capabilities, "scoped", "path-opaque-sources"],
27762
+ datasourceId,
27763
+ tags
27764
+ };
27765
+ }
27766
+ async retrieve(query, options) {
27767
+ const trimmed = query.trim();
27768
+ if (trimmed.length === 0)
27769
+ return [];
27770
+ const topK = options.topK ?? DEFAULT_TOP_K5;
27771
+ const { sidecar, mode } = this.options;
27772
+ const hits = mode === "bm25" ? sidecar.searchLexical(trimmed, topK) : mode === "semantic" ? await sidecar.searchSemantic(trimmed, topK) : await sidecar.searchHybrid(trimmed, topK);
27773
+ const results = [];
27774
+ for (const hit of hits) {
27775
+ const result = this.toResult(hit, options);
27776
+ if (result !== undefined)
27777
+ results.push(result);
27778
+ }
27779
+ return results;
27780
+ }
27781
+ toResult(hit, options) {
27782
+ const { descriptor, datasourceId, instanceId, mode } = this.options;
27783
+ const chunkId = descriptor.chunkId(hit.key, hit.metadata);
27784
+ const source = datasourceSourcePath(datasourceId, instanceId, chunkId);
27785
+ if (!matchesDatasourceScope(source, options.scope))
27786
+ return;
27787
+ const method = sidecarMethodName(datasourceId, mode);
27788
+ return {
27789
+ id: `${datasourceId}:${instanceId}:${chunkId}`,
27790
+ content: hit.content,
27791
+ source,
27792
+ score: hit.score,
27793
+ metadata: {
27794
+ ...hit.metadata,
27795
+ method,
27796
+ datasourceId,
27797
+ instanceId,
27798
+ backend: "sidecar",
27799
+ ...hit.title !== undefined ? { title: hit.title } : {},
27800
+ hierarchy: descriptor.hierarchy(hit.metadata).join("/"),
27801
+ ...hit.publishedAt !== undefined ? { publishedAt: hit.publishedAt } : {}
27802
+ }
27803
+ };
27804
+ }
27805
+ }
27806
+ var DEFAULT_TOP_K5 = 20, DESCRIPTIONS;
27807
+ var init_methods5 = __esm(() => {
27808
+ init_scope2();
27809
+ DESCRIPTIONS = {
27810
+ bm25: { type: "bm25", text: "BM25-ranked FTS5 retrieval", capabilities: ["lexical", "fts5", "bm25"] },
27811
+ semantic: { type: "vector", text: "Semantic (embedding) retrieval", capabilities: ["semantic"] },
27812
+ hybrid: {
27813
+ type: "hybrid",
27814
+ text: "Hybrid BM25 + semantic retrieval (reciprocal rank fusion)",
27815
+ capabilities: ["lexical", "semantic", "hybrid"]
27816
+ }
27817
+ };
27818
+ });
27819
+
26126
27820
  // src/datasource/crawler-skill.ts
26127
27821
  class CrawlerDatasourceSkill {
26128
27822
  definition;
@@ -26133,6 +27827,7 @@ class CrawlerDatasourceSkill {
26133
27827
  tags;
26134
27828
  channelIds;
26135
27829
  channelNames;
27830
+ sidecar;
26136
27831
  lastIndexedAt;
26137
27832
  constructor(definition, options) {
26138
27833
  this.definition = options.datasourceId === undefined ? definition : {
@@ -26148,6 +27843,11 @@ class CrawlerDatasourceSkill {
26148
27843
  this.lastIndexedAt = options.lastIndexedAt;
26149
27844
  this.channelIds = new Set(options.channelIds ?? []);
26150
27845
  this.channelNames = new Set(options.channelNames ?? []);
27846
+ this.sidecar = options.sidecar === undefined ? undefined : new CrawlerSidecar({
27847
+ ...options.sidecar,
27848
+ channelIds: options.channelIds ?? [],
27849
+ channelNames: options.channelNames ?? []
27850
+ });
26151
27851
  }
26152
27852
  describe() {
26153
27853
  return {
@@ -26183,26 +27883,89 @@ class CrawlerDatasourceSkill {
26183
27883
  skill: this.definition.datasourceId,
26184
27884
  chunkCount: result.count,
26185
27885
  indexedAt: this.lastIndexedAt,
26186
- diagnostics: []
27886
+ diagnostics: await this.syncSidecar()
26187
27887
  };
26188
27888
  } catch {
26189
27889
  return this.fail("datasource-unavailable", "spawn-error");
26190
27890
  }
26191
27891
  }
27892
+ async syncSidecar() {
27893
+ if (this.sidecar === undefined || !this.sidecar.semanticEnabled)
27894
+ return [];
27895
+ const base = { instanceId: this.instanceId, source: this.definition.datasourceId };
27896
+ let result;
27897
+ try {
27898
+ result = await this.sidecar.sync();
27899
+ } catch (error) {
27900
+ result = { ok: false, message: error instanceof Error ? error.message : String(error) };
27901
+ }
27902
+ if (!result.ok) {
27903
+ return [
27904
+ {
27905
+ code: "datasource-index-failed",
27906
+ severity: "warning",
27907
+ message: `semantic-unavailable: ${result.message}`.slice(0, 300),
27908
+ ...base
27909
+ }
27910
+ ];
27911
+ }
27912
+ const diagnostics = [];
27913
+ if (result.skipped > 0) {
27914
+ diagnostics.push({
27915
+ code: "datasource-index-partial",
27916
+ severity: "warning",
27917
+ message: `semantic index skipped ${result.skipped} message(s) the embedder rejected; they stay searchable by BM25 and are retried when edited`,
27918
+ ...base
27919
+ });
27920
+ }
27921
+ if (result.remaining > 0) {
27922
+ diagnostics.push({
27923
+ code: "datasource-index-partial",
27924
+ severity: "info",
27925
+ message: `semantic index catching up: ${result.embedded} embedded this refresh, ${result.remaining} remaining; hybrid search uses BM25 for the rest until a later refresh`,
27926
+ ...base
27927
+ });
27928
+ }
27929
+ return diagnostics;
27930
+ }
27931
+ close() {
27932
+ this.sidecar?.close();
27933
+ }
26192
27934
  retrievalMethods() {
27935
+ const cli = this.cliMethod();
27936
+ const { sidecar } = this;
27937
+ if (sidecar === undefined)
27938
+ return [cli];
27939
+ const shared = {
27940
+ sidecar,
27941
+ descriptor: sidecar.descriptor,
27942
+ datasourceId: this.definition.datasourceId,
27943
+ skillType: this.definition.skillType,
27944
+ instanceId: this.instanceId,
27945
+ tags: this.tags
27946
+ };
27947
+ const bm25 = new SidecarRetrievalMethod({ ...shared, mode: "bm25" });
27948
+ if (!sidecar.semanticEnabled)
27949
+ return [cli, bm25];
26193
27950
  return [
26194
- new CrawlerLexicalMethod({
26195
- client: this.client,
26196
- datasourceId: this.definition.datasourceId,
26197
- skillType: this.definition.skillType,
26198
- instanceId: this.instanceId,
26199
- tags: this.tags,
26200
- backendName: this.definition.backendName,
26201
- channelIds: this.channelIds,
26202
- channelNames: this.channelNames
26203
- })
27951
+ cli,
27952
+ bm25,
27953
+ new SidecarRetrievalMethod({ ...shared, mode: "semantic" }),
27954
+ new SidecarRetrievalMethod({ ...shared, mode: "hybrid" })
26204
27955
  ];
26205
27956
  }
27957
+ cliMethod() {
27958
+ return new CrawlerLexicalMethod({
27959
+ client: this.client,
27960
+ datasourceId: this.definition.datasourceId,
27961
+ skillType: this.definition.skillType,
27962
+ instanceId: this.instanceId,
27963
+ tags: this.tags,
27964
+ backendName: this.definition.backendName,
27965
+ channelIds: this.channelIds,
27966
+ channelNames: this.channelNames
27967
+ });
27968
+ }
26206
27969
  describeSources() {
26207
27970
  return this.instances.map((instanceId) => ({
26208
27971
  source: datasourceSourcePath(this.definition.datasourceId, instanceId),
@@ -26333,6 +28096,8 @@ var DEFAULT_INSTANCE_ID5 = "default", DEFAULT_POLLING_INTERVAL_MS4;
26333
28096
  var init_crawler_skill = __esm(() => {
26334
28097
  init_errors4();
26335
28098
  init_scope2();
28099
+ init_sidecar();
28100
+ init_methods5();
26336
28101
  DEFAULT_POLLING_INTERVAL_MS4 = 15 * 60 * 1000;
26337
28102
  });
26338
28103
 
@@ -26485,15 +28250,15 @@ var init_notion = __esm(() => {
26485
28250
  });
26486
28251
 
26487
28252
  // src/datasource/skills/obsidian/paths.ts
26488
- import { join as join35 } from "node:path";
28253
+ import { join as join36 } from "node:path";
26489
28254
  function obsidianDatasourceRoot(workspaceRoot, instanceId) {
26490
- return join35(workspaceRoot, AUTORAG_DIRNAME, DATASOURCES_DIRNAME, OBSIDIAN_DATASOURCE_DIRNAME, instanceId);
28255
+ return join36(workspaceRoot, AUTORAG_DIRNAME, DATASOURCES_DIRNAME, OBSIDIAN_DATASOURCE_DIRNAME, instanceId);
26491
28256
  }
26492
28257
  function obsidianQmdConfigDir(workspaceRoot, instanceId) {
26493
- return join35(obsidianDatasourceRoot(workspaceRoot, instanceId), "config");
28258
+ return join36(obsidianDatasourceRoot(workspaceRoot, instanceId), "config");
26494
28259
  }
26495
28260
  function obsidianQmdCacheDir(workspaceRoot, instanceId) {
26496
- return join35(obsidianDatasourceRoot(workspaceRoot, instanceId), "cache");
28261
+ return join36(obsidianDatasourceRoot(workspaceRoot, instanceId), "cache");
26497
28262
  }
26498
28263
  function stripEdgeDashes(value) {
26499
28264
  let start = 0;
@@ -26516,8 +28281,8 @@ var DEFAULT_QMD_BINARY = "qmd", DEFAULT_QMD_TIMEOUT_MS = 60000, DEFAULT_QMD_MAX_
26516
28281
 
26517
28282
  // src/datasource/skills/obsidian/client.ts
26518
28283
  import { spawn as spawn16 } from "node:child_process";
26519
- import { mkdirSync as mkdirSync24, writeFileSync as writeFileSync22 } from "node:fs";
26520
- import { isAbsolute as isAbsolute8, resolve as resolve14 } from "node:path";
28284
+ import { mkdirSync as mkdirSync25, writeFileSync as writeFileSync22 } from "node:fs";
28285
+ import { isAbsolute as isAbsolute9, resolve as resolve15 } from "node:path";
26521
28286
 
26522
28287
  class QmdClient {
26523
28288
  options;
@@ -26536,10 +28301,10 @@ class QmdClient {
26536
28301
  const cacheDir = obsidianQmdCacheDir(workspaceRoot, instanceId);
26537
28302
  try {
26538
28303
  if (this.options.configPath === undefined) {
26539
- mkdirSync24(configDir, { recursive: true });
26540
- writeFileSync22(resolve14(configDir, "index.yml"), renderIndexYaml(collectionName, vaultPath), "utf8");
28304
+ mkdirSync25(configDir, { recursive: true });
28305
+ writeFileSync22(resolve15(configDir, "index.yml"), renderIndexYaml(collectionName, vaultPath), "utf8");
26541
28306
  }
26542
- mkdirSync24(cacheDir, { recursive: true });
28307
+ mkdirSync25(cacheDir, { recursive: true });
26543
28308
  } catch (error) {
26544
28309
  return failure4("spawn-error", error instanceof Error ? error.message : "failed to write qmd config");
26545
28310
  }
@@ -26613,7 +28378,7 @@ function resolveVaultPath(vaultPath) {
26613
28378
  if (vaultPath === undefined || vaultPath.trim().length === 0)
26614
28379
  return;
26615
28380
  const trimmed = vaultPath.trim();
26616
- return isAbsolute8(trimmed) ? trimmed : resolve14(trimmed);
28381
+ return isAbsolute9(trimmed) ? trimmed : resolve15(trimmed);
26617
28382
  }
26618
28383
  function controlledEnv5(options) {
26619
28384
  const instanceId = options.instanceId ?? "default";
@@ -26932,7 +28697,7 @@ async function retrieveObsidian(client, mode, methodName, instanceId, query, opt
26932
28697
  const trimmed = query.trim();
26933
28698
  if (trimmed.length === 0)
26934
28699
  return [];
26935
- const topK = options.topK ?? DEFAULT_TOP_K5;
28700
+ const topK = options.topK ?? DEFAULT_TOP_K6;
26936
28701
  const searchOptions = { topK, signal: options.signal };
26937
28702
  const result = await client.search(mode, trimmed, searchOptions);
26938
28703
  if (!result.ok)
@@ -26967,8 +28732,8 @@ function toRetrievalResult3(hit, source, methodName, instanceId, mode) {
26967
28732
  }
26968
28733
  };
26969
28734
  }
26970
- var OBSIDIAN_DATASOURCE_ID = "obsidian", DEFAULT_OBSIDIAN_TAGS, DEFAULT_TOP_K5 = 20;
26971
- var init_methods5 = __esm(() => {
28735
+ var OBSIDIAN_DATASOURCE_ID = "obsidian", DEFAULT_OBSIDIAN_TAGS, DEFAULT_TOP_K6 = 20;
28736
+ var init_methods6 = __esm(() => {
26972
28737
  init_errors4();
26973
28738
  init_scope2();
26974
28739
  DEFAULT_OBSIDIAN_TAGS = ["obsidian", "notes"];
@@ -27158,7 +28923,7 @@ var OBSIDIAN_DATASOURCE_ID2 = "obsidian", OBSIDIAN_SKILL_TYPE = "obsidian-vault"
27158
28923
  var init_skill10 = __esm(() => {
27159
28924
  init_scope2();
27160
28925
  init_client10();
27161
- init_methods5();
28926
+ init_methods6();
27162
28927
  init_paths5();
27163
28928
  DEFAULT_POLLING_INTERVAL_MS5 = 15 * 60 * 1000;
27164
28929
  DEFAULT_OBSIDIAN_TAGS2 = ["obsidian", "notes"];
@@ -27167,7 +28932,7 @@ var init_skill10 = __esm(() => {
27167
28932
  // src/datasource/skills/obsidian/index.ts
27168
28933
  var init_obsidian = __esm(() => {
27169
28934
  init_client10();
27170
- init_methods5();
28935
+ init_methods6();
27171
28936
  init_paths5();
27172
28937
  init_skill10();
27173
28938
  });
@@ -27507,7 +29272,7 @@ var init_slack = __esm(() => {
27507
29272
 
27508
29273
  // src/datasource/skills/spotlight/connector.ts
27509
29274
  import { spawn as spawn17 } from "node:child_process";
27510
- import { createHash as createHash14 } from "node:crypto";
29275
+ import { createHash as createHash15 } from "node:crypto";
27511
29276
  import { readFile as fsReadFile, stat as stat7 } from "node:fs/promises";
27512
29277
  import { basename as basename13, extname as extname8 } from "node:path";
27513
29278
 
@@ -27530,7 +29295,7 @@ class SpotlightConnector {
27530
29295
  if (queries.length === 0) {
27531
29296
  return { ok: false, reason: "not-configured", message: "no spotlight queries configured" };
27532
29297
  }
27533
- const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS9;
29298
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS10;
27534
29299
  const status = await this.run("mdutil", ["-s", "/"], timeoutMs);
27535
29300
  if (status.ok && /indexing disabled/iu.test(status.stdout)) {
27536
29301
  return {
@@ -27592,7 +29357,7 @@ class SpotlightConnector {
27592
29357
  if (content.length === 0)
27593
29358
  continue;
27594
29359
  documents.push({
27595
- docId: createHash14("sha256").update(path).digest("hex").slice(0, 16),
29360
+ docId: createHash15("sha256").update(path).digest("hex").slice(0, 16),
27596
29361
  hierarchy: ["files"],
27597
29362
  title: basename13(path),
27598
29363
  content,
@@ -27625,7 +29390,7 @@ class SpotlightConnector {
27625
29390
  return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
27626
29391
  }
27627
29392
  }
27628
- var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS5 = 300, DEFAULT_MAX_BYTES_PER_FILE3, DEFAULT_TIMEOUT_MS9 = 15000, MAX_CONTENT_CHARS4 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve) => {
29393
+ var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS5 = 300, DEFAULT_MAX_BYTES_PER_FILE3, DEFAULT_TIMEOUT_MS10 = 15000, MAX_CONTENT_CHARS4 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve) => {
27629
29394
  const child = spawn17(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
27630
29395
  let stdout = "";
27631
29396
  let stderr = "";
@@ -27742,10 +29507,12 @@ function parseHits3(stdout) {
27742
29507
  for (const [index, row] of rows.entries()) {
27743
29508
  if (!isRecord18(row))
27744
29509
  return;
27745
- const id = stringValue4(row, ["message_id", "event_id", "id"]);
27746
- const content = stringValue4(row, ["snippet", "text", "content"]);
27747
- if (id === undefined || content === undefined)
29510
+ const id = stringValue4(row, ["event_id", "message_id", "id"]);
29511
+ if (id === undefined)
27748
29512
  return;
29513
+ const content = stringValue4(row, ["snippet", "text", "content"]);
29514
+ if (content === undefined)
29515
+ continue;
27749
29516
  const chatId = stringValue4(row, ["chat_jid", "chat_id"]);
27750
29517
  const chatName = stringValue4(row, ["chat_name"]);
27751
29518
  const senderName = stringValue4(row, ["sender_name"]);
@@ -27862,10 +29629,12 @@ function parseHits4(stdout) {
27862
29629
  for (const [index, row] of rows.entries()) {
27863
29630
  if (!isRecord19(row))
27864
29631
  return;
27865
- const id = stringValue5(row, ["message_id", "event_id", "id"]);
27866
- const content = stringValue5(row, ["snippet", "text", "content"]);
27867
- if (id === undefined || content === undefined)
29632
+ const id = stringValue5(row, ["event_id", "message_id", "id"]);
29633
+ if (id === undefined)
27868
29634
  return;
29635
+ const content = stringValue5(row, ["snippet", "text", "content"]);
29636
+ if (content === undefined)
29637
+ continue;
27869
29638
  const chatId = stringValue5(row, ["chat_jid", "chat_id"]);
27870
29639
  const chatName = stringValue5(row, ["chat_name"]);
27871
29640
  const senderName = stringValue5(row, ["sender_name"]);
@@ -27955,6 +29724,31 @@ var init_wacrawl = __esm(() => {
27955
29724
  });
27956
29725
 
27957
29726
  // src/datasource/skills/factory.ts
29727
+ import { homedir as homedir4 } from "node:os";
29728
+ import { join as join37 } from "node:path";
29729
+ function chatCrawlerArgs(descriptor, defaultCrawlerDb, config, workspaceRoot, registrationName) {
29730
+ const sidecar = resolveSidecarOptions({
29731
+ descriptor,
29732
+ connector: config.connector,
29733
+ workspaceRoot,
29734
+ registrationName,
29735
+ instanceId: config.instanceId ?? "default",
29736
+ defaultCrawlerDb
29737
+ });
29738
+ return {
29739
+ datasourceId: registrationName,
29740
+ ...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
29741
+ ...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
29742
+ ...config.tags !== undefined ? { tags: config.tags } : {},
29743
+ channelIds: config.channels?.ids,
29744
+ channelNames: config.channels?.names,
29745
+ ...sidecar === undefined ? {} : { sidecar },
29746
+ connectorOptions: {
29747
+ ...config.connector,
29748
+ ...workspaceRoot === undefined ? {} : { workspacePath: workspaceRoot }
29749
+ }
29750
+ };
29751
+ }
27958
29752
  function larkConnectorOptions(connector) {
27959
29753
  if (connector === undefined)
27960
29754
  return {};
@@ -28015,6 +29809,8 @@ function buildDatasourceSkills(config, workspaceRoot, embeddingRuntime) {
28015
29809
  var BUILDERS, BUILTIN_DATASOURCE_SKILL_NAMES;
28016
29810
  var init_factory = __esm(() => {
28017
29811
  init_aliased_skill();
29812
+ init_config2();
29813
+ init_descriptors();
28018
29814
  init_clawgallery();
28019
29815
  init_cloud_drive();
28020
29816
  init_discrawl();
@@ -28032,42 +29828,9 @@ var init_factory = __esm(() => {
28032
29828
  init_telecrawl();
28033
29829
  init_wacrawl();
28034
29830
  BUILDERS = {
28035
- telegram: (config, _workspaceRoot, registrationName) => new TelecrawlSkill({
28036
- datasourceId: registrationName,
28037
- ...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
28038
- ...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
28039
- ...config.tags !== undefined ? { tags: config.tags } : {},
28040
- channelIds: config.channels?.ids,
28041
- channelNames: config.channels?.names,
28042
- connectorOptions: {
28043
- ...config.connector,
28044
- ..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
28045
- }
28046
- }),
28047
- whatsapp: (config, _workspaceRoot, registrationName) => new WacrawlSkill({
28048
- datasourceId: registrationName,
28049
- ...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
28050
- ...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
28051
- ...config.tags !== undefined ? { tags: config.tags } : {},
28052
- channelIds: config.channels?.ids,
28053
- channelNames: config.channels?.names,
28054
- connectorOptions: {
28055
- ...config.connector,
28056
- ..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
28057
- }
28058
- }),
28059
- slack: (config, _workspaceRoot, registrationName) => new SlackSkill({
28060
- datasourceId: registrationName,
28061
- ...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
28062
- ...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
28063
- ...config.tags !== undefined ? { tags: config.tags } : {},
28064
- channelIds: config.channels?.ids,
28065
- channelNames: config.channels?.names,
28066
- connectorOptions: {
28067
- ...config.connector,
28068
- ..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
28069
- }
28070
- }),
29831
+ telegram: (config, workspaceRoot, registrationName) => new TelecrawlSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.telegram, join37(homedir4(), ".telecrawl", "telecrawl.db"), config, workspaceRoot, registrationName)),
29832
+ whatsapp: (config, workspaceRoot, registrationName) => new WacrawlSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.whatsapp, join37(homedir4(), ".wacrawl", "wacrawl.db"), config, workspaceRoot, registrationName)),
29833
+ slack: (config, workspaceRoot, registrationName) => new SlackSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.slack, join37(homedir4(), ".slacrawl", "slacrawl.db"), config, workspaceRoot, registrationName)),
28071
29834
  discord: (config, workspaceRoot, registrationName, embeddingRuntime) => {
28072
29835
  const connector = config.connector ?? {};
28073
29836
  const clientOptions = {
@@ -28186,8 +29949,8 @@ var init_factory = __esm(() => {
28186
29949
 
28187
29950
  // src/cli/config.ts
28188
29951
  import { randomUUID as randomUUID9 } from "node:crypto";
28189
- import { existsSync as existsSync28, mkdirSync as mkdirSync25, readFileSync as readFileSync24, renameSync as renameSync11, unlinkSync as unlinkSync5, writeFileSync as writeFileSync23 } from "node:fs";
28190
- import { basename as basename14, dirname as dirname23, isAbsolute as isAbsolute9, join as join36, resolve as resolve15 } from "node:path";
29952
+ import { existsSync as existsSync28, mkdirSync as mkdirSync26, readFileSync as readFileSync25, renameSync as renameSync11, unlinkSync as unlinkSync5, writeFileSync as writeFileSync23 } from "node:fs";
29953
+ import { basename as basename14, dirname as dirname25, isAbsolute as isAbsolute10, join as join38, resolve as resolve16 } from "node:path";
28191
29954
  import { findEnvKeys, getEnvApiKey } from "@earendil-works/pi-ai/compat";
28192
29955
  import { getAgentDir as getAgentDir3, ModelRuntime as ModelRuntime2, SettingsManager as SettingsManager2 } from "@earendil-works/pi-coding-agent";
28193
29956
  function resolveConfigPath(input) {
@@ -28203,9 +29966,9 @@ function resolveConfigPath(input) {
28203
29966
  return { configPath: envConfig, explicit: true };
28204
29967
  }
28205
29968
  return {
28206
- configPath: join36(resolveAutoRAGHome(env), DEFAULT_CONFIG_FILENAME),
29969
+ configPath: join38(resolveAutoRAGHome(env), DEFAULT_CONFIG_FILENAME),
28207
29970
  explicit: false,
28208
- legacyPath: join36(cwd, LEGACY_CONFIG_FILENAME)
29971
+ legacyPath: join38(cwd, LEGACY_CONFIG_FILENAME)
28209
29972
  };
28210
29973
  }
28211
29974
  function readConfigFile(configPath, explicit) {
@@ -28223,7 +29986,7 @@ function readConfigFile(configPath, explicit) {
28223
29986
  }
28224
29987
  let text;
28225
29988
  try {
28226
- text = readFileSync24(configPath, "utf8");
29989
+ text = readFileSync25(configPath, "utf8");
28227
29990
  } catch (err) {
28228
29991
  throw new ConfigError(`Failed to read config file: ${err.message}`);
28229
29992
  }
@@ -28261,7 +30024,7 @@ function acquireConfigWriteLock(configPath) {
28261
30024
  });
28262
30025
  }
28263
30026
  function replaceFileAtomically(path, contents, assertCommitAllowed) {
28264
- const temporaryPath = join36(dirname23(path), `.${basename14(path)}.${process.pid}.${randomUUID9()}.tmp`);
30027
+ const temporaryPath = join38(dirname25(path), `.${basename14(path)}.${process.pid}.${randomUUID9()}.tmp`);
28265
30028
  try {
28266
30029
  writeFileSync23(temporaryPath, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
28267
30030
  assertCommitAllowed?.();
@@ -28274,12 +30037,12 @@ function migrateLegacyConfig(configPath, legacyPath) {
28274
30037
  if (existsSync28(configPath) || !existsSync28(legacyPath))
28275
30038
  return;
28276
30039
  const legacy = readConfigFile(legacyPath, true);
28277
- const legacyBytes = readFileSync24(legacyPath);
28278
- const migrated = normalizeLegacyConfigPaths(legacy ?? {}, dirname23(legacyPath));
30040
+ const legacyBytes = readFileSync25(legacyPath);
30041
+ const migrated = normalizeLegacyConfigPaths(legacy ?? {}, dirname25(legacyPath));
28279
30042
  const migratedPathsUnchanged = legacy?.workspacePath === migrated.workspacePath && legacy?.memoryPath === migrated.memoryPath && JSON.stringify(legacy?.searchPaths) === JSON.stringify(migrated.searchPaths);
28280
30043
  const migratedBytes = migratedPathsUnchanged ? legacyBytes : `${JSON.stringify(migrated, null, 2)}
28281
30044
  `;
28282
- mkdirSync25(dirname23(configPath), { recursive: true });
30045
+ mkdirSync26(dirname25(configPath), { recursive: true });
28283
30046
  const lock = acquireConfigWriteLock(configPath);
28284
30047
  try {
28285
30048
  const winner = readConfigFile(configPath, false);
@@ -28309,13 +30072,13 @@ function resolveConfigFileReadOnly(configPath, explicit, legacyPath) {
28309
30072
  const legacy = readConfigFile(legacyPath, false);
28310
30073
  if (legacy === undefined)
28311
30074
  return {};
28312
- return normalizeLegacyConfigPaths(legacy, dirname23(legacyPath));
30075
+ return normalizeLegacyConfigPaths(legacy, dirname25(legacyPath));
28313
30076
  }
28314
30077
  function resolveSearchPaths(searchPaths, origin) {
28315
30078
  return searchPaths.map((searchPath) => resolvePersistedPath(searchPath, origin));
28316
30079
  }
28317
30080
  function resolvePersistedPath(path, origin) {
28318
- return isAbsolute9(path) ? path : resolve15(origin, path);
30081
+ return isAbsolute10(path) ? path : resolve16(origin, path);
28319
30082
  }
28320
30083
  function normalizeLegacyConfigPaths(partial, origin) {
28321
30084
  const workspacePath = resolvePersistedPath(partial.workspacePath ?? ".", origin);
@@ -28502,7 +30265,7 @@ function normalizeEmbedder(raw, path) {
28502
30265
  out.passagePrefix = record.passagePrefix;
28503
30266
  }
28504
30267
  if (record.profile !== undefined) {
28505
- if (record.profile !== "qwen3-embedding-0.6b" && record.profile !== "embeddinggemma-300m") {
30268
+ if (record.profile !== "qwen3-embedding-0.6b") {
28506
30269
  throw new ConfigError(`${path}.profile must be a supported runtime profile`);
28507
30270
  }
28508
30271
  out.profile = record.profile;
@@ -28745,10 +30508,10 @@ function resolveConfig(input) {
28745
30508
  const file = readOnly ? resolveConfigFileReadOnly(configPath, explicit, legacyPath) : !explicit && legacyPath ? migrateLegacyConfig(configPath, legacyPath) ?? readConfigFile(configPath, explicit) ?? {} : readConfigFile(configPath, explicit) ?? {};
28746
30509
  const defaultSearchPaths = ["."];
28747
30510
  const defaultWorkspacePath = cwd;
28748
- const defaultMemoryPath = join36(resolveAutoRAGHome(env), "memory.json");
30511
+ const defaultMemoryPath = join38(resolveAutoRAGHome(env), "memory.json");
28749
30512
  const flagSearchPaths = parseCsv(flagString(flags, "search-paths"));
28750
30513
  const envSearchPaths = parseCsv(envString(env, "AUTORAG_SEARCH_PATHS"));
28751
- const configOrigin = dirname23(resolve15(configPath));
30514
+ const configOrigin = dirname25(resolve16(configPath));
28752
30515
  const fileWorkspacePath = typeof file.workspacePath === "string" ? resolvePersistedPath(file.workspacePath, configOrigin) : undefined;
28753
30516
  const fileSearchPaths = file.searchPaths ? resolveSearchPaths(file.searchPaths, fileWorkspacePath ?? configOrigin) : undefined;
28754
30517
  if (file.excludePaths !== undefined && (!Array.isArray(file.excludePaths) || file.excludePaths.some((path) => typeof path !== "string" || path.length === 0))) {
@@ -28771,7 +30534,7 @@ function resolveConfig(input) {
28771
30534
  const flagModelId = pickString(flags, env, "model-id", "AUTORAG_MODEL_ID", undefined);
28772
30535
  const model = applyRoleFlagOverrides(fileModel, flagModelProvider, flagModelId, "model");
28773
30536
  const config = {
28774
- configPath: resolve15(configPath),
30537
+ configPath: resolve16(configPath),
28775
30538
  searchPaths,
28776
30539
  workspacePath,
28777
30540
  memoryPath,
@@ -28947,36 +30710,38 @@ function buildWebSearchAgentOption(raw) {
28947
30710
  function normalizeRerankConfig(raw, path) {
28948
30711
  if (raw === false)
28949
30712
  return false;
28950
- const out = {
28951
- provider: DEFAULT_RERANK_PROVIDER,
28952
- model: DEFAULT_RERANK_MODEL,
28953
- apiKeyEnv: DEFAULT_RERANK_API_KEY_ENV,
28954
- topN: DEFAULT_RERANK_TOP_N
28955
- };
28956
- if (raw === undefined || raw === null)
28957
- return out;
28958
- if (typeof raw !== "object" || Array.isArray(raw)) {
28959
- throw new ConfigError(`${path} must be an object or false`);
28960
- }
28961
- const record = raw;
28962
- for (const key of Object.keys(record)) {
28963
- if (!RERANK_ALLOWLIST.has(key))
28964
- throw new ConfigError(`${path}.${key} is not a recognized field`);
28965
- }
28966
- if (record.enabled !== undefined) {
28967
- if (typeof record.enabled !== "boolean")
28968
- throw new ConfigError(`${path}.enabled must be a boolean`);
28969
- out.enabled = record.enabled;
30713
+ let record = {};
30714
+ if (raw !== undefined && raw !== null) {
30715
+ if (typeof raw !== "object" || Array.isArray(raw)) {
30716
+ throw new ConfigError(`${path} must be an object or false`);
30717
+ }
30718
+ record = raw;
30719
+ for (const key of Object.keys(record)) {
30720
+ if (!RERANK_ALLOWLIST.has(key))
30721
+ throw new ConfigError(`${path}.${key} is not a recognized field`);
30722
+ }
28970
30723
  }
30724
+ let provider = DEFAULT_RERANK_PROVIDER;
28971
30725
  if (record.provider !== undefined) {
28972
30726
  if (typeof record.provider !== "string" || record.provider.trim() === "") {
28973
30727
  throw new ConfigError(`${path}.provider must be a non-empty string`);
28974
30728
  }
28975
- out.provider = record.provider.trim();
30729
+ provider = record.provider.trim();
28976
30730
  }
28977
- if (!SUPPORTED_RERANK_PROVIDERS.includes(out.provider ?? "")) {
30731
+ if (!SUPPORTED_RERANK_PROVIDERS.includes(provider)) {
28978
30732
  throw new ConfigError(`${path}.provider must be one of: ${SUPPORTED_RERANK_PROVIDERS.join(", ")}`);
28979
30733
  }
30734
+ const out = {
30735
+ provider,
30736
+ model: defaultRerankModel(provider),
30737
+ apiKeyEnv: defaultRerankApiKeyEnv(provider),
30738
+ topN: DEFAULT_RERANK_TOP_N
30739
+ };
30740
+ if (record.enabled !== undefined) {
30741
+ if (typeof record.enabled !== "boolean")
30742
+ throw new ConfigError(`${path}.enabled must be a boolean`);
30743
+ out.enabled = record.enabled;
30744
+ }
28980
30745
  if (record.model !== undefined) {
28981
30746
  if (typeof record.model !== "string" || record.model.trim() === "") {
28982
30747
  throw new ConfigError(`${path}.model must be a non-empty string`);
@@ -29106,7 +30871,8 @@ function buildModelFromConfiguredEndpoint(reference) {
29106
30871
  input: reference.input ?? ["text"],
29107
30872
  cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
29108
30873
  contextWindow: reference.contextWindow ?? 128000,
29109
- maxTokens: reference.maxTokens ?? 16384
30874
+ maxTokens: reference.maxTokens ?? 16384,
30875
+ ...reference.provider === PROVIDER_ID ? { inputLimits: AUTORAG_INPUT_LIMITS } : {}
29110
30876
  };
29111
30877
  }
29112
30878
  function mergeCatalogModel(catalog, reference) {
@@ -29157,8 +30923,8 @@ function getModelRuntime(agentDir) {
29157
30923
  return cached;
29158
30924
  const created = (async () => {
29159
30925
  const runtime = await ModelRuntime2.create({
29160
- authPath: join36(agentDir, "auth.json"),
29161
- modelsPath: join36(agentDir, "models.json"),
30926
+ authPath: join38(agentDir, "auth.json"),
30927
+ modelsPath: join38(agentDir, "models.json"),
29162
30928
  allowModelNetwork: false
29163
30929
  });
29164
30930
  await registerAutoRAGProvider(runtime);
@@ -29417,7 +31183,7 @@ function writeConfigObject(path, config) {
29417
31183
  if (config === null || typeof config !== "object" || Array.isArray(config)) {
29418
31184
  throw new ConfigError("Config file must be a JSON object");
29419
31185
  }
29420
- mkdirSync25(dirname23(path), { recursive: true });
31186
+ mkdirSync26(dirname25(path), { recursive: true });
29421
31187
  const contents = `${JSON.stringify(config, null, 2)}
29422
31188
  `;
29423
31189
  const lock = acquireConfigWriteLock(path);
@@ -29428,9 +31194,9 @@ function writeConfigObject(path, config) {
29428
31194
  }
29429
31195
  }
29430
31196
  function writeDefaultConfig(path, partial, opts = {}) {
29431
- const cwd = resolve15(opts.cwd ?? process.cwd());
31197
+ const cwd = resolve16(opts.cwd ?? process.cwd());
29432
31198
  const workspacePath = resolvePersistedPath(partial.workspacePath ?? ".", cwd);
29433
- const memoryPath = partial.memoryPath === undefined ? join36(resolveAutoRAGHome(opts.env), "memory.json") : resolvePersistedPath(partial.memoryPath, workspacePath);
31199
+ const memoryPath = partial.memoryPath === undefined ? join38(resolveAutoRAGHome(opts.env), "memory.json") : resolvePersistedPath(partial.memoryPath, workspacePath);
29434
31200
  const full = {
29435
31201
  searchPaths: resolveSearchPaths(partial.searchPaths ?? ["."], cwd),
29436
31202
  workspacePath,
@@ -29464,7 +31230,7 @@ function writeDefaultConfig(path, partial, opts = {}) {
29464
31230
  full.p2p = { enabled: false };
29465
31231
  full.jev = partial.jev ?? { backend: DEFAULT_JEV_BACKEND };
29466
31232
  full.queryDecomposition = partial.queryDecomposition ?? { model: { ...DEFAULT_QUERY_DECOMPOSITION_MODEL } };
29467
- mkdirSync25(dirname23(path), { recursive: true });
31233
+ mkdirSync26(dirname25(path), { recursive: true });
29468
31234
  const contents = `${JSON.stringify(full, null, 2)}
29469
31235
  `;
29470
31236
  const lock = acquireConfigWriteLock(path);
@@ -29492,8 +31258,9 @@ function writeDefaultConfig(path, partial, opts = {}) {
29492
31258
  }
29493
31259
  }
29494
31260
  var DEFAULT_CONFIG_FILENAME = "config.json", LEGACY_CONFIG_FILENAME = "autorag.config.json", CONFIG_LOCK_RETRY_MS = 10, CONFIG_LOCK_TIMEOUT_MS = 1e4, CONFIG_LOCK_STALE_MS = 30000, ConfigError, MINSYNC_REQUIRED_MESSAGE = "MinSync is required and cannot be disabled; remove `minSync: false` / `minSync.enabled: false` from the config.", AGENT_MODEL_APIS, AGENT_MODEL_INPUTS, API_KEY_ENV_PATTERN2, POSITIVE_INT_FIELDS, EMBEDDER_ALLOWLIST, MINSYNC_ALLOWLIST, MINSYNC_LEGACY_IGNORED, P2P_ALLOWLIST, P2P_QUOTA_ALLOWLIST, LIMITS_ALLOWLIST, LIMITS_PREFETCH_ALLOWLIST, JEV_CONFIG_FIELDS, JEV_BACKEND_NAMES, DEFAULT_JEV_BACKEND = "openrouter", DEFAULT_QUERY_DECOMPOSITION_MODEL, RERANK_ALLOWLIST, UNKNOWN_MODEL_HINT = "Add baseUrl (and optional api/apiKeyEnv) for an OpenAI-compatible endpoint outside the pi-ai catalog, or use a pi-ai catalog model id.", modelRuntimeCache;
29495
- var init_config3 = __esm(() => {
31261
+ var init_config4 = __esm(() => {
29496
31262
  init_local_model();
31263
+ init_config();
29497
31264
  init_provider();
29498
31265
  init_home();
29499
31266
  init_factory();
@@ -29575,7 +31342,7 @@ var init_config3 = __esm(() => {
29575
31342
  model: true,
29576
31343
  confidenceThreshold: true
29577
31344
  };
29578
- JEV_BACKEND_NAMES = ["typesafe", "openrouter", "vercel"];
31345
+ JEV_BACKEND_NAMES = ["typesafe", "openrouter", "vercel", "autorag"];
29579
31346
  DEFAULT_QUERY_DECOMPOSITION_MODEL = { provider: "openrouter", id: "qwen/qwen3.7-flash" };
29580
31347
  RERANK_ALLOWLIST = new Set(["enabled", "provider", "model", "apiKeyEnv", "baseUrl", "topN", "timeoutMs"]);
29581
31348
  modelRuntimeCache = new Map;
@@ -29665,9 +31432,9 @@ function createAutoRAGLite(options = {}) {
29665
31432
  var init_core = __esm(() => {
29666
31433
  init_agent();
29667
31434
  init_search_documents();
29668
- init_config3();
31435
+ init_config4();
29669
31436
  init_memory();
29670
- init_config3();
31437
+ init_config4();
29671
31438
  });
29672
31439
 
29673
31440
  // src/cli/output.ts
@@ -30112,7 +31879,7 @@ async function runEvidence(ctx) {
30112
31879
  }
30113
31880
  var init_evidence = __esm(() => {
30114
31881
  init_memory();
30115
- init_config3();
31882
+ init_config4();
30116
31883
  });
30117
31884
 
30118
31885
  // src/agent/results.ts
@@ -30170,12 +31937,12 @@ var init_results = __esm(() => {
30170
31937
  });
30171
31938
 
30172
31939
  // src/cli/commands/report.ts
30173
- import { readFileSync as readFileSync25 } from "node:fs";
31940
+ import { readFileSync as readFileSync26 } from "node:fs";
30174
31941
  import { Value as Value3 } from "typebox/value";
30175
31942
  function readReportInput(input) {
30176
31943
  if (typeof input === "string" && input.length > 0)
30177
- return readFileSync25(input, "utf8");
30178
- return readFileSync25(0, "utf8");
31944
+ return readFileSync26(input, "utf8");
31945
+ return readFileSync26(0, "utf8");
30179
31946
  }
30180
31947
  function describeSchemaErrors(value) {
30181
31948
  const errors = Value3.Errors(reportSchema, value);
@@ -30266,7 +32033,7 @@ var MAX_REPORTED_SCHEMA_ERRORS = 8;
30266
32033
  var init_report = __esm(() => {
30267
32034
  init_results();
30268
32035
  init_core();
30269
- init_config3();
32036
+ init_config4();
30270
32037
  });
30271
32038
 
30272
32039
  // src/mcp/index.ts
@@ -30279,8 +32046,8 @@ import { StdioServerTransport } from "@modelcontextprotocol/server/stdio";
30279
32046
  init_evidence();
30280
32047
  init_report();
30281
32048
  init_dupey();
30282
- import { readFileSync as readFileSync26 } from "node:fs";
30283
- import { resolve as resolve17 } from "node:path";
32049
+ import { readFileSync as readFileSync27 } from "node:fs";
32050
+ import { resolve as resolve18 } from "node:path";
30284
32051
  import { fileURLToPath as fileURLToPath3 } from "node:url";
30285
32052
  import { McpServer } from "@modelcontextprotocol/server";
30286
32053
  import { z as z2 } from "zod";
@@ -30288,7 +32055,7 @@ import { z as z2 } from "zod";
30288
32055
  // src/filesystem/name-search.ts
30289
32056
  import { realpathSync as realpathSync6 } from "node:fs";
30290
32057
  import { lstat, readdir as readdir4 } from "node:fs/promises";
30291
- import { basename as basename15, dirname as dirname24, isAbsolute as isAbsolute10, join as join37, relative as relative6, resolve as resolve16, sep as sep7 } from "node:path";
32058
+ import { basename as basename15, dirname as dirname26, isAbsolute as isAbsolute11, join as join39, relative as relative6, resolve as resolve17, sep as sep8 } from "node:path";
30292
32059
  var SKIP_DIR_NAMES2 = {
30293
32060
  ".git": true,
30294
32061
  ".autorag": true,
@@ -30316,27 +32083,27 @@ function fsErrorCode(error) {
30316
32083
  return typeof code === "string" ? code : undefined;
30317
32084
  }
30318
32085
  function isContained(childReal, parentReal) {
30319
- const prefix = parentReal.endsWith(sep7) ? parentReal : `${parentReal}${sep7}`;
32086
+ const prefix = parentReal.endsWith(sep8) ? parentReal : `${parentReal}${sep8}`;
30320
32087
  return childReal === parentReal || childReal.startsWith(prefix);
30321
32088
  }
30322
32089
  function pinExcludedPath2(path) {
30323
- const resolvedPath = resolve16(path);
32090
+ const resolvedPath = resolve17(path);
30324
32091
  try {
30325
- return join37(realpathSync6(dirname24(resolvedPath)), basename15(resolvedPath));
32092
+ return join39(realpathSync6(dirname26(resolvedPath)), basename15(resolvedPath));
30326
32093
  } catch {
30327
32094
  return resolvedPath;
30328
32095
  }
30329
32096
  }
30330
32097
  function isExcluded2(absPath, excluded) {
30331
32098
  for (const excludedPath of excluded) {
30332
- const prefix = excludedPath.endsWith(sep7) ? excludedPath : `${excludedPath}${sep7}`;
32099
+ const prefix = excludedPath.endsWith(sep8) ? excludedPath : `${excludedPath}${sep8}`;
30333
32100
  if (absPath === excludedPath || absPath.startsWith(prefix))
30334
32101
  return true;
30335
32102
  }
30336
32103
  return false;
30337
32104
  }
30338
32105
  function matchesEntry(context, absPath, name) {
30339
- const haystackRaw = context.matchPath ? relative6(context.rootReal, absPath).split(sep7).join("/") : name;
32106
+ const haystackRaw = context.matchPath ? relative6(context.rootReal, absPath).split(sep8).join("/") : name;
30340
32107
  const haystack = context.matchCase ? haystackRaw : haystackRaw.toLowerCase();
30341
32108
  const needle = context.matchCase ? context.query : context.query.toLowerCase();
30342
32109
  return haystack.includes(needle);
@@ -30376,7 +32143,7 @@ async function walkDirectory(context, directory) {
30376
32143
  }
30377
32144
  if (entry.isSymbolicLink())
30378
32145
  continue;
30379
- const absPath = join37(directory, entry.name);
32146
+ const absPath = join39(directory, entry.name);
30380
32147
  if (entry.isDirectory()) {
30381
32148
  if (SKIP_DIR_NAMES2[entry.name] === true)
30382
32149
  continue;
@@ -30413,7 +32180,7 @@ async function pinConfiguredRoots(roots, diagnostics) {
30413
32180
  for (const root of roots) {
30414
32181
  if (typeof root !== "string" || root.length === 0)
30415
32182
  continue;
30416
- const resolvedRoot = resolve16(root);
32183
+ const resolvedRoot = resolve17(root);
30417
32184
  let real;
30418
32185
  try {
30419
32186
  real = realpathSync6(resolvedRoot);
@@ -30435,7 +32202,7 @@ async function pinConfiguredRoots(roots, diagnostics) {
30435
32202
  return outermost;
30436
32203
  }
30437
32204
  async function resolveRequestRoots(requestRoot, pinnedRoots) {
30438
- const candidates = isAbsolute10(requestRoot) ? [requestRoot] : pinnedRoots.map((root) => resolve16(root, requestRoot));
32205
+ const candidates = isAbsolute11(requestRoot) ? [requestRoot] : pinnedRoots.map((root) => resolve17(root, requestRoot));
30439
32206
  const allowed = [];
30440
32207
  for (const candidate of candidates) {
30441
32208
  let real;
@@ -30477,7 +32244,7 @@ async function filterFileNameSearchMatches(roots, candidates, excludePaths = [])
30477
32244
  continue;
30478
32245
  if (isExcluded2(real, excluded))
30479
32246
  continue;
30480
- const segments = real.split(sep7);
32247
+ const segments = real.split(sep8);
30481
32248
  if (segments.some((segment) => SKIP_DIR_NAMES2[segment] === true))
30482
32249
  continue;
30483
32250
  matches.push({ path: real, type: stat.isDirectory() ? "folder" : "file" });
@@ -30626,7 +32393,7 @@ function isToolEnabled(name, options) {
30626
32393
  }
30627
32394
  function readPackageVersion() {
30628
32395
  try {
30629
- const manifest = JSON.parse(readFileSync26(fileURLToPath3(new URL("../../package.json", import.meta.url)), "utf8"));
32396
+ const manifest = JSON.parse(readFileSync27(fileURLToPath3(new URL("../../package.json", import.meta.url)), "utf8"));
30630
32397
  if (typeof manifest === "object" && manifest !== null && "version" in manifest && typeof manifest.version === "string") {
30631
32398
  return manifest.version;
30632
32399
  }
@@ -30865,7 +32632,7 @@ function createAutoRAGMcpServer(lite, options = {}) {
30865
32632
  ...configuredDupey.timeoutMs !== undefined ? { timeoutMs: configuredDupey.timeoutMs } : {},
30866
32633
  ...configuredDupey.run !== undefined ? { run: configuredDupey.run } : {}
30867
32634
  };
30868
- const roots = lite.config.searchPaths.map((path) => resolve17(path));
32635
+ const roots = lite.config.searchPaths.map((path) => resolve18(path));
30869
32636
  const scans = await Promise.all(roots.map((root) => scanWithDupey(root, dupeyOptions)));
30870
32637
  return jsonResult({
30871
32638
  ok: true,