@autorag/librarian 2.10.0 → 2.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/LICENSE +195 -21
  2. package/NOTICE +17 -21
  3. package/README.md +335 -161
  4. package/dist/agent/agent.d.ts +67 -2
  5. package/dist/agent/context-token-guard.d.ts +23 -0
  6. package/dist/agent/jev-extension.d.ts +5 -4
  7. package/dist/agent/jikji-find-tool.d.ts +1 -0
  8. package/dist/agent/pi-session.d.ts +35 -2
  9. package/dist/agent/request-limits.d.ts +20 -0
  10. package/dist/agent/search-all-tool.d.ts +1 -0
  11. package/dist/agent/search-single-datasource-tool.d.ts +1 -0
  12. package/dist/cli/config.d.ts +13 -11
  13. package/dist/cli/index.js +2573 -800
  14. package/dist/cloud/config.d.ts +18 -0
  15. package/dist/cloud/jev-backend.d.ts +90 -0
  16. package/dist/core.js +2185 -418
  17. package/dist/datasource/crawler-skill.d.ts +17 -0
  18. package/dist/datasource/sidecar/config.d.ts +26 -0
  19. package/dist/datasource/sidecar/descriptors.d.ts +67 -0
  20. package/dist/datasource/sidecar/fusion.d.ts +25 -0
  21. package/dist/datasource/sidecar/index.d.ts +131 -0
  22. package/dist/datasource/sidecar/methods.d.ts +27 -0
  23. package/dist/datasource/types.d.ts +3 -1
  24. package/dist/embedding-runtime/index.d.ts +6 -6
  25. package/dist/embedding-runtime/manifest.d.ts +0 -12
  26. package/dist/embedding-runtime/types.d.ts +1 -1
  27. package/dist/index.js +2210 -439
  28. package/dist/mcp/index.js +2205 -438
  29. package/dist/minsync/client.d.ts +3 -3
  30. package/dist/minsync/installer.d.ts +1 -1
  31. package/dist/minsync/process.d.ts +1 -0
  32. package/dist/minsync/types.d.ts +1 -1
  33. package/dist/parser/defaults.d.ts +6 -4
  34. package/dist/parser/ocr-engines.d.ts +6 -0
  35. package/dist/retrieval/index.d.ts +1 -1
  36. package/dist/retrieval/rerank.d.ts +20 -3
  37. package/licenses/embedding-assets.json +0 -35
  38. package/package.json +24 -27
  39. package/skills/autorag/SKILL.md +1 -1
  40. package/skills/autorag-doctor/SKILL.md +1 -1
  41. package/skills/autorag-lite-setup/SKILL.md +2 -2
  42. package/skills/autorag-setup/SKILL.md +2 -2
  43. package/licenses/gemma-notice.txt +0 -24
  44. package/licenses/gemma-section-3.2-flow-down.txt +0 -9
  45. package/licenses/gemma-terms-of-use.html +0 -4750
  46. package/licenses/gemma-terms-of-use.txt +0 -158
package/dist/index.js CHANGED
@@ -61,6 +61,7 @@ function datasourceCodeToRetrieval(code) {
61
61
  case "datasource-auth-error":
62
62
  case "datasource-embedding-egress-rejected":
63
63
  case "datasource-index-failed":
64
+ case "datasource-index-partial":
64
65
  case "datasource-permission-denied":
65
66
  case "datasource-remote-embedding-rejected":
66
67
  return "retrieval-method-failed";
@@ -1112,19 +1113,6 @@ var init_manifest = __esm(() => {
1112
1113
  artifactSha256: "06507c7b42688469c4e7298b0a1e16deff06caf291cf0a5b278c308249c3e439",
1113
1114
  backend: "auto",
1114
1115
  protocolCapabilities
1115
- },
1116
- "embeddinggemma-300m": {
1117
- profileId: "embeddinggemma-300m",
1118
- provider: "google",
1119
- model: "embeddinggemma-300M-Q8_0.gguf",
1120
- dimension: 768,
1121
- queryPrefix: "task: search result | query: ",
1122
- passagePrefix: "title: none | text: ",
1123
- runtimeBuild: RUNTIME_BUILD,
1124
- modelRevision: "0f741b5a6585bd53aeb15cd1372c56f2a0f65e12",
1125
- artifactSha256: "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63",
1126
- backend: "auto",
1127
- protocolCapabilities
1128
1116
  }
1129
1117
  };
1130
1118
  runtimeAssets = {
@@ -1170,17 +1158,6 @@ var init_manifest = __esm(() => {
1170
1158
  unsuitableUse: "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256.",
1171
1159
  upstreamCardUrl: "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B",
1172
1160
  modelCard: "docs/model-cards/qwen3-embedding-0.6b.md"
1173
- },
1174
- embeddinggemma: {
1175
- url: "https://huggingface.co/ggml-org/embeddinggemma-300M-GGUF/resolve/0f741b5a6585bd53aeb15cd1372c56f2a0f65e12/embeddinggemma-300M-Q8_0.gguf",
1176
- revision: "0f741b5a6585bd53aeb15cd1372c56f2a0f65e12",
1177
- sha256: "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63",
1178
- license: "Gemma Terms of Use",
1179
- noticeReference: "Google EmbeddingGemma model card and Gemma Terms of Use",
1180
- intendedUse: "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.",
1181
- unsuitableUse: "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256, uses restricted by the Gemma Prohibited Use Policy.",
1182
- upstreamCardUrl: "https://ai.google.dev/gemma/docs/embeddinggemma/model_card",
1183
- modelCard: "docs/model-cards/embeddinggemma-300m.md"
1184
1161
  }
1185
1162
  };
1186
1163
  });
@@ -1618,7 +1595,7 @@ var init_supervisor = __esm(() => {
1618
1595
  // src/embedding-runtime/index.ts
1619
1596
  import { basename as basename4 } from "node:path";
1620
1597
  function modelAsset(profile) {
1621
- return profile.profileId === "qwen3-embedding-0.6b" ? { id: profile.profileId, filename: basename4(profile.model), ...MODEL_ASSETS.qwen3 } : { id: profile.profileId, filename: basename4(profile.model), ...MODEL_ASSETS.embeddinggemma };
1598
+ return { id: profile.profileId, filename: basename4(profile.model), ...MODEL_ASSETS.qwen3 };
1622
1599
  }
1623
1600
  function runtimeAsset(platform, backend) {
1624
1601
  const selected = platform ?? "darwin-arm64-metal";
@@ -5020,7 +4997,12 @@ var init_local_embedder = __esm(() => {
5020
4997
  // src/minsync/process.ts
5021
4998
  import { spawn as spawn7 } from "node:child_process";
5022
4999
  function spawnProcess(command, args, cwd, options = {}) {
5023
- return new Promise((resolve) => {
5000
+ return new Promise((resolve, reject) => {
5001
+ const { signal } = options;
5002
+ if (signal?.aborted) {
5003
+ reject(abortError(signal));
5004
+ return;
5005
+ }
5024
5006
  const portable = portableSpawnCommand(command, args);
5025
5007
  const child = spawn7(portable.command, [...portable.args], { cwd, stdio: ["ignore", "pipe", "pipe"] });
5026
5008
  let stdout = "";
@@ -5038,15 +5020,26 @@ function spawnProcess(command, args, cwd, options = {}) {
5038
5020
  timedOut = true;
5039
5021
  child.kill("SIGTERM");
5040
5022
  }, options.timeoutMs) : undefined;
5023
+ const onAbort = () => {
5024
+ child.kill("SIGTERM");
5025
+ };
5026
+ signal?.addEventListener("abort", onAbort, { once: true });
5027
+ const settle = () => {
5028
+ clearTimeout(timer);
5029
+ signal?.removeEventListener("abort", onAbort);
5030
+ };
5041
5031
  child.on("error", (error) => {
5042
- if (timer)
5043
- clearTimeout(timer);
5044
- resolve({ ok: false, stdout, stderr: error.message, code: null });
5032
+ settle();
5033
+ if (signal?.aborted)
5034
+ reject(abortError(signal));
5035
+ else
5036
+ resolve({ ok: false, stdout, stderr: error.message, code: null });
5045
5037
  });
5046
5038
  child.on("close", (code) => {
5047
- if (timer)
5048
- clearTimeout(timer);
5049
- if (timedOut) {
5039
+ settle();
5040
+ if (signal?.aborted) {
5041
+ reject(abortError(signal));
5042
+ } else if (timedOut) {
5050
5043
  resolve({ ok: false, stdout, stderr: "process timed out", code: null, timedOut: true });
5051
5044
  } else {
5052
5045
  resolve({ ok: code === 0, stdout, stderr, code });
@@ -5054,6 +5047,9 @@ function spawnProcess(command, args, cwd, options = {}) {
5054
5047
  });
5055
5048
  });
5056
5049
  }
5050
+ function abortError(signal) {
5051
+ return signal.reason instanceof Error ? signal.reason : new DOMException("The operation was aborted", "AbortError");
5052
+ }
5057
5053
  var init_process = __esm(() => {
5058
5054
  init_portable_spawn();
5059
5055
  });
@@ -5286,10 +5282,12 @@ class MinSyncClient {
5286
5282
  } : {}
5287
5283
  };
5288
5284
  }
5289
- async query(text, topK, mode = "vector") {
5285
+ async query(text, topK, mode = "vector", signal) {
5290
5286
  if (!existsSync11(this.binaryPath))
5291
5287
  throw minSyncBinaryMissingError();
5292
- const result = await this.spawn(["query", "--format", "json", "--mode", mode, "-k", String(topK), text]);
5288
+ const result = await this.spawn(["query", "--format", "json", "--mode", mode, "-k", String(topK), text], {
5289
+ ...signal === undefined ? {} : { signal }
5290
+ });
5293
5291
  if (!result.ok)
5294
5292
  throw new MinSyncQueryError(result.code, result.stderr);
5295
5293
  return parseQueryHits(result.stdout);
@@ -5362,7 +5360,7 @@ function isIdentity(value) {
5362
5360
  function isRecord4(value) {
5363
5361
  return typeof value === "object" && value !== null;
5364
5362
  }
5365
- var MINSYNC_OLLAMA_MIGRATION_MESSAGE = "This workspace uses the legacy 768-dimensional Ollama/TEI embedding path. Reindex explicitly, or pin an explicit profile config before using the new default runtime.", API_KEY_ENV_PATTERN, MinSyncQueryError;
5363
+ var MINSYNC_OLLAMA_MIGRATION_MESSAGE = "This workspace uses the legacy 768-dimensional Ollama/TEI embedding path. Reindex explicitly, or pin an explicit Ollama/TEI embedder config before using the new default runtime.", API_KEY_ENV_PATTERN, MinSyncQueryError;
5366
5364
  var init_client4 = __esm(() => {
5367
5365
  init_embedder_config();
5368
5366
  init_errors();
@@ -5606,7 +5604,7 @@ function isGitHubAsset(value) {
5606
5604
  function isRecord5(value) {
5607
5605
  return typeof value === "object" && value !== null;
5608
5606
  }
5609
- var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.4.6", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
5607
+ var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.4.7", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
5610
5608
  var init_installer2 = __esm(() => {
5611
5609
  init_process();
5612
5610
  SHA256_HEX_PATTERN = /^[a-f0-9]{64}$/;
@@ -5915,11 +5913,11 @@ class MinSyncVectorMethod {
5915
5913
  });
5916
5914
  let hits;
5917
5915
  try {
5918
- hits = await client.query(query, queryK, this.mode);
5916
+ hits = await client.query(query, queryK, this.mode, options.signal);
5919
5917
  } catch (error) {
5920
- if (this.mode !== "hybrid")
5918
+ if (this.mode !== "hybrid" || options.signal?.aborted)
5921
5919
  throw error;
5922
- hits = await client.query(query, queryK, "bm25");
5920
+ hits = await client.query(query, queryK, "bm25", options.signal);
5923
5921
  }
5924
5922
  const results = [];
5925
5923
  for (const hit of hits) {
@@ -6242,10 +6240,16 @@ async function runTesseract(recognition, signal, cleanupResolve, cleanupReject)
6242
6240
  };
6243
6241
  signal.addEventListener("abort", abort, { once: true });
6244
6242
  try {
6243
+ const tooSmall = ocrImageTooSmallMessage(recognition.bytes);
6244
+ if (tooSmall !== undefined)
6245
+ throw new Error(tooSmall);
6245
6246
  const { cachePath } = recognition;
6246
6247
  if (cachePath !== undefined)
6247
6248
  await mkdir3(cachePath, { recursive: true });
6248
- worker = await createWorker(recognition.language, undefined, cachePath === undefined ? {} : { cachePath });
6249
+ worker = await createWorker(recognition.language, undefined, {
6250
+ ...cachePath === undefined ? {} : { cachePath },
6251
+ errorHandler: containTesseractWorkerError
6252
+ });
6249
6253
  if (signal.aborted)
6250
6254
  throw new Error("OCR aborted before worker was ready");
6251
6255
  const result = await worker.recognize(Buffer.from(recognition.bytes));
@@ -6255,6 +6259,129 @@ async function runTesseract(recognition, signal, cleanupResolve, cleanupReject)
6255
6259
  await terminate().then(cleanupResolve, cleanupReject);
6256
6260
  }
6257
6261
  }
6262
+ function containTesseractWorkerError(_payload) {}
6263
+ function ocrImageTooSmallMessage(bytes) {
6264
+ const size = readImagePixelSize(bytes);
6265
+ if (size === undefined || size.width >= MIN_LEPTONICA_SIDE_PIXELS && size.height >= MIN_LEPTONICA_SIDE_PIXELS) {
6266
+ return;
6267
+ }
6268
+ return `Image is too small to scale (${size.width}x${size.height} vs minimum of ${MIN_LEPTONICA_SIDE_PIXELS} pixels).`;
6269
+ }
6270
+ function readImagePixelSize(bytes) {
6271
+ if (bytes.length >= 8 && bytes[0] === 137 && bytes[1] === 80)
6272
+ return readPngSize(bytes);
6273
+ if (bytes.length >= 2 && bytes[0] === 255 && bytes[1] === 216)
6274
+ return readJpegSize(bytes);
6275
+ if (bytes.length >= 12 && bytes[0] === 82 && bytes[1] === 73)
6276
+ return readWebpSize(bytes);
6277
+ return;
6278
+ }
6279
+ function readPngSize(bytes) {
6280
+ if (bytes.length < 24)
6281
+ return;
6282
+ if (bytes[0] !== 137 || bytes[1] !== 80 || bytes[2] !== 78 || bytes[3] !== 71 || bytes[4] !== 13 || bytes[5] !== 10 || bytes[6] !== 26 || bytes[7] !== 10) {
6283
+ return;
6284
+ }
6285
+ if (bytes[12] !== 73 || bytes[13] !== 72 || bytes[14] !== 68 || bytes[15] !== 82)
6286
+ return;
6287
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
6288
+ return { width: view.getUint32(16), height: view.getUint32(20) };
6289
+ }
6290
+ function readJpegSize(bytes) {
6291
+ if (bytes.length < 4 || bytes[0] !== 255 || bytes[1] !== 216)
6292
+ return;
6293
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
6294
+ let offset = 2;
6295
+ while (offset + 1 < bytes.length) {
6296
+ if (bytes[offset] !== 255)
6297
+ return;
6298
+ offset += 1;
6299
+ while (offset < bytes.length && bytes[offset] === 255)
6300
+ offset += 1;
6301
+ if (offset >= bytes.length)
6302
+ return;
6303
+ const marker = bytes[offset] ?? 0;
6304
+ offset += 1;
6305
+ if (marker === 0)
6306
+ return;
6307
+ if (marker === 1 || marker === 217 || marker >= 208 && marker <= 216)
6308
+ continue;
6309
+ if (offset + 2 > bytes.length)
6310
+ return;
6311
+ const segmentLength = view.getUint16(offset);
6312
+ if (segmentLength < 2)
6313
+ return;
6314
+ if (isJpegStartOfFrame(marker)) {
6315
+ if (segmentLength < 7 || offset + 7 > bytes.length)
6316
+ return;
6317
+ return { width: view.getUint16(offset + 5), height: view.getUint16(offset + 3) };
6318
+ }
6319
+ if (marker === 218)
6320
+ return;
6321
+ const next = offset + segmentLength;
6322
+ if (next <= offset || next > bytes.length)
6323
+ return;
6324
+ offset = next;
6325
+ }
6326
+ return;
6327
+ }
6328
+ function isJpegStartOfFrame(marker) {
6329
+ return marker >= 192 && marker <= 207 && marker !== 196 && marker !== 200 && marker !== 204;
6330
+ }
6331
+ function readWebpSize(bytes) {
6332
+ if (bytes.length < 20 || !asciiAt(bytes, 0, "RIFF") || !asciiAt(bytes, 8, "WEBP"))
6333
+ return;
6334
+ const chunkSize = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(16, true);
6335
+ if (asciiAt(bytes, 12, "VP8X"))
6336
+ return readVp8xSize(bytes, chunkSize);
6337
+ if (asciiAt(bytes, 12, "VP8L"))
6338
+ return readVp8lSize(bytes, chunkSize);
6339
+ if (asciiAt(bytes, 12, "VP8 "))
6340
+ return readVp8Size(bytes, chunkSize);
6341
+ return;
6342
+ }
6343
+ function readVp8xSize(bytes, chunkSize) {
6344
+ if (!webpPayloadFits(bytes, chunkSize, 10))
6345
+ return;
6346
+ const payload = 20;
6347
+ const width = 1 + readU24LE(bytes, payload + 4);
6348
+ const height = 1 + readU24LE(bytes, payload + 7);
6349
+ return { width, height };
6350
+ }
6351
+ function readVp8lSize(bytes, chunkSize) {
6352
+ if (!webpPayloadFits(bytes, chunkSize, 5) || bytes[20] !== 47)
6353
+ return;
6354
+ const b1 = bytes[21] ?? 0;
6355
+ const b2 = bytes[22] ?? 0;
6356
+ const b3 = bytes[23] ?? 0;
6357
+ const b4 = bytes[24] ?? 0;
6358
+ const width = 1 + (b1 | (b2 & 63) << 8);
6359
+ const height = 1 + (b2 >> 6 | b3 << 2 | (b4 & 15) << 10);
6360
+ return { width, height };
6361
+ }
6362
+ function readVp8Size(bytes, chunkSize) {
6363
+ if (!webpPayloadFits(bytes, chunkSize, 10) || (bytes[20] & 1) !== 0)
6364
+ return;
6365
+ if (bytes[23] !== 157 || bytes[24] !== 1 || bytes[25] !== 42)
6366
+ return;
6367
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
6368
+ return { width: view.getUint16(26, true) & 16383, height: view.getUint16(28, true) & 16383 };
6369
+ }
6370
+ function webpPayloadFits(bytes, chunkSize, needed) {
6371
+ return chunkSize >= needed && bytes.length >= 20 + needed;
6372
+ }
6373
+ function readU24LE(bytes, offset) {
6374
+ return (bytes[offset] ?? 0) | (bytes[offset + 1] ?? 0) << 8 | (bytes[offset + 2] ?? 0) << 16;
6375
+ }
6376
+ function asciiAt(bytes, offset, text) {
6377
+ if (offset + text.length > bytes.length)
6378
+ return false;
6379
+ for (let index = 0;index < text.length; index += 1) {
6380
+ if (bytes[offset + index] !== text.charCodeAt(index))
6381
+ return false;
6382
+ }
6383
+ return true;
6384
+ }
6258
6385
  function withTimeout(operation, timeoutMs, onTimeout) {
6259
6386
  return new Promise((resolve, reject) => {
6260
6387
  const timeout = setTimeout(() => {
@@ -6270,7 +6397,7 @@ function withTimeout(operation, timeoutMs, onTimeout) {
6270
6397
  });
6271
6398
  });
6272
6399
  }
6273
- var TESSERACT_LANGUAGES, BUILTIN_OCR_LANGUAGES, DEFAULT_TESSERACT_TIMEOUT_MS = 30000;
6400
+ var TESSERACT_LANGUAGES, BUILTIN_OCR_LANGUAGES, DEFAULT_TESSERACT_TIMEOUT_MS = 30000, MIN_LEPTONICA_SIDE_PIXELS = 3;
6274
6401
  var init_ocr_engines = __esm(() => {
6275
6402
  TESSERACT_LANGUAGES = {
6276
6403
  ko: "kor",
@@ -6479,8 +6606,8 @@ function createDefaultParserRegistry(options = {}) {
6479
6606
  const kordocOptions = {
6480
6607
  ...options.kordoc,
6481
6608
  images: ocrEnabled,
6482
- ...options.kordoc?.ocr === undefined && ocrEnabled ? {
6483
- ocr: builtinCovers ? true : createTesseractOcrProvider({
6609
+ ...options.kordoc?.ocr === undefined ? {
6610
+ ocr: !ocrEnabled ? false : builtinCovers ? true : createTesseractOcrProvider({
6484
6611
  languages,
6485
6612
  timeoutMs: options.ocr?.timeoutMs,
6486
6613
  cachePath: options.ocr?.cachePath
@@ -6618,6 +6745,23 @@ async function syncParsedMirrors(options) {
6618
6745
  checkpoint();
6619
6746
  continue;
6620
6747
  }
6748
+ const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
6749
+ if (tooSmallImage !== undefined) {
6750
+ deleted += removePrevious(options.root, previous, entry.virtualPath);
6751
+ handledPrevious.add(entry.virtualPath);
6752
+ skipped += 1;
6753
+ recordSkip(entry, "parser-skipped");
6754
+ diagnostics.push({
6755
+ code: "parser-skipped",
6756
+ severity: "warning",
6757
+ message: tooSmallImage,
6758
+ source: entry.virtualPath
6759
+ });
6760
+ sinceCheckpoint += 1;
6761
+ if (sinceCheckpoint >= MIRROR_CHECKPOINT_EVERY)
6762
+ checkpoint();
6763
+ continue;
6764
+ }
6621
6765
  let parsed;
6622
6766
  try {
6623
6767
  parsed = await parser.parse({ virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes });
@@ -6719,6 +6863,11 @@ async function detectMirrorStaleness(options) {
6719
6863
  }
6720
6864
  return diagnostics;
6721
6865
  }
6866
+ function standaloneImageTooSmallMessage(virtualPath, bytes) {
6867
+ if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname4(virtualPath))))
6868
+ return;
6869
+ return ocrImageTooSmallMessage(bytes);
6870
+ }
6722
6871
  function sourceReadFailure(error) {
6723
6872
  if (!(error instanceof Error))
6724
6873
  return;
@@ -6814,11 +6963,13 @@ function removeFile(path) {
6814
6963
  if (existsSync15(path))
6815
6964
  rmSync6(path, { force: true });
6816
6965
  }
6817
- var DEFAULT_MAX_SOURCE_BYTES, MIRROR_CHECKPOINT_EVERY = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.";
6966
+ var DEFAULT_MAX_SOURCE_BYTES, MIRROR_CHECKPOINT_EVERY = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", STANDALONE_OCR_IMAGE_EXTENSIONS;
6818
6967
  var init_sync = __esm(() => {
6819
6968
  init_source_paths();
6820
6969
  init_defaults();
6821
6970
  init_errors2();
6971
+ init_kordoc();
6972
+ init_ocr_engines();
6822
6973
  init_text();
6823
6974
  init_index_store();
6824
6975
  init_paths();
@@ -6846,6 +6997,7 @@ var init_sync = __esm(() => {
6846
6997
  "GrShaderCache",
6847
6998
  "ShaderCache"
6848
6999
  ]);
7000
+ STANDALONE_OCR_IMAGE_EXTENSIONS = new Set(KORDOC_IMAGE_EXTENSIONS);
6849
7001
  });
6850
7002
 
6851
7003
  // src/observability/run-log.ts
@@ -7711,8 +7863,36 @@ class RetrievalMethodRegistry {
7711
7863
  }
7712
7864
  }
7713
7865
 
7866
+ // src/cloud/config.ts
7867
+ function normalizeBaseUrl(value) {
7868
+ const trimmed = value.trim();
7869
+ let end = trimmed.length;
7870
+ while (end > 0 && trimmed.charCodeAt(end - 1) === 47)
7871
+ end--;
7872
+ return trimmed.slice(0, end);
7873
+ }
7874
+ function resolveBaseUrl(env) {
7875
+ const configured = env[AUTORAG_BASE_URL_ENV];
7876
+ const value = typeof configured === "string" && configured.trim() !== "" ? configured : DEFAULT_BASE_URL;
7877
+ return normalizeBaseUrl(value);
7878
+ }
7879
+ var PROVIDER_ID = "autorag", PROVIDER_NAME = "AutoRAG", DEFAULT_BASE_URL = "https://api.dazziapp.com", AUTORAG_BASE_URL_ENV = "AUTORAG_BASE_URL", AUTORAG_API_KEY_ENV = "AUTORAG_API_KEY", API_PATH = "/v1", MODELS_PATH = "/v1/models", AUTHORIZE_PATH = "/desktop/authorize", TOKEN_PATH = "/api/desktop/token", AUTORAG_INPUT_LIMITS, OAUTH_CLIENT_ID = "agent", API_KEY_CONFIG_VALUE;
7880
+ var init_config = __esm(() => {
7881
+ AUTORAG_INPUT_LIMITS = {
7882
+ maxRequestBytes: 3500000,
7883
+ images: { resize: { maxBytes: 1e6 } }
7884
+ };
7885
+ API_KEY_CONFIG_VALUE = `$${AUTORAG_API_KEY_ENV}`;
7886
+ });
7887
+
7714
7888
  // src/retrieval/rerank.ts
7715
7889
  import { OpenRouter } from "@openrouter/sdk";
7890
+ function defaultRerankModel(provider) {
7891
+ return provider === PROVIDER_ID ? DEFAULT_AUTORAG_RERANK_MODEL : DEFAULT_RERANK_MODEL;
7892
+ }
7893
+ function defaultRerankApiKeyEnv(provider) {
7894
+ return provider === PROVIDER_ID ? DEFAULT_AUTORAG_RERANK_API_KEY_ENV : DEFAULT_RERANK_API_KEY_ENV;
7895
+ }
7716
7896
  function createReranker(config) {
7717
7897
  if (config === false || config === undefined)
7718
7898
  return;
@@ -7720,30 +7900,36 @@ function createReranker(config) {
7720
7900
  if (!SUPPORTED_RERANK_PROVIDERS.includes(provider)) {
7721
7901
  throw new Error(`Unsupported rerank provider "${provider}"; supported providers: ${SUPPORTED_RERANK_PROVIDERS.join(", ")}`);
7722
7902
  }
7723
- return new OpenRouterReranker(config);
7903
+ return new OpenRouterReranker({ ...config, provider });
7904
+ }
7905
+ function autoragApiBaseUrl(env) {
7906
+ const root = resolveBaseUrl(env);
7907
+ return root.endsWith(API_PATH) ? root : `${root}${API_PATH}`;
7724
7908
  }
7725
7909
 
7726
7910
  class OpenRouterReranker {
7727
7911
  model;
7728
- provider = DEFAULT_RERANK_PROVIDER;
7912
+ provider;
7729
7913
  client;
7730
7914
  unavailableReason;
7731
7915
  constructor(options = {}) {
7732
- this.model = options.model ?? DEFAULT_RERANK_MODEL;
7916
+ this.provider = options.provider ?? DEFAULT_RERANK_PROVIDER;
7917
+ this.model = options.model ?? defaultRerankModel(this.provider);
7733
7918
  if (options.client !== undefined) {
7734
7919
  this.client = options.client;
7735
7920
  return;
7736
7921
  }
7737
- const apiKeyEnv = options.apiKeyEnv ?? DEFAULT_RERANK_API_KEY_ENV;
7738
7922
  const env = options.env ?? process.env;
7923
+ const apiKeyEnv = options.apiKeyEnv ?? defaultRerankApiKeyEnv(this.provider);
7739
7924
  const apiKey = options.apiKey ?? env[apiKeyEnv];
7740
7925
  if (apiKey === undefined || apiKey.length === 0) {
7741
7926
  this.unavailableReason = `no API key: set ${apiKeyEnv} or configure rerank.apiKeyEnv`;
7742
7927
  return;
7743
7928
  }
7929
+ const baseUrl = options.baseUrl ?? (this.provider === PROVIDER_ID ? autoragApiBaseUrl(env) : undefined);
7744
7930
  const sdkOptions = { apiKey };
7745
- if (options.baseUrl !== undefined)
7746
- sdkOptions.serverURL = options.baseUrl;
7931
+ if (baseUrl !== undefined)
7932
+ sdkOptions.serverURL = baseUrl;
7747
7933
  if (options.timeoutMs !== undefined)
7748
7934
  sdkOptions.timeoutMs = options.timeoutMs;
7749
7935
  const sdk = new OpenRouter(sdkOptions);
@@ -7808,9 +7994,10 @@ class OpenRouterReranker {
7808
7994
  return reranked;
7809
7995
  }
7810
7996
  }
7811
- var DEFAULT_RERANK_MODEL = "voyageai/rerank-3-lite", DEFAULT_RERANK_PROVIDER = "openrouter", DEFAULT_RERANK_API_KEY_ENV = "OPENROUTER_API_KEY", DEFAULT_RERANK_TOP_N = 25, SUPPORTED_RERANK_PROVIDERS;
7997
+ var DEFAULT_RERANK_MODEL = "voyageai/rerank-3-lite", DEFAULT_RERANK_PROVIDER = "openrouter", DEFAULT_RERANK_API_KEY_ENV = "OPENROUTER_API_KEY", DEFAULT_AUTORAG_RERANK_MODEL = "qwen/qwen3-reranker-8b", DEFAULT_AUTORAG_RERANK_API_KEY_ENV = "AUTORAG_API_KEY", DEFAULT_RERANK_TOP_N = 25, SUPPORTED_RERANK_PROVIDERS;
7812
7998
  var init_rerank = __esm(() => {
7813
- SUPPORTED_RERANK_PROVIDERS = [DEFAULT_RERANK_PROVIDER];
7999
+ init_config();
8000
+ SUPPORTED_RERANK_PROVIDERS = [DEFAULT_RERANK_PROVIDER, PROVIDER_ID];
7814
8001
  });
7815
8002
 
7816
8003
  // src/retrieval/selection.ts
@@ -9029,22 +9216,219 @@ var init_fsearch_search_tool = __esm(() => {
9029
9216
  });
9030
9217
  });
9031
9218
 
9032
- // src/cloud/config.ts
9033
- function normalizeBaseUrl(value) {
9034
- const trimmed = value.trim();
9035
- let end = trimmed.length;
9036
- while (end > 0 && trimmed.charCodeAt(end - 1) === 47)
9037
- end--;
9038
- return trimmed.slice(0, end);
9219
+ // src/cloud/jev-backend.ts
9220
+ import {
9221
+ BackendError
9222
+ } from "jev-use";
9223
+
9224
+ class AutoragJevBackend {
9225
+ name = "autorag";
9226
+ apiKey;
9227
+ baseUrl;
9228
+ defaultModel;
9229
+ timeoutMs;
9230
+ maxRetries;
9231
+ fetchImpl;
9232
+ constructor(options = {}) {
9233
+ const env = options.env ?? process.env;
9234
+ const apiKey = options.apiKey ?? env[AUTORAG_API_KEY_ENV];
9235
+ if (apiKey === undefined || apiKey.trim() === "") {
9236
+ throw new Error(`jev.backend "autorag" needs ${AUTORAG_API_KEY_ENV}.`);
9237
+ }
9238
+ this.apiKey = apiKey;
9239
+ this.baseUrl = options.baseUrl ?? resolveBaseUrl(env);
9240
+ this.defaultModel = options.defaultModel ?? AUTORAG_DEFAULT_JEV_MODEL;
9241
+ this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS6;
9242
+ this.maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
9243
+ this.fetchImpl = options.fetchImpl;
9244
+ }
9245
+ async judge(request) {
9246
+ const model = request.model ?? this.defaultModel;
9247
+ const body = toNativeBody(request.state, request.questions, model);
9248
+ const started = Date.now();
9249
+ const response = await this.postJson(`${this.baseUrl}${JEV_PATH}`, body);
9250
+ return {
9251
+ answers: parseNativeAnswers(this.name, response.answers, request.questions),
9252
+ model: response.model ?? model,
9253
+ latencyMs: Date.now() - started,
9254
+ usage: {
9255
+ inputTokens: response.usage?.input_tokens,
9256
+ outputTokens: response.usage?.output_tokens
9257
+ }
9258
+ };
9259
+ }
9260
+ async postJson(url, body) {
9261
+ const doFetch = this.fetchImpl ?? fetch;
9262
+ let lastError;
9263
+ for (let attempt = 0;attempt <= this.maxRetries; attempt += 1) {
9264
+ if (attempt > 0)
9265
+ await sleep3(retryDelay(attempt, lastError));
9266
+ const controller = new AbortController;
9267
+ const timer = setTimeout(() => controller.abort(), this.timeoutMs);
9268
+ try {
9269
+ const response = await doFetch(url, {
9270
+ method: "POST",
9271
+ headers: {
9272
+ "Content-Type": "application/json",
9273
+ Accept: "application/json",
9274
+ Authorization: `Bearer ${this.apiKey}`
9275
+ },
9276
+ body: JSON.stringify(body),
9277
+ signal: controller.signal
9278
+ });
9279
+ if (response.ok)
9280
+ return await response.json();
9281
+ const message = await extractErrorMessage(response);
9282
+ lastError = new BackendError(this.name, `HTTP ${response.status}: ${message}`, response.status);
9283
+ lastError.retryAfterMs = parseRetryAfter(response);
9284
+ if (!isRetryable(response.status) || attempt === this.maxRetries)
9285
+ throw lastError;
9286
+ } catch (error) {
9287
+ if (error instanceof BackendError) {
9288
+ if (attempt === this.maxRetries || !isRetryable(error.status))
9289
+ throw error;
9290
+ continue;
9291
+ }
9292
+ const aborted = error?.name === "AbortError";
9293
+ lastError = new BackendError(this.name, aborted ? `request timed out after ${this.timeoutMs}ms` : `network error: ${String(error)}`, undefined, error);
9294
+ if (attempt === this.maxRetries)
9295
+ throw lastError;
9296
+ } finally {
9297
+ clearTimeout(timer);
9298
+ }
9299
+ }
9300
+ throw lastError ?? new BackendError(this.name, "unreachable");
9301
+ }
9039
9302
  }
9040
- function resolveBaseUrl(env) {
9041
- const configured = env[AUTORAG_BASE_URL_ENV];
9042
- const value = typeof configured === "string" && configured.trim() !== "" ? configured : DEFAULT_BASE_URL;
9043
- return normalizeBaseUrl(value);
9303
+ function serializeState(state) {
9304
+ return typeof state === "string" ? state : JSON.stringify(state);
9044
9305
  }
9045
- var PROVIDER_ID = "autorag", PROVIDER_NAME = "AutoRAG", DEFAULT_BASE_URL = "https://api.dazziapp.com", AUTORAG_BASE_URL_ENV = "AUTORAG_BASE_URL", AUTORAG_API_KEY_ENV = "AUTORAG_API_KEY", API_PATH = "/v1", MODELS_PATH = "/v1/models", AUTHORIZE_PATH = "/desktop/authorize", TOKEN_PATH = "/api/desktop/token", OAUTH_CLIENT_ID = "agent", API_KEY_CONFIG_VALUE;
9046
- var init_config = __esm(() => {
9047
- API_KEY_CONFIG_VALUE = `$${AUTORAG_API_KEY_ENV}`;
9306
+ function toNativeQuestion(question) {
9307
+ switch (question.type) {
9308
+ case "noul":
9309
+ return {
9310
+ type: "noul",
9311
+ instructions: question.question,
9312
+ ...question.criteria ? { criteria: question.criteria } : {}
9313
+ };
9314
+ case "choice":
9315
+ return {
9316
+ type: "choice",
9317
+ instructions: question.question,
9318
+ criteria: Object.fromEntries(optionEntries(question.options ?? []).map(([label, meaning]) => [label, meaning || label]))
9319
+ };
9320
+ case "score":
9321
+ return {
9322
+ type: "score",
9323
+ instructions: question.question,
9324
+ criteria: question.levels ?? []
9325
+ };
9326
+ }
9327
+ }
9328
+ function toNativeBody(state, questions, model) {
9329
+ return {
9330
+ model,
9331
+ state: serializeState(state),
9332
+ questions: Object.fromEntries(questions.map((question) => [question.id, toNativeQuestion(question)]))
9333
+ };
9334
+ }
9335
+ function parseNativeAnswers(backend, answers, questions) {
9336
+ if (!answers || typeof answers !== "object") {
9337
+ throw new BackendError(backend, "response has no answers object");
9338
+ }
9339
+ const byId = answers;
9340
+ return questions.map((question) => parseNativeAnswer(backend, question, byId[question.id]));
9341
+ }
9342
+ function parseNativeAnswer(backend, question, answer) {
9343
+ if (!answer) {
9344
+ throw new BackendError(backend, `response missing answer for question "${question.id}"`);
9345
+ }
9346
+ switch (question.type) {
9347
+ case "noul":
9348
+ return {
9349
+ answer: numberOrThrow(backend, question.id, answer.noul),
9350
+ confidence: answer.confidence
9351
+ };
9352
+ case "choice":
9353
+ return {
9354
+ answer: stringOrThrow(backend, question.id, answer.choice),
9355
+ confidence: answer.confidence,
9356
+ distribution: answer.probabilities
9357
+ };
9358
+ case "score":
9359
+ return {
9360
+ answer: numberOrThrow(backend, question.id, answer.score),
9361
+ confidence: answer.confidence,
9362
+ distribution: answer.probabilities,
9363
+ legend: answer.legend
9364
+ };
9365
+ }
9366
+ }
9367
+ function optionEntries(options) {
9368
+ return Array.isArray(options) ? options.map((label) => [label, ""]) : Object.entries(options);
9369
+ }
9370
+ function numberOrThrow(backend, id, value) {
9371
+ if (typeof value !== "number" || !Number.isFinite(value)) {
9372
+ throw new BackendError(backend, `answer "${id}" has no numeric value`);
9373
+ }
9374
+ return value;
9375
+ }
9376
+ function stringOrThrow(backend, id, value) {
9377
+ if (typeof value !== "string" || !value) {
9378
+ throw new BackendError(backend, `answer "${id}" has no choice value`);
9379
+ }
9380
+ return value;
9381
+ }
9382
+ function isRetryable(status) {
9383
+ if (status === undefined)
9384
+ return true;
9385
+ return status === 408 || status === 429 || status >= 500;
9386
+ }
9387
+ function retryDelay(attempt, lastError) {
9388
+ if (lastError?.retryAfterMs !== undefined)
9389
+ return Math.min(lastError.retryAfterMs, 60000);
9390
+ const base = Math.min(RETRY_BASE_MS * 2 ** (attempt - 1), RETRY_MAX_MS);
9391
+ return base * (1 + Math.random() * 0.25);
9392
+ }
9393
+ function parseRetryAfter(response) {
9394
+ const milliseconds = response.headers.get("retry-after-ms");
9395
+ if (milliseconds && Number.isFinite(Number(milliseconds)))
9396
+ return Number(milliseconds);
9397
+ const seconds = response.headers.get("retry-after");
9398
+ if (seconds && Number.isFinite(Number(seconds)))
9399
+ return Number(seconds) * 1000;
9400
+ return;
9401
+ }
9402
+ async function extractErrorMessage(response) {
9403
+ try {
9404
+ const data = await response.json();
9405
+ for (const key of ["error", "message", "detail"]) {
9406
+ const value = data[key];
9407
+ if (typeof value === "string")
9408
+ return value;
9409
+ if (value && typeof value === "object") {
9410
+ const message = value.message;
9411
+ if (typeof message === "string")
9412
+ return message;
9413
+ }
9414
+ if (Array.isArray(value) && value.length > 0) {
9415
+ const first = value[0];
9416
+ if (typeof first?.msg === "string")
9417
+ return first.msg;
9418
+ }
9419
+ }
9420
+ return JSON.stringify(data).slice(0, 300);
9421
+ } catch {
9422
+ return response.statusText || "request failed";
9423
+ }
9424
+ }
9425
+ function sleep3(ms) {
9426
+ return new Promise((resolve) => setTimeout(resolve, ms));
9427
+ }
9428
+ var JEV_PATH, AUTORAG_DEFAULT_JEV_MODEL = "jev-1.13", DEFAULT_TIMEOUT_MS6 = 1e4, DEFAULT_MAX_RETRIES = 2, RETRY_BASE_MS = 500, RETRY_MAX_MS = 5000;
9429
+ var init_jev_backend = __esm(() => {
9430
+ init_config();
9431
+ JEV_PATH = `${API_PATH}/jev`;
9048
9432
  });
9049
9433
 
9050
9434
  // src/cloud/errors.ts
@@ -9175,7 +9559,8 @@ function mapServerModel(entry) {
9175
9559
  cost: readPricing(entry.pricing),
9176
9560
  reasoning: entry.reasoning === true,
9177
9561
  contextWindow: readPositiveNumber(entry.context_window) ?? DEFAULT_CONTEXT_WINDOW,
9178
- maxTokens: readPositiveNumber(entry.max_output_tokens) ?? DEFAULT_MAX_TOKENS
9562
+ maxTokens: readPositiveNumber(entry.max_output_tokens) ?? DEFAULT_MAX_TOKENS,
9563
+ inputLimits: AUTORAG_INPUT_LIMITS
9179
9564
  };
9180
9565
  }
9181
9566
  function buildStoreEntry(models, providerId, baseUrl) {
@@ -9214,7 +9599,8 @@ function modelsFromStore(stored) {
9214
9599
  cost: entry.cost,
9215
9600
  reasoning: entry.reasoning,
9216
9601
  contextWindow: entry.contextWindow,
9217
- maxTokens: entry.maxTokens
9602
+ maxTokens: entry.maxTokens,
9603
+ inputLimits: AUTORAG_INPUT_LIMITS
9218
9604
  });
9219
9605
  }
9220
9606
  return models;
@@ -9622,69 +10008,456 @@ var init_provider = __esm(() => {
9622
10008
  init_oauth_login();
9623
10009
  });
9624
10010
 
9625
- // src/agent/pi-session.ts
9626
- import { join as join21 } from "node:path";
9627
- import { streamSimple as compatStreamSimple } from "@earendil-works/pi-ai/compat";
9628
- import {
9629
- createAgentSession,
9630
- createAgentSessionFromServices,
9631
- createAgentSessionRuntime,
9632
- createAgentSessionServices,
9633
- DefaultResourceLoader,
9634
- getAgentDir,
9635
- ModelRuntime,
9636
- SessionManager,
9637
- SettingsManager
9638
- } from "@earendil-works/pi-coding-agent";
9639
- function resolveInteractiveTuiMode(settingsManager) {
9640
- const saved = settingsManager.getProjectSettings().tuiMode ?? settingsManager.getGlobalSettings().tuiMode;
9641
- return saved ?? "regular";
9642
- }
9643
- function toToolDefinition(tool) {
9644
- return {
9645
- name: tool.name,
9646
- label: tool.label,
9647
- description: tool.description,
9648
- parameters: tool.parameters,
9649
- ...tool.prepareArguments === undefined ? {} : { prepareArguments: tool.prepareArguments },
9650
- execute: async (toolCallId, params, signal, onUpdate) => {
9651
- return tool.execute(toolCallId, params, signal, onUpdate);
9652
- }
10011
+ // src/agent/context-token-guard.ts
10012
+ import { calculateContextTokens, estimateTokens } from "@earendil-works/pi-coding-agent";
10013
+ function createContextTokenGuardedTransform(model, base, fixedOverheadTokens = () => 0) {
10014
+ return async (messages) => {
10015
+ const transformed = base === undefined ? messages : await base(messages);
10016
+ const currentModel = typeof model === "function" ? model() : model;
10017
+ return guardAgentContextMessages(currentModel, transformed, transformed === messages, fixedOverheadTokens());
9653
10018
  };
9654
10019
  }
9655
- function modelConfig(model, configuredApiKey) {
9656
- return {
9657
- name: model.name,
9658
- baseUrl: model.baseUrl,
9659
- api: model.api,
9660
- ...configuredApiKey === undefined ? {} : { apiKey: configuredApiKey },
9661
- authHeader: false,
9662
- streamSimple: (requestModel, context, options) => compatStreamSimple(requestModel, context, {
9663
- ...options,
9664
- ...configuredApiKey === EMBEDDED_AUTH_MARKER ? { apiKey: undefined } : {}
9665
- }),
9666
- models: [
9667
- {
9668
- id: model.id,
9669
- name: model.name,
9670
- reasoning: model.reasoning,
9671
- input: [...model.input],
9672
- cost: model.cost,
9673
- contextWindow: model.contextWindow,
9674
- maxTokens: model.maxTokens,
9675
- ...model.thinkingLevelMap === undefined ? {} : { thinkingLevelMap: model.thinkingLevelMap }
10020
+ function guardAgentContextMessages(model, messages, useReportedUsage = true, fixedOverheadTokens = 0) {
10021
+ if (model === undefined || !Number.isFinite(model.contextWindow) || model.contextWindow <= 0 || messages.length === 0) {
10022
+ return messages;
10023
+ }
10024
+ const usableContext = Math.floor(model.contextWindow / CONTEXT_HEADROOM);
10025
+ const outputReserve = Math.min(usableContext, Number.isFinite(model.maxTokens) ? Math.max(0, model.maxTokens) : 0);
10026
+ const inputBudget = Math.max(0, usableContext - outputReserve);
10027
+ const budget = Math.max(0, inputBudget - fixedOverheadTokens);
10028
+ const reported = useReportedUsage && findLastAssistantUsage(messages) !== undefined;
10029
+ if (estimateContextMessageTokens(messages, useReportedUsage) <= (reported ? inputBudget : budget)) {
10030
+ return messages;
10031
+ }
10032
+ const candidates = [];
10033
+ const candidateIndexes = [];
10034
+ const base = messages.slice();
10035
+ for (const [index, message] of base.entries()) {
10036
+ const text = textContent(message);
10037
+ if (text === undefined)
10038
+ continue;
10039
+ const section = parseCandidateSection(message, text);
10040
+ if (section === undefined)
10041
+ continue;
10042
+ candidates.push(section);
10043
+ candidateIndexes.push(index);
10044
+ base[index] = replaceTextContent(message, `${section.prefix}${section.suffix}`);
10045
+ }
10046
+ let guarded = messages;
10047
+ if (candidates.length > 0) {
10048
+ const fixedTokens = estimateContextMessageTokens(base, false);
10049
+ const maximumOmitted = candidates.reduce((total, section) => total + section.blocks.length, 0);
10050
+ const omissionMarker = (count) => `
10051
+
10052
+ ${GUARD_MARKER} Retrieval candidates omitted: ${count}.
10053
+ `;
10054
+ let remaining = Math.max(0, budget - fixedTokens - Math.ceil(omissionMarker(maximumOmitted).length / 4));
10055
+ const kept = candidates.map(() => []);
10056
+ let omitted = 0;
10057
+ let firstOmitted = -1;
10058
+ for (const [sectionIndex, section] of candidates.entries()) {
10059
+ for (const block of section.blocks) {
10060
+ const tokens = Math.max(1, Math.ceil(block.length / 4));
10061
+ if (tokens <= remaining) {
10062
+ kept[sectionIndex]?.push(block);
10063
+ remaining -= tokens;
10064
+ continue;
10065
+ }
10066
+ omitted += section.blocks.length - (kept[sectionIndex]?.length ?? 0);
10067
+ for (const later of candidates.slice(sectionIndex + 1))
10068
+ omitted += later.blocks.length;
10069
+ firstOmitted = sectionIndex;
10070
+ break;
9676
10071
  }
9677
- ]
9678
- };
10072
+ if (firstOmitted !== -1)
10073
+ break;
10074
+ }
10075
+ if (omitted > 0) {
10076
+ const candidateByMessage = new Map;
10077
+ for (const [sectionIndex, messageIndex] of candidateIndexes.entries())
10078
+ candidateByMessage.set(messageIndex, sectionIndex);
10079
+ guarded = base.map((message, index) => {
10080
+ const sectionIndex = candidateByMessage.get(index);
10081
+ if (sectionIndex === undefined)
10082
+ return message;
10083
+ const section = candidates[sectionIndex];
10084
+ const marker = sectionIndex === firstOmitted ? omissionMarker(omitted) : "";
10085
+ return replaceTextContent(message, `${section.prefix}${kept[sectionIndex]?.join("") ?? ""}${marker}${section.suffix}`);
10086
+ });
10087
+ }
10088
+ }
10089
+ if (estimateContextMessageTokens(guarded, false) <= budget && guarded !== messages)
10090
+ return guarded;
10091
+ return trimFixedContext(guarded, budget);
9679
10092
  }
9680
- async function configureModelRuntime(modelRuntime, model, apiKey, providerApiKeys) {
9681
- if (model !== undefined) {
9682
- const configuredKey = apiKey ?? providerApiKeys?.[model.provider];
9683
- if (modelRuntime.getModel(model.provider, model.id) === undefined) {
9684
- modelRuntime.registerProvider(model.provider, modelConfig(model, configuredKey ?? EMBEDDED_AUTH_MARKER));
10093
+ function estimateContextMessageTokens(messages, useReportedUsage = false) {
10094
+ if (useReportedUsage) {
10095
+ const usage = findLastAssistantUsage(messages);
10096
+ if (usage !== undefined) {
10097
+ return calculateContextTokens(usage.usage) + messages.slice(usage.index + 1).reduce((total, message) => total + estimateMessageTokens(message), 0);
9685
10098
  }
9686
10099
  }
9687
- const keys = new Map;
10100
+ return messages.reduce((total, message) => total + estimateMessageTokens(message), 0);
10101
+ }
10102
+ function trimFixedContext(messages, inputBudget) {
10103
+ const trimmed = messages.slice();
10104
+ const lastIndex = trimmed.length - 1;
10105
+ const shrinkable = trimmed.map((message, index) => ({ message, index })).filter(({ index, message }) => index !== lastIndex && message.role !== "system");
10106
+ for (const { message, index } of shrinkable) {
10107
+ const replacement = replaceMessageWithMarker(message);
10108
+ if (replacement === message || estimateMessageTokens(replacement) >= estimateMessageTokens(message))
10109
+ continue;
10110
+ trimmed[index] = replacement;
10111
+ if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
10112
+ return trimmed;
10113
+ }
10114
+ const lastMessage = trimmed[lastIndex];
10115
+ if (lastMessage !== undefined && lastMessage.role !== "system") {
10116
+ const replacement = replaceMessageWithMarker(lastMessage);
10117
+ if (replacement !== lastMessage) {
10118
+ trimmed[lastIndex] = replacement;
10119
+ if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
10120
+ return trimmed;
10121
+ }
10122
+ const fixedWithoutLast = trimmed.reduce((total, message, index) => index === lastIndex ? total : total + estimateMessageTokens(message), 0);
10123
+ if (fixedWithoutLast < inputBudget) {
10124
+ const fitted = fitMessageToTokens(lastMessage, inputBudget - fixedWithoutLast);
10125
+ if (fitted !== lastMessage) {
10126
+ trimmed[lastIndex] = fitted;
10127
+ if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
10128
+ return trimmed;
10129
+ }
10130
+ }
10131
+ }
10132
+ for (const [index, message] of trimmed.entries()) {
10133
+ if (message.role !== "system")
10134
+ continue;
10135
+ const replacement = replaceMessageWithMarker(message);
10136
+ if (replacement === message || estimateMessageTokens(replacement) >= estimateMessageTokens(message))
10137
+ continue;
10138
+ trimmed[index] = replacement;
10139
+ if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
10140
+ return trimmed;
10141
+ }
10142
+ return trimmed;
10143
+ }
10144
+ function fitMessageToTokens(message, maxTokens) {
10145
+ const original = textContent(message);
10146
+ if (original === undefined)
10147
+ return message;
10148
+ const marker = `
10149
+
10150
+ ${GUARD_MARKER}
10151
+ `;
10152
+ let low = 0;
10153
+ let high = original.length;
10154
+ let best = replaceTextContent(message, marker);
10155
+ while (low <= high) {
10156
+ const middle = Math.floor((low + high) / 2);
10157
+ const candidate = replaceTextContent(message, excerpt(original, middle, marker));
10158
+ if (estimateMessageTokens(candidate) <= maxTokens) {
10159
+ best = candidate;
10160
+ low = middle + 1;
10161
+ } else {
10162
+ high = middle - 1;
10163
+ }
10164
+ }
10165
+ return best;
10166
+ }
10167
+ function excerpt(text, maxChars, marker) {
10168
+ if (text.length <= maxChars)
10169
+ return text;
10170
+ const available = Math.max(0, maxChars - marker.length);
10171
+ const head = Math.ceil(available * 0.7);
10172
+ const tail = Math.max(0, available - head);
10173
+ return `${text.slice(0, head)}${marker}${tail > 0 ? text.slice(-tail) : ""}`;
10174
+ }
10175
+ function replaceMessageWithMarker(message) {
10176
+ return replaceTextContent(message, `
10177
+
10178
+ ${GUARD_MARKER}
10179
+ `);
10180
+ }
10181
+ function estimateMessageTokens(message) {
10182
+ if (message.role === "system")
10183
+ return estimateSystemMessageTokens(message);
10184
+ const estimate = estimateTokens(message);
10185
+ return message.role === "toolResult" ? estimate + Math.ceil(message.toolName.length / 4) : estimate;
10186
+ }
10187
+ function estimateSystemMessageTokens(message) {
10188
+ let chars = typeof message.content === "string" ? message.content.length : 0;
10189
+ if (Array.isArray(message.content)) {
10190
+ for (const block of message.content) {
10191
+ if (block.type === "text")
10192
+ chars += block.text.length;
10193
+ else if (block.type === "image")
10194
+ chars += 4800;
10195
+ }
10196
+ }
10197
+ chars += JSON.stringify(message.sections ?? {}).length;
10198
+ chars += JSON.stringify(message.toolsAdded ?? []).length;
10199
+ chars += JSON.stringify(message.toolsRemoved ?? []).length;
10200
+ return Math.ceil(chars / 4);
10201
+ }
10202
+ function findLastAssistantUsage(messages) {
10203
+ for (let index = messages.length - 1;index >= 0; index--) {
10204
+ const message = messages[index];
10205
+ if (message?.role !== "assistant" || message.stopReason === "aborted" || message.stopReason === "error")
10206
+ continue;
10207
+ if (calculateContextTokens(message.usage) > 0)
10208
+ return { index, usage: message.usage };
10209
+ }
10210
+ return;
10211
+ }
10212
+ function textContent(message) {
10213
+ if (message.role === "system") {
10214
+ if (typeof message.content === "string")
10215
+ return message.content;
10216
+ return message.content.filter((block) => block.type === "text").map((block) => block.text).join("");
10217
+ }
10218
+ if (message.role === "user" || message.role === "toolResult") {
10219
+ if (typeof message.content === "string")
10220
+ return message.content;
10221
+ let text = "";
10222
+ for (const block of message.content) {
10223
+ if (block.type !== "text")
10224
+ return;
10225
+ text += block.text;
10226
+ }
10227
+ return text;
10228
+ }
10229
+ if (message.role === "assistant") {
10230
+ let text = "";
10231
+ for (const block of message.content)
10232
+ if (block.type === "text")
10233
+ text += block.text;
10234
+ return text.length > 0 ? text : undefined;
10235
+ }
10236
+ return;
10237
+ }
10238
+ function replaceTextContent(message, text) {
10239
+ if (message.role === "system")
10240
+ return { ...message, content: text };
10241
+ if (message.role === "user") {
10242
+ if (typeof message.content === "string")
10243
+ return { ...message, content: text };
10244
+ return { ...message, content: [{ type: "text", text }] };
10245
+ }
10246
+ if (message.role === "toolResult")
10247
+ return { ...message, content: [{ type: "text", text }] };
10248
+ if (message.role === "assistant") {
10249
+ const content = [];
10250
+ let replaced = false;
10251
+ for (const block of message.content) {
10252
+ if (block.type !== "text") {
10253
+ content.push(block);
10254
+ continue;
10255
+ }
10256
+ if (replaced)
10257
+ continue;
10258
+ replaced = true;
10259
+ content.push({ ...block, text });
10260
+ }
10261
+ return replaced ? { ...message, content } : message;
10262
+ }
10263
+ return message;
10264
+ }
10265
+ function parseCandidateSection(message, text) {
10266
+ if (message.role === "user") {
10267
+ const markerMatch = /(?:Baseline retrieval (?:evidence \(already gathered for you\)|context):|initial candidates:)/u.exec(text);
10268
+ if (markerMatch === null)
10269
+ return;
10270
+ const sectionStart = (markerMatch.index ?? 0) + markerMatch[0].length;
10271
+ const sectionEnd = findBaselineSectionEnd(text, sectionStart);
10272
+ if (!/^\[e?\d+\] /mu.test(text.slice(sectionStart, sectionEnd)))
10273
+ return;
10274
+ return splitCandidates(text, sectionStart, sectionEnd);
10275
+ }
10276
+ if (message.role !== "toolResult")
10277
+ return;
10278
+ const isRetrieval = RETRIEVAL_TOOL_NAMES[message.toolName] === true || message.toolName.startsWith(RETRIEVAL_TOOL_PREFIX);
10279
+ if (!isRetrieval)
10280
+ return;
10281
+ if (message.toolName === "jikji_find") {
10282
+ const directiveStart = text.indexOf(`
10283
+
10284
+ directive:`);
10285
+ return splitCandidates(text, 0, directiveStart >= 0 ? directiveStart : text.length, /^- [^\n]*(?:\n|$)/gmu);
10286
+ }
10287
+ const diagnosticsStart = text.indexOf(`
10288
+
10289
+ Diagnostics:`);
10290
+ return splitCandidates(text, 0, diagnosticsStart >= 0 ? diagnosticsStart : text.length);
10291
+ }
10292
+ function findBaselineSectionEnd(text, startIndex) {
10293
+ const endMarkers = [
10294
+ `
10295
+
10296
+ Produce the best`,
10297
+ `
10298
+
10299
+ Treat candidates as unverified evidence`,
10300
+ `
10301
+
10302
+ Formatting and content rules`
10303
+ ];
10304
+ const positions = endMarkers.map((marker) => text.indexOf(marker, startIndex)).filter((index) => index >= 0);
10305
+ return positions.length > 0 ? Math.min(...positions) : text.length;
10306
+ }
10307
+ function splitCandidates(text, sectionStart, sectionEnd, pattern = /^\[e?\d+\] [^\n]*(?:\n|$)/gmu) {
10308
+ const region = text.slice(sectionStart, sectionEnd);
10309
+ const matches = [...region.matchAll(pattern)];
10310
+ const firstStart = matches[0]?.index;
10311
+ if (firstStart === undefined)
10312
+ return;
10313
+ const blocks = [];
10314
+ for (const [index, match] of matches.entries()) {
10315
+ const start = match.index ?? 0;
10316
+ const end = index + 1 < matches.length ? matches[index + 1]?.index ?? region.length : region.length;
10317
+ blocks.push(region.slice(start, end));
10318
+ }
10319
+ return {
10320
+ prefix: text.slice(0, sectionStart) + region.slice(0, firstStart),
10321
+ suffix: text.slice(sectionEnd),
10322
+ blocks
10323
+ };
10324
+ }
10325
+ var CONTEXT_HEADROOM = 1.2, GUARD_MARKER = "[AutoRAG context guard: omitted context to stay within the model context budget.]", RETRIEVAL_TOOL_PREFIX = "search_datasource_", RETRIEVAL_TOOL_NAMES;
10326
+ var init_context_token_guard = __esm(() => {
10327
+ RETRIEVAL_TOOL_NAMES = {
10328
+ jikji_find: true,
10329
+ search_all_documents: true,
10330
+ semantic_search_local_docs: true,
10331
+ search_single_datasource_documents: true,
10332
+ everything_search: true,
10333
+ fsearch_search: true
10334
+ };
10335
+ });
10336
+
10337
+ // src/agent/request-limits.ts
10338
+ function fitRequestLimits(messages, limits) {
10339
+ const maxBytes = limits?.maxRequestBytes;
10340
+ const maxImages = limits?.images?.maxPerRequest;
10341
+ if (maxBytes === undefined && maxImages === undefined)
10342
+ return;
10343
+ const images = imageLocations(messages);
10344
+ if (images.length === 0)
10345
+ return;
10346
+ let keptImages = images.length;
10347
+ let bytes = maxBytes === undefined ? 0 : Buffer.byteLength(JSON.stringify(messages));
10348
+ const dropped = [];
10349
+ for (const image of images) {
10350
+ const overCount = maxImages !== undefined && keptImages > maxImages;
10351
+ const overBytes = maxBytes !== undefined && bytes > maxBytes;
10352
+ if (!overCount && !overBytes)
10353
+ break;
10354
+ dropped.push(image);
10355
+ keptImages--;
10356
+ bytes -= image.bytes - NOTE_BYTES;
10357
+ }
10358
+ if (dropped.length === 0)
10359
+ return;
10360
+ const byMessage = new Map;
10361
+ for (const { message, block } of dropped) {
10362
+ const blocks = byMessage.get(message) ?? new Set;
10363
+ blocks.add(block);
10364
+ byMessage.set(message, blocks);
10365
+ }
10366
+ return messages.map((message, index) => {
10367
+ const blocks = byMessage.get(index);
10368
+ if (blocks === undefined)
10369
+ return message;
10370
+ const content = message.content.map((part, block) => blocks.has(block) ? { type: "text", text: IMAGE_OMITTED_NOTE } : part);
10371
+ return { ...message, content };
10372
+ });
10373
+ }
10374
+ function imageLocations(messages) {
10375
+ const locations = [];
10376
+ messages.forEach((message, index) => {
10377
+ const content = message.content;
10378
+ if (!Array.isArray(content))
10379
+ return;
10380
+ content.forEach((part, block) => {
10381
+ if (!isImage(part))
10382
+ return;
10383
+ locations.push({ message: index, block, bytes: Buffer.byteLength(JSON.stringify(part)) });
10384
+ });
10385
+ });
10386
+ return locations;
10387
+ }
10388
+ function isImage(part) {
10389
+ return typeof part === "object" && part !== null && part.type === "image";
10390
+ }
10391
+ var IMAGE_OMITTED_NOTE = "[Image omitted from this request to stay within the model provider's request limits. Read the file again if you still need to see it.]", NOTE_BYTES;
10392
+ var init_request_limits = __esm(() => {
10393
+ NOTE_BYTES = Buffer.byteLength(JSON.stringify({ type: "text", text: IMAGE_OMITTED_NOTE }));
10394
+ });
10395
+
10396
+ // src/agent/pi-session.ts
10397
+ import { join as join21 } from "node:path";
10398
+ import { streamSimple as compatStreamSimple } from "@earendil-works/pi-ai/compat";
10399
+ import {
10400
+ createAgentSession,
10401
+ createAgentSessionFromServices,
10402
+ createAgentSessionRuntime,
10403
+ createAgentSessionServices,
10404
+ DefaultResourceLoader,
10405
+ getAgentDir,
10406
+ ModelRuntime,
10407
+ SessionManager,
10408
+ SettingsManager
10409
+ } from "@earendil-works/pi-coding-agent";
10410
+ import { getKeybindings, matchesKey } from "@earendil-works/pi-tui";
10411
+ function resolveInteractiveTuiMode(settingsManager) {
10412
+ const saved = settingsManager.getProjectSettings().tuiMode ?? settingsManager.getGlobalSettings().tuiMode;
10413
+ return saved ?? "regular";
10414
+ }
10415
+ function toToolDefinition(tool) {
10416
+ return {
10417
+ name: tool.name,
10418
+ label: tool.label,
10419
+ description: tool.description,
10420
+ parameters: tool.parameters,
10421
+ ...tool.prepareArguments === undefined ? {} : { prepareArguments: tool.prepareArguments },
10422
+ execute: async (toolCallId, params, signal, onUpdate) => {
10423
+ return tool.execute(toolCallId, params, signal, onUpdate);
10424
+ }
10425
+ };
10426
+ }
10427
+ function modelConfig(model, configuredApiKey) {
10428
+ return {
10429
+ name: model.name,
10430
+ baseUrl: model.baseUrl,
10431
+ api: model.api,
10432
+ ...configuredApiKey === undefined ? {} : { apiKey: configuredApiKey },
10433
+ authHeader: false,
10434
+ streamSimple: (requestModel, context, options) => compatStreamSimple(requestModel, context, {
10435
+ ...options,
10436
+ ...configuredApiKey === EMBEDDED_AUTH_MARKER ? { apiKey: undefined } : {}
10437
+ }),
10438
+ models: [
10439
+ {
10440
+ id: model.id,
10441
+ name: model.name,
10442
+ reasoning: model.reasoning,
10443
+ input: [...model.input],
10444
+ cost: model.cost,
10445
+ contextWindow: model.contextWindow,
10446
+ maxTokens: model.maxTokens,
10447
+ ...model.inputLimits === undefined ? {} : { inputLimits: model.inputLimits },
10448
+ ...model.thinkingLevelMap === undefined ? {} : { thinkingLevelMap: model.thinkingLevelMap }
10449
+ }
10450
+ ]
10451
+ };
10452
+ }
10453
+ async function configureModelRuntime(modelRuntime, model, apiKey, providerApiKeys) {
10454
+ if (model !== undefined) {
10455
+ const configuredKey = apiKey ?? providerApiKeys?.[model.provider];
10456
+ if (modelRuntime.getModel(model.provider, model.id) === undefined) {
10457
+ modelRuntime.registerProvider(model.provider, modelConfig(model, configuredKey ?? EMBEDDED_AUTH_MARKER));
10458
+ }
10459
+ }
10460
+ const keys = new Map;
9688
10461
  if (model !== undefined && apiKey !== undefined)
9689
10462
  keys.set(model.provider, apiKey);
9690
10463
  for (const [provider, key] of Object.entries(providerApiKeys ?? {}))
@@ -9692,20 +10465,47 @@ async function configureModelRuntime(modelRuntime, model, apiKey, providerApiKey
9692
10465
  for (const [provider, key] of keys)
9693
10466
  await modelRuntime.setRuntimeApiKey(provider, key);
9694
10467
  }
10468
+ function enforceRequestLimits(pi) {
10469
+ pi.on("context", (event, ctx) => {
10470
+ const messages = fitRequestLimits(event.messages, ctx.model?.inputLimits);
10471
+ return messages === undefined ? undefined : { messages };
10472
+ });
10473
+ }
10474
+ function guardContext(pi, contextTransform) {
10475
+ pi.on("context", async (event, ctx) => {
10476
+ const activeTools = new Set(pi.getActiveTools());
10477
+ const toolDeclarations = pi.getAllTools().filter((tool) => activeTools.has(tool.name)).map(({ name, description, parameters }) => ({ name, description, parameters }));
10478
+ const fixedOverheadTokens = Math.ceil((ctx.getSystemPrompt().length + JSON.stringify(toolDeclarations).length) / 4);
10479
+ return {
10480
+ messages: await createContextTokenGuardedTransform(ctx.model, contextTransform, () => fixedOverheadTokens)(event.messages)
10481
+ };
10482
+ });
10483
+ }
9695
10484
  function createAutoRAGExtension(getSystemPrompt, contextTransform) {
9696
10485
  return (pi) => {
9697
10486
  pi.on("before_agent_start", () => ({ systemPrompt: getSystemPrompt() }));
9698
- if (contextTransform !== undefined) {
9699
- pi.on("context", async (event) => ({ messages: await contextTransform(event.messages) }));
9700
- }
10487
+ guardContext(pi, contextTransform);
10488
+ enforceRequestLimits(pi);
10489
+ };
10490
+ }
10491
+ function interactiveKeyOf(data) {
10492
+ const bindings = getKeybindings();
10493
+ const keysOf = (action, fallback) => {
10494
+ const keys = bindings.getKeys(action);
10495
+ return keys.length > 0 ? keys : [fallback];
9701
10496
  };
10497
+ const windows = process.platform === "win32" || process.platform === "linux" && Boolean(process.env.WSL_DISTRO_NAME || process.env.WSL_INTEROP);
10498
+ if (keysOf("app.interrupt", "escape").some((key) => matchesKey(data, key)))
10499
+ return "interrupt";
10500
+ if (keysOf("app.message.followUp", windows ? "ctrl+q" : "alt+enter").some((key) => matchesKey(data, key)))
10501
+ return "followUp";
10502
+ return;
9702
10503
  }
9703
- function createAutoRAGInteractiveExtension(getSystemPrompt, contextTransform, onQuery, updateNotice) {
10504
+ function createAutoRAGInteractiveExtension(getSystemPrompt, contextTransform, options, updateNotice) {
9704
10505
  return (pi) => {
9705
10506
  pi.on("before_agent_start", () => ({ systemPrompt: getSystemPrompt() }));
9706
- if (contextTransform !== undefined) {
9707
- pi.on("context", async (event) => ({ messages: await contextTransform(event.messages) }));
9708
- }
10507
+ guardContext(pi, contextTransform);
10508
+ enforceRequestLimits(pi);
9709
10509
  if (updateNotice !== undefined) {
9710
10510
  pi.on("session_start", () => {
9711
10511
  updateNotice().then((text) => {
@@ -9721,20 +10521,37 @@ function createAutoRAGInteractiveExtension(getSystemPrompt, contextTransform, on
9721
10521
  });
9722
10522
  });
9723
10523
  }
9724
- pi.on("input", async (event) => {
9725
- const query = event.text.trim();
9726
- if (event.source !== "interactive" || query.length === 0 || query.startsWith("/"))
10524
+ const { onInput, isSearchRunning, onKey } = options;
10525
+ if (onKey !== undefined) {
10526
+ pi.on("session_start", (_event, ctx) => {
10527
+ ctx.ui.onTerminalInput((data) => {
10528
+ const key = interactiveKeyOf(data);
10529
+ return key !== undefined && onKey(key) ? { consume: true } : undefined;
10530
+ });
10531
+ });
10532
+ }
10533
+ pi.on("input", (event) => {
10534
+ const text = event.text.trim();
10535
+ if (event.source !== "interactive" || text.length === 0)
10536
+ return;
10537
+ if (text.startsWith("/") && isSearchRunning?.() !== true)
9727
10538
  return;
10539
+ let disposition;
9728
10540
  try {
9729
- await onQuery(query, pi);
10541
+ disposition = onInput({ text, images: event.images, streamingBehavior: event.streamingBehavior }, pi);
9730
10542
  } catch (error) {
9731
10543
  pi.sendMessage({
9732
10544
  customType: "autorag.error",
9733
10545
  content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
9734
10546
  display: true
9735
- });
9736
- }
9737
- return { action: "handled" };
10547
+ }, { triggerTurn: false });
10548
+ return { action: "handled" };
10549
+ }
10550
+ return disposition.kind === "steer" ? {
10551
+ action: "transform",
10552
+ text: disposition.text,
10553
+ ...event.images === undefined ? {} : { images: event.images }
10554
+ } : { action: "handled" };
9738
10555
  });
9739
10556
  };
9740
10557
  }
@@ -9814,7 +10631,7 @@ async function createAutoRAGPiInteractiveRuntime(options) {
9814
10631
  settingsManager,
9815
10632
  resourceLoaderOptions: {
9816
10633
  extensionFactories: [
9817
- createAutoRAGInteractiveExtension(options.getSystemPrompt, options.contextTransform, options.onQuery, options.updateNotice),
10634
+ createAutoRAGInteractiveExtension(options.getSystemPrompt, options.contextTransform, { onInput: options.onInput, isSearchRunning: options.isSearchRunning, onKey: options.onKey }, options.updateNotice),
9818
10635
  ...options.extensionFactories ?? []
9819
10636
  ],
9820
10637
  systemPrompt: options.getSystemPrompt(),
@@ -9849,6 +10666,8 @@ async function createAutoRAGPiInteractiveRuntime(options) {
9849
10666
  var PI_BUILTIN_TOOL_NAMES, EMBEDDED_AUTH_MARKER = "<autorag-embedded-model>";
9850
10667
  var init_pi_session = __esm(() => {
9851
10668
  init_provider();
10669
+ init_context_token_guard();
10670
+ init_request_limits();
9852
10671
  PI_BUILTIN_TOOL_NAMES = ["read", "bash", "edit", "write", "grep", "find", "ls"];
9853
10672
  });
9854
10673
 
@@ -9858,12 +10677,26 @@ import { Type as Type6 } from "typebox";
9858
10677
  function createJevJudge(options = {}) {
9859
10678
  let client;
9860
10679
  return async (state, questions, callOptions = {}) => {
9861
- client ??= new Jev({
9862
- ...options.backend !== undefined ? { backend: options.backend } : {},
9863
- ...options.env !== undefined ? { env: options.env } : {},
9864
- ...options.model !== undefined ? { model: options.model } : {},
9865
- ...options.confidenceThreshold !== undefined ? { confidenceThreshold: options.confidenceThreshold } : {}
9866
- });
10680
+ if (client === undefined) {
10681
+ const env = options.env ?? process.env;
10682
+ const explicit = options.backend;
10683
+ let backend;
10684
+ if (typeof explicit === "string") {
10685
+ backend = explicit === "autorag" ? new AutoragJevBackend({ env }) : explicit;
10686
+ } else if (explicit !== undefined) {
10687
+ backend = explicit;
10688
+ } else if (env.JEV_BACKEND?.toLowerCase() === "autorag") {
10689
+ backend = new AutoragJevBackend({ env });
10690
+ } else {
10691
+ backend = undefined;
10692
+ }
10693
+ client = new Jev({
10694
+ ...backend !== undefined ? { backend } : {},
10695
+ ...options.env !== undefined ? { env: options.env } : {},
10696
+ ...options.model !== undefined ? { model: options.model } : {},
10697
+ ...options.confidenceThreshold !== undefined ? { confidenceThreshold: options.confidenceThreshold } : {}
10698
+ });
10699
+ }
9867
10700
  const model = options.model ?? BACKEND_DEFAULT_MODEL[client.backend.name];
9868
10701
  return client.judge(state, questions, {
9869
10702
  ...model !== undefined ? { model } : {},
@@ -9905,8 +10738,12 @@ function createJevExtension(judgeOrOptions = {}) {
9905
10738
  }
9906
10739
  var JEV_TOOL_NAME = "jev", OPENROUTER_DEFAULT_MODEL = "typesafe/jev-1.13", BACKEND_DEFAULT_MODEL, jevQuestionSchema, jevParameters;
9907
10740
  var init_jev_extension = __esm(() => {
10741
+ init_jev_backend();
9908
10742
  init_pi_session();
9909
- BACKEND_DEFAULT_MODEL = { openrouter: OPENROUTER_DEFAULT_MODEL };
10743
+ BACKEND_DEFAULT_MODEL = {
10744
+ openrouter: OPENROUTER_DEFAULT_MODEL,
10745
+ autorag: AUTORAG_DEFAULT_JEV_MODEL
10746
+ };
9910
10747
  jevQuestionSchema = Type6.Object({
9911
10748
  id: Type6.Optional(Type6.String({ description: "Caller-assigned id, echoed back in the verdict" })),
9912
10749
  type: Type6.Union([Type6.Literal("noul"), Type6.Literal("choice"), Type6.Literal("score")], {
@@ -9948,7 +10785,7 @@ function createJikjiFindTool(provider, ledger = new EvidenceLedger) {
9948
10785
  label: "Jikji Find",
9949
10786
  description: "Discover local files via Jikji. Returns answer_paths with per-candidate next_read hints and a tool-call policy directive. Call this FIRST for local file discovery when Jikji is configured.",
9950
10787
  parameters: jikjiFindSchema,
9951
- async execute(_toolCallId, params) {
10788
+ async execute(_toolCallId, params, signal) {
9952
10789
  const query = String(params.query ?? "").trim();
9953
10790
  if (query.length === 0) {
9954
10791
  return {
@@ -9971,7 +10808,8 @@ function createJikjiFindTool(provider, ledger = new EvidenceLedger) {
9971
10808
  }
9972
10809
  const { answerPack, policy, diagnostics, perRoot } = await provider.findJikji(query, {
9973
10810
  topK: params.topK,
9974
- first: params.first
10811
+ first: params.first,
10812
+ ...signal === undefined ? {} : { signal }
9975
10813
  });
9976
10814
  if (answerPack === undefined) {
9977
10815
  const diagMsg = diagnostics.length > 0 ? diagnostics.map((d) => d.message).join(" ") : "Jikji is unavailable or all roots failed.";
@@ -12285,7 +13123,7 @@ function createSearchAllDocumentsTool(provider, ledger = new EvidenceLedger) {
12285
13123
  label: "Search All Documents",
12286
13124
  description: "Search across all configured retrieval methods (posix, MinSync, datasources) and return every distinct chunk they found, with only pure duplicates removed. This is the exhaustive retrieval surface: one call can return many candidates, including several passages of the same document, so judge the evidence yourself rather than assuming it was pre-filtered. Authority is server-configured; tool arguments can only provide query, topK, and an optional narrowing scope.",
12287
13125
  parameters: searchAllSchema,
12288
- async execute(_toolCallId, params) {
13126
+ async execute(_toolCallId, params, signal) {
12289
13127
  const query = params.query.trim();
12290
13128
  if (query.length === 0) {
12291
13129
  return {
@@ -12295,7 +13133,8 @@ function createSearchAllDocumentsTool(provider, ledger = new EvidenceLedger) {
12295
13133
  }
12296
13134
  const { results, diagnostics, perMethodCounts } = await provider.searchAllDocuments(query, {
12297
13135
  topK: params.topK,
12298
- scope: params.scope
13136
+ scope: params.scope,
13137
+ ...signal === undefined ? {} : { signal }
12299
13138
  });
12300
13139
  const evidenceIds = results.map((result) => ledger.registerResult(SEARCH_ALL_DOCUMENTS_TOOL_NAME, result));
12301
13140
  return {
@@ -12319,7 +13158,7 @@ Diagnostics: ${diagnostics.map((d) => `${d.source ?? "all"}:${d.code}`).join(",
12319
13158
  if (results.length === 0)
12320
13159
  return `No results.${diagnosticSummary}`;
12321
13160
  const rows = results.map((result, index) => {
12322
- const line = result.content.replace(/\s+/gu, " ").slice(0, 500);
13161
+ const line = result.content.replace(/\s+/gu, " ");
12323
13162
  return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
12324
13163
  ${line}`;
12325
13164
  });
@@ -12350,7 +13189,7 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
12350
13189
  label: "Search MinSync Documents",
12351
13190
  description: "Search parsed document mirrors with MinSync semantic vector retrieval. Use for conceptual and meaning-based search.",
12352
13191
  parameters: searchMinSyncSchema,
12353
- async execute(_toolCallId, params) {
13192
+ async execute(_toolCallId, params, signal) {
12354
13193
  if (params.query.trim().length === 0) {
12355
13194
  return {
12356
13195
  content: [{ type: "text", text: "MinSync query was empty; no documents searched." }],
@@ -12360,7 +13199,8 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
12360
13199
  const scope = resolveScope(params.scope);
12361
13200
  const results = await getMethod().retrieve(params.query, {
12362
13201
  topK: params.topK,
12363
- scope
13202
+ scope,
13203
+ ...signal === undefined ? {} : { signal }
12364
13204
  });
12365
13205
  const evidenceIds = results.map((result) => ledger.registerResult(SEARCH_MINSYNC_DOCUMENTS_TOOL_NAME, result));
12366
13206
  return {
@@ -12379,7 +13219,7 @@ function formatResults2(results, evidenceIds) {
12379
13219
  if (results.length === 0)
12380
13220
  return "No MinSync results.";
12381
13221
  const rows = results.map((result, index) => {
12382
- const line = result.content.replace(/\s+/gu, " ").slice(0, 500);
13222
+ const line = result.content.replace(/\s+/gu, " ");
12383
13223
  return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
12384
13224
  ${line}`;
12385
13225
  });
@@ -12427,7 +13267,7 @@ function createTool(provider, spec, ledger) {
12427
13267
  label: `Search ${spec.datasourceId}`,
12428
13268
  description: `Search only the "${spec.datasourceId}" datasource connection: ${spec.description} ` + `This tool spawns only this connection's CLIs and returns only this connection's hits.${scopeLine} ` + `Authority is server-configured; tool arguments can only provide query, topK, and an optional narrowing scope.`,
12429
13269
  parameters: searchSingleDatasourceSchema,
12430
- async execute(_toolCallId, params) {
13270
+ async execute(_toolCallId, params, signal) {
12431
13271
  const query = params.query.trim();
12432
13272
  if (query.length === 0) {
12433
13273
  return {
@@ -12443,7 +13283,8 @@ function createTool(provider, spec, ledger) {
12443
13283
  }
12444
13284
  const { results, diagnostics } = await provider.searchSingleDatasourceDocuments(spec.datasourceId, query, {
12445
13285
  topK: params.topK,
12446
- scope: params.scope
13286
+ scope: params.scope,
13287
+ ...signal === undefined ? {} : { signal }
12447
13288
  });
12448
13289
  const evidenceIds = results.map((result) => ledger.registerResult(`search_datasource_${spec.datasourceId}`, result));
12449
13290
  return {
@@ -12467,7 +13308,7 @@ Diagnostics: ${diagnostics.map((d) => `${d.source ?? "datasource"}:${d.code}`).j
12467
13308
  if (results.length === 0)
12468
13309
  return `No datasource results.${diagnosticSummary}`;
12469
13310
  const rows = results.map((result, index) => {
12470
- const line = result.content.replace(/\s+/gu, " ").slice(0, 500);
13311
+ const line = result.content.replace(/\s+/gu, " ");
12471
13312
  return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
12472
13313
  ${line}`;
12473
13314
  });
@@ -18030,7 +18871,7 @@ function mergeSources(merged, sources) {
18030
18871
  existing.source.ageSeconds ??= source.ageSeconds;
18031
18872
  }
18032
18873
  }
18033
- function sleep3(ms, signal) {
18874
+ function sleep4(ms, signal) {
18034
18875
  if (ms <= 0)
18035
18876
  return Promise.resolve();
18036
18877
  let resolveFn;
@@ -18080,9 +18921,9 @@ async function searchPublicWeb(params, deadlines = {}) {
18080
18921
  failures.push({ provider: { id, label: id }, error });
18081
18922
  }
18082
18923
  }));
18083
- await Promise.race([all, sleep3(softMs, signal)]);
18924
+ await Promise.race([all, sleep4(softMs, signal)]);
18084
18925
  if (!responses.some(isUseful) && failures.length < engineIds.length) {
18085
- await Promise.race([all, firstSuccess, sleep3(Math.max(0, hardMs - softMs), signal)]);
18926
+ await Promise.race([all, firstSuccess, sleep4(Math.max(0, hardMs - softMs), signal)]);
18086
18927
  }
18087
18928
  straggler.abort();
18088
18929
  const merged = new Map;
@@ -18498,6 +19339,20 @@ class AutoRAGAgent {
18498
19339
  lastQuery;
18499
19340
  lastSessionId;
18500
19341
  activeRun = false;
19342
+ interactive = {
19343
+ run: undefined,
19344
+ followUps: [],
19345
+ steers: [],
19346
+ queuedSteers: [],
19347
+ preSessionSteers: [],
19348
+ acceptingSteers: false,
19349
+ harnessAbort: false,
19350
+ userAborted: false,
19351
+ steerInterrupt: new AbortController,
19352
+ nextSubmitIsFollowUp: false,
19353
+ steerTexts: new Set
19354
+ };
19355
+ activeRunAbort;
18501
19356
  modelNativeSearchAuth;
18502
19357
  retrievalTrace = [];
18503
19358
  evidenceLedger = new EvidenceLedger;
@@ -18974,9 +19829,9 @@ ${context.text}
18974
19829
  apiKey: resolved.apiKey,
18975
19830
  providerApiKeys: resolved.providerApiKeys,
18976
19831
  getSystemPrompt: () => systemPrompt,
18977
- customTools: this.tools.filter((tool) => !PI_BUILTIN_TOOL_NAMES.includes(tool.name)),
19832
+ customTools: this.steerableTools(),
18978
19833
  remoteSession: this.remoteSession,
18979
- contextTransform: (messages) => this.withMemoryContext(messages),
19834
+ contextTransform: (messages) => this.withMemoryContext(frameSteers(messages, this.interactive.steerTexts)),
18980
19835
  ...this.jevExtension !== undefined ? { extensionFactories: [this.jevExtension], extensionToolNames: [JEV_TOOL_NAME] } : {}
18981
19836
  });
18982
19837
  const agent = piSession.session.agent;
@@ -18992,6 +19847,8 @@ ${context.text}
18992
19847
  const unsubscribers = [...this.listeners].map((listener) => session.agent.subscribe(listener));
18993
19848
  unsubscribers.push(session.agent.subscribe((event) => {
18994
19849
  this.recordSearchToolEvent(event);
19850
+ if (event.type === "message_start" && event.message.role === "user")
19851
+ this.steersReachedModel();
18995
19852
  }));
18996
19853
  return unsubscribers;
18997
19854
  }
@@ -19008,6 +19865,7 @@ ${context.text}
19008
19865
  return;
19009
19866
  this.searchToolCallCount += 1;
19010
19867
  if (this.searchToolCallCount >= this.maxSearchToolCalls) {
19868
+ this.interactive.harnessAbort = true;
19011
19869
  this.activeSession?.abort();
19012
19870
  }
19013
19871
  const details = event.result.details;
@@ -19054,9 +19912,11 @@ ${context.text}
19054
19912
  getSystemPrompt: () => buildSystemPrompt(this.currentSystemPromptConfig({
19055
19913
  modelId: this.boundPiRuntime?.session.model?.id ?? resolved?.model.id
19056
19914
  })),
19057
- contextTransform: (messages) => this.withMemoryContext(messages),
19058
- customTools: this.tools.filter((tool) => !PI_BUILTIN_TOOL_NAMES.includes(tool.name)),
19059
- onQuery: (query, pi) => this.runInteractivePiQuery(query, pi),
19915
+ contextTransform: async (messages) => this.withMemoryContext(frameSteers(messages, this.interactive.steerTexts)),
19916
+ customTools: this.steerableTools(),
19917
+ onInput: (input, pi) => this.routeInteractiveInput(input, pi),
19918
+ isSearchRunning: () => this.interactive.run !== undefined,
19919
+ onKey: (key) => this.handleInteractiveKey(key),
19060
19920
  ...this.jevExtension !== undefined ? { extensionFactories: [this.jevExtension], extensionToolNames: [JEV_TOOL_NAME] } : {},
19061
19921
  ...this.updateNotice === undefined ? {} : { updateNotice: this.updateNotice }
19062
19922
  });
@@ -19071,18 +19931,154 @@ ${context.text}
19071
19931
  }
19072
19932
  };
19073
19933
  }
19934
+ steerableTools() {
19935
+ return this.tools.filter((tool) => !PI_BUILTIN_TOOL_NAMES.includes(tool.name)).map((tool) => this.searchToolNames.has(tool.name) || tool.name === WEB_SEARCH_TOOL_NAME || tool.name === WEB_FETCH_TOOL_NAME ? this.interruptibleBySteer(tool) : tool);
19936
+ }
19937
+ interruptibleBySteer(tool) {
19938
+ return {
19939
+ ...tool,
19940
+ execute: async (toolCallId, params, signal, onUpdate) => {
19941
+ const steer = this.interactive.steerInterrupt.signal;
19942
+ if (steer.aborted)
19943
+ return skippedBySteer(tool.name);
19944
+ const combined = signal === undefined ? steer : AbortSignal.any([signal, steer]);
19945
+ let onSteer = () => {};
19946
+ const steered = new Promise((resolve) => {
19947
+ onSteer = () => resolve("steered");
19948
+ steer.addEventListener("abort", onSteer, { once: true });
19949
+ });
19950
+ try {
19951
+ const outcome = await Promise.race([tool.execute(toolCallId, params, combined, onUpdate), steered]);
19952
+ return outcome === "steered" ? skippedBySteer(tool.name) : outcome;
19953
+ } catch (error) {
19954
+ if (steer.aborted && signal?.aborted !== true)
19955
+ return skippedBySteer(tool.name);
19956
+ throw error;
19957
+ } finally {
19958
+ steer.removeEventListener("abort", onSteer);
19959
+ }
19960
+ }
19961
+ };
19962
+ }
19074
19963
  deriveAnswer(text) {
19075
19964
  const derived = deriveResultsFromAnswer(text, this.evidenceLedger, { allowLocalFiles: !this.remoteSession });
19076
19965
  this.routingDiagnostics.push(...derived.diagnostics);
19077
19966
  return derived.details;
19078
19967
  }
19968
+ routeInteractiveInput(input, pi) {
19969
+ const state = this.interactive;
19970
+ const followUp = input.streamingBehavior === "followUp" || state.nextSubmitIsFollowUp;
19971
+ state.nextSubmitIsFollowUp = false;
19972
+ if (state.run === undefined) {
19973
+ this.startInteractiveQuery(input.text, pi);
19974
+ return { kind: "handled" };
19975
+ }
19976
+ if (followUp) {
19977
+ state.followUps.push(input.text);
19978
+ return { kind: "handled" };
19979
+ }
19980
+ if (input.streamingBehavior === "steer") {
19981
+ state.steers.push(input.text);
19982
+ state.steerTexts.add(input.text);
19983
+ state.steerInterrupt.abort();
19984
+ return { kind: "steer", text: input.text };
19985
+ }
19986
+ if (!this.steer(input.text, input.images))
19987
+ state.followUps.push(input.text);
19988
+ return { kind: "handled" };
19989
+ }
19990
+ handleInteractiveKey(key) {
19991
+ const state = this.interactive;
19992
+ if (state.run === undefined || this.boundPiRuntime?.session.isStreaming === true)
19993
+ return false;
19994
+ if (key === "followUp") {
19995
+ state.nextSubmitIsFollowUp = true;
19996
+ return false;
19997
+ }
19998
+ state.userAborted = true;
19999
+ state.steerInterrupt.abort();
20000
+ return true;
20001
+ }
20002
+ startInteractiveQuery(query, pi) {
20003
+ const state = this.interactive;
20004
+ const run = (async () => {
20005
+ let next = query;
20006
+ while (next !== undefined) {
20007
+ try {
20008
+ await this.runInteractivePiQuery(next, pi);
20009
+ } catch (error) {
20010
+ pi.sendMessage({
20011
+ customType: "autorag.error",
20012
+ content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
20013
+ display: true
20014
+ }, { triggerTurn: false });
20015
+ }
20016
+ next = state.followUps.shift();
20017
+ }
20018
+ })().finally(() => {
20019
+ state.run = undefined;
20020
+ });
20021
+ state.run = run;
20022
+ }
19079
20023
  async runInteractivePiQuery(query, pi) {
19080
20024
  pi.setSessionName(query.slice(0, 80));
19081
- for await (const _event of this.searchDocumentsStream(query)) {}
20025
+ const state = this.interactive;
20026
+ const session = this.boundPiRuntime?.session;
20027
+ const unsubscribe = session?.subscribe((event) => {
20028
+ if (event.type === "queue_update" && state.queuedSteers.length > 0 && !session.agent.hasQueuedMessages()) {
20029
+ state.queuedSteers.length = 0;
20030
+ }
20031
+ if (event.type === "agent_settled" && event.aborted && !state.harnessAbort)
20032
+ state.userAborted = true;
20033
+ });
20034
+ try {
20035
+ for await (const _event of this.searchDocumentsStream(query)) {}
20036
+ } finally {
20037
+ unsubscribe?.();
20038
+ if (state.queuedSteers.length > 0) {
20039
+ session?.agent.clearSteeringQueue();
20040
+ if (!state.userAborted)
20041
+ state.followUps.unshift(...state.queuedSteers);
20042
+ state.queuedSteers.length = 0;
20043
+ }
20044
+ }
20045
+ }
20046
+ async waitForIdle() {
20047
+ while (this.interactive.run !== undefined)
20048
+ await this.interactive.run;
20049
+ }
20050
+ steer(text, images = []) {
20051
+ const state = this.interactive;
20052
+ if (!this.activeRun || !state.acceptingSteers || state.userAborted || text.trim() === "")
20053
+ return false;
20054
+ state.steers.push(text);
20055
+ state.steerTexts.add(text);
20056
+ state.steerInterrupt.abort();
20057
+ const content = [{ type: "text", text }, ...images];
20058
+ const message = { role: "user", content, timestamp: Date.now() };
20059
+ const agent = this.activeSession?.agent;
20060
+ if (agent === undefined)
20061
+ state.preSessionSteers.push(message);
20062
+ else
20063
+ agent.steer(message);
20064
+ state.queuedSteers.push(text);
20065
+ return true;
19082
20066
  }
19083
20067
  abort() {
20068
+ if (!this.activeRun)
20069
+ return;
20070
+ const state = this.interactive;
20071
+ state.userAborted = true;
20072
+ state.steerInterrupt.abort();
20073
+ this.activeRunAbort?.abort();
19084
20074
  this.activeSession?.abort();
19085
20075
  }
20076
+ steersReachedModel() {
20077
+ const state = this.interactive;
20078
+ state.queuedSteers.length = 0;
20079
+ if (state.steerInterrupt.signal.aborted && !state.userAborted)
20080
+ state.steerInterrupt = new AbortController;
20081
+ }
19086
20082
  startAutoRefresh(intervalMs, options = {}) {
19087
20083
  this.stopAutoRefresh();
19088
20084
  const tick = () => {
@@ -19142,6 +20138,15 @@ ${context.text}
19142
20138
  const publishOnCapture = this.jevJudge === undefined;
19143
20139
  let published = false;
19144
20140
  const planAbort = new AbortController;
20141
+ this.activeRunAbort = planAbort;
20142
+ const steering = this.interactive;
20143
+ steering.steers.length = 0;
20144
+ steering.queuedSteers.length = 0;
20145
+ steering.preSessionSteers.length = 0;
20146
+ steering.harnessAbort = false;
20147
+ steering.userAborted = false;
20148
+ steering.steerInterrupt = new AbortController;
20149
+ steering.acceptingSteers = true;
19145
20150
  const publishPreliminary = (details) => {
19146
20151
  if (planAbort.signal.aborted || published)
19147
20152
  return;
@@ -19177,6 +20182,9 @@ ${context.text}
19177
20182
  session = await this.createSearchSession(resolved, buildSystemPrompt(this.currentSystemPromptConfig({ modelId: resolved.model.id })));
19178
20183
  this.activeSession = session;
19179
20184
  unsubscribers = this.configureSearchSession(session);
20185
+ for (const message of steering.preSessionSteers)
20186
+ session.agent.steer(message);
20187
+ steering.preSessionSteers.length = 0;
19180
20188
  let timeout;
19181
20189
  let timedOutAfterFastAnswer = false;
19182
20190
  try {
@@ -19237,25 +20245,42 @@ Warning: the edited config did not validate, so the previous config was restored
19237
20245
  }
19238
20246
  return;
19239
20247
  }
19240
- if (plan.route === "direct") {
20248
+ const steered = () => this.interactive.steers.length > 0;
20249
+ const stopped = () => this.interactive.userAborted;
20250
+ if (stopped())
20251
+ return;
20252
+ let directText;
20253
+ if (plan.route === "direct" && !steered()) {
19241
20254
  setThinking(this.fastThinkingLevel);
19242
20255
  await activeSession.prompt(this.buildDirectAnswerPrompt(trimmedQuery));
19243
- const answer = finalAnswerText(transcript());
19244
- if (answer !== undefined)
19245
- captured = this.deriveAnswer(answer);
20256
+ if (stopped())
20257
+ return;
20258
+ directText = finalAnswerText(transcript());
20259
+ if (!steered()) {
20260
+ if (directText !== undefined)
20261
+ captured = this.deriveAnswer(directText);
20262
+ return;
20263
+ }
20264
+ }
20265
+ const baseline = plan.route === "direct" ? "" : await this.prefetchInitialRetrievalContext(trimmedQuery, plan.queries, options, plan.datasources);
20266
+ if (stopped())
19246
20267
  return;
20268
+ let fastText = directText;
20269
+ let preliminary;
20270
+ if (plan.route !== "direct") {
20271
+ setThinking(this.fastThinkingLevel);
20272
+ await activeSession.prompt(this.buildFastAnswerPrompt(trimmedQuery, options, baseline));
20273
+ if (stopped())
20274
+ return;
20275
+ fastText = finalAnswerText(transcript());
20276
+ preliminary = fastText === undefined ? undefined : this.deriveAnswer(fastText);
19247
20277
  }
19248
- const baseline = await this.prefetchInitialRetrievalContext(trimmedQuery, plan.queries, options, plan.datasources);
19249
- setThinking(this.fastThinkingLevel);
19250
- await activeSession.prompt(this.buildFastAnswerPrompt(trimmedQuery, options, baseline));
19251
- const fastText = finalAnswerText(transcript());
19252
- const preliminary = fastText === undefined ? undefined : this.deriveAnswer(fastText);
19253
20278
  if (preliminary !== undefined) {
19254
20279
  emitPreliminary(preliminary);
19255
- const followUp = await this.shouldFollowUp(trimmedQuery, preliminary);
19256
- if (planAbort.signal.aborted)
20280
+ const followUp = steered() || await this.shouldFollowUp(trimmedQuery, preliminary);
20281
+ if (planAbort.signal.aborted || stopped())
19257
20282
  return;
19258
- if (!followUp) {
20283
+ if (!followUp && !steered()) {
19259
20284
  captured = preliminary;
19260
20285
  return;
19261
20286
  }
@@ -19263,14 +20288,23 @@ Warning: the edited config did not validate, so the previous config was restored
19263
20288
  }
19264
20289
  setThinking(this.finalThinkingLevel);
19265
20290
  const fastAnswerDelivered = preliminary !== undefined && this.preliminaryCallback !== undefined;
19266
- await activeSession.prompt(this.buildRefinementPrompt(trimmedQuery, options, fastText, fastAnswerDelivered));
20291
+ await activeSession.prompt(this.buildRefinementPrompt(trimmedQuery, options, fastText, fastAnswerDelivered, this.interactive.steers));
20292
+ while (!stopped() && !planAbort.signal.aborted && this.interactive.queuedSteers.length > 0) {
20293
+ await activeSession.prompt(buildSteerContinuationPrompt(this.interactive.steers));
20294
+ }
20295
+ steering.acceptingSteers = false;
20296
+ if (stopped())
20297
+ return;
19267
20298
  const finalText = finalAnswerText(transcript());
19268
20299
  if (finalText !== undefined)
19269
20300
  captured = this.deriveAnswer(finalText);
19270
- })(),
20301
+ })().finally(() => {
20302
+ steering.acceptingSteers = false;
20303
+ }),
19271
20304
  new Promise((resolve, reject) => {
19272
20305
  timeout = setTimeout(() => {
19273
20306
  planAbort.abort();
20307
+ this.interactive.harnessAbort = true;
19274
20308
  Promise.resolve(session?.abort());
19275
20309
  if (fastCaptured !== undefined && !this.remoteSession) {
19276
20310
  timedOutAfterFastAnswer = true;
@@ -19717,12 +20751,13 @@ Warning: the edited config did not validate, so the previous config was restored
19717
20751
  return selection.selected;
19718
20752
  }
19719
20753
  async prefetchInitialRetrievalContext(query, searchQueries, options, datasources = []) {
19720
- const retrieveOptions = { topK: this.limits.prefetch.minSyncTopK, scope: options.scope };
20754
+ const signal = this.interactive.steerInterrupt.signal;
20755
+ const retrieveOptions = { topK: this.limits.prefetch.minSyncTopK, scope: options.scope, signal };
19721
20756
  const formatSearchQueries = (list) => `Search queries (decomposed from the original question):
19722
20757
  ${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
19723
20758
  `)}`;
19724
20759
  const perQuery = await Promise.all(searchQueries.map((searchQuery) => Promise.all([
19725
- this.jikjiClient === undefined ? Promise.resolve(undefined) : this.findJikji(searchQuery, { topK: this.limits.prefetch.jikjiTopK }).catch(() => {
20760
+ this.jikjiClient === undefined ? Promise.resolve(undefined) : this.findJikji(searchQuery, { topK: this.limits.prefetch.jikjiTopK, signal }).catch(() => {
19726
20761
  return;
19727
20762
  }),
19728
20763
  this.minSyncMethod.retrieve(searchQuery, retrieveOptions).catch((error) => {
@@ -19732,7 +20767,8 @@ ${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
19732
20767
  }),
19733
20768
  Promise.all(datasources.map((datasourceId) => this.searchSingleDatasourceDocuments(datasourceId, searchQuery, {
19734
20769
  topK: this.limits.singleDatasourceTopK,
19735
- scope: options.scope
20770
+ scope: options.scope,
20771
+ signal
19736
20772
  }).then((outcome) => outcome.results).catch(() => [])))
19737
20773
  ])));
19738
20774
  for (const [index, [jikji, vector, perDatasource]] of perQuery.entries()) {
@@ -19782,7 +20818,7 @@ ${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
19782
20818
  options.observedSources?.add(result.source);
19783
20819
  for (const result of datasourceResults)
19784
20820
  options.observedSources?.add(result.source);
19785
- const reranked = await this.rerankPrefetchPool(query, jikjiPaths, minSyncResults, datasourceResults);
20821
+ const reranked = await this.rerankPrefetchPool(query, jikjiPaths, minSyncResults, datasourceResults, signal);
19786
20822
  if (reranked !== undefined) {
19787
20823
  const baseline = formatRerankedBaseline(reranked, reranked.map((result) => this.evidenceLedger.registerResult("baseline", result)));
19788
20824
  return searchQueries.length > 1 ? `${formatSearchQueries(searchQueries)}
@@ -19814,7 +20850,7 @@ ${result.content.replace(/\s+/gu, " ")}`).join(`
19814
20850
 
19815
20851
  `);
19816
20852
  }
19817
- async rerankPrefetchPool(query, answerPaths, minSyncResults, datasourceResults) {
20853
+ async rerankPrefetchPool(query, answerPaths, minSyncResults, datasourceResults, signal) {
19818
20854
  const reranker = this.reranker;
19819
20855
  if (reranker === undefined)
19820
20856
  return;
@@ -19832,7 +20868,7 @@ ${result.content.replace(/\s+/gu, " ")}`).join(`
19832
20868
  if (!reranker.describe().available)
19833
20869
  return;
19834
20870
  try {
19835
- const reranked = await reranker.rerank(query, candidates, { topN: this.rerankTopN });
20871
+ const reranked = await reranker.rerank(query, candidates, { topN: this.rerankTopN, signal });
19836
20872
  return reranked.length > 0 ? reranked : undefined;
19837
20873
  } catch {
19838
20874
  return;
@@ -19875,7 +20911,7 @@ ${baseline}
19875
20911
  const fetch2 = names.has(WEB_FETCH_TOOL_NAME) ? ` (and ${WEB_FETCH_TOOL_NAME} to read a promising page)` : "";
19876
20912
  return `When local evidence is missing, thin, or possibly outdated — or the question concerns current events, recent releases, prices, or other public facts — use ${WEB_SEARCH_TOOL_NAME}${fetch2} to confirm or correct claims and fill gaps; use URLs as result sources for web-derived evidence. `;
19877
20913
  }
19878
- buildRefinementPrompt(query, options, fastAnswer, fastAnswerDelivered) {
20914
+ buildRefinementPrompt(query, options, fastAnswer, fastAnswerDelivered, steers = []) {
19879
20915
  const limit = typeof options.topK === "number" ? ` Cite at most ${options.topK} sources.` : "";
19880
20916
  const scope = options.scope ? ` Restrict search to virtual path scope ${options.scope}.` : "";
19881
20917
  const firstAnswer = formatFirstAnswerContext(fastAnswer, fastAnswerDelivered);
@@ -19898,7 +20934,7 @@ ${baseline}
19898
20934
 
19899
20935
  ` + `${firstAnswer}
19900
20936
 
19901
- ` + `Now verify it rigorously. ${this.discoveryHint((tools) => `Actively use ${tools} when discovering or exploring local files and folders. `)}Check important claims against source files with bash when needed, correct anything wrong or unsupported, fill gaps with retrieval tools, and resolve conflicts and freshness. ${this.webFallbackHint()}
20937
+ ` + formatSteerContext(steers) + `Now verify it rigorously. ${this.discoveryHint((tools) => `Actively use ${tools} when discovering or exploring local files and folders. `)}Check important claims against source files with bash when needed, correct anything wrong or unsupported, fill gaps with retrieval tools, and resolve conflicts and freshness. ${this.webFallbackHint()}
19902
20938
 
19903
20939
  ` + `${answerRules}
19904
20940
 
@@ -20301,7 +21337,8 @@ ${initialRetrievalContext ?? "Pending; continue only with a brief progress state
20301
21337
  const sourceRoots = planJikjiSourceRoots(this.searchPaths);
20302
21338
  const findOpts = {
20303
21339
  topK: opts?.topK,
20304
- first: opts?.first
21340
+ first: opts?.first,
21341
+ ...opts?.signal === undefined ? {} : { signal: opts.signal }
20305
21342
  };
20306
21343
  const diagnostics = [];
20307
21344
  const okPacks = [];
@@ -20452,13 +21489,18 @@ ${initialRetrievalContext ?? "Pending; continue only with a brief progress state
20452
21489
  return { results, diagnostics };
20453
21490
  }
20454
21491
  async searchAllDocuments(query, options = {}) {
20455
- return this.retrieveWithDiagnostics(query, { topK: options.topK, scope: options.scope });
21492
+ return this.retrieveWithDiagnostics(query, {
21493
+ topK: options.topK,
21494
+ scope: options.scope,
21495
+ ...options.signal === undefined ? {} : { signal: options.signal }
21496
+ });
20456
21497
  }
20457
21498
  async searchSingleDatasourceDocuments(datasourceId, query, options = {}) {
20458
21499
  const retrievalOptions = {
20459
21500
  ...this.activeRetrievalOptions,
20460
21501
  topK: options.topK,
20461
- scope: options.scope
21502
+ scope: options.scope,
21503
+ ...options.signal === undefined ? {} : { signal: options.signal }
20462
21504
  };
20463
21505
  const methods = this.methodRegistry.list().filter((method) => {
20464
21506
  const descriptor = method.describe();
@@ -20561,6 +21603,49 @@ ${result.content.replace(/\s+/gu, " ")}`);
20561
21603
  ${lines.join(`
20562
21604
  `)}`;
20563
21605
  }
21606
+ function skippedBySteer(toolName) {
21607
+ return {
21608
+ content: [
21609
+ {
21610
+ type: "text",
21611
+ text: `Skipped due to the user's steering message. ${toolName} was stopped before it returned; do not count it as completed work. Apply the steering message, then retry the search if it is still needed.`
21612
+ }
21613
+ ],
21614
+ details: { method: toolName, skipped: "steer" }
21615
+ };
21616
+ }
21617
+ function buildSteerMessage(text) {
21618
+ return `The user interrupted the running search with this correction:
21619
+
21620
+ ${text}
21621
+
21622
+ ` + `Treat it as direction for the search in progress, not as a new question: adjust which sources, queries, or claims you pursue, then continue. ` + `When finished, reply with the final answer to the original question, updated for this correction, and no tool call.`;
21623
+ }
21624
+ function frameSteers(messages, steerTexts) {
21625
+ if (steerTexts.size === 0)
21626
+ return messages;
21627
+ return messages.map((message) => {
21628
+ if (message.role !== "user" || typeof message.content === "string")
21629
+ return message;
21630
+ const [first, ...rest] = message.content;
21631
+ if (first?.type !== "text" || !steerTexts.has(first.text))
21632
+ return message;
21633
+ return { ...message, content: [{ ...first, text: buildSteerMessage(first.text) }, ...rest] };
21634
+ });
21635
+ }
21636
+ function formatSteerContext(steers) {
21637
+ if (steers.length === 0)
21638
+ return "";
21639
+ return `While the search ran, the user corrected it:
21640
+ ${steers.map((steer) => `- ${steer}`).join(`
21641
+ `)}
21642
+ ` + `The first answer predates these corrections; follow them over it.
21643
+
21644
+ `;
21645
+ }
21646
+ function buildSteerContinuationPrompt(steers) {
21647
+ return `${formatSteerContext(steers)}Apply the latest correction: search or verify further if it needs that, ` + `then reply with the complete final answer to the original question and no tool call.`;
21648
+ }
20564
21649
  function formatFirstAnswerContext(fastAnswer, delivered) {
20565
21650
  if (fastAnswer === undefined) {
20566
21651
  return "(the fast phase produced no answer — write the complete answer)";
@@ -21023,23 +22108,175 @@ ${this.description}`
21023
22108
  }
21024
22109
  }
21025
22110
 
21026
- // src/datasource/skills/clawgallery/client.ts
21027
- import { spawn as spawn9 } from "node:child_process";
21028
- import { existsSync as existsSync23 } from "node:fs";
21029
-
21030
- class ClawGalleryClient {
21031
- options;
21032
- constructor(options = {}) {
21033
- this.options = options;
21034
- }
21035
- async bootstrap(signal) {
21036
- if (this.options.path !== undefined && !existsSync23(this.options.path)) {
21037
- return {
21038
- ok: false,
21039
- reason: "nonzero-exit",
21040
- stdout: "",
21041
- stderr: "configured ClawGallery folder does not exist",
21042
- code: null
22111
+ // src/datasource/connector.ts
22112
+ import { createHash as createHash12 } from "node:crypto";
22113
+ function connectorFailureToDiagnosticCode(reason) {
22114
+ switch (reason) {
22115
+ case "auth":
22116
+ return "datasource-auth-error";
22117
+ case "permission":
22118
+ return "datasource-permission-denied";
22119
+ case "rate-limited":
22120
+ return "datasource-rate-limited";
22121
+ case "unavailable":
22122
+ case "not-configured":
22123
+ return "datasource-unavailable";
22124
+ case "empty":
22125
+ return "datasource-empty";
22126
+ case "api-error":
22127
+ case "invalid-data":
22128
+ return "datasource-index-failed";
22129
+ }
22130
+ }
22131
+ function boundDiagnosticText(value) {
22132
+ const trimmed = value.trim();
22133
+ return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
22134
+ }
22135
+ function sanitizeIdSegment(value) {
22136
+ const normalized = value.replace(/[^\p{L}\p{N}._-]+/gu, "-");
22137
+ let start = 0;
22138
+ while (start < normalized.length && (normalized[start] === "-" || normalized[start] === "."))
22139
+ start += 1;
22140
+ let end = normalized.length;
22141
+ while (end > start && (normalized[end - 1] === "-" || normalized[end - 1] === "."))
22142
+ end -= 1;
22143
+ const cleaned = normalized.slice(start, end);
22144
+ if (cleaned === value && cleaned.length > 0 && cleaned.length <= 80)
22145
+ return cleaned;
22146
+ const hash = createHash12("sha256").update(value).digest("hex").slice(0, 8);
22147
+ const stem = cleaned.slice(0, 60);
22148
+ return stem.length > 0 ? `${stem}-${hash}` : hash;
22149
+ }
22150
+ var init_connector = () => {};
22151
+
22152
+ // src/datasource/sidecar/config.ts
22153
+ import { readFileSync as readFileSync19 } from "node:fs";
22154
+ import { homedir as homedir3 } from "node:os";
22155
+ import { dirname as dirname17, isAbsolute as isAbsolute8, join as join30, resolve as resolve14, sep as sep7 } from "node:path";
22156
+ import { parse as parseToml2 } from "smol-toml";
22157
+ function resolveSidecarOptions(input) {
22158
+ const { connector, workspaceRoot } = input;
22159
+ const setting = connector?.sidecar;
22160
+ if (setting === false || workspaceRoot === undefined)
22161
+ return;
22162
+ const object = typeof setting === "object" && setting !== null && !Array.isArray(setting) ? setting : {};
22163
+ if (object.enabled === false)
22164
+ return;
22165
+ const configured = [object.crawlerDbPath, connector?.databasePath].find((value) => typeof value === "string" && value.length > 0);
22166
+ const crawlerDbPath = expandHome(configured ?? archivePathFromCrawlerConfig(input) ?? input.defaultCrawlerDb);
22167
+ const workspaceId = typeof connector?.workspace === "string" && connector.workspace.length > 0 ? connector.workspace : undefined;
22168
+ return {
22169
+ descriptor: input.descriptor,
22170
+ crawlerDbPath,
22171
+ sidecarDbPath: join30(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(input.registrationName), sanitizeIdSegment(input.instanceId), "sidecar.sqlite"),
22172
+ ...workspaceId === undefined ? {} : { workspaceId },
22173
+ ...object.semantic === false ? {} : { embedder: createGatewayEmbedder() }
22174
+ };
22175
+ }
22176
+ function archivePathFromCrawlerConfig(input) {
22177
+ const configPath = input.connector?.configPath;
22178
+ if (input.descriptor.archivePathInConfig !== true || typeof configPath !== "string" || configPath.length === 0) {
22179
+ return;
22180
+ }
22181
+ const file = expandHome(configPath);
22182
+ try {
22183
+ const dbPath = parseToml2(readFileSync19(file, "utf8")).db_path;
22184
+ if (typeof dbPath !== "string" || dbPath.length === 0)
22185
+ return;
22186
+ const expanded = expandHome(dbPath);
22187
+ return isAbsolute8(expanded) ? expanded : resolve14(dirname17(file), expanded);
22188
+ } catch {
22189
+ return;
22190
+ }
22191
+ }
22192
+ function expandHome(path) {
22193
+ if (path === "~")
22194
+ return homedir3();
22195
+ const prefixed = path.startsWith("~/") || sep7 === "\\" && path.startsWith("~\\");
22196
+ return prefixed ? join30(homedir3(), path.slice(2)) : path;
22197
+ }
22198
+ var init_config2 = __esm(() => {
22199
+ init_connector();
22200
+ });
22201
+
22202
+ // src/datasource/sidecar/descriptors.ts
22203
+ function chatCrawler(id) {
22204
+ return {
22205
+ id,
22206
+ keyExpression: "m.event_id",
22207
+ chunkId: (key) => key,
22208
+ fromClause: "messages m",
22209
+ liveWhere: "(m.deleted_at is null or m.deleted_at = 0)",
22210
+ textExpression: "coalesce(m.text, '')",
22211
+ titleExpression: "m.chat_name",
22212
+ epochSecondsExpression: "m.ts",
22213
+ fts: { table: "messages_fts", joinOn: "messages_fts.rowid = m.rowid" },
22214
+ metadata: [
22215
+ { sql: "m.chat_jid", key: "chatId" },
22216
+ { sql: "m.chat_name", key: "chatName" },
22217
+ { sql: "m.sender_name", key: "senderName" },
22218
+ { sql: "m.ts", key: "timestamp" }
22219
+ ],
22220
+ channelIdExpression: "m.chat_jid",
22221
+ channelNameExpression: "m.chat_name",
22222
+ hierarchy: (metadata) => ["chats", str(metadata.chatName) ?? str(metadata.chatId) ?? "unknown"]
22223
+ };
22224
+ }
22225
+ var str = (value) => typeof value === "string" && value.length > 0 ? value : undefined, slack, CRAWLER_DESCRIPTORS;
22226
+ var init_descriptors = __esm(() => {
22227
+ slack = {
22228
+ id: "slack",
22229
+ archivePathInConfig: true,
22230
+ keyExpression: "m.channel_id || '|' || m.ts",
22231
+ chunkId: (_key, metadata) => `${metadata.channelId ?? ""}-${(metadata.timestamp ?? "").replaceAll(".", "-")}`,
22232
+ fromClause: "messages m left join channels c on c.id = m.channel_id and c.workspace_id = m.workspace_id " + "left join users u on u.id = m.user_id",
22233
+ liveWhere: "(m.deleted_ts is null or m.deleted_ts = '')",
22234
+ scopeExpression: "m.workspace_id",
22235
+ textExpression: "coalesce(nullif(m.normalized_text, ''), m.text)",
22236
+ titleExpression: "c.name",
22237
+ epochSecondsExpression: "cast(m.ts as real)",
22238
+ fts: { table: "message_fts", joinOn: "message_fts.message_key = m.channel_id || '|' || m.ts" },
22239
+ metadata: [
22240
+ { sql: "m.channel_id", key: "channelId" },
22241
+ { sql: "c.name", key: "channelName" },
22242
+ { sql: "m.workspace_id", key: "workspaceId" },
22243
+ { sql: "m.user_id", key: "userId" },
22244
+ { sql: "coalesce(nullif(u.real_name, ''), u.name)", key: "senderName" },
22245
+ { sql: "m.ts", key: "timestamp" }
22246
+ ],
22247
+ channelIdExpression: "m.channel_id",
22248
+ channelNameExpression: "c.name",
22249
+ hierarchy: (metadata) => [
22250
+ "workspaces",
22251
+ str(metadata.workspaceId) ?? "unknown",
22252
+ "channels",
22253
+ str(metadata.channelName) ?? str(metadata.channelId) ?? "unknown"
22254
+ ]
22255
+ };
22256
+ CRAWLER_DESCRIPTORS = {
22257
+ slack,
22258
+ telegram: chatCrawler("telegram"),
22259
+ whatsapp: chatCrawler("whatsapp")
22260
+ };
22261
+ });
22262
+
22263
+ // src/datasource/skills/clawgallery/client.ts
22264
+ import { spawn as spawn9 } from "node:child_process";
22265
+ import { existsSync as existsSync23 } from "node:fs";
22266
+
22267
+ class ClawGalleryClient {
22268
+ options;
22269
+ constructor(options = {}) {
22270
+ this.options = options;
22271
+ }
22272
+ async bootstrap(signal) {
22273
+ if (this.options.path !== undefined && !existsSync23(this.options.path)) {
22274
+ return {
22275
+ ok: false,
22276
+ reason: "nonzero-exit",
22277
+ stdout: "",
22278
+ stderr: "configured ClawGallery folder does not exist",
22279
+ code: null
21043
22280
  };
21044
22281
  }
21045
22282
  const args = ["bootstrap"];
@@ -21138,7 +22375,7 @@ function spawnCli(binary, args, env, options, signal) {
21138
22375
  const timeout = setTimeout(() => {
21139
22376
  reason = "timeout";
21140
22377
  terminate2(child);
21141
- }, options.timeoutMs ?? DEFAULT_TIMEOUT_MS6);
22378
+ }, options.timeoutMs ?? DEFAULT_TIMEOUT_MS7);
21142
22379
  const abort = () => {
21143
22380
  reason = "aborted";
21144
22381
  terminate2(child);
@@ -21244,16 +22481,16 @@ function failure2(result, reason) {
21244
22481
  ok: false,
21245
22482
  reason: reason ?? result.reason ?? "nonzero-exit",
21246
22483
  stdout: result.stdout,
21247
- stderr: boundDiagnosticText(result.stderr),
22484
+ stderr: boundDiagnosticText2(result.stderr),
21248
22485
  code: result.code
21249
22486
  };
21250
22487
  }
21251
- function boundDiagnosticText(value) {
22488
+ function boundDiagnosticText2(value) {
21252
22489
  if (value.length === 0)
21253
22490
  return "";
21254
22491
  return value.trim().slice(0, 4000);
21255
22492
  }
21256
- var DEFAULT_BINARY2 = "clawgallery", DEFAULT_TIMEOUT_MS6 = 60000, DEFAULT_MAX_BUFFER_BYTES2, SAFE_ENV;
22493
+ var DEFAULT_BINARY2 = "clawgallery", DEFAULT_TIMEOUT_MS7 = 60000, DEFAULT_MAX_BUFFER_BYTES2, SAFE_ENV;
21257
22494
  var init_client5 = __esm(() => {
21258
22495
  init_portable_spawn();
21259
22496
  DEFAULT_MAX_BUFFER_BYTES2 = 4 * 1024 * 1024;
@@ -21547,7 +22784,7 @@ function statusToReason(status) {
21547
22784
  }
21548
22785
  async function requestText(url, options) {
21549
22786
  const fetchImpl = options.fetchImpl ?? globalThis.fetch;
21550
- const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS7;
22787
+ const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
21551
22788
  let response;
21552
22789
  try {
21553
22790
  response = await fetchImpl(url, {
@@ -21611,13 +22848,13 @@ function resolveToken(token, tokenEnv) {
21611
22848
  }
21612
22849
  return;
21613
22850
  }
21614
- var DEFAULT_TIMEOUT_MS7 = 15000;
22851
+ var DEFAULT_TIMEOUT_MS8 = 15000;
21615
22852
 
21616
22853
  // src/datasource/skills/cloud-drive/rclone-connector.ts
21617
22854
  import { spawn as spawn10 } from "node:child_process";
21618
- import { mkdirSync as mkdirSync18, readFileSync as readFileSync19, renameSync as renameSync10, rmSync as rmSync9, writeFileSync as writeFileSync17 } from "node:fs";
22855
+ import { mkdirSync as mkdirSync18, readFileSync as readFileSync20, renameSync as renameSync10, rmSync as rmSync9, writeFileSync as writeFileSync17 } from "node:fs";
21619
22856
  import { readFile as readFile4 } from "node:fs/promises";
21620
- import { dirname as dirname17, join as join30, normalize as normalize4 } from "node:path";
22857
+ import { dirname as dirname18, join as join31, normalize as normalize4 } from "node:path";
21621
22858
 
21622
22859
  class RcloneConnector {
21623
22860
  options;
@@ -21633,7 +22870,7 @@ class RcloneConnector {
21633
22870
  if (remote === undefined || remote.length === 0) {
21634
22871
  return { ok: false, reason: "not-configured", message: "rclone remote not configured" };
21635
22872
  }
21636
- const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
22873
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS9;
21637
22874
  const exportFormats = this.options.exportFormats ?? DEFAULT_EXPORT_FORMATS;
21638
22875
  let listResult;
21639
22876
  try {
@@ -21704,7 +22941,7 @@ class RcloneConnector {
21704
22941
  return;
21705
22942
  const destination = mirrorFilePath(paths.mirrorRoot, entry.path);
21706
22943
  const temp = `${destination}.autorag-tmp`;
21707
- mkdirSync18(dirname17(temp), { recursive: true });
22944
+ mkdirSync18(dirname18(temp), { recursive: true });
21708
22945
  let copied;
21709
22946
  try {
21710
22947
  copied = await this.runner(copyArgs(remote, entry.path, temp, exportFormats, this.options.concurrency, this.options.bandwidthLimit), timeoutMs);
@@ -21809,12 +23046,12 @@ function sameEntry(previous, current) {
21809
23046
  return previous?.path === current.path && previous.size === current.size && previous.modTime === current.modTime && previous.remoteId === current.remoteId && JSON.stringify(previous.hashes) === JSON.stringify(current.hashes);
21810
23047
  }
21811
23048
  function mirrorPaths(workspaceRoot, skillName, instanceId) {
21812
- const base = join30(workspaceRoot, ".autorag", "datasources", skillName, instanceId);
21813
- return { mirrorRoot: join30(base, "mirror"), manifestPath: join30(base, "manifest.json") };
23049
+ const base = join31(workspaceRoot, ".autorag", "datasources", skillName, instanceId);
23050
+ return { mirrorRoot: join31(base, "mirror"), manifestPath: join31(base, "manifest.json") };
21814
23051
  }
21815
23052
  function mirrorFilePath(root, path) {
21816
23053
  const safe = normalize4(path).replace(/^(\.\.(?:[/\\]|$))+|^[/\\]+/gu, "");
21817
- return join30(root, safe);
23054
+ return join31(root, safe);
21818
23055
  }
21819
23056
  function remoteTarget(remote, path) {
21820
23057
  return remote.endsWith(":") || remote.endsWith("/") ? `${remote}${path}` : `${remote}/${path}`;
@@ -21833,14 +23070,14 @@ function copyArgs(remote, path, destination, exportFormats, concurrency, bandwid
21833
23070
  }
21834
23071
  function loadManifest2(path) {
21835
23072
  try {
21836
- const parsed = JSON.parse(readFileSync19(path, "utf8"));
23073
+ const parsed = JSON.parse(readFileSync20(path, "utf8"));
21837
23074
  return parsed.version === 1 && Array.isArray(parsed.entries) ? parsed : undefined;
21838
23075
  } catch {
21839
23076
  return;
21840
23077
  }
21841
23078
  }
21842
23079
  function saveManifest(path, manifest) {
21843
- mkdirSync18(dirname17(path), { recursive: true });
23080
+ mkdirSync18(dirname18(path), { recursive: true });
21844
23081
  const temp = `${path}.autorag-tmp`;
21845
23082
  writeFileSync17(temp, `${JSON.stringify(manifest)}
21846
23083
  `, "utf8");
@@ -21942,7 +23179,7 @@ function runBinary(binary, args, timeoutMs, env, cwd) {
21942
23179
  });
21943
23180
  });
21944
23181
  }
21945
- var DEFAULT_BINARY3 = "rclone", DEFAULT_EXTENSIONS, DEFAULT_EXPORT_FORMATS = "txt,csv", DEFAULT_MAX_DOCUMENTS = 200, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_TIMEOUT_MS8 = 60000, MAX_CONTENT_CHARS2 = 1e5;
23182
+ var DEFAULT_BINARY3 = "rclone", DEFAULT_EXTENSIONS, DEFAULT_EXPORT_FORMATS = "txt,csv", DEFAULT_MAX_DOCUMENTS = 200, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_TIMEOUT_MS9 = 60000, MAX_CONTENT_CHARS2 = 1e5;
21946
23183
  var init_rclone_connector = __esm(() => {
21947
23184
  init_defaults();
21948
23185
  init_portable_spawn();
@@ -21950,52 +23187,11 @@ var init_rclone_connector = __esm(() => {
21950
23187
  DEFAULT_MAX_BYTES_PER_FILE = 2 * 1024 * 1024;
21951
23188
  });
21952
23189
 
21953
- // src/datasource/connector.ts
21954
- import { createHash as createHash12 } from "node:crypto";
21955
- function connectorFailureToDiagnosticCode(reason) {
21956
- switch (reason) {
21957
- case "auth":
21958
- return "datasource-auth-error";
21959
- case "permission":
21960
- return "datasource-permission-denied";
21961
- case "rate-limited":
21962
- return "datasource-rate-limited";
21963
- case "unavailable":
21964
- case "not-configured":
21965
- return "datasource-unavailable";
21966
- case "empty":
21967
- return "datasource-empty";
21968
- case "api-error":
21969
- case "invalid-data":
21970
- return "datasource-index-failed";
21971
- }
21972
- }
21973
- function boundDiagnosticText2(value) {
21974
- const trimmed = value.trim();
21975
- return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
21976
- }
21977
- function sanitizeIdSegment(value) {
21978
- const normalized = value.replace(/[^\p{L}\p{N}._-]+/gu, "-");
21979
- let start = 0;
21980
- while (start < normalized.length && (normalized[start] === "-" || normalized[start] === "."))
21981
- start += 1;
21982
- let end = normalized.length;
21983
- while (end > start && (normalized[end - 1] === "-" || normalized[end - 1] === "."))
21984
- end -= 1;
21985
- const cleaned = normalized.slice(start, end);
21986
- if (cleaned === value && cleaned.length > 0 && cleaned.length <= 80)
21987
- return cleaned;
21988
- const hash = createHash12("sha256").update(value).digest("hex").slice(0, 8);
21989
- const stem = cleaned.slice(0, 60);
21990
- return stem.length > 0 ? `${stem}-${hash}` : hash;
21991
- }
21992
- var init_connector = () => {};
21993
-
21994
23190
  // src/datasource/chunk-store.ts
21995
- import { existsSync as existsSync24, mkdirSync as mkdirSync19, readFileSync as readFileSync20, writeFileSync as writeFileSync18 } from "node:fs";
21996
- import { dirname as dirname18, join as join31 } from "node:path";
23191
+ import { existsSync as existsSync24, mkdirSync as mkdirSync19, readFileSync as readFileSync21, writeFileSync as writeFileSync18 } from "node:fs";
23192
+ import { dirname as dirname19, join as join32 } from "node:path";
21997
23193
  function datasourceChunkStorePath(workspaceRoot, skillName, instanceId) {
21998
- return join31(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId), "chunks.json");
23194
+ return join32(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId), "chunks.json");
21999
23195
  }
22000
23196
 
22001
23197
  class DatasourceChunkStore {
@@ -22034,7 +23230,7 @@ class DatasourceChunkStore {
22034
23230
  try {
22035
23231
  if (!existsSync24(this.storePath))
22036
23232
  return false;
22037
- const parsed = JSON.parse(readFileSync20(this.storePath, "utf8"));
23233
+ const parsed = JSON.parse(readFileSync21(this.storePath, "utf8"));
22038
23234
  if (parsed.version !== STORE_VERSION || !Array.isArray(parsed.chunks))
22039
23235
  return false;
22040
23236
  this.chunkList = parsed.chunks.filter((chunk) => typeof chunk?.chunkId === "string" && typeof chunk?.content === "string");
@@ -22080,7 +23276,7 @@ class DatasourceChunkStore {
22080
23276
  if (this.storePath === undefined)
22081
23277
  return;
22082
23278
  try {
22083
- mkdirSync19(dirname18(this.storePath), { recursive: true });
23279
+ mkdirSync19(dirname19(this.storePath), { recursive: true });
22084
23280
  const payload = {
22085
23281
  version: STORE_VERSION,
22086
23282
  skill: this.skillName,
@@ -22222,7 +23418,7 @@ class ConnectorDatasourceSkill {
22222
23418
  }
22223
23419
  if (!fetched.ok) {
22224
23420
  const code = connectorFailureToDiagnosticCode(fetched.reason);
22225
- const message = boundDiagnosticText2(fetched.message ?? fetched.reason);
23421
+ const message = boundDiagnosticText(fetched.message ?? fetched.reason);
22226
23422
  return this.fail(code, `${fetched.reason}: ${message}`);
22227
23423
  }
22228
23424
  const documents = this.applyDedupeWindow(fetched.documents);
@@ -22234,7 +23430,7 @@ class ConnectorDatasourceSkill {
22234
23430
  const diagnostics = (fetched.warnings ?? []).map((warning) => ({
22235
23431
  code: "datasource-index-failed",
22236
23432
  severity: "warning",
22237
- message: boundDiagnosticText2(warning),
23433
+ message: boundDiagnosticText(warning),
22238
23434
  instanceId: this.instanceId,
22239
23435
  source: this.definition.skillName
22240
23436
  }));
@@ -22351,7 +23547,7 @@ class ConnectorDatasourceSkill {
22351
23547
  return out;
22352
23548
  }
22353
23549
  fail(code, message) {
22354
- const sanitized = boundDiagnosticText2(message);
23550
+ const sanitized = boundDiagnosticText(message);
22355
23551
  this.lastError = sanitized;
22356
23552
  const diagnostic = {
22357
23553
  code,
@@ -22543,8 +23739,8 @@ var init_types4 = __esm(() => {
22543
23739
 
22544
23740
  // src/datasource/skills/discrawl/client.ts
22545
23741
  import { spawn as spawn12 } from "node:child_process";
22546
- import { mkdirSync as mkdirSync20, readFileSync as readFileSync21, writeFileSync as writeFileSync19 } from "node:fs";
22547
- import { dirname as dirname19 } from "node:path";
23742
+ import { mkdirSync as mkdirSync20, readFileSync as readFileSync22, writeFileSync as writeFileSync19 } from "node:fs";
23743
+ import { dirname as dirname20 } from "node:path";
22548
23744
 
22549
23745
  class DiscrawlClient {
22550
23746
  options;
@@ -22560,7 +23756,7 @@ class DiscrawlClient {
22560
23756
  const path = `${workspace}/.autorag/datasources/discrawl/config.toml`;
22561
23757
  let existing = "";
22562
23758
  try {
22563
- existing = readFileSync21(path, "utf8");
23759
+ existing = readFileSync22(path, "utf8");
22564
23760
  } catch {}
22565
23761
  const managed = existing.startsWith(`# AutoRAG managed discrawl embeddings v1
22566
23762
  `);
@@ -22568,7 +23764,7 @@ class DiscrawlClient {
22568
23764
  return { configured: false, rebuildRequired: false };
22569
23765
  const previous = managed ? parseManagedEmbedding(existing) : undefined;
22570
23766
  const config = managedDiscrawlConfig({ ...runtime, baseUrl: nativeGatewayBaseUrl(runtime.baseUrl) });
22571
- mkdirSync20(dirname19(path), { recursive: true });
23767
+ mkdirSync20(dirname20(path), { recursive: true });
22572
23768
  if (existing !== config)
22573
23769
  writeFileSync19(path, config, "utf8");
22574
23770
  return {
@@ -23593,8 +24789,8 @@ var init_github = __esm(() => {
23593
24789
 
23594
24790
  // src/datasource/skills/github-gist/connector.ts
23595
24791
  import { execFile as execFile2 } from "node:child_process";
23596
- import { existsSync as existsSync25, mkdirSync as mkdirSync21, readFileSync as readFileSync22, writeFileSync as writeFileSync20 } from "node:fs";
23597
- import { dirname as dirname20 } from "node:path";
24792
+ import { existsSync as existsSync25, mkdirSync as mkdirSync21, readFileSync as readFileSync23, writeFileSync as writeFileSync20 } from "node:fs";
24793
+ import { dirname as dirname21 } from "node:path";
23598
24794
  async function defaultGhTokenRunner() {
23599
24795
  return new Promise((resolve) => {
23600
24796
  execFile2("gh", ["auth", "token"], { timeout: GH_TOKEN_TIMEOUT_MS }, (error, stdout) => {
@@ -23716,7 +24912,7 @@ class GitHubGistConnector {
23716
24912
  if (statePath === undefined || !existsSync25(statePath))
23717
24913
  return { version: STATE_VERSION, gists: {} };
23718
24914
  try {
23719
- const parsed = JSON.parse(readFileSync22(statePath, "utf8"));
24915
+ const parsed = JSON.parse(readFileSync23(statePath, "utf8"));
23720
24916
  if (parsed.version !== STATE_VERSION || typeof parsed.gists !== "object" || parsed.gists === null) {
23721
24917
  return { version: STATE_VERSION, gists: {} };
23722
24918
  }
@@ -23730,7 +24926,7 @@ class GitHubGistConnector {
23730
24926
  if (statePath === undefined)
23731
24927
  return;
23732
24928
  try {
23733
- mkdirSync21(dirname20(statePath), { recursive: true });
24929
+ mkdirSync21(dirname21(statePath), { recursive: true });
23734
24930
  writeFileSync20(statePath, `${JSON.stringify(state)}
23735
24931
  `, "utf8");
23736
24932
  } catch {}
@@ -23776,8 +24972,8 @@ var init_connector3 = () => {};
23776
24972
 
23777
24973
  // src/datasource/skills/github-gist/semantic.ts
23778
24974
  import { createHash as createHash13 } from "node:crypto";
23779
- import { existsSync as existsSync26, mkdirSync as mkdirSync22, readFileSync as readFileSync23, writeFileSync as writeFileSync21 } from "node:fs";
23780
- import { dirname as dirname21 } from "node:path";
24975
+ import { existsSync as existsSync26, mkdirSync as mkdirSync22, readFileSync as readFileSync24, writeFileSync as writeFileSync21 } from "node:fs";
24976
+ import { dirname as dirname22 } from "node:path";
23781
24977
  function contentHash(chunk) {
23782
24978
  return createHash13("sha256").update(`${chunk.title ?? ""}
23783
24979
  ${chunk.content}`).digest("hex");
@@ -23870,7 +25066,7 @@ ${chunk.content}`));
23870
25066
  if (this.statePath === undefined || !existsSync26(this.statePath))
23871
25067
  return;
23872
25068
  try {
23873
- const parsed = JSON.parse(readFileSync23(this.statePath, "utf8"));
25069
+ const parsed = JSON.parse(readFileSync24(this.statePath, "utf8"));
23874
25070
  if (parsed.version !== VECTORS_VERSION || typeof parsed.entries !== "object" || parsed.entries === null)
23875
25071
  return;
23876
25072
  if (typeof parsed.identity?.dimension !== "number")
@@ -23885,7 +25081,7 @@ ${chunk.content}`));
23885
25081
  if (this.identity === undefined)
23886
25082
  return;
23887
25083
  try {
23888
- mkdirSync22(dirname21(this.statePath), { recursive: true });
25084
+ mkdirSync22(dirname22(this.statePath), { recursive: true });
23889
25085
  const payload = {
23890
25086
  version: VECTORS_VERSION,
23891
25087
  identity: this.identity,
@@ -23968,9 +25164,9 @@ var init_semantic = __esm(() => {
23968
25164
  });
23969
25165
 
23970
25166
  // src/datasource/skills/github-gist/skill.ts
23971
- import { join as join32 } from "node:path";
25167
+ import { join as join33 } from "node:path";
23972
25168
  function gistDatasourceDir(workspaceRoot, skillName, instanceId) {
23973
- return join32(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId));
25169
+ return join33(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId));
23974
25170
  }
23975
25171
  var GITHUB_GIST_SKILL_DEFINITION, DEFAULT_INSTANCE_ID3 = "default", GitHubGistSkill;
23976
25172
  var init_skill5 = __esm(() => {
@@ -23998,7 +25194,7 @@ var init_skill5 = __esm(() => {
23998
25194
  constructor(options = {}) {
23999
25195
  const { connector, connectorOptions, semantic, ...rest } = options;
24000
25196
  const instanceId = options.instanceId ?? DEFAULT_INSTANCE_ID3;
24001
- const statePath = connectorOptions?.statePath ?? (options.workspaceRoot === undefined ? undefined : join32(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "state.json"));
25197
+ const statePath = connectorOptions?.statePath ?? (options.workspaceRoot === undefined ? undefined : join33(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "state.json"));
24002
25198
  super(GITHUB_GIST_SKILL_DEFINITION, {
24003
25199
  ...rest,
24004
25200
  connector: connector ?? new GitHubGistConnector({
@@ -24008,7 +25204,7 @@ var init_skill5 = __esm(() => {
24008
25204
  });
24009
25205
  if (semantic?.enabled !== false) {
24010
25206
  this.semanticIndex = new GistSemanticIndex({
24011
- statePath: semantic?.statePath ?? (options.workspaceRoot === undefined ? undefined : join32(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "vectors.json"))
25207
+ statePath: semantic?.statePath ?? (options.workspaceRoot === undefined ? undefined : join33(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "vectors.json"))
24012
25208
  });
24013
25209
  this.semanticEmbedder = semantic?.embedder ?? createGatewayEmbedder(semantic?.embedderOptions ?? {});
24014
25210
  this.semanticMethod = new GitHubGistSemanticMethod({
@@ -24036,7 +25232,7 @@ var init_skill5 = __esm(() => {
24036
25232
  {
24037
25233
  code: "datasource-index-failed",
24038
25234
  severity: "warning",
24039
- message: boundDiagnosticText2(`semantic-unavailable: ${sync.message}`),
25235
+ message: boundDiagnosticText(`semantic-unavailable: ${sync.message}`),
24040
25236
  instanceId: this.instanceId,
24041
25237
  source: this.describe().name
24042
25238
  }
@@ -24731,7 +25927,7 @@ var init_types5 = __esm(() => {
24731
25927
  // src/datasource/skills/lazykatok/client.ts
24732
25928
  import { spawn as spawn14 } from "node:child_process";
24733
25929
  import { existsSync as existsSync27 } from "node:fs";
24734
- import { delimiter as delimiter4, join as join33 } from "node:path";
25930
+ import { delimiter as delimiter4, join as join34 } from "node:path";
24735
25931
 
24736
25932
  class LazykatokClient {
24737
25933
  options;
@@ -24919,7 +26115,7 @@ function lookupInPath2(executable, env) {
24919
26115
  for (const dir of pathEnv.split(delimiter4)) {
24920
26116
  if (dir.length === 0)
24921
26117
  continue;
24922
- const candidate = join33(dir, executable);
26118
+ const candidate = join34(dir, executable);
24923
26119
  if (existsSync27(candidate))
24924
26120
  return candidate;
24925
26121
  }
@@ -25409,7 +26605,7 @@ var init_lazykatok = __esm(() => {
25409
26605
 
25410
26606
  // src/datasource/skills/mail-export/connector.ts
25411
26607
  import { readdir as readdir3, readFile as readFile5, stat as stat6 } from "node:fs/promises";
25412
- import { basename as basename12, dirname as dirname22, extname as extname7, join as join34 } from "node:path";
26608
+ import { basename as basename12, dirname as dirname23, extname as extname7, join as join35 } from "node:path";
25413
26609
  import { simpleParser as simpleParser2 } from "mailparser";
25414
26610
 
25415
26611
  class MailExportConnector {
@@ -25435,7 +26631,7 @@ class MailExportConnector {
25435
26631
  continue;
25436
26632
  const extension = extname7(entry.name).toLowerCase();
25437
26633
  if (extension === ".eml" || extension === ".mbox") {
25438
- files.push(join34(entry.parentPath, entry.name));
26634
+ files.push(join35(entry.parentPath, entry.name));
25439
26635
  }
25440
26636
  }
25441
26637
  } else if (info.isFile()) {
@@ -25470,7 +26666,7 @@ class MailExportConnector {
25470
26666
  continue;
25471
26667
  }
25472
26668
  const stem = basename12(file, extname7(file));
25473
- const mailboxDir = basename12(dirname22(file));
26669
+ const mailboxDir = basename12(dirname23(file));
25474
26670
  const hierarchy = ["mailboxes", mailboxDir, stem];
25475
26671
  const extension = extname7(file).toLowerCase();
25476
26672
  const rawMessages = extension === ".mbox" ? splitMbox(bytes.toString("utf8")) : [bytes.toString("utf8")];
@@ -25591,7 +26787,7 @@ function ensurePrivateMailcrawlDataDir(path) {
25591
26787
  if (process.platform !== "win32")
25592
26788
  chmodSync(path, 448);
25593
26789
  }
25594
- var init_config2 = () => {};
26790
+ var init_config3 = () => {};
25595
26791
 
25596
26792
  // src/datasource/skills/mailcrawl/client.ts
25597
26793
  import { spawn as spawn15 } from "node:child_process";
@@ -25917,7 +27113,7 @@ function truncateUtf8(value, maxBytes) {
25917
27113
  var DEFAULT_BINARY4 = "mailcrawl", DEFAULT_SEARCH_TIMEOUT_MS3 = 60000, DEFAULT_INDEX_TIMEOUT_MS3 = 1800000, DEFAULT_MAX_BUFFER_BYTES4 = 1048576, LOOPBACK_HOSTS2, SAFE_ENV_KEYS2, SAFE_MAILCRAWL_ENV_KEYS, REMOTE_EMBEDDING_ENV_KEYS;
25918
27114
  var init_client9 = __esm(() => {
25919
27115
  init_portable_spawn();
25920
- init_config2();
27116
+ init_config3();
25921
27117
  LOOPBACK_HOSTS2 = new Set(["127.0.0.1", "localhost", "[::1]", "::1"]);
25922
27118
  SAFE_ENV_KEYS2 = new Set(["HOME", "LANG", "LC_ALL", "PATH", "TMPDIR", "TMP", "TEMP", "NO_COLOR"]);
25923
27119
  SAFE_MAILCRAWL_ENV_KEYS = new Set([
@@ -26164,7 +27360,7 @@ var TAGS;
26164
27360
  var init_skill9 = __esm(() => {
26165
27361
  init_scope2();
26166
27362
  init_client9();
26167
- init_config2();
27363
+ init_config3();
26168
27364
  init_methods4();
26169
27365
  TAGS = ["mailcrawl", "email", "pii"];
26170
27366
  });
@@ -26176,6 +27372,504 @@ var init_mailcrawl = __esm(() => {
26176
27372
  init_skill9();
26177
27373
  });
26178
27374
 
27375
+ // src/datasource/sidecar/fusion.ts
27376
+ function encodeVector(vector) {
27377
+ const floats = Float32Array.from(vector);
27378
+ return new Uint8Array(floats.buffer, floats.byteOffset, floats.byteLength);
27379
+ }
27380
+ function decodeVector(blob) {
27381
+ if (blob.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0)
27382
+ return;
27383
+ return new Float32Array(new Uint8Array(blob).buffer);
27384
+ }
27385
+ function cosine3(a, b) {
27386
+ if (a.length === 0 || a.length !== b.length)
27387
+ return 0;
27388
+ let dot = 0;
27389
+ let normA = 0;
27390
+ let normB = 0;
27391
+ for (let index = 0;index < a.length; index += 1) {
27392
+ const x = a[index] ?? 0;
27393
+ const y = b[index] ?? 0;
27394
+ dot += x * y;
27395
+ normA += x * x;
27396
+ normB += y * y;
27397
+ }
27398
+ if (normA === 0 || normB === 0)
27399
+ return 0;
27400
+ return dot / Math.sqrt(normA * normB);
27401
+ }
27402
+ function topKByCosine(query, rows, k) {
27403
+ if (k <= 0)
27404
+ return [];
27405
+ const scored = [];
27406
+ for (const row of rows) {
27407
+ const score = cosine3(query, row.vector);
27408
+ if (score > 0)
27409
+ scored.push({ key: row.key, score });
27410
+ }
27411
+ scored.sort((a, b) => b.score - a.score || a.key.localeCompare(b.key));
27412
+ return scored.slice(0, k);
27413
+ }
27414
+ function reciprocalRankFusion(lists) {
27415
+ const scores = new Map;
27416
+ for (const list of lists) {
27417
+ const seen = new Set;
27418
+ for (const [index, key] of list.entries()) {
27419
+ if (seen.has(key))
27420
+ continue;
27421
+ seen.add(key);
27422
+ scores.set(key, (scores.get(key) ?? 0) + 1 / (RRF_K2 + index + 1));
27423
+ }
27424
+ }
27425
+ return [...scores.entries()].map(([key, score]) => ({ key, score })).sort((a, b) => b.score - a.score);
27426
+ }
27427
+ var RRF_K2 = 60;
27428
+
27429
+ // src/datasource/sidecar/index.ts
27430
+ import { createHash as createHash14 } from "node:crypto";
27431
+ import { mkdirSync as mkdirSync24, statSync as statSync3 } from "node:fs";
27432
+ import { dirname as dirname24 } from "node:path";
27433
+ import { DatabaseSync as DatabaseSync2 } from "node:sqlite";
27434
+ import { pathToFileURL } from "node:url";
27435
+
27436
+ class CrawlerSidecar {
27437
+ descriptor;
27438
+ options;
27439
+ select;
27440
+ where;
27441
+ opened;
27442
+ attached;
27443
+ lastSemantic;
27444
+ constructor(options) {
27445
+ this.options = options;
27446
+ const { descriptor } = options;
27447
+ this.descriptor = descriptor;
27448
+ const metadataColumns = descriptor.metadata.map((column, index) => `${column.sql} as m${index}`);
27449
+ this.select = [
27450
+ `${descriptor.keyExpression} as k`,
27451
+ `${descriptor.textExpression} as content`,
27452
+ `${descriptor.titleExpression ?? "null"} as title`,
27453
+ `${descriptor.epochSecondsExpression ?? "null"} as epoch`,
27454
+ ...metadataColumns
27455
+ ].join(", ");
27456
+ const scope = descriptor.scopeExpression !== undefined && options.workspaceId !== undefined;
27457
+ const allowlist = [
27458
+ ...(options.channelIds?.length ?? 0) > 0 ? [`${descriptor.channelIdExpression} in (${options.channelIds?.map(() => "?").join(", ")})`] : [],
27459
+ ...(options.channelNames?.length ?? 0) > 0 ? [`${descriptor.channelNameExpression} in (${options.channelNames?.map(() => "?").join(", ")})`] : []
27460
+ ];
27461
+ this.where = [
27462
+ descriptor.liveWhere,
27463
+ ...scope ? [`${descriptor.scopeExpression} = ?`] : [],
27464
+ ...allowlist.length > 0 ? [`(${allowlist.join(" or ")})`] : []
27465
+ ].join(" and ");
27466
+ }
27467
+ get db() {
27468
+ if (this.opened !== undefined)
27469
+ return this.opened;
27470
+ mkdirSync24(dirname24(this.options.sidecarDbPath), { recursive: true });
27471
+ const db = new DatabaseSync2(this.options.sidecarDbPath);
27472
+ db.exec(`pragma busy_timeout = ${BUSY_TIMEOUT_MS}`);
27473
+ db.exec("pragma journal_mode = wal");
27474
+ db.exec(`
27475
+ create table if not exists sidecar_meta (k text primary key, v text not null);
27476
+ create table if not exists sidecar_vec (message_key text primary key, hash text not null, vec blob not null);
27477
+ `);
27478
+ this.opened = db;
27479
+ return db;
27480
+ }
27481
+ attachCrawler() {
27482
+ let file;
27483
+ try {
27484
+ const { dev, ino } = statSync3(this.options.crawlerDbPath);
27485
+ file = { dev, ino };
27486
+ } catch {
27487
+ this.detachCrawler();
27488
+ return false;
27489
+ }
27490
+ if (this.attached !== undefined) {
27491
+ if (this.attached.dev === file.dev && this.attached.ino === file.ino)
27492
+ return true;
27493
+ this.detachCrawler();
27494
+ }
27495
+ const crawlerUri = `${pathToFileURL(this.options.crawlerDbPath).href}?mode=ro`.replaceAll("'", "''");
27496
+ try {
27497
+ this.db.exec(`attach database '${crawlerUri}' as crawler`);
27498
+ } catch {
27499
+ return false;
27500
+ }
27501
+ const { descriptor } = this.options;
27502
+ try {
27503
+ this.db.prepare(`select ${this.select} from ${descriptor.fromClause} where ${this.where} limit 0`).all(...this.whereParams());
27504
+ this.db.prepare(`select 1 from ${descriptor.fts.table} limit 0`).all();
27505
+ } catch {
27506
+ this.db.exec("detach database crawler");
27507
+ return false;
27508
+ }
27509
+ this.attached = file;
27510
+ this.lastSemantic = undefined;
27511
+ return true;
27512
+ }
27513
+ detachCrawler() {
27514
+ if (this.attached === undefined)
27515
+ return;
27516
+ this.attached = undefined;
27517
+ this.lastSemantic = undefined;
27518
+ try {
27519
+ this.opened?.exec("detach database crawler");
27520
+ } catch {}
27521
+ }
27522
+ get semanticEnabled() {
27523
+ return this.options.embedder !== undefined;
27524
+ }
27525
+ close() {
27526
+ this.opened?.close();
27527
+ this.opened = undefined;
27528
+ this.attached = undefined;
27529
+ this.lastSemantic = undefined;
27530
+ }
27531
+ whereParams() {
27532
+ const { descriptor, workspaceId, channelIds = [], channelNames = [] } = this.options;
27533
+ return [
27534
+ ...descriptor.scopeExpression !== undefined && workspaceId !== undefined ? [workspaceId] : [],
27535
+ ...channelIds,
27536
+ ...channelNames
27537
+ ];
27538
+ }
27539
+ searchLexical(query, topK) {
27540
+ const match = toMatchQuery(query);
27541
+ if (match === undefined || topK <= 0 || !this.attachCrawler())
27542
+ return [];
27543
+ const { descriptor } = this.options;
27544
+ const rows = this.db.prepare(`select ${this.select}, bm25(${descriptor.fts.table}) as rank
27545
+ from ${descriptor.fromClause} join ${descriptor.fts.table} on ${descriptor.fts.joinOn}
27546
+ where ${descriptor.fts.table} match ? and ${this.where}
27547
+ order by rank limit ?`).all(match, ...this.whereParams(), topK);
27548
+ return rows.map((row) => this.toHit(this.toRow(row), -Number(row.rank)));
27549
+ }
27550
+ async searchSemantic(query, topK) {
27551
+ const trimmed = query.trim();
27552
+ if (this.options.embedder === undefined || trimmed.length === 0 || topK <= 0 || !this.attachCrawler())
27553
+ return [];
27554
+ const ranked = await this.rankSemantic(trimmed, topK * POOL_FACTOR);
27555
+ if (ranked.length === 0)
27556
+ return [];
27557
+ const live = this.hydrate(ranked.map((entry) => entry.key));
27558
+ const hits = [];
27559
+ for (const entry of ranked) {
27560
+ const row = live.get(entry.key);
27561
+ if (row !== undefined)
27562
+ hits.push(this.toHit(row, entry.score));
27563
+ if (hits.length >= topK)
27564
+ break;
27565
+ }
27566
+ return hits;
27567
+ }
27568
+ async rankSemantic(query, pool) {
27569
+ const now = Date.now();
27570
+ let cached = this.lastSemantic;
27571
+ if (cached === undefined || cached.query !== query || now - cached.at > SEMANTIC_REUSE_MS) {
27572
+ const embedder = this.options.embedder;
27573
+ if (embedder === undefined)
27574
+ return [];
27575
+ cached = {
27576
+ query,
27577
+ at: now,
27578
+ vector: embedder.embed([query]).then((vectors) => vectors[0], () => {
27579
+ return;
27580
+ }),
27581
+ ranked: undefined
27582
+ };
27583
+ this.lastSemantic = cached;
27584
+ }
27585
+ const vector = await cached.vector;
27586
+ if (vector === undefined) {
27587
+ if (this.lastSemantic === cached)
27588
+ this.lastSemantic = undefined;
27589
+ return [];
27590
+ }
27591
+ if (cached.ranked === undefined || cached.ranked.pool < pool) {
27592
+ const wide = pool * POOL_FACTOR;
27593
+ cached.ranked = { pool: wide, keys: topKByCosine(vector, this.storedVectors(), wide) };
27594
+ }
27595
+ return cached.ranked.keys.slice(0, pool);
27596
+ }
27597
+ async searchHybrid(query, topK) {
27598
+ if (topK <= 0)
27599
+ return [];
27600
+ const pool = topK * POOL_FACTOR;
27601
+ const lexical = this.searchLexical(query, pool);
27602
+ const semantic = await this.searchSemantic(query, pool);
27603
+ if (semantic.length === 0)
27604
+ return lexical.slice(0, topK);
27605
+ const byKey = new Map;
27606
+ for (const hit of [...semantic, ...lexical])
27607
+ byKey.set(hit.key, hit);
27608
+ const fused = [];
27609
+ for (const entry of reciprocalRankFusion([lexical.map((hit) => hit.key), semantic.map((hit) => hit.key)])) {
27610
+ const hit = byKey.get(entry.key);
27611
+ if (hit !== undefined)
27612
+ fused.push({ ...hit, score: entry.score });
27613
+ if (fused.length >= topK)
27614
+ break;
27615
+ }
27616
+ return fused;
27617
+ }
27618
+ async sync() {
27619
+ const { embedder } = this.options;
27620
+ if (embedder === undefined)
27621
+ return { ok: false, message: "no embedder configured" };
27622
+ if (!this.attachCrawler())
27623
+ return { ok: false, message: "crawler archive not available yet" };
27624
+ let identity;
27625
+ try {
27626
+ identity = await embedder.identity();
27627
+ } catch (error) {
27628
+ return { ok: false, message: errorMessage2(error) };
27629
+ }
27630
+ this.adoptIdentity(identity);
27631
+ const pruned = this.prune();
27632
+ const pending = this.pendingKeys();
27633
+ const batchSize = Math.max(1, this.options.batchSize ?? DEFAULT_BATCH_SIZE);
27634
+ const budget = Math.max(1, this.options.maxEmbedPerSync ?? DEFAULT_MAX_EMBED_PER_SYNC);
27635
+ const target = Math.min(pending.length, budget);
27636
+ const embed = async (rows) => {
27637
+ const vectors = await embedder.embed(rows.map((row) => row.input));
27638
+ if (vectors.length !== rows.length) {
27639
+ throw new Error(`embedder returned ${vectors.length} vectors for ${rows.length} messages`);
27640
+ }
27641
+ const wrong = vectors.find((vector) => vector.length !== identity.dimension);
27642
+ if (wrong !== undefined) {
27643
+ throw new Error(`embedder returned a ${wrong.length}-dimension vector, expected ${identity.dimension}`);
27644
+ }
27645
+ return vectors;
27646
+ };
27647
+ let embedded = 0;
27648
+ let skipped = 0;
27649
+ for (let offset = 0;offset < target; offset += batchSize) {
27650
+ const rows = [...this.hydrate(pending.slice(offset, Math.min(offset + batchSize, target))).values()];
27651
+ if (rows.length === 0)
27652
+ continue;
27653
+ try {
27654
+ this.store(rows, await embed(rows));
27655
+ embedded += rows.length;
27656
+ } catch (batchError) {
27657
+ const rejected = [];
27658
+ let lastError = batchError;
27659
+ for (const row of rows) {
27660
+ try {
27661
+ this.store([row], await embed([row]));
27662
+ embedded += 1;
27663
+ } catch (error) {
27664
+ rejected.push(row);
27665
+ lastError = error;
27666
+ }
27667
+ }
27668
+ if (rejected.length === 0)
27669
+ continue;
27670
+ if (embedded === 0)
27671
+ return { ok: false, message: errorMessage2(lastError) };
27672
+ this.store(rejected, undefined);
27673
+ skipped += rejected.length;
27674
+ }
27675
+ }
27676
+ return { ok: true, embedded, skipped, pruned, remaining: pending.length - embedded - skipped };
27677
+ }
27678
+ adoptIdentity(identity) {
27679
+ const serialized = JSON.stringify(identity);
27680
+ const current = this.db.prepare("select v from sidecar_meta where k = ?").get(IDENTITY_KEY);
27681
+ if (current?.v === serialized)
27682
+ return;
27683
+ this.transaction(() => {
27684
+ this.db.exec("delete from sidecar_vec");
27685
+ this.db.prepare("insert or replace into sidecar_meta (k, v) values (?, ?)").run(IDENTITY_KEY, serialized);
27686
+ });
27687
+ this.lastSemantic = undefined;
27688
+ }
27689
+ prune() {
27690
+ const { descriptor } = this.options;
27691
+ const result = this.db.prepare(`delete from sidecar_vec where message_key not in (
27692
+ select ${descriptor.keyExpression} from ${descriptor.fromClause} where ${this.where}
27693
+ )`).run(...this.whereParams());
27694
+ if (Number(result.changes) > 0)
27695
+ this.lastSemantic = undefined;
27696
+ return Number(result.changes);
27697
+ }
27698
+ pendingKeys() {
27699
+ const { descriptor } = this.options;
27700
+ const rows = this.db.prepare(`select ${this.select}, v.hash as stored_hash
27701
+ from ${descriptor.fromClause}
27702
+ left join sidecar_vec v on v.message_key = ${descriptor.keyExpression}
27703
+ where ${this.where}`).iterate(...this.whereParams());
27704
+ const pending = [];
27705
+ for (const row of rows) {
27706
+ if (row.stored_hash !== this.toRow(row).hash)
27707
+ pending.push(String(row.k));
27708
+ }
27709
+ return pending;
27710
+ }
27711
+ hydrate(keys) {
27712
+ const { descriptor } = this.options;
27713
+ const out = new Map;
27714
+ if (keys.length === 0)
27715
+ return out;
27716
+ const rows = this.db.prepare(`select ${this.select} from ${descriptor.fromClause}
27717
+ where ${descriptor.keyExpression} in (${keys.map(() => "?").join(", ")}) and ${this.where}`).all(...keys, ...this.whereParams());
27718
+ const byKey = new Map(rows.map((row) => [String(row.k), this.toRow(row)]));
27719
+ for (const key of keys) {
27720
+ const row = byKey.get(key);
27721
+ if (row !== undefined)
27722
+ out.set(key, row);
27723
+ }
27724
+ return out;
27725
+ }
27726
+ store(rows, vectors) {
27727
+ const upsert = this.db.prepare("insert or replace into sidecar_vec (message_key, hash, vec) values (?, ?, ?)");
27728
+ this.transaction(() => {
27729
+ for (const [index, row] of rows.entries()) {
27730
+ upsert.run(row.key, row.hash, encodeVector(vectors?.[index] ?? []));
27731
+ }
27732
+ });
27733
+ this.lastSemantic = undefined;
27734
+ }
27735
+ *storedVectors() {
27736
+ const rows = this.db.prepare("select message_key, vec from sidecar_vec where length(vec) > 0").iterate();
27737
+ for (const row of rows) {
27738
+ const vector = row.vec instanceof Uint8Array ? decodeVector(row.vec) : undefined;
27739
+ if (vector !== undefined)
27740
+ yield { key: String(row.message_key), vector };
27741
+ }
27742
+ }
27743
+ transaction(work) {
27744
+ this.db.exec("begin immediate");
27745
+ try {
27746
+ work();
27747
+ this.db.exec("commit");
27748
+ } catch (error) {
27749
+ this.db.exec("rollback");
27750
+ throw error;
27751
+ }
27752
+ }
27753
+ toRow(row) {
27754
+ const metadata = {};
27755
+ for (const [index, column] of this.options.descriptor.metadata.entries()) {
27756
+ const value = row[`m${index}`];
27757
+ if (typeof value === "string" && value.length > 0 || typeof value === "number")
27758
+ metadata[column.key] = String(value);
27759
+ }
27760
+ const title = typeof row.title === "string" && row.title.length > 0 ? row.title : undefined;
27761
+ const epoch = typeof row.epoch === "number" && Number.isFinite(row.epoch) ? row.epoch : undefined;
27762
+ const content = String(row.content ?? "");
27763
+ const input = `${title ?? ""}
27764
+ ${content}`.slice(0, MAX_EMBED_INPUT_CHARS);
27765
+ return {
27766
+ key: String(row.k),
27767
+ content,
27768
+ title,
27769
+ epoch,
27770
+ metadata,
27771
+ input,
27772
+ hash: createHash14("sha1").update(input).digest("hex")
27773
+ };
27774
+ }
27775
+ toHit(row, score) {
27776
+ return {
27777
+ key: row.key,
27778
+ content: row.content,
27779
+ score,
27780
+ ...row.title !== undefined ? { title: row.title } : {},
27781
+ ...row.epoch !== undefined ? { publishedAt: Math.round(row.epoch * 1000) } : {},
27782
+ metadata: row.metadata
27783
+ };
27784
+ }
27785
+ }
27786
+ function toMatchQuery(text) {
27787
+ const terms = [...new Set(text.match(/[\p{L}\p{N}_]+/gu) ?? [])];
27788
+ return terms.length === 0 ? undefined : terms.map((term) => `"${term}"`).join(" OR ");
27789
+ }
27790
+ function errorMessage2(error) {
27791
+ return error instanceof Error ? error.message : String(error);
27792
+ }
27793
+ var DEFAULT_BATCH_SIZE = 256, DEFAULT_MAX_EMBED_PER_SYNC = 2048, MAX_EMBED_INPUT_CHARS = 8192, BUSY_TIMEOUT_MS = 5000, POOL_FACTOR = 3, SEMANTIC_REUSE_MS = 1e4, IDENTITY_KEY = "identity";
27794
+ var init_sidecar = () => {};
27795
+
27796
+ // src/datasource/sidecar/methods.ts
27797
+ function sidecarMethodName(datasourceId, mode) {
27798
+ return mode === "bm25" ? `${datasourceId}-sidecar-bm25` : `${datasourceId}-${mode}`;
27799
+ }
27800
+
27801
+ class SidecarRetrievalMethod {
27802
+ options;
27803
+ constructor(options) {
27804
+ this.options = options;
27805
+ }
27806
+ describe() {
27807
+ const { datasourceId, skillType, mode, tags } = this.options;
27808
+ const { type, text, capabilities } = DESCRIPTIONS[mode];
27809
+ return {
27810
+ name: sidecarMethodName(datasourceId, mode),
27811
+ type,
27812
+ description: `${text} over ${skillType} via a read-only sidecar index`,
27813
+ status: "active",
27814
+ capabilities: [...capabilities, "scoped", "path-opaque-sources"],
27815
+ datasourceId,
27816
+ tags
27817
+ };
27818
+ }
27819
+ async retrieve(query, options) {
27820
+ const trimmed = query.trim();
27821
+ if (trimmed.length === 0)
27822
+ return [];
27823
+ const topK = options.topK ?? DEFAULT_TOP_K5;
27824
+ const { sidecar, mode } = this.options;
27825
+ const hits = mode === "bm25" ? sidecar.searchLexical(trimmed, topK) : mode === "semantic" ? await sidecar.searchSemantic(trimmed, topK) : await sidecar.searchHybrid(trimmed, topK);
27826
+ const results = [];
27827
+ for (const hit of hits) {
27828
+ const result = this.toResult(hit, options);
27829
+ if (result !== undefined)
27830
+ results.push(result);
27831
+ }
27832
+ return results;
27833
+ }
27834
+ toResult(hit, options) {
27835
+ const { descriptor, datasourceId, instanceId, mode } = this.options;
27836
+ const chunkId = descriptor.chunkId(hit.key, hit.metadata);
27837
+ const source = datasourceSourcePath(datasourceId, instanceId, chunkId);
27838
+ if (!matchesDatasourceScope(source, options.scope))
27839
+ return;
27840
+ const method = sidecarMethodName(datasourceId, mode);
27841
+ return {
27842
+ id: `${datasourceId}:${instanceId}:${chunkId}`,
27843
+ content: hit.content,
27844
+ source,
27845
+ score: hit.score,
27846
+ metadata: {
27847
+ ...hit.metadata,
27848
+ method,
27849
+ datasourceId,
27850
+ instanceId,
27851
+ backend: "sidecar",
27852
+ ...hit.title !== undefined ? { title: hit.title } : {},
27853
+ hierarchy: descriptor.hierarchy(hit.metadata).join("/"),
27854
+ ...hit.publishedAt !== undefined ? { publishedAt: hit.publishedAt } : {}
27855
+ }
27856
+ };
27857
+ }
27858
+ }
27859
+ var DEFAULT_TOP_K5 = 20, DESCRIPTIONS;
27860
+ var init_methods5 = __esm(() => {
27861
+ init_scope2();
27862
+ DESCRIPTIONS = {
27863
+ bm25: { type: "bm25", text: "BM25-ranked FTS5 retrieval", capabilities: ["lexical", "fts5", "bm25"] },
27864
+ semantic: { type: "vector", text: "Semantic (embedding) retrieval", capabilities: ["semantic"] },
27865
+ hybrid: {
27866
+ type: "hybrid",
27867
+ text: "Hybrid BM25 + semantic retrieval (reciprocal rank fusion)",
27868
+ capabilities: ["lexical", "semantic", "hybrid"]
27869
+ }
27870
+ };
27871
+ });
27872
+
26179
27873
  // src/datasource/crawler-skill.ts
26180
27874
  class CrawlerDatasourceSkill {
26181
27875
  definition;
@@ -26186,6 +27880,7 @@ class CrawlerDatasourceSkill {
26186
27880
  tags;
26187
27881
  channelIds;
26188
27882
  channelNames;
27883
+ sidecar;
26189
27884
  lastIndexedAt;
26190
27885
  constructor(definition, options) {
26191
27886
  this.definition = options.datasourceId === undefined ? definition : {
@@ -26201,6 +27896,11 @@ class CrawlerDatasourceSkill {
26201
27896
  this.lastIndexedAt = options.lastIndexedAt;
26202
27897
  this.channelIds = new Set(options.channelIds ?? []);
26203
27898
  this.channelNames = new Set(options.channelNames ?? []);
27899
+ this.sidecar = options.sidecar === undefined ? undefined : new CrawlerSidecar({
27900
+ ...options.sidecar,
27901
+ channelIds: options.channelIds ?? [],
27902
+ channelNames: options.channelNames ?? []
27903
+ });
26204
27904
  }
26205
27905
  describe() {
26206
27906
  return {
@@ -26236,26 +27936,89 @@ class CrawlerDatasourceSkill {
26236
27936
  skill: this.definition.datasourceId,
26237
27937
  chunkCount: result.count,
26238
27938
  indexedAt: this.lastIndexedAt,
26239
- diagnostics: []
27939
+ diagnostics: await this.syncSidecar()
26240
27940
  };
26241
27941
  } catch {
26242
27942
  return this.fail("datasource-unavailable", "spawn-error");
26243
27943
  }
26244
27944
  }
27945
+ async syncSidecar() {
27946
+ if (this.sidecar === undefined || !this.sidecar.semanticEnabled)
27947
+ return [];
27948
+ const base = { instanceId: this.instanceId, source: this.definition.datasourceId };
27949
+ let result;
27950
+ try {
27951
+ result = await this.sidecar.sync();
27952
+ } catch (error) {
27953
+ result = { ok: false, message: error instanceof Error ? error.message : String(error) };
27954
+ }
27955
+ if (!result.ok) {
27956
+ return [
27957
+ {
27958
+ code: "datasource-index-failed",
27959
+ severity: "warning",
27960
+ message: `semantic-unavailable: ${result.message}`.slice(0, 300),
27961
+ ...base
27962
+ }
27963
+ ];
27964
+ }
27965
+ const diagnostics = [];
27966
+ if (result.skipped > 0) {
27967
+ diagnostics.push({
27968
+ code: "datasource-index-partial",
27969
+ severity: "warning",
27970
+ message: `semantic index skipped ${result.skipped} message(s) the embedder rejected; they stay searchable by BM25 and are retried when edited`,
27971
+ ...base
27972
+ });
27973
+ }
27974
+ if (result.remaining > 0) {
27975
+ diagnostics.push({
27976
+ code: "datasource-index-partial",
27977
+ severity: "info",
27978
+ message: `semantic index catching up: ${result.embedded} embedded this refresh, ${result.remaining} remaining; hybrid search uses BM25 for the rest until a later refresh`,
27979
+ ...base
27980
+ });
27981
+ }
27982
+ return diagnostics;
27983
+ }
27984
+ close() {
27985
+ this.sidecar?.close();
27986
+ }
26245
27987
  retrievalMethods() {
27988
+ const cli = this.cliMethod();
27989
+ const { sidecar } = this;
27990
+ if (sidecar === undefined)
27991
+ return [cli];
27992
+ const shared = {
27993
+ sidecar,
27994
+ descriptor: sidecar.descriptor,
27995
+ datasourceId: this.definition.datasourceId,
27996
+ skillType: this.definition.skillType,
27997
+ instanceId: this.instanceId,
27998
+ tags: this.tags
27999
+ };
28000
+ const bm25 = new SidecarRetrievalMethod({ ...shared, mode: "bm25" });
28001
+ if (!sidecar.semanticEnabled)
28002
+ return [cli, bm25];
26246
28003
  return [
26247
- new CrawlerLexicalMethod({
26248
- client: this.client,
26249
- datasourceId: this.definition.datasourceId,
26250
- skillType: this.definition.skillType,
26251
- instanceId: this.instanceId,
26252
- tags: this.tags,
26253
- backendName: this.definition.backendName,
26254
- channelIds: this.channelIds,
26255
- channelNames: this.channelNames
26256
- })
28004
+ cli,
28005
+ bm25,
28006
+ new SidecarRetrievalMethod({ ...shared, mode: "semantic" }),
28007
+ new SidecarRetrievalMethod({ ...shared, mode: "hybrid" })
26257
28008
  ];
26258
28009
  }
28010
+ cliMethod() {
28011
+ return new CrawlerLexicalMethod({
28012
+ client: this.client,
28013
+ datasourceId: this.definition.datasourceId,
28014
+ skillType: this.definition.skillType,
28015
+ instanceId: this.instanceId,
28016
+ tags: this.tags,
28017
+ backendName: this.definition.backendName,
28018
+ channelIds: this.channelIds,
28019
+ channelNames: this.channelNames
28020
+ });
28021
+ }
26259
28022
  describeSources() {
26260
28023
  return this.instances.map((instanceId) => ({
26261
28024
  source: datasourceSourcePath(this.definition.datasourceId, instanceId),
@@ -26386,6 +28149,8 @@ var DEFAULT_INSTANCE_ID5 = "default", DEFAULT_POLLING_INTERVAL_MS4;
26386
28149
  var init_crawler_skill = __esm(() => {
26387
28150
  init_errors4();
26388
28151
  init_scope2();
28152
+ init_sidecar();
28153
+ init_methods5();
26389
28154
  DEFAULT_POLLING_INTERVAL_MS4 = 15 * 60 * 1000;
26390
28155
  });
26391
28156
 
@@ -26538,15 +28303,15 @@ var init_notion = __esm(() => {
26538
28303
  });
26539
28304
 
26540
28305
  // src/datasource/skills/obsidian/paths.ts
26541
- import { join as join35 } from "node:path";
28306
+ import { join as join36 } from "node:path";
26542
28307
  function obsidianDatasourceRoot(workspaceRoot, instanceId) {
26543
- return join35(workspaceRoot, AUTORAG_DIRNAME, DATASOURCES_DIRNAME, OBSIDIAN_DATASOURCE_DIRNAME, instanceId);
28308
+ return join36(workspaceRoot, AUTORAG_DIRNAME, DATASOURCES_DIRNAME, OBSIDIAN_DATASOURCE_DIRNAME, instanceId);
26544
28309
  }
26545
28310
  function obsidianQmdConfigDir(workspaceRoot, instanceId) {
26546
- return join35(obsidianDatasourceRoot(workspaceRoot, instanceId), "config");
28311
+ return join36(obsidianDatasourceRoot(workspaceRoot, instanceId), "config");
26547
28312
  }
26548
28313
  function obsidianQmdCacheDir(workspaceRoot, instanceId) {
26549
- return join35(obsidianDatasourceRoot(workspaceRoot, instanceId), "cache");
28314
+ return join36(obsidianDatasourceRoot(workspaceRoot, instanceId), "cache");
26550
28315
  }
26551
28316
  function stripEdgeDashes(value) {
26552
28317
  let start = 0;
@@ -26569,8 +28334,8 @@ var DEFAULT_QMD_BINARY = "qmd", DEFAULT_QMD_TIMEOUT_MS = 60000, DEFAULT_QMD_MAX_
26569
28334
 
26570
28335
  // src/datasource/skills/obsidian/client.ts
26571
28336
  import { spawn as spawn16 } from "node:child_process";
26572
- import { mkdirSync as mkdirSync24, writeFileSync as writeFileSync22 } from "node:fs";
26573
- import { isAbsolute as isAbsolute8, resolve as resolve14 } from "node:path";
28337
+ import { mkdirSync as mkdirSync25, writeFileSync as writeFileSync22 } from "node:fs";
28338
+ import { isAbsolute as isAbsolute9, resolve as resolve15 } from "node:path";
26574
28339
 
26575
28340
  class QmdClient {
26576
28341
  options;
@@ -26589,10 +28354,10 @@ class QmdClient {
26589
28354
  const cacheDir = obsidianQmdCacheDir(workspaceRoot, instanceId);
26590
28355
  try {
26591
28356
  if (this.options.configPath === undefined) {
26592
- mkdirSync24(configDir, { recursive: true });
26593
- writeFileSync22(resolve14(configDir, "index.yml"), renderIndexYaml(collectionName, vaultPath), "utf8");
28357
+ mkdirSync25(configDir, { recursive: true });
28358
+ writeFileSync22(resolve15(configDir, "index.yml"), renderIndexYaml(collectionName, vaultPath), "utf8");
26594
28359
  }
26595
- mkdirSync24(cacheDir, { recursive: true });
28360
+ mkdirSync25(cacheDir, { recursive: true });
26596
28361
  } catch (error) {
26597
28362
  return failure4("spawn-error", error instanceof Error ? error.message : "failed to write qmd config");
26598
28363
  }
@@ -26666,7 +28431,7 @@ function resolveVaultPath(vaultPath) {
26666
28431
  if (vaultPath === undefined || vaultPath.trim().length === 0)
26667
28432
  return;
26668
28433
  const trimmed = vaultPath.trim();
26669
- return isAbsolute8(trimmed) ? trimmed : resolve14(trimmed);
28434
+ return isAbsolute9(trimmed) ? trimmed : resolve15(trimmed);
26670
28435
  }
26671
28436
  function controlledEnv5(options) {
26672
28437
  const instanceId = options.instanceId ?? "default";
@@ -26985,7 +28750,7 @@ async function retrieveObsidian(client, mode, methodName, instanceId, query, opt
26985
28750
  const trimmed = query.trim();
26986
28751
  if (trimmed.length === 0)
26987
28752
  return [];
26988
- const topK = options.topK ?? DEFAULT_TOP_K5;
28753
+ const topK = options.topK ?? DEFAULT_TOP_K6;
26989
28754
  const searchOptions = { topK, signal: options.signal };
26990
28755
  const result = await client.search(mode, trimmed, searchOptions);
26991
28756
  if (!result.ok)
@@ -27020,8 +28785,8 @@ function toRetrievalResult3(hit, source, methodName, instanceId, mode) {
27020
28785
  }
27021
28786
  };
27022
28787
  }
27023
- var OBSIDIAN_DATASOURCE_ID = "obsidian", DEFAULT_OBSIDIAN_TAGS, DEFAULT_TOP_K5 = 20;
27024
- var init_methods5 = __esm(() => {
28788
+ var OBSIDIAN_DATASOURCE_ID = "obsidian", DEFAULT_OBSIDIAN_TAGS, DEFAULT_TOP_K6 = 20;
28789
+ var init_methods6 = __esm(() => {
27025
28790
  init_errors4();
27026
28791
  init_scope2();
27027
28792
  DEFAULT_OBSIDIAN_TAGS = ["obsidian", "notes"];
@@ -27211,7 +28976,7 @@ var OBSIDIAN_DATASOURCE_ID2 = "obsidian", OBSIDIAN_SKILL_TYPE = "obsidian-vault"
27211
28976
  var init_skill10 = __esm(() => {
27212
28977
  init_scope2();
27213
28978
  init_client10();
27214
- init_methods5();
28979
+ init_methods6();
27215
28980
  init_paths5();
27216
28981
  DEFAULT_POLLING_INTERVAL_MS5 = 15 * 60 * 1000;
27217
28982
  DEFAULT_OBSIDIAN_TAGS2 = ["obsidian", "notes"];
@@ -27220,7 +28985,7 @@ var init_skill10 = __esm(() => {
27220
28985
  // src/datasource/skills/obsidian/index.ts
27221
28986
  var init_obsidian = __esm(() => {
27222
28987
  init_client10();
27223
- init_methods5();
28988
+ init_methods6();
27224
28989
  init_paths5();
27225
28990
  init_skill10();
27226
28991
  });
@@ -27560,7 +29325,7 @@ var init_slack = __esm(() => {
27560
29325
 
27561
29326
  // src/datasource/skills/spotlight/connector.ts
27562
29327
  import { spawn as spawn17 } from "node:child_process";
27563
- import { createHash as createHash14 } from "node:crypto";
29328
+ import { createHash as createHash15 } from "node:crypto";
27564
29329
  import { readFile as fsReadFile, stat as stat7 } from "node:fs/promises";
27565
29330
  import { basename as basename13, extname as extname8 } from "node:path";
27566
29331
 
@@ -27583,7 +29348,7 @@ class SpotlightConnector {
27583
29348
  if (queries.length === 0) {
27584
29349
  return { ok: false, reason: "not-configured", message: "no spotlight queries configured" };
27585
29350
  }
27586
- const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS9;
29351
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS10;
27587
29352
  const status = await this.run("mdutil", ["-s", "/"], timeoutMs);
27588
29353
  if (status.ok && /indexing disabled/iu.test(status.stdout)) {
27589
29354
  return {
@@ -27645,7 +29410,7 @@ class SpotlightConnector {
27645
29410
  if (content.length === 0)
27646
29411
  continue;
27647
29412
  documents.push({
27648
- docId: createHash14("sha256").update(path).digest("hex").slice(0, 16),
29413
+ docId: createHash15("sha256").update(path).digest("hex").slice(0, 16),
27649
29414
  hierarchy: ["files"],
27650
29415
  title: basename13(path),
27651
29416
  content,
@@ -27678,7 +29443,7 @@ class SpotlightConnector {
27678
29443
  return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
27679
29444
  }
27680
29445
  }
27681
- var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS5 = 300, DEFAULT_MAX_BYTES_PER_FILE3, DEFAULT_TIMEOUT_MS9 = 15000, MAX_CONTENT_CHARS4 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve) => {
29446
+ var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS5 = 300, DEFAULT_MAX_BYTES_PER_FILE3, DEFAULT_TIMEOUT_MS10 = 15000, MAX_CONTENT_CHARS4 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve) => {
27682
29447
  const child = spawn17(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
27683
29448
  let stdout = "";
27684
29449
  let stderr = "";
@@ -27795,10 +29560,12 @@ function parseHits3(stdout) {
27795
29560
  for (const [index, row] of rows.entries()) {
27796
29561
  if (!isRecord18(row))
27797
29562
  return;
27798
- const id = stringValue4(row, ["message_id", "event_id", "id"]);
27799
- const content = stringValue4(row, ["snippet", "text", "content"]);
27800
- if (id === undefined || content === undefined)
29563
+ const id = stringValue4(row, ["event_id", "message_id", "id"]);
29564
+ if (id === undefined)
27801
29565
  return;
29566
+ const content = stringValue4(row, ["snippet", "text", "content"]);
29567
+ if (content === undefined)
29568
+ continue;
27802
29569
  const chatId = stringValue4(row, ["chat_jid", "chat_id"]);
27803
29570
  const chatName = stringValue4(row, ["chat_name"]);
27804
29571
  const senderName = stringValue4(row, ["sender_name"]);
@@ -27915,10 +29682,12 @@ function parseHits4(stdout) {
27915
29682
  for (const [index, row] of rows.entries()) {
27916
29683
  if (!isRecord19(row))
27917
29684
  return;
27918
- const id = stringValue5(row, ["message_id", "event_id", "id"]);
27919
- const content = stringValue5(row, ["snippet", "text", "content"]);
27920
- if (id === undefined || content === undefined)
29685
+ const id = stringValue5(row, ["event_id", "message_id", "id"]);
29686
+ if (id === undefined)
27921
29687
  return;
29688
+ const content = stringValue5(row, ["snippet", "text", "content"]);
29689
+ if (content === undefined)
29690
+ continue;
27922
29691
  const chatId = stringValue5(row, ["chat_jid", "chat_id"]);
27923
29692
  const chatName = stringValue5(row, ["chat_name"]);
27924
29693
  const senderName = stringValue5(row, ["sender_name"]);
@@ -28008,6 +29777,31 @@ var init_wacrawl = __esm(() => {
28008
29777
  });
28009
29778
 
28010
29779
  // src/datasource/skills/factory.ts
29780
+ import { homedir as homedir4 } from "node:os";
29781
+ import { join as join37 } from "node:path";
29782
+ function chatCrawlerArgs(descriptor, defaultCrawlerDb, config, workspaceRoot, registrationName) {
29783
+ const sidecar = resolveSidecarOptions({
29784
+ descriptor,
29785
+ connector: config.connector,
29786
+ workspaceRoot,
29787
+ registrationName,
29788
+ instanceId: config.instanceId ?? "default",
29789
+ defaultCrawlerDb
29790
+ });
29791
+ return {
29792
+ datasourceId: registrationName,
29793
+ ...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
29794
+ ...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
29795
+ ...config.tags !== undefined ? { tags: config.tags } : {},
29796
+ channelIds: config.channels?.ids,
29797
+ channelNames: config.channels?.names,
29798
+ ...sidecar === undefined ? {} : { sidecar },
29799
+ connectorOptions: {
29800
+ ...config.connector,
29801
+ ...workspaceRoot === undefined ? {} : { workspacePath: workspaceRoot }
29802
+ }
29803
+ };
29804
+ }
28011
29805
  function larkConnectorOptions(connector) {
28012
29806
  if (connector === undefined)
28013
29807
  return {};
@@ -28068,6 +29862,8 @@ function buildDatasourceSkills(config, workspaceRoot, embeddingRuntime) {
28068
29862
  var BUILDERS, BUILTIN_DATASOURCE_SKILL_NAMES;
28069
29863
  var init_factory = __esm(() => {
28070
29864
  init_aliased_skill();
29865
+ init_config2();
29866
+ init_descriptors();
28071
29867
  init_clawgallery();
28072
29868
  init_cloud_drive();
28073
29869
  init_discrawl();
@@ -28085,42 +29881,9 @@ var init_factory = __esm(() => {
28085
29881
  init_telecrawl();
28086
29882
  init_wacrawl();
28087
29883
  BUILDERS = {
28088
- telegram: (config, _workspaceRoot, registrationName) => new TelecrawlSkill({
28089
- datasourceId: registrationName,
28090
- ...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
28091
- ...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
28092
- ...config.tags !== undefined ? { tags: config.tags } : {},
28093
- channelIds: config.channels?.ids,
28094
- channelNames: config.channels?.names,
28095
- connectorOptions: {
28096
- ...config.connector,
28097
- ..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
28098
- }
28099
- }),
28100
- whatsapp: (config, _workspaceRoot, registrationName) => new WacrawlSkill({
28101
- datasourceId: registrationName,
28102
- ...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
28103
- ...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
28104
- ...config.tags !== undefined ? { tags: config.tags } : {},
28105
- channelIds: config.channels?.ids,
28106
- channelNames: config.channels?.names,
28107
- connectorOptions: {
28108
- ...config.connector,
28109
- ..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
28110
- }
28111
- }),
28112
- slack: (config, _workspaceRoot, registrationName) => new SlackSkill({
28113
- datasourceId: registrationName,
28114
- ...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
28115
- ...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
28116
- ...config.tags !== undefined ? { tags: config.tags } : {},
28117
- channelIds: config.channels?.ids,
28118
- channelNames: config.channels?.names,
28119
- connectorOptions: {
28120
- ...config.connector,
28121
- ..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
28122
- }
28123
- }),
29884
+ telegram: (config, workspaceRoot, registrationName) => new TelecrawlSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.telegram, join37(homedir4(), ".telecrawl", "telecrawl.db"), config, workspaceRoot, registrationName)),
29885
+ whatsapp: (config, workspaceRoot, registrationName) => new WacrawlSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.whatsapp, join37(homedir4(), ".wacrawl", "wacrawl.db"), config, workspaceRoot, registrationName)),
29886
+ slack: (config, workspaceRoot, registrationName) => new SlackSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.slack, join37(homedir4(), ".slacrawl", "slacrawl.db"), config, workspaceRoot, registrationName)),
28124
29887
  discord: (config, workspaceRoot, registrationName, embeddingRuntime) => {
28125
29888
  const connector = config.connector ?? {};
28126
29889
  const clientOptions = {
@@ -28239,8 +30002,8 @@ var init_factory = __esm(() => {
28239
30002
 
28240
30003
  // src/cli/config.ts
28241
30004
  import { randomUUID as randomUUID9 } from "node:crypto";
28242
- import { existsSync as existsSync28, mkdirSync as mkdirSync25, readFileSync as readFileSync24, renameSync as renameSync11, unlinkSync as unlinkSync5, writeFileSync as writeFileSync23 } from "node:fs";
28243
- import { basename as basename14, dirname as dirname23, isAbsolute as isAbsolute9, join as join36, resolve as resolve15 } from "node:path";
30005
+ import { existsSync as existsSync28, mkdirSync as mkdirSync26, readFileSync as readFileSync25, renameSync as renameSync11, unlinkSync as unlinkSync5, writeFileSync as writeFileSync23 } from "node:fs";
30006
+ import { basename as basename14, dirname as dirname25, isAbsolute as isAbsolute10, join as join38, resolve as resolve16 } from "node:path";
28244
30007
  import { findEnvKeys, getEnvApiKey } from "@earendil-works/pi-ai/compat";
28245
30008
  import { getAgentDir as getAgentDir3, ModelRuntime as ModelRuntime2, SettingsManager as SettingsManager2 } from "@earendil-works/pi-coding-agent";
28246
30009
  function resolveConfigPath(input) {
@@ -28256,9 +30019,9 @@ function resolveConfigPath(input) {
28256
30019
  return { configPath: envConfig, explicit: true };
28257
30020
  }
28258
30021
  return {
28259
- configPath: join36(resolveAutoRAGHome(env), DEFAULT_CONFIG_FILENAME),
30022
+ configPath: join38(resolveAutoRAGHome(env), DEFAULT_CONFIG_FILENAME),
28260
30023
  explicit: false,
28261
- legacyPath: join36(cwd, LEGACY_CONFIG_FILENAME)
30024
+ legacyPath: join38(cwd, LEGACY_CONFIG_FILENAME)
28262
30025
  };
28263
30026
  }
28264
30027
  function readConfigFile(configPath, explicit) {
@@ -28276,7 +30039,7 @@ function readConfigFile(configPath, explicit) {
28276
30039
  }
28277
30040
  let text;
28278
30041
  try {
28279
- text = readFileSync24(configPath, "utf8");
30042
+ text = readFileSync25(configPath, "utf8");
28280
30043
  } catch (err) {
28281
30044
  throw new ConfigError(`Failed to read config file: ${err.message}`);
28282
30045
  }
@@ -28314,7 +30077,7 @@ function acquireConfigWriteLock(configPath) {
28314
30077
  });
28315
30078
  }
28316
30079
  function replaceFileAtomically(path, contents, assertCommitAllowed) {
28317
- const temporaryPath = join36(dirname23(path), `.${basename14(path)}.${process.pid}.${randomUUID9()}.tmp`);
30080
+ const temporaryPath = join38(dirname25(path), `.${basename14(path)}.${process.pid}.${randomUUID9()}.tmp`);
28318
30081
  try {
28319
30082
  writeFileSync23(temporaryPath, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
28320
30083
  assertCommitAllowed?.();
@@ -28327,12 +30090,12 @@ function migrateLegacyConfig(configPath, legacyPath) {
28327
30090
  if (existsSync28(configPath) || !existsSync28(legacyPath))
28328
30091
  return;
28329
30092
  const legacy = readConfigFile(legacyPath, true);
28330
- const legacyBytes = readFileSync24(legacyPath);
28331
- const migrated = normalizeLegacyConfigPaths(legacy ?? {}, dirname23(legacyPath));
30093
+ const legacyBytes = readFileSync25(legacyPath);
30094
+ const migrated = normalizeLegacyConfigPaths(legacy ?? {}, dirname25(legacyPath));
28332
30095
  const migratedPathsUnchanged = legacy?.workspacePath === migrated.workspacePath && legacy?.memoryPath === migrated.memoryPath && JSON.stringify(legacy?.searchPaths) === JSON.stringify(migrated.searchPaths);
28333
30096
  const migratedBytes = migratedPathsUnchanged ? legacyBytes : `${JSON.stringify(migrated, null, 2)}
28334
30097
  `;
28335
- mkdirSync25(dirname23(configPath), { recursive: true });
30098
+ mkdirSync26(dirname25(configPath), { recursive: true });
28336
30099
  const lock = acquireConfigWriteLock(configPath);
28337
30100
  try {
28338
30101
  const winner = readConfigFile(configPath, false);
@@ -28362,13 +30125,13 @@ function resolveConfigFileReadOnly(configPath, explicit, legacyPath) {
28362
30125
  const legacy = readConfigFile(legacyPath, false);
28363
30126
  if (legacy === undefined)
28364
30127
  return {};
28365
- return normalizeLegacyConfigPaths(legacy, dirname23(legacyPath));
30128
+ return normalizeLegacyConfigPaths(legacy, dirname25(legacyPath));
28366
30129
  }
28367
30130
  function resolveSearchPaths(searchPaths, origin) {
28368
30131
  return searchPaths.map((searchPath) => resolvePersistedPath(searchPath, origin));
28369
30132
  }
28370
30133
  function resolvePersistedPath(path, origin) {
28371
- return isAbsolute9(path) ? path : resolve15(origin, path);
30134
+ return isAbsolute10(path) ? path : resolve16(origin, path);
28372
30135
  }
28373
30136
  function normalizeLegacyConfigPaths(partial, origin) {
28374
30137
  const workspacePath = resolvePersistedPath(partial.workspacePath ?? ".", origin);
@@ -28555,7 +30318,7 @@ function normalizeEmbedder(raw, path) {
28555
30318
  out.passagePrefix = record.passagePrefix;
28556
30319
  }
28557
30320
  if (record.profile !== undefined) {
28558
- if (record.profile !== "qwen3-embedding-0.6b" && record.profile !== "embeddinggemma-300m") {
30321
+ if (record.profile !== "qwen3-embedding-0.6b") {
28559
30322
  throw new ConfigError(`${path}.profile must be a supported runtime profile`);
28560
30323
  }
28561
30324
  out.profile = record.profile;
@@ -28798,10 +30561,10 @@ function resolveConfig(input) {
28798
30561
  const file = readOnly ? resolveConfigFileReadOnly(configPath, explicit, legacyPath) : !explicit && legacyPath ? migrateLegacyConfig(configPath, legacyPath) ?? readConfigFile(configPath, explicit) ?? {} : readConfigFile(configPath, explicit) ?? {};
28799
30562
  const defaultSearchPaths = ["."];
28800
30563
  const defaultWorkspacePath = cwd;
28801
- const defaultMemoryPath = join36(resolveAutoRAGHome(env), "memory.json");
30564
+ const defaultMemoryPath = join38(resolveAutoRAGHome(env), "memory.json");
28802
30565
  const flagSearchPaths = parseCsv(flagString(flags, "search-paths"));
28803
30566
  const envSearchPaths = parseCsv(envString(env, "AUTORAG_SEARCH_PATHS"));
28804
- const configOrigin = dirname23(resolve15(configPath));
30567
+ const configOrigin = dirname25(resolve16(configPath));
28805
30568
  const fileWorkspacePath = typeof file.workspacePath === "string" ? resolvePersistedPath(file.workspacePath, configOrigin) : undefined;
28806
30569
  const fileSearchPaths = file.searchPaths ? resolveSearchPaths(file.searchPaths, fileWorkspacePath ?? configOrigin) : undefined;
28807
30570
  if (file.excludePaths !== undefined && (!Array.isArray(file.excludePaths) || file.excludePaths.some((path) => typeof path !== "string" || path.length === 0))) {
@@ -28824,7 +30587,7 @@ function resolveConfig(input) {
28824
30587
  const flagModelId = pickString(flags, env, "model-id", "AUTORAG_MODEL_ID", undefined);
28825
30588
  const model = applyRoleFlagOverrides(fileModel, flagModelProvider, flagModelId, "model");
28826
30589
  const config = {
28827
- configPath: resolve15(configPath),
30590
+ configPath: resolve16(configPath),
28828
30591
  searchPaths,
28829
30592
  workspacePath,
28830
30593
  memoryPath,
@@ -29000,36 +30763,38 @@ function buildWebSearchAgentOption(raw) {
29000
30763
  function normalizeRerankConfig(raw, path) {
29001
30764
  if (raw === false)
29002
30765
  return false;
29003
- const out = {
29004
- provider: DEFAULT_RERANK_PROVIDER,
29005
- model: DEFAULT_RERANK_MODEL,
29006
- apiKeyEnv: DEFAULT_RERANK_API_KEY_ENV,
29007
- topN: DEFAULT_RERANK_TOP_N
29008
- };
29009
- if (raw === undefined || raw === null)
29010
- return out;
29011
- if (typeof raw !== "object" || Array.isArray(raw)) {
29012
- throw new ConfigError(`${path} must be an object or false`);
29013
- }
29014
- const record = raw;
29015
- for (const key of Object.keys(record)) {
29016
- if (!RERANK_ALLOWLIST.has(key))
29017
- throw new ConfigError(`${path}.${key} is not a recognized field`);
29018
- }
29019
- if (record.enabled !== undefined) {
29020
- if (typeof record.enabled !== "boolean")
29021
- throw new ConfigError(`${path}.enabled must be a boolean`);
29022
- out.enabled = record.enabled;
30766
+ let record = {};
30767
+ if (raw !== undefined && raw !== null) {
30768
+ if (typeof raw !== "object" || Array.isArray(raw)) {
30769
+ throw new ConfigError(`${path} must be an object or false`);
30770
+ }
30771
+ record = raw;
30772
+ for (const key of Object.keys(record)) {
30773
+ if (!RERANK_ALLOWLIST.has(key))
30774
+ throw new ConfigError(`${path}.${key} is not a recognized field`);
30775
+ }
29023
30776
  }
30777
+ let provider = DEFAULT_RERANK_PROVIDER;
29024
30778
  if (record.provider !== undefined) {
29025
30779
  if (typeof record.provider !== "string" || record.provider.trim() === "") {
29026
30780
  throw new ConfigError(`${path}.provider must be a non-empty string`);
29027
30781
  }
29028
- out.provider = record.provider.trim();
30782
+ provider = record.provider.trim();
29029
30783
  }
29030
- if (!SUPPORTED_RERANK_PROVIDERS.includes(out.provider ?? "")) {
30784
+ if (!SUPPORTED_RERANK_PROVIDERS.includes(provider)) {
29031
30785
  throw new ConfigError(`${path}.provider must be one of: ${SUPPORTED_RERANK_PROVIDERS.join(", ")}`);
29032
30786
  }
30787
+ const out = {
30788
+ provider,
30789
+ model: defaultRerankModel(provider),
30790
+ apiKeyEnv: defaultRerankApiKeyEnv(provider),
30791
+ topN: DEFAULT_RERANK_TOP_N
30792
+ };
30793
+ if (record.enabled !== undefined) {
30794
+ if (typeof record.enabled !== "boolean")
30795
+ throw new ConfigError(`${path}.enabled must be a boolean`);
30796
+ out.enabled = record.enabled;
30797
+ }
29033
30798
  if (record.model !== undefined) {
29034
30799
  if (typeof record.model !== "string" || record.model.trim() === "") {
29035
30800
  throw new ConfigError(`${path}.model must be a non-empty string`);
@@ -29159,7 +30924,8 @@ function buildModelFromConfiguredEndpoint(reference) {
29159
30924
  input: reference.input ?? ["text"],
29160
30925
  cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
29161
30926
  contextWindow: reference.contextWindow ?? 128000,
29162
- maxTokens: reference.maxTokens ?? 16384
30927
+ maxTokens: reference.maxTokens ?? 16384,
30928
+ ...reference.provider === PROVIDER_ID ? { inputLimits: AUTORAG_INPUT_LIMITS } : {}
29163
30929
  };
29164
30930
  }
29165
30931
  function mergeCatalogModel(catalog, reference) {
@@ -29210,8 +30976,8 @@ function getModelRuntime(agentDir) {
29210
30976
  return cached;
29211
30977
  const created = (async () => {
29212
30978
  const runtime = await ModelRuntime2.create({
29213
- authPath: join36(agentDir, "auth.json"),
29214
- modelsPath: join36(agentDir, "models.json"),
30979
+ authPath: join38(agentDir, "auth.json"),
30980
+ modelsPath: join38(agentDir, "models.json"),
29215
30981
  allowModelNetwork: false
29216
30982
  });
29217
30983
  await registerAutoRAGProvider(runtime);
@@ -29470,7 +31236,7 @@ function writeConfigObject(path, config) {
29470
31236
  if (config === null || typeof config !== "object" || Array.isArray(config)) {
29471
31237
  throw new ConfigError("Config file must be a JSON object");
29472
31238
  }
29473
- mkdirSync25(dirname23(path), { recursive: true });
31239
+ mkdirSync26(dirname25(path), { recursive: true });
29474
31240
  const contents = `${JSON.stringify(config, null, 2)}
29475
31241
  `;
29476
31242
  const lock = acquireConfigWriteLock(path);
@@ -29481,9 +31247,9 @@ function writeConfigObject(path, config) {
29481
31247
  }
29482
31248
  }
29483
31249
  function writeDefaultConfig(path, partial, opts = {}) {
29484
- const cwd = resolve15(opts.cwd ?? process.cwd());
31250
+ const cwd = resolve16(opts.cwd ?? process.cwd());
29485
31251
  const workspacePath = resolvePersistedPath(partial.workspacePath ?? ".", cwd);
29486
- const memoryPath = partial.memoryPath === undefined ? join36(resolveAutoRAGHome(opts.env), "memory.json") : resolvePersistedPath(partial.memoryPath, workspacePath);
31252
+ const memoryPath = partial.memoryPath === undefined ? join38(resolveAutoRAGHome(opts.env), "memory.json") : resolvePersistedPath(partial.memoryPath, workspacePath);
29487
31253
  const full = {
29488
31254
  searchPaths: resolveSearchPaths(partial.searchPaths ?? ["."], cwd),
29489
31255
  workspacePath,
@@ -29517,7 +31283,7 @@ function writeDefaultConfig(path, partial, opts = {}) {
29517
31283
  full.p2p = { enabled: false };
29518
31284
  full.jev = partial.jev ?? { backend: DEFAULT_JEV_BACKEND };
29519
31285
  full.queryDecomposition = partial.queryDecomposition ?? { model: { ...DEFAULT_QUERY_DECOMPOSITION_MODEL } };
29520
- mkdirSync25(dirname23(path), { recursive: true });
31286
+ mkdirSync26(dirname25(path), { recursive: true });
29521
31287
  const contents = `${JSON.stringify(full, null, 2)}
29522
31288
  `;
29523
31289
  const lock = acquireConfigWriteLock(path);
@@ -29545,8 +31311,9 @@ function writeDefaultConfig(path, partial, opts = {}) {
29545
31311
  }
29546
31312
  }
29547
31313
  var DEFAULT_CONFIG_FILENAME = "config.json", LEGACY_CONFIG_FILENAME = "autorag.config.json", CONFIG_LOCK_RETRY_MS = 10, CONFIG_LOCK_TIMEOUT_MS = 1e4, CONFIG_LOCK_STALE_MS = 30000, ConfigError, MINSYNC_REQUIRED_MESSAGE = "MinSync is required and cannot be disabled; remove `minSync: false` / `minSync.enabled: false` from the config.", AGENT_MODEL_APIS, AGENT_MODEL_INPUTS, API_KEY_ENV_PATTERN2, POSITIVE_INT_FIELDS, EMBEDDER_ALLOWLIST, MINSYNC_ALLOWLIST, MINSYNC_LEGACY_IGNORED, P2P_ALLOWLIST, P2P_QUOTA_ALLOWLIST, LIMITS_ALLOWLIST, LIMITS_PREFETCH_ALLOWLIST, JEV_CONFIG_FIELDS, JEV_BACKEND_NAMES, DEFAULT_JEV_BACKEND = "openrouter", DEFAULT_QUERY_DECOMPOSITION_MODEL, RERANK_ALLOWLIST, UNKNOWN_MODEL_HINT = "Add baseUrl (and optional api/apiKeyEnv) for an OpenAI-compatible endpoint outside the pi-ai catalog, or use a pi-ai catalog model id.", modelRuntimeCache;
29548
- var init_config3 = __esm(() => {
31314
+ var init_config4 = __esm(() => {
29549
31315
  init_local_model();
31316
+ init_config();
29550
31317
  init_provider();
29551
31318
  init_home();
29552
31319
  init_factory();
@@ -29628,7 +31395,7 @@ var init_config3 = __esm(() => {
29628
31395
  model: true,
29629
31396
  confidenceThreshold: true
29630
31397
  };
29631
- JEV_BACKEND_NAMES = ["typesafe", "openrouter", "vercel"];
31398
+ JEV_BACKEND_NAMES = ["typesafe", "openrouter", "vercel", "autorag"];
29632
31399
  DEFAULT_QUERY_DECOMPOSITION_MODEL = { provider: "openrouter", id: "qwen/qwen3.7-flash" };
29633
31400
  RERANK_ALLOWLIST = new Set(["enabled", "provider", "model", "apiKeyEnv", "baseUrl", "topN", "timeoutMs"]);
29634
31401
  modelRuntimeCache = new Map;
@@ -29718,9 +31485,9 @@ function createAutoRAGLite(options = {}) {
29718
31485
  var init_core = __esm(() => {
29719
31486
  init_agent();
29720
31487
  init_search_documents();
29721
- init_config3();
31488
+ init_config4();
29722
31489
  init_memory();
29723
- init_config3();
31490
+ init_config4();
29724
31491
  });
29725
31492
 
29726
31493
  // src/cli/output.ts
@@ -30165,16 +31932,16 @@ async function runEvidence(ctx) {
30165
31932
  }
30166
31933
  var init_evidence = __esm(() => {
30167
31934
  init_memory();
30168
- init_config3();
31935
+ init_config4();
30169
31936
  });
30170
31937
 
30171
31938
  // src/cli/commands/report.ts
30172
- import { readFileSync as readFileSync25 } from "node:fs";
31939
+ import { readFileSync as readFileSync26 } from "node:fs";
30173
31940
  import { Value as Value3 } from "typebox/value";
30174
31941
  function readReportInput(input) {
30175
31942
  if (typeof input === "string" && input.length > 0)
30176
- return readFileSync25(input, "utf8");
30177
- return readFileSync25(0, "utf8");
31943
+ return readFileSync26(input, "utf8");
31944
+ return readFileSync26(0, "utf8");
30178
31945
  }
30179
31946
  function describeSchemaErrors(value) {
30180
31947
  const errors = Value3.Errors(reportSchema, value);
@@ -30265,7 +32032,7 @@ var MAX_REPORTED_SCHEMA_ERRORS = 8;
30265
32032
  var init_report = __esm(() => {
30266
32033
  init_results();
30267
32034
  init_core();
30268
- init_config3();
32035
+ init_config4();
30269
32036
  });
30270
32037
 
30271
32038
  // src/mirror/index.ts
@@ -30504,8 +32271,8 @@ init_loader();
30504
32271
  init_evidence();
30505
32272
  init_report();
30506
32273
  init_dupey();
30507
- import { readFileSync as readFileSync26 } from "node:fs";
30508
- import { resolve as resolve17 } from "node:path";
32274
+ import { readFileSync as readFileSync27 } from "node:fs";
32275
+ import { resolve as resolve18 } from "node:path";
30509
32276
  import { fileURLToPath as fileURLToPath3 } from "node:url";
30510
32277
  import { McpServer } from "@modelcontextprotocol/server";
30511
32278
  import { z as z2 } from "zod";
@@ -30513,7 +32280,7 @@ import { z as z2 } from "zod";
30513
32280
  // src/filesystem/name-search.ts
30514
32281
  import { realpathSync as realpathSync6 } from "node:fs";
30515
32282
  import { lstat, readdir as readdir4 } from "node:fs/promises";
30516
- import { basename as basename15, dirname as dirname24, isAbsolute as isAbsolute10, join as join37, relative as relative6, resolve as resolve16, sep as sep7 } from "node:path";
32283
+ import { basename as basename15, dirname as dirname26, isAbsolute as isAbsolute11, join as join39, relative as relative6, resolve as resolve17, sep as sep8 } from "node:path";
30517
32284
  var SKIP_DIR_NAMES2 = {
30518
32285
  ".git": true,
30519
32286
  ".autorag": true,
@@ -30541,27 +32308,27 @@ function fsErrorCode(error) {
30541
32308
  return typeof code === "string" ? code : undefined;
30542
32309
  }
30543
32310
  function isContained(childReal, parentReal) {
30544
- const prefix = parentReal.endsWith(sep7) ? parentReal : `${parentReal}${sep7}`;
32311
+ const prefix = parentReal.endsWith(sep8) ? parentReal : `${parentReal}${sep8}`;
30545
32312
  return childReal === parentReal || childReal.startsWith(prefix);
30546
32313
  }
30547
32314
  function pinExcludedPath2(path) {
30548
- const resolvedPath = resolve16(path);
32315
+ const resolvedPath = resolve17(path);
30549
32316
  try {
30550
- return join37(realpathSync6(dirname24(resolvedPath)), basename15(resolvedPath));
32317
+ return join39(realpathSync6(dirname26(resolvedPath)), basename15(resolvedPath));
30551
32318
  } catch {
30552
32319
  return resolvedPath;
30553
32320
  }
30554
32321
  }
30555
32322
  function isExcluded2(absPath, excluded) {
30556
32323
  for (const excludedPath of excluded) {
30557
- const prefix = excludedPath.endsWith(sep7) ? excludedPath : `${excludedPath}${sep7}`;
32324
+ const prefix = excludedPath.endsWith(sep8) ? excludedPath : `${excludedPath}${sep8}`;
30558
32325
  if (absPath === excludedPath || absPath.startsWith(prefix))
30559
32326
  return true;
30560
32327
  }
30561
32328
  return false;
30562
32329
  }
30563
32330
  function matchesEntry(context, absPath, name) {
30564
- const haystackRaw = context.matchPath ? relative6(context.rootReal, absPath).split(sep7).join("/") : name;
32331
+ const haystackRaw = context.matchPath ? relative6(context.rootReal, absPath).split(sep8).join("/") : name;
30565
32332
  const haystack = context.matchCase ? haystackRaw : haystackRaw.toLowerCase();
30566
32333
  const needle = context.matchCase ? context.query : context.query.toLowerCase();
30567
32334
  return haystack.includes(needle);
@@ -30601,7 +32368,7 @@ async function walkDirectory(context, directory) {
30601
32368
  }
30602
32369
  if (entry.isSymbolicLink())
30603
32370
  continue;
30604
- const absPath = join37(directory, entry.name);
32371
+ const absPath = join39(directory, entry.name);
30605
32372
  if (entry.isDirectory()) {
30606
32373
  if (SKIP_DIR_NAMES2[entry.name] === true)
30607
32374
  continue;
@@ -30638,7 +32405,7 @@ async function pinConfiguredRoots(roots, diagnostics) {
30638
32405
  for (const root of roots) {
30639
32406
  if (typeof root !== "string" || root.length === 0)
30640
32407
  continue;
30641
- const resolvedRoot = resolve16(root);
32408
+ const resolvedRoot = resolve17(root);
30642
32409
  let real;
30643
32410
  try {
30644
32411
  real = realpathSync6(resolvedRoot);
@@ -30660,7 +32427,7 @@ async function pinConfiguredRoots(roots, diagnostics) {
30660
32427
  return outermost;
30661
32428
  }
30662
32429
  async function resolveRequestRoots(requestRoot, pinnedRoots) {
30663
- const candidates = isAbsolute10(requestRoot) ? [requestRoot] : pinnedRoots.map((root) => resolve16(root, requestRoot));
32430
+ const candidates = isAbsolute11(requestRoot) ? [requestRoot] : pinnedRoots.map((root) => resolve17(root, requestRoot));
30664
32431
  const allowed = [];
30665
32432
  for (const candidate of candidates) {
30666
32433
  let real;
@@ -30702,7 +32469,7 @@ async function filterFileNameSearchMatches(roots, candidates, excludePaths = [])
30702
32469
  continue;
30703
32470
  if (isExcluded2(real, excluded))
30704
32471
  continue;
30705
- const segments = real.split(sep7);
32472
+ const segments = real.split(sep8);
30706
32473
  if (segments.some((segment) => SKIP_DIR_NAMES2[segment] === true))
30707
32474
  continue;
30708
32475
  matches.push({ path: real, type: stat.isDirectory() ? "folder" : "file" });
@@ -30851,7 +32618,7 @@ function isToolEnabled(name, options) {
30851
32618
  }
30852
32619
  function readPackageVersion() {
30853
32620
  try {
30854
- const manifest = JSON.parse(readFileSync26(fileURLToPath3(new URL("../../package.json", import.meta.url)), "utf8"));
32621
+ const manifest = JSON.parse(readFileSync27(fileURLToPath3(new URL("../../package.json", import.meta.url)), "utf8"));
30855
32622
  if (typeof manifest === "object" && manifest !== null && "version" in manifest && typeof manifest.version === "string") {
30856
32623
  return manifest.version;
30857
32624
  }
@@ -31090,7 +32857,7 @@ function createAutoRAGMcpServer(lite, options = {}) {
31090
32857
  ...configuredDupey.timeoutMs !== undefined ? { timeoutMs: configuredDupey.timeoutMs } : {},
31091
32858
  ...configuredDupey.run !== undefined ? { run: configuredDupey.run } : {}
31092
32859
  };
31093
- const roots = lite.config.searchPaths.map((path) => resolve17(path));
32860
+ const roots = lite.config.searchPaths.map((path) => resolve18(path));
31094
32861
  const scans = await Promise.all(roots.map((root) => scanWithDupey(root, dupeyOptions)));
31095
32862
  return jsonResult({
31096
32863
  ok: true,
@@ -31232,6 +32999,8 @@ export {
31232
32999
  ConnectorDatasourceSkill,
31233
33000
  ConnectorLexicalMethod,
31234
33001
  DATASOURCE_CHUNKS_SEGMENT,
33002
+ DEFAULT_AUTORAG_RERANK_API_KEY_ENV,
33003
+ DEFAULT_AUTORAG_RERANK_MODEL,
31235
33004
  DEFAULT_BASE_URL,
31236
33005
  DEFAULT_CONTEXT_WINDOW,
31237
33006
  DEFAULT_DISCRAWL_EMBEDDING_MODEL,
@@ -31331,7 +33100,7 @@ export {
31331
33100
  WacrawlSkill,
31332
33101
  base64UrlEncode,
31333
33102
  bm25Scores,
31334
- boundDiagnosticText2 as boundDiagnosticText,
33103
+ boundDiagnosticText,
31335
33104
  buildAgentOptions,
31336
33105
  buildAuthorizeUrl,
31337
33106
  buildDatasourceChunkSource,
@@ -31361,6 +33130,8 @@ export {
31361
33130
  datasourceChunkStorePath,
31362
33131
  datasourceSourceHasFragment,
31363
33132
  datasourceSourcePath,
33133
+ defaultRerankApiKeyEnv,
33134
+ defaultRerankModel,
31364
33135
  derivedDatasourceIds,
31365
33136
  describeRetrievalError,
31366
33137
  detectMirrorStaleness,