@autorag/librarian 2.10.0 → 2.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/LICENSE +195 -21
  2. package/NOTICE +17 -21
  3. package/README.md +335 -161
  4. package/dist/agent/agent.d.ts +67 -2
  5. package/dist/agent/context-token-guard.d.ts +23 -0
  6. package/dist/agent/jev-extension.d.ts +5 -4
  7. package/dist/agent/jikji-find-tool.d.ts +1 -0
  8. package/dist/agent/pi-session.d.ts +35 -2
  9. package/dist/agent/request-limits.d.ts +20 -0
  10. package/dist/agent/search-all-tool.d.ts +1 -0
  11. package/dist/agent/search-single-datasource-tool.d.ts +1 -0
  12. package/dist/cli/config.d.ts +13 -11
  13. package/dist/cli/index.js +2573 -800
  14. package/dist/cloud/config.d.ts +18 -0
  15. package/dist/cloud/jev-backend.d.ts +90 -0
  16. package/dist/core.js +2185 -418
  17. package/dist/datasource/crawler-skill.d.ts +17 -0
  18. package/dist/datasource/sidecar/config.d.ts +26 -0
  19. package/dist/datasource/sidecar/descriptors.d.ts +67 -0
  20. package/dist/datasource/sidecar/fusion.d.ts +25 -0
  21. package/dist/datasource/sidecar/index.d.ts +131 -0
  22. package/dist/datasource/sidecar/methods.d.ts +27 -0
  23. package/dist/datasource/types.d.ts +3 -1
  24. package/dist/embedding-runtime/index.d.ts +6 -6
  25. package/dist/embedding-runtime/manifest.d.ts +0 -12
  26. package/dist/embedding-runtime/types.d.ts +1 -1
  27. package/dist/index.js +2210 -439
  28. package/dist/mcp/index.js +2205 -438
  29. package/dist/minsync/client.d.ts +3 -3
  30. package/dist/minsync/installer.d.ts +1 -1
  31. package/dist/minsync/process.d.ts +1 -0
  32. package/dist/minsync/types.d.ts +1 -1
  33. package/dist/parser/defaults.d.ts +6 -4
  34. package/dist/parser/ocr-engines.d.ts +6 -0
  35. package/dist/retrieval/index.d.ts +1 -1
  36. package/dist/retrieval/rerank.d.ts +20 -3
  37. package/licenses/embedding-assets.json +0 -35
  38. package/package.json +24 -27
  39. package/skills/autorag/SKILL.md +1 -1
  40. package/skills/autorag-doctor/SKILL.md +1 -1
  41. package/skills/autorag-lite-setup/SKILL.md +2 -2
  42. package/skills/autorag-setup/SKILL.md +2 -2
  43. package/licenses/gemma-notice.txt +0 -24
  44. package/licenses/gemma-section-3.2-flow-down.txt +0 -9
  45. package/licenses/gemma-terms-of-use.html +0 -4750
  46. package/licenses/gemma-terms-of-use.txt +0 -158
package/dist/core.js CHANGED
@@ -61,6 +61,7 @@ function datasourceCodeToRetrieval(code) {
61
61
  case "datasource-auth-error":
62
62
  case "datasource-embedding-egress-rejected":
63
63
  case "datasource-index-failed":
64
+ case "datasource-index-partial":
64
65
  case "datasource-permission-denied":
65
66
  case "datasource-remote-embedding-rejected":
66
67
  return "retrieval-method-failed";
@@ -1112,19 +1113,6 @@ var init_manifest = __esm(() => {
1112
1113
  artifactSha256: "06507c7b42688469c4e7298b0a1e16deff06caf291cf0a5b278c308249c3e439",
1113
1114
  backend: "auto",
1114
1115
  protocolCapabilities
1115
- },
1116
- "embeddinggemma-300m": {
1117
- profileId: "embeddinggemma-300m",
1118
- provider: "google",
1119
- model: "embeddinggemma-300M-Q8_0.gguf",
1120
- dimension: 768,
1121
- queryPrefix: "task: search result | query: ",
1122
- passagePrefix: "title: none | text: ",
1123
- runtimeBuild: RUNTIME_BUILD,
1124
- modelRevision: "0f741b5a6585bd53aeb15cd1372c56f2a0f65e12",
1125
- artifactSha256: "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63",
1126
- backend: "auto",
1127
- protocolCapabilities
1128
1116
  }
1129
1117
  };
1130
1118
  runtimeAssets = {
@@ -1170,17 +1158,6 @@ var init_manifest = __esm(() => {
1170
1158
  unsuitableUse: "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256.",
1171
1159
  upstreamCardUrl: "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B",
1172
1160
  modelCard: "docs/model-cards/qwen3-embedding-0.6b.md"
1173
- },
1174
- embeddinggemma: {
1175
- url: "https://huggingface.co/ggml-org/embeddinggemma-300M-GGUF/resolve/0f741b5a6585bd53aeb15cd1372c56f2a0f65e12/embeddinggemma-300M-Q8_0.gguf",
1176
- revision: "0f741b5a6585bd53aeb15cd1372c56f2a0f65e12",
1177
- sha256: "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63",
1178
- license: "Gemma Terms of Use",
1179
- noticeReference: "Google EmbeddingGemma model card and Gemma Terms of Use",
1180
- intendedUse: "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.",
1181
- unsuitableUse: "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256, uses restricted by the Gemma Prohibited Use Policy.",
1182
- upstreamCardUrl: "https://ai.google.dev/gemma/docs/embeddinggemma/model_card",
1183
- modelCard: "docs/model-cards/embeddinggemma-300m.md"
1184
1161
  }
1185
1162
  };
1186
1163
  });
@@ -1618,7 +1595,7 @@ var init_supervisor = __esm(() => {
1618
1595
  // src/embedding-runtime/index.ts
1619
1596
  import { basename as basename4 } from "node:path";
1620
1597
  function modelAsset(profile) {
1621
- return profile.profileId === "qwen3-embedding-0.6b" ? { id: profile.profileId, filename: basename4(profile.model), ...MODEL_ASSETS.qwen3 } : { id: profile.profileId, filename: basename4(profile.model), ...MODEL_ASSETS.embeddinggemma };
1598
+ return { id: profile.profileId, filename: basename4(profile.model), ...MODEL_ASSETS.qwen3 };
1622
1599
  }
1623
1600
  function runtimeAsset(platform, backend) {
1624
1601
  const selected = platform ?? "darwin-arm64-metal";
@@ -5020,7 +4997,12 @@ var init_local_embedder = __esm(() => {
5020
4997
  // src/minsync/process.ts
5021
4998
  import { spawn as spawn7 } from "node:child_process";
5022
4999
  function spawnProcess(command, args, cwd, options = {}) {
5023
- return new Promise((resolve) => {
5000
+ return new Promise((resolve, reject) => {
5001
+ const { signal } = options;
5002
+ if (signal?.aborted) {
5003
+ reject(abortError(signal));
5004
+ return;
5005
+ }
5024
5006
  const portable = portableSpawnCommand(command, args);
5025
5007
  const child = spawn7(portable.command, [...portable.args], { cwd, stdio: ["ignore", "pipe", "pipe"] });
5026
5008
  let stdout = "";
@@ -5038,15 +5020,26 @@ function spawnProcess(command, args, cwd, options = {}) {
5038
5020
  timedOut = true;
5039
5021
  child.kill("SIGTERM");
5040
5022
  }, options.timeoutMs) : undefined;
5023
+ const onAbort = () => {
5024
+ child.kill("SIGTERM");
5025
+ };
5026
+ signal?.addEventListener("abort", onAbort, { once: true });
5027
+ const settle = () => {
5028
+ clearTimeout(timer);
5029
+ signal?.removeEventListener("abort", onAbort);
5030
+ };
5041
5031
  child.on("error", (error) => {
5042
- if (timer)
5043
- clearTimeout(timer);
5044
- resolve({ ok: false, stdout, stderr: error.message, code: null });
5032
+ settle();
5033
+ if (signal?.aborted)
5034
+ reject(abortError(signal));
5035
+ else
5036
+ resolve({ ok: false, stdout, stderr: error.message, code: null });
5045
5037
  });
5046
5038
  child.on("close", (code) => {
5047
- if (timer)
5048
- clearTimeout(timer);
5049
- if (timedOut) {
5039
+ settle();
5040
+ if (signal?.aborted) {
5041
+ reject(abortError(signal));
5042
+ } else if (timedOut) {
5050
5043
  resolve({ ok: false, stdout, stderr: "process timed out", code: null, timedOut: true });
5051
5044
  } else {
5052
5045
  resolve({ ok: code === 0, stdout, stderr, code });
@@ -5054,6 +5047,9 @@ function spawnProcess(command, args, cwd, options = {}) {
5054
5047
  });
5055
5048
  });
5056
5049
  }
5050
+ function abortError(signal) {
5051
+ return signal.reason instanceof Error ? signal.reason : new DOMException("The operation was aborted", "AbortError");
5052
+ }
5057
5053
  var init_process = __esm(() => {
5058
5054
  init_portable_spawn();
5059
5055
  });
@@ -5286,10 +5282,12 @@ class MinSyncClient {
5286
5282
  } : {}
5287
5283
  };
5288
5284
  }
5289
- async query(text, topK, mode = "vector") {
5285
+ async query(text, topK, mode = "vector", signal) {
5290
5286
  if (!existsSync11(this.binaryPath))
5291
5287
  throw minSyncBinaryMissingError();
5292
- const result = await this.spawn(["query", "--format", "json", "--mode", mode, "-k", String(topK), text]);
5288
+ const result = await this.spawn(["query", "--format", "json", "--mode", mode, "-k", String(topK), text], {
5289
+ ...signal === undefined ? {} : { signal }
5290
+ });
5293
5291
  if (!result.ok)
5294
5292
  throw new MinSyncQueryError(result.code, result.stderr);
5295
5293
  return parseQueryHits(result.stdout);
@@ -5362,7 +5360,7 @@ function isIdentity(value) {
5362
5360
  function isRecord4(value) {
5363
5361
  return typeof value === "object" && value !== null;
5364
5362
  }
5365
- var MINSYNC_OLLAMA_MIGRATION_MESSAGE = "This workspace uses the legacy 768-dimensional Ollama/TEI embedding path. Reindex explicitly, or pin an explicit profile config before using the new default runtime.", API_KEY_ENV_PATTERN, MinSyncQueryError;
5363
+ var MINSYNC_OLLAMA_MIGRATION_MESSAGE = "This workspace uses the legacy 768-dimensional Ollama/TEI embedding path. Reindex explicitly, or pin an explicit Ollama/TEI embedder config before using the new default runtime.", API_KEY_ENV_PATTERN, MinSyncQueryError;
5366
5364
  var init_client4 = __esm(() => {
5367
5365
  init_embedder_config();
5368
5366
  init_errors();
@@ -5606,7 +5604,7 @@ function isGitHubAsset(value) {
5606
5604
  function isRecord5(value) {
5607
5605
  return typeof value === "object" && value !== null;
5608
5606
  }
5609
- var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.4.6", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
5607
+ var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.4.7", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
5610
5608
  var init_installer2 = __esm(() => {
5611
5609
  init_process();
5612
5610
  SHA256_HEX_PATTERN = /^[a-f0-9]{64}$/;
@@ -5915,11 +5913,11 @@ class MinSyncVectorMethod {
5915
5913
  });
5916
5914
  let hits;
5917
5915
  try {
5918
- hits = await client.query(query, queryK, this.mode);
5916
+ hits = await client.query(query, queryK, this.mode, options.signal);
5919
5917
  } catch (error) {
5920
- if (this.mode !== "hybrid")
5918
+ if (this.mode !== "hybrid" || options.signal?.aborted)
5921
5919
  throw error;
5922
- hits = await client.query(query, queryK, "bm25");
5920
+ hits = await client.query(query, queryK, "bm25", options.signal);
5923
5921
  }
5924
5922
  const results = [];
5925
5923
  for (const hit of hits) {
@@ -6242,10 +6240,16 @@ async function runTesseract(recognition, signal, cleanupResolve, cleanupReject)
6242
6240
  };
6243
6241
  signal.addEventListener("abort", abort, { once: true });
6244
6242
  try {
6243
+ const tooSmall = ocrImageTooSmallMessage(recognition.bytes);
6244
+ if (tooSmall !== undefined)
6245
+ throw new Error(tooSmall);
6245
6246
  const { cachePath } = recognition;
6246
6247
  if (cachePath !== undefined)
6247
6248
  await mkdir3(cachePath, { recursive: true });
6248
- worker = await createWorker(recognition.language, undefined, cachePath === undefined ? {} : { cachePath });
6249
+ worker = await createWorker(recognition.language, undefined, {
6250
+ ...cachePath === undefined ? {} : { cachePath },
6251
+ errorHandler: containTesseractWorkerError
6252
+ });
6249
6253
  if (signal.aborted)
6250
6254
  throw new Error("OCR aborted before worker was ready");
6251
6255
  const result = await worker.recognize(Buffer.from(recognition.bytes));
@@ -6255,6 +6259,129 @@ async function runTesseract(recognition, signal, cleanupResolve, cleanupReject)
6255
6259
  await terminate().then(cleanupResolve, cleanupReject);
6256
6260
  }
6257
6261
  }
6262
+ function containTesseractWorkerError(_payload) {}
6263
+ function ocrImageTooSmallMessage(bytes) {
6264
+ const size = readImagePixelSize(bytes);
6265
+ if (size === undefined || size.width >= MIN_LEPTONICA_SIDE_PIXELS && size.height >= MIN_LEPTONICA_SIDE_PIXELS) {
6266
+ return;
6267
+ }
6268
+ return `Image is too small to scale (${size.width}x${size.height} vs minimum of ${MIN_LEPTONICA_SIDE_PIXELS} pixels).`;
6269
+ }
6270
+ function readImagePixelSize(bytes) {
6271
+ if (bytes.length >= 8 && bytes[0] === 137 && bytes[1] === 80)
6272
+ return readPngSize(bytes);
6273
+ if (bytes.length >= 2 && bytes[0] === 255 && bytes[1] === 216)
6274
+ return readJpegSize(bytes);
6275
+ if (bytes.length >= 12 && bytes[0] === 82 && bytes[1] === 73)
6276
+ return readWebpSize(bytes);
6277
+ return;
6278
+ }
6279
+ function readPngSize(bytes) {
6280
+ if (bytes.length < 24)
6281
+ return;
6282
+ if (bytes[0] !== 137 || bytes[1] !== 80 || bytes[2] !== 78 || bytes[3] !== 71 || bytes[4] !== 13 || bytes[5] !== 10 || bytes[6] !== 26 || bytes[7] !== 10) {
6283
+ return;
6284
+ }
6285
+ if (bytes[12] !== 73 || bytes[13] !== 72 || bytes[14] !== 68 || bytes[15] !== 82)
6286
+ return;
6287
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
6288
+ return { width: view.getUint32(16), height: view.getUint32(20) };
6289
+ }
6290
+ function readJpegSize(bytes) {
6291
+ if (bytes.length < 4 || bytes[0] !== 255 || bytes[1] !== 216)
6292
+ return;
6293
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
6294
+ let offset = 2;
6295
+ while (offset + 1 < bytes.length) {
6296
+ if (bytes[offset] !== 255)
6297
+ return;
6298
+ offset += 1;
6299
+ while (offset < bytes.length && bytes[offset] === 255)
6300
+ offset += 1;
6301
+ if (offset >= bytes.length)
6302
+ return;
6303
+ const marker = bytes[offset] ?? 0;
6304
+ offset += 1;
6305
+ if (marker === 0)
6306
+ return;
6307
+ if (marker === 1 || marker === 217 || marker >= 208 && marker <= 216)
6308
+ continue;
6309
+ if (offset + 2 > bytes.length)
6310
+ return;
6311
+ const segmentLength = view.getUint16(offset);
6312
+ if (segmentLength < 2)
6313
+ return;
6314
+ if (isJpegStartOfFrame(marker)) {
6315
+ if (segmentLength < 7 || offset + 7 > bytes.length)
6316
+ return;
6317
+ return { width: view.getUint16(offset + 5), height: view.getUint16(offset + 3) };
6318
+ }
6319
+ if (marker === 218)
6320
+ return;
6321
+ const next = offset + segmentLength;
6322
+ if (next <= offset || next > bytes.length)
6323
+ return;
6324
+ offset = next;
6325
+ }
6326
+ return;
6327
+ }
6328
+ function isJpegStartOfFrame(marker) {
6329
+ return marker >= 192 && marker <= 207 && marker !== 196 && marker !== 200 && marker !== 204;
6330
+ }
6331
+ function readWebpSize(bytes) {
6332
+ if (bytes.length < 20 || !asciiAt(bytes, 0, "RIFF") || !asciiAt(bytes, 8, "WEBP"))
6333
+ return;
6334
+ const chunkSize = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(16, true);
6335
+ if (asciiAt(bytes, 12, "VP8X"))
6336
+ return readVp8xSize(bytes, chunkSize);
6337
+ if (asciiAt(bytes, 12, "VP8L"))
6338
+ return readVp8lSize(bytes, chunkSize);
6339
+ if (asciiAt(bytes, 12, "VP8 "))
6340
+ return readVp8Size(bytes, chunkSize);
6341
+ return;
6342
+ }
6343
+ function readVp8xSize(bytes, chunkSize) {
6344
+ if (!webpPayloadFits(bytes, chunkSize, 10))
6345
+ return;
6346
+ const payload = 20;
6347
+ const width = 1 + readU24LE(bytes, payload + 4);
6348
+ const height = 1 + readU24LE(bytes, payload + 7);
6349
+ return { width, height };
6350
+ }
6351
+ function readVp8lSize(bytes, chunkSize) {
6352
+ if (!webpPayloadFits(bytes, chunkSize, 5) || bytes[20] !== 47)
6353
+ return;
6354
+ const b1 = bytes[21] ?? 0;
6355
+ const b2 = bytes[22] ?? 0;
6356
+ const b3 = bytes[23] ?? 0;
6357
+ const b4 = bytes[24] ?? 0;
6358
+ const width = 1 + (b1 | (b2 & 63) << 8);
6359
+ const height = 1 + (b2 >> 6 | b3 << 2 | (b4 & 15) << 10);
6360
+ return { width, height };
6361
+ }
6362
+ function readVp8Size(bytes, chunkSize) {
6363
+ if (!webpPayloadFits(bytes, chunkSize, 10) || (bytes[20] & 1) !== 0)
6364
+ return;
6365
+ if (bytes[23] !== 157 || bytes[24] !== 1 || bytes[25] !== 42)
6366
+ return;
6367
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
6368
+ return { width: view.getUint16(26, true) & 16383, height: view.getUint16(28, true) & 16383 };
6369
+ }
6370
+ function webpPayloadFits(bytes, chunkSize, needed) {
6371
+ return chunkSize >= needed && bytes.length >= 20 + needed;
6372
+ }
6373
+ function readU24LE(bytes, offset) {
6374
+ return (bytes[offset] ?? 0) | (bytes[offset + 1] ?? 0) << 8 | (bytes[offset + 2] ?? 0) << 16;
6375
+ }
6376
+ function asciiAt(bytes, offset, text) {
6377
+ if (offset + text.length > bytes.length)
6378
+ return false;
6379
+ for (let index = 0;index < text.length; index += 1) {
6380
+ if (bytes[offset + index] !== text.charCodeAt(index))
6381
+ return false;
6382
+ }
6383
+ return true;
6384
+ }
6258
6385
  function withTimeout(operation, timeoutMs, onTimeout) {
6259
6386
  return new Promise((resolve, reject) => {
6260
6387
  const timeout = setTimeout(() => {
@@ -6270,7 +6397,7 @@ function withTimeout(operation, timeoutMs, onTimeout) {
6270
6397
  });
6271
6398
  });
6272
6399
  }
6273
- var TESSERACT_LANGUAGES, BUILTIN_OCR_LANGUAGES, DEFAULT_TESSERACT_TIMEOUT_MS = 30000;
6400
+ var TESSERACT_LANGUAGES, BUILTIN_OCR_LANGUAGES, DEFAULT_TESSERACT_TIMEOUT_MS = 30000, MIN_LEPTONICA_SIDE_PIXELS = 3;
6274
6401
  var init_ocr_engines = __esm(() => {
6275
6402
  TESSERACT_LANGUAGES = {
6276
6403
  ko: "kor",
@@ -6479,8 +6606,8 @@ function createDefaultParserRegistry(options = {}) {
6479
6606
  const kordocOptions = {
6480
6607
  ...options.kordoc,
6481
6608
  images: ocrEnabled,
6482
- ...options.kordoc?.ocr === undefined && ocrEnabled ? {
6483
- ocr: builtinCovers ? true : createTesseractOcrProvider({
6609
+ ...options.kordoc?.ocr === undefined ? {
6610
+ ocr: !ocrEnabled ? false : builtinCovers ? true : createTesseractOcrProvider({
6484
6611
  languages,
6485
6612
  timeoutMs: options.ocr?.timeoutMs,
6486
6613
  cachePath: options.ocr?.cachePath
@@ -6618,6 +6745,23 @@ async function syncParsedMirrors(options) {
6618
6745
  checkpoint();
6619
6746
  continue;
6620
6747
  }
6748
+ const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
6749
+ if (tooSmallImage !== undefined) {
6750
+ deleted += removePrevious(options.root, previous, entry.virtualPath);
6751
+ handledPrevious.add(entry.virtualPath);
6752
+ skipped += 1;
6753
+ recordSkip(entry, "parser-skipped");
6754
+ diagnostics.push({
6755
+ code: "parser-skipped",
6756
+ severity: "warning",
6757
+ message: tooSmallImage,
6758
+ source: entry.virtualPath
6759
+ });
6760
+ sinceCheckpoint += 1;
6761
+ if (sinceCheckpoint >= MIRROR_CHECKPOINT_EVERY)
6762
+ checkpoint();
6763
+ continue;
6764
+ }
6621
6765
  let parsed;
6622
6766
  try {
6623
6767
  parsed = await parser.parse({ virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes });
@@ -6719,6 +6863,11 @@ async function detectMirrorStaleness(options) {
6719
6863
  }
6720
6864
  return diagnostics;
6721
6865
  }
6866
+ function standaloneImageTooSmallMessage(virtualPath, bytes) {
6867
+ if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname4(virtualPath))))
6868
+ return;
6869
+ return ocrImageTooSmallMessage(bytes);
6870
+ }
6722
6871
  function sourceReadFailure(error) {
6723
6872
  if (!(error instanceof Error))
6724
6873
  return;
@@ -6814,11 +6963,13 @@ function removeFile(path) {
6814
6963
  if (existsSync15(path))
6815
6964
  rmSync6(path, { force: true });
6816
6965
  }
6817
- var DEFAULT_MAX_SOURCE_BYTES, MIRROR_CHECKPOINT_EVERY = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.";
6966
+ var DEFAULT_MAX_SOURCE_BYTES, MIRROR_CHECKPOINT_EVERY = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", STANDALONE_OCR_IMAGE_EXTENSIONS;
6818
6967
  var init_sync = __esm(() => {
6819
6968
  init_source_paths();
6820
6969
  init_defaults();
6821
6970
  init_errors2();
6971
+ init_kordoc();
6972
+ init_ocr_engines();
6822
6973
  init_text();
6823
6974
  init_index_store();
6824
6975
  init_paths();
@@ -6846,6 +6997,7 @@ var init_sync = __esm(() => {
6846
6997
  "GrShaderCache",
6847
6998
  "ShaderCache"
6848
6999
  ]);
7000
+ STANDALONE_OCR_IMAGE_EXTENSIONS = new Set(KORDOC_IMAGE_EXTENSIONS);
6849
7001
  });
6850
7002
 
6851
7003
  // src/observability/run-log.ts
@@ -7711,8 +7863,36 @@ class RetrievalMethodRegistry {
7711
7863
  }
7712
7864
  }
7713
7865
 
7866
+ // src/cloud/config.ts
7867
+ function normalizeBaseUrl(value) {
7868
+ const trimmed = value.trim();
7869
+ let end = trimmed.length;
7870
+ while (end > 0 && trimmed.charCodeAt(end - 1) === 47)
7871
+ end--;
7872
+ return trimmed.slice(0, end);
7873
+ }
7874
+ function resolveBaseUrl(env) {
7875
+ const configured = env[AUTORAG_BASE_URL_ENV];
7876
+ const value = typeof configured === "string" && configured.trim() !== "" ? configured : DEFAULT_BASE_URL;
7877
+ return normalizeBaseUrl(value);
7878
+ }
7879
+ var PROVIDER_ID = "autorag", PROVIDER_NAME = "AutoRAG", DEFAULT_BASE_URL = "https://api.dazziapp.com", AUTORAG_BASE_URL_ENV = "AUTORAG_BASE_URL", AUTORAG_API_KEY_ENV = "AUTORAG_API_KEY", API_PATH = "/v1", MODELS_PATH = "/v1/models", AUTHORIZE_PATH = "/desktop/authorize", TOKEN_PATH = "/api/desktop/token", AUTORAG_INPUT_LIMITS, OAUTH_CLIENT_ID = "agent", API_KEY_CONFIG_VALUE;
7880
+ var init_config = __esm(() => {
7881
+ AUTORAG_INPUT_LIMITS = {
7882
+ maxRequestBytes: 3500000,
7883
+ images: { resize: { maxBytes: 1e6 } }
7884
+ };
7885
+ API_KEY_CONFIG_VALUE = `$${AUTORAG_API_KEY_ENV}`;
7886
+ });
7887
+
7714
7888
  // src/retrieval/rerank.ts
7715
7889
  import { OpenRouter } from "@openrouter/sdk";
7890
+ function defaultRerankModel(provider) {
7891
+ return provider === PROVIDER_ID ? DEFAULT_AUTORAG_RERANK_MODEL : DEFAULT_RERANK_MODEL;
7892
+ }
7893
+ function defaultRerankApiKeyEnv(provider) {
7894
+ return provider === PROVIDER_ID ? DEFAULT_AUTORAG_RERANK_API_KEY_ENV : DEFAULT_RERANK_API_KEY_ENV;
7895
+ }
7716
7896
  function createReranker(config) {
7717
7897
  if (config === false || config === undefined)
7718
7898
  return;
@@ -7720,30 +7900,36 @@ function createReranker(config) {
7720
7900
  if (!SUPPORTED_RERANK_PROVIDERS.includes(provider)) {
7721
7901
  throw new Error(`Unsupported rerank provider "${provider}"; supported providers: ${SUPPORTED_RERANK_PROVIDERS.join(", ")}`);
7722
7902
  }
7723
- return new OpenRouterReranker(config);
7903
+ return new OpenRouterReranker({ ...config, provider });
7904
+ }
7905
+ function autoragApiBaseUrl(env) {
7906
+ const root = resolveBaseUrl(env);
7907
+ return root.endsWith(API_PATH) ? root : `${root}${API_PATH}`;
7724
7908
  }
7725
7909
 
7726
7910
  class OpenRouterReranker {
7727
7911
  model;
7728
- provider = DEFAULT_RERANK_PROVIDER;
7912
+ provider;
7729
7913
  client;
7730
7914
  unavailableReason;
7731
7915
  constructor(options = {}) {
7732
- this.model = options.model ?? DEFAULT_RERANK_MODEL;
7916
+ this.provider = options.provider ?? DEFAULT_RERANK_PROVIDER;
7917
+ this.model = options.model ?? defaultRerankModel(this.provider);
7733
7918
  if (options.client !== undefined) {
7734
7919
  this.client = options.client;
7735
7920
  return;
7736
7921
  }
7737
- const apiKeyEnv = options.apiKeyEnv ?? DEFAULT_RERANK_API_KEY_ENV;
7738
7922
  const env = options.env ?? process.env;
7923
+ const apiKeyEnv = options.apiKeyEnv ?? defaultRerankApiKeyEnv(this.provider);
7739
7924
  const apiKey = options.apiKey ?? env[apiKeyEnv];
7740
7925
  if (apiKey === undefined || apiKey.length === 0) {
7741
7926
  this.unavailableReason = `no API key: set ${apiKeyEnv} or configure rerank.apiKeyEnv`;
7742
7927
  return;
7743
7928
  }
7929
+ const baseUrl = options.baseUrl ?? (this.provider === PROVIDER_ID ? autoragApiBaseUrl(env) : undefined);
7744
7930
  const sdkOptions = { apiKey };
7745
- if (options.baseUrl !== undefined)
7746
- sdkOptions.serverURL = options.baseUrl;
7931
+ if (baseUrl !== undefined)
7932
+ sdkOptions.serverURL = baseUrl;
7747
7933
  if (options.timeoutMs !== undefined)
7748
7934
  sdkOptions.timeoutMs = options.timeoutMs;
7749
7935
  const sdk = new OpenRouter(sdkOptions);
@@ -7808,9 +7994,10 @@ class OpenRouterReranker {
7808
7994
  return reranked;
7809
7995
  }
7810
7996
  }
7811
- var DEFAULT_RERANK_MODEL = "voyageai/rerank-3-lite", DEFAULT_RERANK_PROVIDER = "openrouter", DEFAULT_RERANK_API_KEY_ENV = "OPENROUTER_API_KEY", DEFAULT_RERANK_TOP_N = 25, SUPPORTED_RERANK_PROVIDERS;
7997
+ var DEFAULT_RERANK_MODEL = "voyageai/rerank-3-lite", DEFAULT_RERANK_PROVIDER = "openrouter", DEFAULT_RERANK_API_KEY_ENV = "OPENROUTER_API_KEY", DEFAULT_AUTORAG_RERANK_MODEL = "qwen/qwen3-reranker-8b", DEFAULT_AUTORAG_RERANK_API_KEY_ENV = "AUTORAG_API_KEY", DEFAULT_RERANK_TOP_N = 25, SUPPORTED_RERANK_PROVIDERS;
7812
7998
  var init_rerank = __esm(() => {
7813
- SUPPORTED_RERANK_PROVIDERS = [DEFAULT_RERANK_PROVIDER];
7999
+ init_config();
8000
+ SUPPORTED_RERANK_PROVIDERS = [DEFAULT_RERANK_PROVIDER, PROVIDER_ID];
7814
8001
  });
7815
8002
 
7816
8003
  // src/retrieval/selection.ts
@@ -9029,22 +9216,219 @@ var init_fsearch_search_tool = __esm(() => {
9029
9216
  });
9030
9217
  });
9031
9218
 
9032
- // src/cloud/config.ts
9033
- function normalizeBaseUrl(value) {
9034
- const trimmed = value.trim();
9035
- let end = trimmed.length;
9036
- while (end > 0 && trimmed.charCodeAt(end - 1) === 47)
9037
- end--;
9038
- return trimmed.slice(0, end);
9219
+ // src/cloud/jev-backend.ts
9220
+ import {
9221
+ BackendError
9222
+ } from "jev-use";
9223
+
9224
+ class AutoragJevBackend {
9225
+ name = "autorag";
9226
+ apiKey;
9227
+ baseUrl;
9228
+ defaultModel;
9229
+ timeoutMs;
9230
+ maxRetries;
9231
+ fetchImpl;
9232
+ constructor(options = {}) {
9233
+ const env = options.env ?? process.env;
9234
+ const apiKey = options.apiKey ?? env[AUTORAG_API_KEY_ENV];
9235
+ if (apiKey === undefined || apiKey.trim() === "") {
9236
+ throw new Error(`jev.backend "autorag" needs ${AUTORAG_API_KEY_ENV}.`);
9237
+ }
9238
+ this.apiKey = apiKey;
9239
+ this.baseUrl = options.baseUrl ?? resolveBaseUrl(env);
9240
+ this.defaultModel = options.defaultModel ?? AUTORAG_DEFAULT_JEV_MODEL;
9241
+ this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS6;
9242
+ this.maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
9243
+ this.fetchImpl = options.fetchImpl;
9244
+ }
9245
+ async judge(request) {
9246
+ const model = request.model ?? this.defaultModel;
9247
+ const body = toNativeBody(request.state, request.questions, model);
9248
+ const started = Date.now();
9249
+ const response = await this.postJson(`${this.baseUrl}${JEV_PATH}`, body);
9250
+ return {
9251
+ answers: parseNativeAnswers(this.name, response.answers, request.questions),
9252
+ model: response.model ?? model,
9253
+ latencyMs: Date.now() - started,
9254
+ usage: {
9255
+ inputTokens: response.usage?.input_tokens,
9256
+ outputTokens: response.usage?.output_tokens
9257
+ }
9258
+ };
9259
+ }
9260
+ async postJson(url, body) {
9261
+ const doFetch = this.fetchImpl ?? fetch;
9262
+ let lastError;
9263
+ for (let attempt = 0;attempt <= this.maxRetries; attempt += 1) {
9264
+ if (attempt > 0)
9265
+ await sleep3(retryDelay(attempt, lastError));
9266
+ const controller = new AbortController;
9267
+ const timer = setTimeout(() => controller.abort(), this.timeoutMs);
9268
+ try {
9269
+ const response = await doFetch(url, {
9270
+ method: "POST",
9271
+ headers: {
9272
+ "Content-Type": "application/json",
9273
+ Accept: "application/json",
9274
+ Authorization: `Bearer ${this.apiKey}`
9275
+ },
9276
+ body: JSON.stringify(body),
9277
+ signal: controller.signal
9278
+ });
9279
+ if (response.ok)
9280
+ return await response.json();
9281
+ const message = await extractErrorMessage(response);
9282
+ lastError = new BackendError(this.name, `HTTP ${response.status}: ${message}`, response.status);
9283
+ lastError.retryAfterMs = parseRetryAfter(response);
9284
+ if (!isRetryable(response.status) || attempt === this.maxRetries)
9285
+ throw lastError;
9286
+ } catch (error) {
9287
+ if (error instanceof BackendError) {
9288
+ if (attempt === this.maxRetries || !isRetryable(error.status))
9289
+ throw error;
9290
+ continue;
9291
+ }
9292
+ const aborted = error?.name === "AbortError";
9293
+ lastError = new BackendError(this.name, aborted ? `request timed out after ${this.timeoutMs}ms` : `network error: ${String(error)}`, undefined, error);
9294
+ if (attempt === this.maxRetries)
9295
+ throw lastError;
9296
+ } finally {
9297
+ clearTimeout(timer);
9298
+ }
9299
+ }
9300
+ throw lastError ?? new BackendError(this.name, "unreachable");
9301
+ }
9039
9302
  }
9040
- function resolveBaseUrl(env) {
9041
- const configured = env[AUTORAG_BASE_URL_ENV];
9042
- const value = typeof configured === "string" && configured.trim() !== "" ? configured : DEFAULT_BASE_URL;
9043
- return normalizeBaseUrl(value);
9303
+ function serializeState(state) {
9304
+ return typeof state === "string" ? state : JSON.stringify(state);
9044
9305
  }
9045
- var PROVIDER_ID = "autorag", PROVIDER_NAME = "AutoRAG", DEFAULT_BASE_URL = "https://api.dazziapp.com", AUTORAG_BASE_URL_ENV = "AUTORAG_BASE_URL", AUTORAG_API_KEY_ENV = "AUTORAG_API_KEY", API_PATH = "/v1", MODELS_PATH = "/v1/models", AUTHORIZE_PATH = "/desktop/authorize", TOKEN_PATH = "/api/desktop/token", OAUTH_CLIENT_ID = "agent", API_KEY_CONFIG_VALUE;
9046
- var init_config = __esm(() => {
9047
- API_KEY_CONFIG_VALUE = `$${AUTORAG_API_KEY_ENV}`;
9306
+ function toNativeQuestion(question) {
9307
+ switch (question.type) {
9308
+ case "noul":
9309
+ return {
9310
+ type: "noul",
9311
+ instructions: question.question,
9312
+ ...question.criteria ? { criteria: question.criteria } : {}
9313
+ };
9314
+ case "choice":
9315
+ return {
9316
+ type: "choice",
9317
+ instructions: question.question,
9318
+ criteria: Object.fromEntries(optionEntries(question.options ?? []).map(([label, meaning]) => [label, meaning || label]))
9319
+ };
9320
+ case "score":
9321
+ return {
9322
+ type: "score",
9323
+ instructions: question.question,
9324
+ criteria: question.levels ?? []
9325
+ };
9326
+ }
9327
+ }
9328
+ function toNativeBody(state, questions, model) {
9329
+ return {
9330
+ model,
9331
+ state: serializeState(state),
9332
+ questions: Object.fromEntries(questions.map((question) => [question.id, toNativeQuestion(question)]))
9333
+ };
9334
+ }
9335
+ function parseNativeAnswers(backend, answers, questions) {
9336
+ if (!answers || typeof answers !== "object") {
9337
+ throw new BackendError(backend, "response has no answers object");
9338
+ }
9339
+ const byId = answers;
9340
+ return questions.map((question) => parseNativeAnswer(backend, question, byId[question.id]));
9341
+ }
9342
+ function parseNativeAnswer(backend, question, answer) {
9343
+ if (!answer) {
9344
+ throw new BackendError(backend, `response missing answer for question "${question.id}"`);
9345
+ }
9346
+ switch (question.type) {
9347
+ case "noul":
9348
+ return {
9349
+ answer: numberOrThrow(backend, question.id, answer.noul),
9350
+ confidence: answer.confidence
9351
+ };
9352
+ case "choice":
9353
+ return {
9354
+ answer: stringOrThrow(backend, question.id, answer.choice),
9355
+ confidence: answer.confidence,
9356
+ distribution: answer.probabilities
9357
+ };
9358
+ case "score":
9359
+ return {
9360
+ answer: numberOrThrow(backend, question.id, answer.score),
9361
+ confidence: answer.confidence,
9362
+ distribution: answer.probabilities,
9363
+ legend: answer.legend
9364
+ };
9365
+ }
9366
+ }
9367
+ function optionEntries(options) {
9368
+ return Array.isArray(options) ? options.map((label) => [label, ""]) : Object.entries(options);
9369
+ }
9370
+ function numberOrThrow(backend, id, value) {
9371
+ if (typeof value !== "number" || !Number.isFinite(value)) {
9372
+ throw new BackendError(backend, `answer "${id}" has no numeric value`);
9373
+ }
9374
+ return value;
9375
+ }
9376
+ function stringOrThrow(backend, id, value) {
9377
+ if (typeof value !== "string" || !value) {
9378
+ throw new BackendError(backend, `answer "${id}" has no choice value`);
9379
+ }
9380
+ return value;
9381
+ }
9382
+ function isRetryable(status) {
9383
+ if (status === undefined)
9384
+ return true;
9385
+ return status === 408 || status === 429 || status >= 500;
9386
+ }
9387
+ function retryDelay(attempt, lastError) {
9388
+ if (lastError?.retryAfterMs !== undefined)
9389
+ return Math.min(lastError.retryAfterMs, 60000);
9390
+ const base = Math.min(RETRY_BASE_MS * 2 ** (attempt - 1), RETRY_MAX_MS);
9391
+ return base * (1 + Math.random() * 0.25);
9392
+ }
9393
+ function parseRetryAfter(response) {
9394
+ const milliseconds = response.headers.get("retry-after-ms");
9395
+ if (milliseconds && Number.isFinite(Number(milliseconds)))
9396
+ return Number(milliseconds);
9397
+ const seconds = response.headers.get("retry-after");
9398
+ if (seconds && Number.isFinite(Number(seconds)))
9399
+ return Number(seconds) * 1000;
9400
+ return;
9401
+ }
9402
+ async function extractErrorMessage(response) {
9403
+ try {
9404
+ const data = await response.json();
9405
+ for (const key of ["error", "message", "detail"]) {
9406
+ const value = data[key];
9407
+ if (typeof value === "string")
9408
+ return value;
9409
+ if (value && typeof value === "object") {
9410
+ const message = value.message;
9411
+ if (typeof message === "string")
9412
+ return message;
9413
+ }
9414
+ if (Array.isArray(value) && value.length > 0) {
9415
+ const first = value[0];
9416
+ if (typeof first?.msg === "string")
9417
+ return first.msg;
9418
+ }
9419
+ }
9420
+ return JSON.stringify(data).slice(0, 300);
9421
+ } catch {
9422
+ return response.statusText || "request failed";
9423
+ }
9424
+ }
9425
+ function sleep3(ms) {
9426
+ return new Promise((resolve) => setTimeout(resolve, ms));
9427
+ }
9428
+ var JEV_PATH, AUTORAG_DEFAULT_JEV_MODEL = "jev-1.13", DEFAULT_TIMEOUT_MS6 = 1e4, DEFAULT_MAX_RETRIES = 2, RETRY_BASE_MS = 500, RETRY_MAX_MS = 5000;
9429
+ var init_jev_backend = __esm(() => {
9430
+ init_config();
9431
+ JEV_PATH = `${API_PATH}/jev`;
9048
9432
  });
9049
9433
 
9050
9434
  // src/cloud/errors.ts
@@ -9175,7 +9559,8 @@ function mapServerModel(entry) {
9175
9559
  cost: readPricing(entry.pricing),
9176
9560
  reasoning: entry.reasoning === true,
9177
9561
  contextWindow: readPositiveNumber(entry.context_window) ?? DEFAULT_CONTEXT_WINDOW,
9178
- maxTokens: readPositiveNumber(entry.max_output_tokens) ?? DEFAULT_MAX_TOKENS
9562
+ maxTokens: readPositiveNumber(entry.max_output_tokens) ?? DEFAULT_MAX_TOKENS,
9563
+ inputLimits: AUTORAG_INPUT_LIMITS
9179
9564
  };
9180
9565
  }
9181
9566
  function buildStoreEntry(models, providerId, baseUrl) {
@@ -9214,7 +9599,8 @@ function modelsFromStore(stored) {
9214
9599
  cost: entry.cost,
9215
9600
  reasoning: entry.reasoning,
9216
9601
  contextWindow: entry.contextWindow,
9217
- maxTokens: entry.maxTokens
9602
+ maxTokens: entry.maxTokens,
9603
+ inputLimits: AUTORAG_INPUT_LIMITS
9218
9604
  });
9219
9605
  }
9220
9606
  return models;
@@ -9622,90 +10008,504 @@ var init_provider = __esm(() => {
9622
10008
  init_oauth_login();
9623
10009
  });
9624
10010
 
9625
- // src/agent/pi-session.ts
9626
- import { join as join21 } from "node:path";
9627
- import { streamSimple as compatStreamSimple } from "@earendil-works/pi-ai/compat";
9628
- import {
9629
- createAgentSession,
9630
- createAgentSessionFromServices,
9631
- createAgentSessionRuntime,
9632
- createAgentSessionServices,
9633
- DefaultResourceLoader,
9634
- getAgentDir,
9635
- ModelRuntime,
9636
- SessionManager,
9637
- SettingsManager
9638
- } from "@earendil-works/pi-coding-agent";
9639
- function resolveInteractiveTuiMode(settingsManager) {
9640
- const saved = settingsManager.getProjectSettings().tuiMode ?? settingsManager.getGlobalSettings().tuiMode;
9641
- return saved ?? "regular";
9642
- }
9643
- function toToolDefinition(tool) {
9644
- return {
9645
- name: tool.name,
9646
- label: tool.label,
9647
- description: tool.description,
9648
- parameters: tool.parameters,
9649
- ...tool.prepareArguments === undefined ? {} : { prepareArguments: tool.prepareArguments },
9650
- execute: async (toolCallId, params, signal, onUpdate) => {
9651
- return tool.execute(toolCallId, params, signal, onUpdate);
9652
- }
10011
+ // src/agent/context-token-guard.ts
10012
+ import { calculateContextTokens, estimateTokens } from "@earendil-works/pi-coding-agent";
10013
+ function createContextTokenGuardedTransform(model, base, fixedOverheadTokens = () => 0) {
10014
+ return async (messages) => {
10015
+ const transformed = base === undefined ? messages : await base(messages);
10016
+ const currentModel = typeof model === "function" ? model() : model;
10017
+ return guardAgentContextMessages(currentModel, transformed, transformed === messages, fixedOverheadTokens());
9653
10018
  };
9654
10019
  }
9655
- function modelConfig(model, configuredApiKey) {
9656
- return {
9657
- name: model.name,
9658
- baseUrl: model.baseUrl,
9659
- api: model.api,
9660
- ...configuredApiKey === undefined ? {} : { apiKey: configuredApiKey },
9661
- authHeader: false,
9662
- streamSimple: (requestModel, context, options) => compatStreamSimple(requestModel, context, {
9663
- ...options,
9664
- ...configuredApiKey === EMBEDDED_AUTH_MARKER ? { apiKey: undefined } : {}
9665
- }),
9666
- models: [
9667
- {
9668
- id: model.id,
9669
- name: model.name,
9670
- reasoning: model.reasoning,
9671
- input: [...model.input],
9672
- cost: model.cost,
9673
- contextWindow: model.contextWindow,
9674
- maxTokens: model.maxTokens,
9675
- ...model.thinkingLevelMap === undefined ? {} : { thinkingLevelMap: model.thinkingLevelMap }
10020
+ function guardAgentContextMessages(model, messages, useReportedUsage = true, fixedOverheadTokens = 0) {
10021
+ if (model === undefined || !Number.isFinite(model.contextWindow) || model.contextWindow <= 0 || messages.length === 0) {
10022
+ return messages;
10023
+ }
10024
+ const usableContext = Math.floor(model.contextWindow / CONTEXT_HEADROOM);
10025
+ const outputReserve = Math.min(usableContext, Number.isFinite(model.maxTokens) ? Math.max(0, model.maxTokens) : 0);
10026
+ const inputBudget = Math.max(0, usableContext - outputReserve);
10027
+ const budget = Math.max(0, inputBudget - fixedOverheadTokens);
10028
+ const reported = useReportedUsage && findLastAssistantUsage(messages) !== undefined;
10029
+ if (estimateContextMessageTokens(messages, useReportedUsage) <= (reported ? inputBudget : budget)) {
10030
+ return messages;
10031
+ }
10032
+ const candidates = [];
10033
+ const candidateIndexes = [];
10034
+ const base = messages.slice();
10035
+ for (const [index, message] of base.entries()) {
10036
+ const text = textContent(message);
10037
+ if (text === undefined)
10038
+ continue;
10039
+ const section = parseCandidateSection(message, text);
10040
+ if (section === undefined)
10041
+ continue;
10042
+ candidates.push(section);
10043
+ candidateIndexes.push(index);
10044
+ base[index] = replaceTextContent(message, `${section.prefix}${section.suffix}`);
10045
+ }
10046
+ let guarded = messages;
10047
+ if (candidates.length > 0) {
10048
+ const fixedTokens = estimateContextMessageTokens(base, false);
10049
+ const maximumOmitted = candidates.reduce((total, section) => total + section.blocks.length, 0);
10050
+ const omissionMarker = (count) => `
10051
+
10052
+ ${GUARD_MARKER} Retrieval candidates omitted: ${count}.
10053
+ `;
10054
+ let remaining = Math.max(0, budget - fixedTokens - Math.ceil(omissionMarker(maximumOmitted).length / 4));
10055
+ const kept = candidates.map(() => []);
10056
+ let omitted = 0;
10057
+ let firstOmitted = -1;
10058
+ for (const [sectionIndex, section] of candidates.entries()) {
10059
+ for (const block of section.blocks) {
10060
+ const tokens = Math.max(1, Math.ceil(block.length / 4));
10061
+ if (tokens <= remaining) {
10062
+ kept[sectionIndex]?.push(block);
10063
+ remaining -= tokens;
10064
+ continue;
10065
+ }
10066
+ omitted += section.blocks.length - (kept[sectionIndex]?.length ?? 0);
10067
+ for (const later of candidates.slice(sectionIndex + 1))
10068
+ omitted += later.blocks.length;
10069
+ firstOmitted = sectionIndex;
10070
+ break;
9676
10071
  }
9677
- ]
9678
- };
10072
+ if (firstOmitted !== -1)
10073
+ break;
10074
+ }
10075
+ if (omitted > 0) {
10076
+ const candidateByMessage = new Map;
10077
+ for (const [sectionIndex, messageIndex] of candidateIndexes.entries())
10078
+ candidateByMessage.set(messageIndex, sectionIndex);
10079
+ guarded = base.map((message, index) => {
10080
+ const sectionIndex = candidateByMessage.get(index);
10081
+ if (sectionIndex === undefined)
10082
+ return message;
10083
+ const section = candidates[sectionIndex];
10084
+ const marker = sectionIndex === firstOmitted ? omissionMarker(omitted) : "";
10085
+ return replaceTextContent(message, `${section.prefix}${kept[sectionIndex]?.join("") ?? ""}${marker}${section.suffix}`);
10086
+ });
10087
+ }
10088
+ }
10089
+ if (estimateContextMessageTokens(guarded, false) <= budget && guarded !== messages)
10090
+ return guarded;
10091
+ return trimFixedContext(guarded, budget);
9679
10092
  }
9680
- async function configureModelRuntime(modelRuntime, model, apiKey, providerApiKeys) {
9681
- if (model !== undefined) {
9682
- const configuredKey = apiKey ?? providerApiKeys?.[model.provider];
9683
- if (modelRuntime.getModel(model.provider, model.id) === undefined) {
9684
- modelRuntime.registerProvider(model.provider, modelConfig(model, configuredKey ?? EMBEDDED_AUTH_MARKER));
10093
+ function estimateContextMessageTokens(messages, useReportedUsage = false) {
10094
+ if (useReportedUsage) {
10095
+ const usage = findLastAssistantUsage(messages);
10096
+ if (usage !== undefined) {
10097
+ return calculateContextTokens(usage.usage) + messages.slice(usage.index + 1).reduce((total, message) => total + estimateMessageTokens(message), 0);
9685
10098
  }
9686
10099
  }
9687
- const keys = new Map;
9688
- if (model !== undefined && apiKey !== undefined)
10100
+ return messages.reduce((total, message) => total + estimateMessageTokens(message), 0);
10101
+ }
10102
+ function trimFixedContext(messages, inputBudget) {
10103
+ const trimmed = messages.slice();
10104
+ const lastIndex = trimmed.length - 1;
10105
+ const shrinkable = trimmed.map((message, index) => ({ message, index })).filter(({ index, message }) => index !== lastIndex && message.role !== "system");
10106
+ for (const { message, index } of shrinkable) {
10107
+ const replacement = replaceMessageWithMarker(message);
10108
+ if (replacement === message || estimateMessageTokens(replacement) >= estimateMessageTokens(message))
10109
+ continue;
10110
+ trimmed[index] = replacement;
10111
+ if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
10112
+ return trimmed;
10113
+ }
10114
+ const lastMessage = trimmed[lastIndex];
10115
+ if (lastMessage !== undefined && lastMessage.role !== "system") {
10116
+ const replacement = replaceMessageWithMarker(lastMessage);
10117
+ if (replacement !== lastMessage) {
10118
+ trimmed[lastIndex] = replacement;
10119
+ if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
10120
+ return trimmed;
10121
+ }
10122
+ const fixedWithoutLast = trimmed.reduce((total, message, index) => index === lastIndex ? total : total + estimateMessageTokens(message), 0);
10123
+ if (fixedWithoutLast < inputBudget) {
10124
+ const fitted = fitMessageToTokens(lastMessage, inputBudget - fixedWithoutLast);
10125
+ if (fitted !== lastMessage) {
10126
+ trimmed[lastIndex] = fitted;
10127
+ if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
10128
+ return trimmed;
10129
+ }
10130
+ }
10131
+ }
10132
+ for (const [index, message] of trimmed.entries()) {
10133
+ if (message.role !== "system")
10134
+ continue;
10135
+ const replacement = replaceMessageWithMarker(message);
10136
+ if (replacement === message || estimateMessageTokens(replacement) >= estimateMessageTokens(message))
10137
+ continue;
10138
+ trimmed[index] = replacement;
10139
+ if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
10140
+ return trimmed;
10141
+ }
10142
+ return trimmed;
10143
+ }
10144
+ function fitMessageToTokens(message, maxTokens) {
10145
+ const original = textContent(message);
10146
+ if (original === undefined)
10147
+ return message;
10148
+ const marker = `
10149
+
10150
+ ${GUARD_MARKER}
10151
+ `;
10152
+ let low = 0;
10153
+ let high = original.length;
10154
+ let best = replaceTextContent(message, marker);
10155
+ while (low <= high) {
10156
+ const middle = Math.floor((low + high) / 2);
10157
+ const candidate = replaceTextContent(message, excerpt(original, middle, marker));
10158
+ if (estimateMessageTokens(candidate) <= maxTokens) {
10159
+ best = candidate;
10160
+ low = middle + 1;
10161
+ } else {
10162
+ high = middle - 1;
10163
+ }
10164
+ }
10165
+ return best;
10166
+ }
10167
+ function excerpt(text, maxChars, marker) {
10168
+ if (text.length <= maxChars)
10169
+ return text;
10170
+ const available = Math.max(0, maxChars - marker.length);
10171
+ const head = Math.ceil(available * 0.7);
10172
+ const tail = Math.max(0, available - head);
10173
+ return `${text.slice(0, head)}${marker}${tail > 0 ? text.slice(-tail) : ""}`;
10174
+ }
10175
+ function replaceMessageWithMarker(message) {
10176
+ return replaceTextContent(message, `
10177
+
10178
+ ${GUARD_MARKER}
10179
+ `);
10180
+ }
10181
+ function estimateMessageTokens(message) {
10182
+ if (message.role === "system")
10183
+ return estimateSystemMessageTokens(message);
10184
+ const estimate = estimateTokens(message);
10185
+ return message.role === "toolResult" ? estimate + Math.ceil(message.toolName.length / 4) : estimate;
10186
+ }
10187
+ function estimateSystemMessageTokens(message) {
10188
+ let chars = typeof message.content === "string" ? message.content.length : 0;
10189
+ if (Array.isArray(message.content)) {
10190
+ for (const block of message.content) {
10191
+ if (block.type === "text")
10192
+ chars += block.text.length;
10193
+ else if (block.type === "image")
10194
+ chars += 4800;
10195
+ }
10196
+ }
10197
+ chars += JSON.stringify(message.sections ?? {}).length;
10198
+ chars += JSON.stringify(message.toolsAdded ?? []).length;
10199
+ chars += JSON.stringify(message.toolsRemoved ?? []).length;
10200
+ return Math.ceil(chars / 4);
10201
+ }
10202
+ function findLastAssistantUsage(messages) {
10203
+ for (let index = messages.length - 1;index >= 0; index--) {
10204
+ const message = messages[index];
10205
+ if (message?.role !== "assistant" || message.stopReason === "aborted" || message.stopReason === "error")
10206
+ continue;
10207
+ if (calculateContextTokens(message.usage) > 0)
10208
+ return { index, usage: message.usage };
10209
+ }
10210
+ return;
10211
+ }
10212
+ function textContent(message) {
10213
+ if (message.role === "system") {
10214
+ if (typeof message.content === "string")
10215
+ return message.content;
10216
+ return message.content.filter((block) => block.type === "text").map((block) => block.text).join("");
10217
+ }
10218
+ if (message.role === "user" || message.role === "toolResult") {
10219
+ if (typeof message.content === "string")
10220
+ return message.content;
10221
+ let text = "";
10222
+ for (const block of message.content) {
10223
+ if (block.type !== "text")
10224
+ return;
10225
+ text += block.text;
10226
+ }
10227
+ return text;
10228
+ }
10229
+ if (message.role === "assistant") {
10230
+ let text = "";
10231
+ for (const block of message.content)
10232
+ if (block.type === "text")
10233
+ text += block.text;
10234
+ return text.length > 0 ? text : undefined;
10235
+ }
10236
+ return;
10237
+ }
10238
+ function replaceTextContent(message, text) {
10239
+ if (message.role === "system")
10240
+ return { ...message, content: text };
10241
+ if (message.role === "user") {
10242
+ if (typeof message.content === "string")
10243
+ return { ...message, content: text };
10244
+ return { ...message, content: [{ type: "text", text }] };
10245
+ }
10246
+ if (message.role === "toolResult")
10247
+ return { ...message, content: [{ type: "text", text }] };
10248
+ if (message.role === "assistant") {
10249
+ const content = [];
10250
+ let replaced = false;
10251
+ for (const block of message.content) {
10252
+ if (block.type !== "text") {
10253
+ content.push(block);
10254
+ continue;
10255
+ }
10256
+ if (replaced)
10257
+ continue;
10258
+ replaced = true;
10259
+ content.push({ ...block, text });
10260
+ }
10261
+ return replaced ? { ...message, content } : message;
10262
+ }
10263
+ return message;
10264
+ }
10265
+ function parseCandidateSection(message, text) {
10266
+ if (message.role === "user") {
10267
+ const markerMatch = /(?:Baseline retrieval (?:evidence \(already gathered for you\)|context):|initial candidates:)/u.exec(text);
10268
+ if (markerMatch === null)
10269
+ return;
10270
+ const sectionStart = (markerMatch.index ?? 0) + markerMatch[0].length;
10271
+ const sectionEnd = findBaselineSectionEnd(text, sectionStart);
10272
+ if (!/^\[e?\d+\] /mu.test(text.slice(sectionStart, sectionEnd)))
10273
+ return;
10274
+ return splitCandidates(text, sectionStart, sectionEnd);
10275
+ }
10276
+ if (message.role !== "toolResult")
10277
+ return;
10278
+ const isRetrieval = RETRIEVAL_TOOL_NAMES[message.toolName] === true || message.toolName.startsWith(RETRIEVAL_TOOL_PREFIX);
10279
+ if (!isRetrieval)
10280
+ return;
10281
+ if (message.toolName === "jikji_find") {
10282
+ const directiveStart = text.indexOf(`
10283
+
10284
+ directive:`);
10285
+ return splitCandidates(text, 0, directiveStart >= 0 ? directiveStart : text.length, /^- [^\n]*(?:\n|$)/gmu);
10286
+ }
10287
+ const diagnosticsStart = text.indexOf(`
10288
+
10289
+ Diagnostics:`);
10290
+ return splitCandidates(text, 0, diagnosticsStart >= 0 ? diagnosticsStart : text.length);
10291
+ }
10292
+ function findBaselineSectionEnd(text, startIndex) {
10293
+ const endMarkers = [
10294
+ `
10295
+
10296
+ Produce the best`,
10297
+ `
10298
+
10299
+ Treat candidates as unverified evidence`,
10300
+ `
10301
+
10302
+ Formatting and content rules`
10303
+ ];
10304
+ const positions = endMarkers.map((marker) => text.indexOf(marker, startIndex)).filter((index) => index >= 0);
10305
+ return positions.length > 0 ? Math.min(...positions) : text.length;
10306
+ }
10307
+ function splitCandidates(text, sectionStart, sectionEnd, pattern = /^\[e?\d+\] [^\n]*(?:\n|$)/gmu) {
10308
+ const region = text.slice(sectionStart, sectionEnd);
10309
+ const matches = [...region.matchAll(pattern)];
10310
+ const firstStart = matches[0]?.index;
10311
+ if (firstStart === undefined)
10312
+ return;
10313
+ const blocks = [];
10314
+ for (const [index, match] of matches.entries()) {
10315
+ const start = match.index ?? 0;
10316
+ const end = index + 1 < matches.length ? matches[index + 1]?.index ?? region.length : region.length;
10317
+ blocks.push(region.slice(start, end));
10318
+ }
10319
+ return {
10320
+ prefix: text.slice(0, sectionStart) + region.slice(0, firstStart),
10321
+ suffix: text.slice(sectionEnd),
10322
+ blocks
10323
+ };
10324
+ }
10325
+ var CONTEXT_HEADROOM = 1.2, GUARD_MARKER = "[AutoRAG context guard: omitted context to stay within the model context budget.]", RETRIEVAL_TOOL_PREFIX = "search_datasource_", RETRIEVAL_TOOL_NAMES;
10326
+ var init_context_token_guard = __esm(() => {
10327
+ RETRIEVAL_TOOL_NAMES = {
10328
+ jikji_find: true,
10329
+ search_all_documents: true,
10330
+ semantic_search_local_docs: true,
10331
+ search_single_datasource_documents: true,
10332
+ everything_search: true,
10333
+ fsearch_search: true
10334
+ };
10335
+ });
10336
+
10337
+ // src/agent/request-limits.ts
10338
+ function fitRequestLimits(messages, limits) {
10339
+ const maxBytes = limits?.maxRequestBytes;
10340
+ const maxImages = limits?.images?.maxPerRequest;
10341
+ if (maxBytes === undefined && maxImages === undefined)
10342
+ return;
10343
+ const images = imageLocations(messages);
10344
+ if (images.length === 0)
10345
+ return;
10346
+ let keptImages = images.length;
10347
+ let bytes = maxBytes === undefined ? 0 : Buffer.byteLength(JSON.stringify(messages));
10348
+ const dropped = [];
10349
+ for (const image of images) {
10350
+ const overCount = maxImages !== undefined && keptImages > maxImages;
10351
+ const overBytes = maxBytes !== undefined && bytes > maxBytes;
10352
+ if (!overCount && !overBytes)
10353
+ break;
10354
+ dropped.push(image);
10355
+ keptImages--;
10356
+ bytes -= image.bytes - NOTE_BYTES;
10357
+ }
10358
+ if (dropped.length === 0)
10359
+ return;
10360
+ const byMessage = new Map;
10361
+ for (const { message, block } of dropped) {
10362
+ const blocks = byMessage.get(message) ?? new Set;
10363
+ blocks.add(block);
10364
+ byMessage.set(message, blocks);
10365
+ }
10366
+ return messages.map((message, index) => {
10367
+ const blocks = byMessage.get(index);
10368
+ if (blocks === undefined)
10369
+ return message;
10370
+ const content = message.content.map((part, block) => blocks.has(block) ? { type: "text", text: IMAGE_OMITTED_NOTE } : part);
10371
+ return { ...message, content };
10372
+ });
10373
+ }
10374
+ function imageLocations(messages) {
10375
+ const locations = [];
10376
+ messages.forEach((message, index) => {
10377
+ const content = message.content;
10378
+ if (!Array.isArray(content))
10379
+ return;
10380
+ content.forEach((part, block) => {
10381
+ if (!isImage(part))
10382
+ return;
10383
+ locations.push({ message: index, block, bytes: Buffer.byteLength(JSON.stringify(part)) });
10384
+ });
10385
+ });
10386
+ return locations;
10387
+ }
10388
+ function isImage(part) {
10389
+ return typeof part === "object" && part !== null && part.type === "image";
10390
+ }
10391
+ var IMAGE_OMITTED_NOTE = "[Image omitted from this request to stay within the model provider's request limits. Read the file again if you still need to see it.]", NOTE_BYTES;
10392
+ var init_request_limits = __esm(() => {
10393
+ NOTE_BYTES = Buffer.byteLength(JSON.stringify({ type: "text", text: IMAGE_OMITTED_NOTE }));
10394
+ });
10395
+
10396
+ // src/agent/pi-session.ts
10397
+ import { join as join21 } from "node:path";
10398
+ import { streamSimple as compatStreamSimple } from "@earendil-works/pi-ai/compat";
10399
+ import {
10400
+ createAgentSession,
10401
+ createAgentSessionFromServices,
10402
+ createAgentSessionRuntime,
10403
+ createAgentSessionServices,
10404
+ DefaultResourceLoader,
10405
+ getAgentDir,
10406
+ ModelRuntime,
10407
+ SessionManager,
10408
+ SettingsManager
10409
+ } from "@earendil-works/pi-coding-agent";
10410
+ import { getKeybindings, matchesKey } from "@earendil-works/pi-tui";
10411
+ function resolveInteractiveTuiMode(settingsManager) {
10412
+ const saved = settingsManager.getProjectSettings().tuiMode ?? settingsManager.getGlobalSettings().tuiMode;
10413
+ return saved ?? "regular";
10414
+ }
10415
+ function toToolDefinition(tool) {
10416
+ return {
10417
+ name: tool.name,
10418
+ label: tool.label,
10419
+ description: tool.description,
10420
+ parameters: tool.parameters,
10421
+ ...tool.prepareArguments === undefined ? {} : { prepareArguments: tool.prepareArguments },
10422
+ execute: async (toolCallId, params, signal, onUpdate) => {
10423
+ return tool.execute(toolCallId, params, signal, onUpdate);
10424
+ }
10425
+ };
10426
+ }
10427
+ function modelConfig(model, configuredApiKey) {
10428
+ return {
10429
+ name: model.name,
10430
+ baseUrl: model.baseUrl,
10431
+ api: model.api,
10432
+ ...configuredApiKey === undefined ? {} : { apiKey: configuredApiKey },
10433
+ authHeader: false,
10434
+ streamSimple: (requestModel, context, options) => compatStreamSimple(requestModel, context, {
10435
+ ...options,
10436
+ ...configuredApiKey === EMBEDDED_AUTH_MARKER ? { apiKey: undefined } : {}
10437
+ }),
10438
+ models: [
10439
+ {
10440
+ id: model.id,
10441
+ name: model.name,
10442
+ reasoning: model.reasoning,
10443
+ input: [...model.input],
10444
+ cost: model.cost,
10445
+ contextWindow: model.contextWindow,
10446
+ maxTokens: model.maxTokens,
10447
+ ...model.inputLimits === undefined ? {} : { inputLimits: model.inputLimits },
10448
+ ...model.thinkingLevelMap === undefined ? {} : { thinkingLevelMap: model.thinkingLevelMap }
10449
+ }
10450
+ ]
10451
+ };
10452
+ }
10453
+ async function configureModelRuntime(modelRuntime, model, apiKey, providerApiKeys) {
10454
+ if (model !== undefined) {
10455
+ const configuredKey = apiKey ?? providerApiKeys?.[model.provider];
10456
+ if (modelRuntime.getModel(model.provider, model.id) === undefined) {
10457
+ modelRuntime.registerProvider(model.provider, modelConfig(model, configuredKey ?? EMBEDDED_AUTH_MARKER));
10458
+ }
10459
+ }
10460
+ const keys = new Map;
10461
+ if (model !== undefined && apiKey !== undefined)
9689
10462
  keys.set(model.provider, apiKey);
9690
10463
  for (const [provider, key] of Object.entries(providerApiKeys ?? {}))
9691
10464
  keys.set(provider, key);
9692
10465
  for (const [provider, key] of keys)
9693
10466
  await modelRuntime.setRuntimeApiKey(provider, key);
9694
10467
  }
10468
+ function enforceRequestLimits(pi) {
10469
+ pi.on("context", (event, ctx) => {
10470
+ const messages = fitRequestLimits(event.messages, ctx.model?.inputLimits);
10471
+ return messages === undefined ? undefined : { messages };
10472
+ });
10473
+ }
10474
+ function guardContext(pi, contextTransform) {
10475
+ pi.on("context", async (event, ctx) => {
10476
+ const activeTools = new Set(pi.getActiveTools());
10477
+ const toolDeclarations = pi.getAllTools().filter((tool) => activeTools.has(tool.name)).map(({ name, description, parameters }) => ({ name, description, parameters }));
10478
+ const fixedOverheadTokens = Math.ceil((ctx.getSystemPrompt().length + JSON.stringify(toolDeclarations).length) / 4);
10479
+ return {
10480
+ messages: await createContextTokenGuardedTransform(ctx.model, contextTransform, () => fixedOverheadTokens)(event.messages)
10481
+ };
10482
+ });
10483
+ }
9695
10484
  function createAutoRAGExtension(getSystemPrompt, contextTransform) {
9696
10485
  return (pi) => {
9697
10486
  pi.on("before_agent_start", () => ({ systemPrompt: getSystemPrompt() }));
9698
- if (contextTransform !== undefined) {
9699
- pi.on("context", async (event) => ({ messages: await contextTransform(event.messages) }));
9700
- }
10487
+ guardContext(pi, contextTransform);
10488
+ enforceRequestLimits(pi);
9701
10489
  };
9702
10490
  }
9703
- function createAutoRAGInteractiveExtension(getSystemPrompt, contextTransform, onQuery, updateNotice) {
10491
+ function interactiveKeyOf(data) {
10492
+ const bindings = getKeybindings();
10493
+ const keysOf = (action, fallback) => {
10494
+ const keys = bindings.getKeys(action);
10495
+ return keys.length > 0 ? keys : [fallback];
10496
+ };
10497
+ const windows = process.platform === "win32" || process.platform === "linux" && Boolean(process.env.WSL_DISTRO_NAME || process.env.WSL_INTEROP);
10498
+ if (keysOf("app.interrupt", "escape").some((key) => matchesKey(data, key)))
10499
+ return "interrupt";
10500
+ if (keysOf("app.message.followUp", windows ? "ctrl+q" : "alt+enter").some((key) => matchesKey(data, key)))
10501
+ return "followUp";
10502
+ return;
10503
+ }
10504
+ function createAutoRAGInteractiveExtension(getSystemPrompt, contextTransform, options, updateNotice) {
9704
10505
  return (pi) => {
9705
10506
  pi.on("before_agent_start", () => ({ systemPrompt: getSystemPrompt() }));
9706
- if (contextTransform !== undefined) {
9707
- pi.on("context", async (event) => ({ messages: await contextTransform(event.messages) }));
9708
- }
10507
+ guardContext(pi, contextTransform);
10508
+ enforceRequestLimits(pi);
9709
10509
  if (updateNotice !== undefined) {
9710
10510
  pi.on("session_start", () => {
9711
10511
  updateNotice().then((text) => {
@@ -9721,20 +10521,37 @@ function createAutoRAGInteractiveExtension(getSystemPrompt, contextTransform, on
9721
10521
  });
9722
10522
  });
9723
10523
  }
9724
- pi.on("input", async (event) => {
9725
- const query = event.text.trim();
9726
- if (event.source !== "interactive" || query.length === 0 || query.startsWith("/"))
10524
+ const { onInput, isSearchRunning, onKey } = options;
10525
+ if (onKey !== undefined) {
10526
+ pi.on("session_start", (_event, ctx) => {
10527
+ ctx.ui.onTerminalInput((data) => {
10528
+ const key = interactiveKeyOf(data);
10529
+ return key !== undefined && onKey(key) ? { consume: true } : undefined;
10530
+ });
10531
+ });
10532
+ }
10533
+ pi.on("input", (event) => {
10534
+ const text = event.text.trim();
10535
+ if (event.source !== "interactive" || text.length === 0)
10536
+ return;
10537
+ if (text.startsWith("/") && isSearchRunning?.() !== true)
9727
10538
  return;
10539
+ let disposition;
9728
10540
  try {
9729
- await onQuery(query, pi);
10541
+ disposition = onInput({ text, images: event.images, streamingBehavior: event.streamingBehavior }, pi);
9730
10542
  } catch (error) {
9731
10543
  pi.sendMessage({
9732
10544
  customType: "autorag.error",
9733
10545
  content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
9734
10546
  display: true
9735
- });
9736
- }
9737
- return { action: "handled" };
10547
+ }, { triggerTurn: false });
10548
+ return { action: "handled" };
10549
+ }
10550
+ return disposition.kind === "steer" ? {
10551
+ action: "transform",
10552
+ text: disposition.text,
10553
+ ...event.images === undefined ? {} : { images: event.images }
10554
+ } : { action: "handled" };
9738
10555
  });
9739
10556
  };
9740
10557
  }
@@ -9814,7 +10631,7 @@ async function createAutoRAGPiInteractiveRuntime(options) {
9814
10631
  settingsManager,
9815
10632
  resourceLoaderOptions: {
9816
10633
  extensionFactories: [
9817
- createAutoRAGInteractiveExtension(options.getSystemPrompt, options.contextTransform, options.onQuery, options.updateNotice),
10634
+ createAutoRAGInteractiveExtension(options.getSystemPrompt, options.contextTransform, { onInput: options.onInput, isSearchRunning: options.isSearchRunning, onKey: options.onKey }, options.updateNotice),
9818
10635
  ...options.extensionFactories ?? []
9819
10636
  ],
9820
10637
  systemPrompt: options.getSystemPrompt(),
@@ -9849,6 +10666,8 @@ async function createAutoRAGPiInteractiveRuntime(options) {
9849
10666
  var PI_BUILTIN_TOOL_NAMES, EMBEDDED_AUTH_MARKER = "<autorag-embedded-model>";
9850
10667
  var init_pi_session = __esm(() => {
9851
10668
  init_provider();
10669
+ init_context_token_guard();
10670
+ init_request_limits();
9852
10671
  PI_BUILTIN_TOOL_NAMES = ["read", "bash", "edit", "write", "grep", "find", "ls"];
9853
10672
  });
9854
10673
 
@@ -9858,12 +10677,26 @@ import { Type as Type6 } from "typebox";
9858
10677
  function createJevJudge(options = {}) {
9859
10678
  let client;
9860
10679
  return async (state, questions, callOptions = {}) => {
9861
- client ??= new Jev({
9862
- ...options.backend !== undefined ? { backend: options.backend } : {},
9863
- ...options.env !== undefined ? { env: options.env } : {},
9864
- ...options.model !== undefined ? { model: options.model } : {},
9865
- ...options.confidenceThreshold !== undefined ? { confidenceThreshold: options.confidenceThreshold } : {}
9866
- });
10680
+ if (client === undefined) {
10681
+ const env = options.env ?? process.env;
10682
+ const explicit = options.backend;
10683
+ let backend;
10684
+ if (typeof explicit === "string") {
10685
+ backend = explicit === "autorag" ? new AutoragJevBackend({ env }) : explicit;
10686
+ } else if (explicit !== undefined) {
10687
+ backend = explicit;
10688
+ } else if (env.JEV_BACKEND?.toLowerCase() === "autorag") {
10689
+ backend = new AutoragJevBackend({ env });
10690
+ } else {
10691
+ backend = undefined;
10692
+ }
10693
+ client = new Jev({
10694
+ ...backend !== undefined ? { backend } : {},
10695
+ ...options.env !== undefined ? { env: options.env } : {},
10696
+ ...options.model !== undefined ? { model: options.model } : {},
10697
+ ...options.confidenceThreshold !== undefined ? { confidenceThreshold: options.confidenceThreshold } : {}
10698
+ });
10699
+ }
9867
10700
  const model = options.model ?? BACKEND_DEFAULT_MODEL[client.backend.name];
9868
10701
  return client.judge(state, questions, {
9869
10702
  ...model !== undefined ? { model } : {},
@@ -9905,8 +10738,12 @@ function createJevExtension(judgeOrOptions = {}) {
9905
10738
  }
9906
10739
  var JEV_TOOL_NAME = "jev", OPENROUTER_DEFAULT_MODEL = "typesafe/jev-1.13", BACKEND_DEFAULT_MODEL, jevQuestionSchema, jevParameters;
9907
10740
  var init_jev_extension = __esm(() => {
10741
+ init_jev_backend();
9908
10742
  init_pi_session();
9909
- BACKEND_DEFAULT_MODEL = { openrouter: OPENROUTER_DEFAULT_MODEL };
10743
+ BACKEND_DEFAULT_MODEL = {
10744
+ openrouter: OPENROUTER_DEFAULT_MODEL,
10745
+ autorag: AUTORAG_DEFAULT_JEV_MODEL
10746
+ };
9910
10747
  jevQuestionSchema = Type6.Object({
9911
10748
  id: Type6.Optional(Type6.String({ description: "Caller-assigned id, echoed back in the verdict" })),
9912
10749
  type: Type6.Union([Type6.Literal("noul"), Type6.Literal("choice"), Type6.Literal("score")], {
@@ -9948,7 +10785,7 @@ function createJikjiFindTool(provider, ledger = new EvidenceLedger) {
9948
10785
  label: "Jikji Find",
9949
10786
  description: "Discover local files via Jikji. Returns answer_paths with per-candidate next_read hints and a tool-call policy directive. Call this FIRST for local file discovery when Jikji is configured.",
9950
10787
  parameters: jikjiFindSchema,
9951
- async execute(_toolCallId, params) {
10788
+ async execute(_toolCallId, params, signal) {
9952
10789
  const query = String(params.query ?? "").trim();
9953
10790
  if (query.length === 0) {
9954
10791
  return {
@@ -9971,7 +10808,8 @@ function createJikjiFindTool(provider, ledger = new EvidenceLedger) {
9971
10808
  }
9972
10809
  const { answerPack, policy, diagnostics, perRoot } = await provider.findJikji(query, {
9973
10810
  topK: params.topK,
9974
- first: params.first
10811
+ first: params.first,
10812
+ ...signal === undefined ? {} : { signal }
9975
10813
  });
9976
10814
  if (answerPack === undefined) {
9977
10815
  const diagMsg = diagnostics.length > 0 ? diagnostics.map((d) => d.message).join(" ") : "Jikji is unavailable or all roots failed.";
@@ -12285,7 +13123,7 @@ function createSearchAllDocumentsTool(provider, ledger = new EvidenceLedger) {
12285
13123
  label: "Search All Documents",
12286
13124
  description: "Search across all configured retrieval methods (posix, MinSync, datasources) and return every distinct chunk they found, with only pure duplicates removed. This is the exhaustive retrieval surface: one call can return many candidates, including several passages of the same document, so judge the evidence yourself rather than assuming it was pre-filtered. Authority is server-configured; tool arguments can only provide query, topK, and an optional narrowing scope.",
12287
13125
  parameters: searchAllSchema,
12288
- async execute(_toolCallId, params) {
13126
+ async execute(_toolCallId, params, signal) {
12289
13127
  const query = params.query.trim();
12290
13128
  if (query.length === 0) {
12291
13129
  return {
@@ -12295,7 +13133,8 @@ function createSearchAllDocumentsTool(provider, ledger = new EvidenceLedger) {
12295
13133
  }
12296
13134
  const { results, diagnostics, perMethodCounts } = await provider.searchAllDocuments(query, {
12297
13135
  topK: params.topK,
12298
- scope: params.scope
13136
+ scope: params.scope,
13137
+ ...signal === undefined ? {} : { signal }
12299
13138
  });
12300
13139
  const evidenceIds = results.map((result) => ledger.registerResult(SEARCH_ALL_DOCUMENTS_TOOL_NAME, result));
12301
13140
  return {
@@ -12319,7 +13158,7 @@ Diagnostics: ${diagnostics.map((d) => `${d.source ?? "all"}:${d.code}`).join(",
12319
13158
  if (results.length === 0)
12320
13159
  return `No results.${diagnosticSummary}`;
12321
13160
  const rows = results.map((result, index) => {
12322
- const line = result.content.replace(/\s+/gu, " ").slice(0, 500);
13161
+ const line = result.content.replace(/\s+/gu, " ");
12323
13162
  return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
12324
13163
  ${line}`;
12325
13164
  });
@@ -12350,7 +13189,7 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
12350
13189
  label: "Search MinSync Documents",
12351
13190
  description: "Search parsed document mirrors with MinSync semantic vector retrieval. Use for conceptual and meaning-based search.",
12352
13191
  parameters: searchMinSyncSchema,
12353
- async execute(_toolCallId, params) {
13192
+ async execute(_toolCallId, params, signal) {
12354
13193
  if (params.query.trim().length === 0) {
12355
13194
  return {
12356
13195
  content: [{ type: "text", text: "MinSync query was empty; no documents searched." }],
@@ -12360,7 +13199,8 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
12360
13199
  const scope = resolveScope(params.scope);
12361
13200
  const results = await getMethod().retrieve(params.query, {
12362
13201
  topK: params.topK,
12363
- scope
13202
+ scope,
13203
+ ...signal === undefined ? {} : { signal }
12364
13204
  });
12365
13205
  const evidenceIds = results.map((result) => ledger.registerResult(SEARCH_MINSYNC_DOCUMENTS_TOOL_NAME, result));
12366
13206
  return {
@@ -12379,7 +13219,7 @@ function formatResults2(results, evidenceIds) {
12379
13219
  if (results.length === 0)
12380
13220
  return "No MinSync results.";
12381
13221
  const rows = results.map((result, index) => {
12382
- const line = result.content.replace(/\s+/gu, " ").slice(0, 500);
13222
+ const line = result.content.replace(/\s+/gu, " ");
12383
13223
  return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
12384
13224
  ${line}`;
12385
13225
  });
@@ -12427,7 +13267,7 @@ function createTool(provider, spec, ledger) {
12427
13267
  label: `Search ${spec.datasourceId}`,
12428
13268
  description: `Search only the "${spec.datasourceId}" datasource connection: ${spec.description} ` + `This tool spawns only this connection's CLIs and returns only this connection's hits.${scopeLine} ` + `Authority is server-configured; tool arguments can only provide query, topK, and an optional narrowing scope.`,
12429
13269
  parameters: searchSingleDatasourceSchema,
12430
- async execute(_toolCallId, params) {
13270
+ async execute(_toolCallId, params, signal) {
12431
13271
  const query = params.query.trim();
12432
13272
  if (query.length === 0) {
12433
13273
  return {
@@ -12443,7 +13283,8 @@ function createTool(provider, spec, ledger) {
12443
13283
  }
12444
13284
  const { results, diagnostics } = await provider.searchSingleDatasourceDocuments(spec.datasourceId, query, {
12445
13285
  topK: params.topK,
12446
- scope: params.scope
13286
+ scope: params.scope,
13287
+ ...signal === undefined ? {} : { signal }
12447
13288
  });
12448
13289
  const evidenceIds = results.map((result) => ledger.registerResult(`search_datasource_${spec.datasourceId}`, result));
12449
13290
  return {
@@ -12467,7 +13308,7 @@ Diagnostics: ${diagnostics.map((d) => `${d.source ?? "datasource"}:${d.code}`).j
12467
13308
  if (results.length === 0)
12468
13309
  return `No datasource results.${diagnosticSummary}`;
12469
13310
  const rows = results.map((result, index) => {
12470
- const line = result.content.replace(/\s+/gu, " ").slice(0, 500);
13311
+ const line = result.content.replace(/\s+/gu, " ");
12471
13312
  return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
12472
13313
  ${line}`;
12473
13314
  });
@@ -18030,7 +18871,7 @@ function mergeSources(merged, sources) {
18030
18871
  existing.source.ageSeconds ??= source.ageSeconds;
18031
18872
  }
18032
18873
  }
18033
- function sleep3(ms, signal) {
18874
+ function sleep4(ms, signal) {
18034
18875
  if (ms <= 0)
18035
18876
  return Promise.resolve();
18036
18877
  let resolveFn;
@@ -18080,9 +18921,9 @@ async function searchPublicWeb(params, deadlines = {}) {
18080
18921
  failures.push({ provider: { id, label: id }, error });
18081
18922
  }
18082
18923
  }));
18083
- await Promise.race([all, sleep3(softMs, signal)]);
18924
+ await Promise.race([all, sleep4(softMs, signal)]);
18084
18925
  if (!responses.some(isUseful) && failures.length < engineIds.length) {
18085
- await Promise.race([all, firstSuccess, sleep3(Math.max(0, hardMs - softMs), signal)]);
18926
+ await Promise.race([all, firstSuccess, sleep4(Math.max(0, hardMs - softMs), signal)]);
18086
18927
  }
18087
18928
  straggler.abort();
18088
18929
  const merged = new Map;
@@ -18498,6 +19339,20 @@ class AutoRAGAgent {
18498
19339
  lastQuery;
18499
19340
  lastSessionId;
18500
19341
  activeRun = false;
19342
+ interactive = {
19343
+ run: undefined,
19344
+ followUps: [],
19345
+ steers: [],
19346
+ queuedSteers: [],
19347
+ preSessionSteers: [],
19348
+ acceptingSteers: false,
19349
+ harnessAbort: false,
19350
+ userAborted: false,
19351
+ steerInterrupt: new AbortController,
19352
+ nextSubmitIsFollowUp: false,
19353
+ steerTexts: new Set
19354
+ };
19355
+ activeRunAbort;
18501
19356
  modelNativeSearchAuth;
18502
19357
  retrievalTrace = [];
18503
19358
  evidenceLedger = new EvidenceLedger;
@@ -18974,9 +19829,9 @@ ${context.text}
18974
19829
  apiKey: resolved.apiKey,
18975
19830
  providerApiKeys: resolved.providerApiKeys,
18976
19831
  getSystemPrompt: () => systemPrompt,
18977
- customTools: this.tools.filter((tool) => !PI_BUILTIN_TOOL_NAMES.includes(tool.name)),
19832
+ customTools: this.steerableTools(),
18978
19833
  remoteSession: this.remoteSession,
18979
- contextTransform: (messages) => this.withMemoryContext(messages),
19834
+ contextTransform: (messages) => this.withMemoryContext(frameSteers(messages, this.interactive.steerTexts)),
18980
19835
  ...this.jevExtension !== undefined ? { extensionFactories: [this.jevExtension], extensionToolNames: [JEV_TOOL_NAME] } : {}
18981
19836
  });
18982
19837
  const agent = piSession.session.agent;
@@ -18992,6 +19847,8 @@ ${context.text}
18992
19847
  const unsubscribers = [...this.listeners].map((listener) => session.agent.subscribe(listener));
18993
19848
  unsubscribers.push(session.agent.subscribe((event) => {
18994
19849
  this.recordSearchToolEvent(event);
19850
+ if (event.type === "message_start" && event.message.role === "user")
19851
+ this.steersReachedModel();
18995
19852
  }));
18996
19853
  return unsubscribers;
18997
19854
  }
@@ -19008,6 +19865,7 @@ ${context.text}
19008
19865
  return;
19009
19866
  this.searchToolCallCount += 1;
19010
19867
  if (this.searchToolCallCount >= this.maxSearchToolCalls) {
19868
+ this.interactive.harnessAbort = true;
19011
19869
  this.activeSession?.abort();
19012
19870
  }
19013
19871
  const details = event.result.details;
@@ -19054,9 +19912,11 @@ ${context.text}
19054
19912
  getSystemPrompt: () => buildSystemPrompt(this.currentSystemPromptConfig({
19055
19913
  modelId: this.boundPiRuntime?.session.model?.id ?? resolved?.model.id
19056
19914
  })),
19057
- contextTransform: (messages) => this.withMemoryContext(messages),
19058
- customTools: this.tools.filter((tool) => !PI_BUILTIN_TOOL_NAMES.includes(tool.name)),
19059
- onQuery: (query, pi) => this.runInteractivePiQuery(query, pi),
19915
+ contextTransform: async (messages) => this.withMemoryContext(frameSteers(messages, this.interactive.steerTexts)),
19916
+ customTools: this.steerableTools(),
19917
+ onInput: (input, pi) => this.routeInteractiveInput(input, pi),
19918
+ isSearchRunning: () => this.interactive.run !== undefined,
19919
+ onKey: (key) => this.handleInteractiveKey(key),
19060
19920
  ...this.jevExtension !== undefined ? { extensionFactories: [this.jevExtension], extensionToolNames: [JEV_TOOL_NAME] } : {},
19061
19921
  ...this.updateNotice === undefined ? {} : { updateNotice: this.updateNotice }
19062
19922
  });
@@ -19071,18 +19931,154 @@ ${context.text}
19071
19931
  }
19072
19932
  };
19073
19933
  }
19934
+ steerableTools() {
19935
+ return this.tools.filter((tool) => !PI_BUILTIN_TOOL_NAMES.includes(tool.name)).map((tool) => this.searchToolNames.has(tool.name) || tool.name === WEB_SEARCH_TOOL_NAME || tool.name === WEB_FETCH_TOOL_NAME ? this.interruptibleBySteer(tool) : tool);
19936
+ }
19937
+ interruptibleBySteer(tool) {
19938
+ return {
19939
+ ...tool,
19940
+ execute: async (toolCallId, params, signal, onUpdate) => {
19941
+ const steer = this.interactive.steerInterrupt.signal;
19942
+ if (steer.aborted)
19943
+ return skippedBySteer(tool.name);
19944
+ const combined = signal === undefined ? steer : AbortSignal.any([signal, steer]);
19945
+ let onSteer = () => {};
19946
+ const steered = new Promise((resolve) => {
19947
+ onSteer = () => resolve("steered");
19948
+ steer.addEventListener("abort", onSteer, { once: true });
19949
+ });
19950
+ try {
19951
+ const outcome = await Promise.race([tool.execute(toolCallId, params, combined, onUpdate), steered]);
19952
+ return outcome === "steered" ? skippedBySteer(tool.name) : outcome;
19953
+ } catch (error) {
19954
+ if (steer.aborted && signal?.aborted !== true)
19955
+ return skippedBySteer(tool.name);
19956
+ throw error;
19957
+ } finally {
19958
+ steer.removeEventListener("abort", onSteer);
19959
+ }
19960
+ }
19961
+ };
19962
+ }
19074
19963
  deriveAnswer(text) {
19075
19964
  const derived = deriveResultsFromAnswer(text, this.evidenceLedger, { allowLocalFiles: !this.remoteSession });
19076
19965
  this.routingDiagnostics.push(...derived.diagnostics);
19077
19966
  return derived.details;
19078
19967
  }
19968
+ routeInteractiveInput(input, pi) {
19969
+ const state = this.interactive;
19970
+ const followUp = input.streamingBehavior === "followUp" || state.nextSubmitIsFollowUp;
19971
+ state.nextSubmitIsFollowUp = false;
19972
+ if (state.run === undefined) {
19973
+ this.startInteractiveQuery(input.text, pi);
19974
+ return { kind: "handled" };
19975
+ }
19976
+ if (followUp) {
19977
+ state.followUps.push(input.text);
19978
+ return { kind: "handled" };
19979
+ }
19980
+ if (input.streamingBehavior === "steer") {
19981
+ state.steers.push(input.text);
19982
+ state.steerTexts.add(input.text);
19983
+ state.steerInterrupt.abort();
19984
+ return { kind: "steer", text: input.text };
19985
+ }
19986
+ if (!this.steer(input.text, input.images))
19987
+ state.followUps.push(input.text);
19988
+ return { kind: "handled" };
19989
+ }
19990
+ handleInteractiveKey(key) {
19991
+ const state = this.interactive;
19992
+ if (state.run === undefined || this.boundPiRuntime?.session.isStreaming === true)
19993
+ return false;
19994
+ if (key === "followUp") {
19995
+ state.nextSubmitIsFollowUp = true;
19996
+ return false;
19997
+ }
19998
+ state.userAborted = true;
19999
+ state.steerInterrupt.abort();
20000
+ return true;
20001
+ }
20002
+ startInteractiveQuery(query, pi) {
20003
+ const state = this.interactive;
20004
+ const run = (async () => {
20005
+ let next = query;
20006
+ while (next !== undefined) {
20007
+ try {
20008
+ await this.runInteractivePiQuery(next, pi);
20009
+ } catch (error) {
20010
+ pi.sendMessage({
20011
+ customType: "autorag.error",
20012
+ content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
20013
+ display: true
20014
+ }, { triggerTurn: false });
20015
+ }
20016
+ next = state.followUps.shift();
20017
+ }
20018
+ })().finally(() => {
20019
+ state.run = undefined;
20020
+ });
20021
+ state.run = run;
20022
+ }
19079
20023
  async runInteractivePiQuery(query, pi) {
19080
20024
  pi.setSessionName(query.slice(0, 80));
19081
- for await (const _event of this.searchDocumentsStream(query)) {}
20025
+ const state = this.interactive;
20026
+ const session = this.boundPiRuntime?.session;
20027
+ const unsubscribe = session?.subscribe((event) => {
20028
+ if (event.type === "queue_update" && state.queuedSteers.length > 0 && !session.agent.hasQueuedMessages()) {
20029
+ state.queuedSteers.length = 0;
20030
+ }
20031
+ if (event.type === "agent_settled" && event.aborted && !state.harnessAbort)
20032
+ state.userAborted = true;
20033
+ });
20034
+ try {
20035
+ for await (const _event of this.searchDocumentsStream(query)) {}
20036
+ } finally {
20037
+ unsubscribe?.();
20038
+ if (state.queuedSteers.length > 0) {
20039
+ session?.agent.clearSteeringQueue();
20040
+ if (!state.userAborted)
20041
+ state.followUps.unshift(...state.queuedSteers);
20042
+ state.queuedSteers.length = 0;
20043
+ }
20044
+ }
20045
+ }
20046
+ async waitForIdle() {
20047
+ while (this.interactive.run !== undefined)
20048
+ await this.interactive.run;
20049
+ }
20050
+ steer(text, images = []) {
20051
+ const state = this.interactive;
20052
+ if (!this.activeRun || !state.acceptingSteers || state.userAborted || text.trim() === "")
20053
+ return false;
20054
+ state.steers.push(text);
20055
+ state.steerTexts.add(text);
20056
+ state.steerInterrupt.abort();
20057
+ const content = [{ type: "text", text }, ...images];
20058
+ const message = { role: "user", content, timestamp: Date.now() };
20059
+ const agent = this.activeSession?.agent;
20060
+ if (agent === undefined)
20061
+ state.preSessionSteers.push(message);
20062
+ else
20063
+ agent.steer(message);
20064
+ state.queuedSteers.push(text);
20065
+ return true;
19082
20066
  }
19083
20067
  abort() {
20068
+ if (!this.activeRun)
20069
+ return;
20070
+ const state = this.interactive;
20071
+ state.userAborted = true;
20072
+ state.steerInterrupt.abort();
20073
+ this.activeRunAbort?.abort();
19084
20074
  this.activeSession?.abort();
19085
20075
  }
20076
+ steersReachedModel() {
20077
+ const state = this.interactive;
20078
+ state.queuedSteers.length = 0;
20079
+ if (state.steerInterrupt.signal.aborted && !state.userAborted)
20080
+ state.steerInterrupt = new AbortController;
20081
+ }
19086
20082
  startAutoRefresh(intervalMs, options = {}) {
19087
20083
  this.stopAutoRefresh();
19088
20084
  const tick = () => {
@@ -19142,6 +20138,15 @@ ${context.text}
19142
20138
  const publishOnCapture = this.jevJudge === undefined;
19143
20139
  let published = false;
19144
20140
  const planAbort = new AbortController;
20141
+ this.activeRunAbort = planAbort;
20142
+ const steering = this.interactive;
20143
+ steering.steers.length = 0;
20144
+ steering.queuedSteers.length = 0;
20145
+ steering.preSessionSteers.length = 0;
20146
+ steering.harnessAbort = false;
20147
+ steering.userAborted = false;
20148
+ steering.steerInterrupt = new AbortController;
20149
+ steering.acceptingSteers = true;
19145
20150
  const publishPreliminary = (details) => {
19146
20151
  if (planAbort.signal.aborted || published)
19147
20152
  return;
@@ -19177,6 +20182,9 @@ ${context.text}
19177
20182
  session = await this.createSearchSession(resolved, buildSystemPrompt(this.currentSystemPromptConfig({ modelId: resolved.model.id })));
19178
20183
  this.activeSession = session;
19179
20184
  unsubscribers = this.configureSearchSession(session);
20185
+ for (const message of steering.preSessionSteers)
20186
+ session.agent.steer(message);
20187
+ steering.preSessionSteers.length = 0;
19180
20188
  let timeout;
19181
20189
  let timedOutAfterFastAnswer = false;
19182
20190
  try {
@@ -19237,25 +20245,42 @@ Warning: the edited config did not validate, so the previous config was restored
19237
20245
  }
19238
20246
  return;
19239
20247
  }
19240
- if (plan.route === "direct") {
20248
+ const steered = () => this.interactive.steers.length > 0;
20249
+ const stopped = () => this.interactive.userAborted;
20250
+ if (stopped())
20251
+ return;
20252
+ let directText;
20253
+ if (plan.route === "direct" && !steered()) {
19241
20254
  setThinking(this.fastThinkingLevel);
19242
20255
  await activeSession.prompt(this.buildDirectAnswerPrompt(trimmedQuery));
19243
- const answer = finalAnswerText(transcript());
19244
- if (answer !== undefined)
19245
- captured = this.deriveAnswer(answer);
20256
+ if (stopped())
20257
+ return;
20258
+ directText = finalAnswerText(transcript());
20259
+ if (!steered()) {
20260
+ if (directText !== undefined)
20261
+ captured = this.deriveAnswer(directText);
20262
+ return;
20263
+ }
20264
+ }
20265
+ const baseline = plan.route === "direct" ? "" : await this.prefetchInitialRetrievalContext(trimmedQuery, plan.queries, options, plan.datasources);
20266
+ if (stopped())
19246
20267
  return;
20268
+ let fastText = directText;
20269
+ let preliminary;
20270
+ if (plan.route !== "direct") {
20271
+ setThinking(this.fastThinkingLevel);
20272
+ await activeSession.prompt(this.buildFastAnswerPrompt(trimmedQuery, options, baseline));
20273
+ if (stopped())
20274
+ return;
20275
+ fastText = finalAnswerText(transcript());
20276
+ preliminary = fastText === undefined ? undefined : this.deriveAnswer(fastText);
19247
20277
  }
19248
- const baseline = await this.prefetchInitialRetrievalContext(trimmedQuery, plan.queries, options, plan.datasources);
19249
- setThinking(this.fastThinkingLevel);
19250
- await activeSession.prompt(this.buildFastAnswerPrompt(trimmedQuery, options, baseline));
19251
- const fastText = finalAnswerText(transcript());
19252
- const preliminary = fastText === undefined ? undefined : this.deriveAnswer(fastText);
19253
20278
  if (preliminary !== undefined) {
19254
20279
  emitPreliminary(preliminary);
19255
- const followUp = await this.shouldFollowUp(trimmedQuery, preliminary);
19256
- if (planAbort.signal.aborted)
20280
+ const followUp = steered() || await this.shouldFollowUp(trimmedQuery, preliminary);
20281
+ if (planAbort.signal.aborted || stopped())
19257
20282
  return;
19258
- if (!followUp) {
20283
+ if (!followUp && !steered()) {
19259
20284
  captured = preliminary;
19260
20285
  return;
19261
20286
  }
@@ -19263,14 +20288,23 @@ Warning: the edited config did not validate, so the previous config was restored
19263
20288
  }
19264
20289
  setThinking(this.finalThinkingLevel);
19265
20290
  const fastAnswerDelivered = preliminary !== undefined && this.preliminaryCallback !== undefined;
19266
- await activeSession.prompt(this.buildRefinementPrompt(trimmedQuery, options, fastText, fastAnswerDelivered));
20291
+ await activeSession.prompt(this.buildRefinementPrompt(trimmedQuery, options, fastText, fastAnswerDelivered, this.interactive.steers));
20292
+ while (!stopped() && !planAbort.signal.aborted && this.interactive.queuedSteers.length > 0) {
20293
+ await activeSession.prompt(buildSteerContinuationPrompt(this.interactive.steers));
20294
+ }
20295
+ steering.acceptingSteers = false;
20296
+ if (stopped())
20297
+ return;
19267
20298
  const finalText = finalAnswerText(transcript());
19268
20299
  if (finalText !== undefined)
19269
20300
  captured = this.deriveAnswer(finalText);
19270
- })(),
20301
+ })().finally(() => {
20302
+ steering.acceptingSteers = false;
20303
+ }),
19271
20304
  new Promise((resolve, reject) => {
19272
20305
  timeout = setTimeout(() => {
19273
20306
  planAbort.abort();
20307
+ this.interactive.harnessAbort = true;
19274
20308
  Promise.resolve(session?.abort());
19275
20309
  if (fastCaptured !== undefined && !this.remoteSession) {
19276
20310
  timedOutAfterFastAnswer = true;
@@ -19717,12 +20751,13 @@ Warning: the edited config did not validate, so the previous config was restored
19717
20751
  return selection.selected;
19718
20752
  }
19719
20753
  async prefetchInitialRetrievalContext(query, searchQueries, options, datasources = []) {
19720
- const retrieveOptions = { topK: this.limits.prefetch.minSyncTopK, scope: options.scope };
20754
+ const signal = this.interactive.steerInterrupt.signal;
20755
+ const retrieveOptions = { topK: this.limits.prefetch.minSyncTopK, scope: options.scope, signal };
19721
20756
  const formatSearchQueries = (list) => `Search queries (decomposed from the original question):
19722
20757
  ${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
19723
20758
  `)}`;
19724
20759
  const perQuery = await Promise.all(searchQueries.map((searchQuery) => Promise.all([
19725
- this.jikjiClient === undefined ? Promise.resolve(undefined) : this.findJikji(searchQuery, { topK: this.limits.prefetch.jikjiTopK }).catch(() => {
20760
+ this.jikjiClient === undefined ? Promise.resolve(undefined) : this.findJikji(searchQuery, { topK: this.limits.prefetch.jikjiTopK, signal }).catch(() => {
19726
20761
  return;
19727
20762
  }),
19728
20763
  this.minSyncMethod.retrieve(searchQuery, retrieveOptions).catch((error) => {
@@ -19732,7 +20767,8 @@ ${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
19732
20767
  }),
19733
20768
  Promise.all(datasources.map((datasourceId) => this.searchSingleDatasourceDocuments(datasourceId, searchQuery, {
19734
20769
  topK: this.limits.singleDatasourceTopK,
19735
- scope: options.scope
20770
+ scope: options.scope,
20771
+ signal
19736
20772
  }).then((outcome) => outcome.results).catch(() => [])))
19737
20773
  ])));
19738
20774
  for (const [index, [jikji, vector, perDatasource]] of perQuery.entries()) {
@@ -19782,7 +20818,7 @@ ${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
19782
20818
  options.observedSources?.add(result.source);
19783
20819
  for (const result of datasourceResults)
19784
20820
  options.observedSources?.add(result.source);
19785
- const reranked = await this.rerankPrefetchPool(query, jikjiPaths, minSyncResults, datasourceResults);
20821
+ const reranked = await this.rerankPrefetchPool(query, jikjiPaths, minSyncResults, datasourceResults, signal);
19786
20822
  if (reranked !== undefined) {
19787
20823
  const baseline = formatRerankedBaseline(reranked, reranked.map((result) => this.evidenceLedger.registerResult("baseline", result)));
19788
20824
  return searchQueries.length > 1 ? `${formatSearchQueries(searchQueries)}
@@ -19814,7 +20850,7 @@ ${result.content.replace(/\s+/gu, " ")}`).join(`
19814
20850
 
19815
20851
  `);
19816
20852
  }
19817
- async rerankPrefetchPool(query, answerPaths, minSyncResults, datasourceResults) {
20853
+ async rerankPrefetchPool(query, answerPaths, minSyncResults, datasourceResults, signal) {
19818
20854
  const reranker = this.reranker;
19819
20855
  if (reranker === undefined)
19820
20856
  return;
@@ -19832,7 +20868,7 @@ ${result.content.replace(/\s+/gu, " ")}`).join(`
19832
20868
  if (!reranker.describe().available)
19833
20869
  return;
19834
20870
  try {
19835
- const reranked = await reranker.rerank(query, candidates, { topN: this.rerankTopN });
20871
+ const reranked = await reranker.rerank(query, candidates, { topN: this.rerankTopN, signal });
19836
20872
  return reranked.length > 0 ? reranked : undefined;
19837
20873
  } catch {
19838
20874
  return;
@@ -19875,7 +20911,7 @@ ${baseline}
19875
20911
  const fetch2 = names.has(WEB_FETCH_TOOL_NAME) ? ` (and ${WEB_FETCH_TOOL_NAME} to read a promising page)` : "";
19876
20912
  return `When local evidence is missing, thin, or possibly outdated — or the question concerns current events, recent releases, prices, or other public facts — use ${WEB_SEARCH_TOOL_NAME}${fetch2} to confirm or correct claims and fill gaps; use URLs as result sources for web-derived evidence. `;
19877
20913
  }
19878
- buildRefinementPrompt(query, options, fastAnswer, fastAnswerDelivered) {
20914
+ buildRefinementPrompt(query, options, fastAnswer, fastAnswerDelivered, steers = []) {
19879
20915
  const limit = typeof options.topK === "number" ? ` Cite at most ${options.topK} sources.` : "";
19880
20916
  const scope = options.scope ? ` Restrict search to virtual path scope ${options.scope}.` : "";
19881
20917
  const firstAnswer = formatFirstAnswerContext(fastAnswer, fastAnswerDelivered);
@@ -19898,7 +20934,7 @@ ${baseline}
19898
20934
 
19899
20935
  ` + `${firstAnswer}
19900
20936
 
19901
- ` + `Now verify it rigorously. ${this.discoveryHint((tools) => `Actively use ${tools} when discovering or exploring local files and folders. `)}Check important claims against source files with bash when needed, correct anything wrong or unsupported, fill gaps with retrieval tools, and resolve conflicts and freshness. ${this.webFallbackHint()}
20937
+ ` + formatSteerContext(steers) + `Now verify it rigorously. ${this.discoveryHint((tools) => `Actively use ${tools} when discovering or exploring local files and folders. `)}Check important claims against source files with bash when needed, correct anything wrong or unsupported, fill gaps with retrieval tools, and resolve conflicts and freshness. ${this.webFallbackHint()}
19902
20938
 
19903
20939
  ` + `${answerRules}
19904
20940
 
@@ -20301,7 +21337,8 @@ ${initialRetrievalContext ?? "Pending; continue only with a brief progress state
20301
21337
  const sourceRoots = planJikjiSourceRoots(this.searchPaths);
20302
21338
  const findOpts = {
20303
21339
  topK: opts?.topK,
20304
- first: opts?.first
21340
+ first: opts?.first,
21341
+ ...opts?.signal === undefined ? {} : { signal: opts.signal }
20305
21342
  };
20306
21343
  const diagnostics = [];
20307
21344
  const okPacks = [];
@@ -20452,13 +21489,18 @@ ${initialRetrievalContext ?? "Pending; continue only with a brief progress state
20452
21489
  return { results, diagnostics };
20453
21490
  }
20454
21491
  async searchAllDocuments(query, options = {}) {
20455
- return this.retrieveWithDiagnostics(query, { topK: options.topK, scope: options.scope });
21492
+ return this.retrieveWithDiagnostics(query, {
21493
+ topK: options.topK,
21494
+ scope: options.scope,
21495
+ ...options.signal === undefined ? {} : { signal: options.signal }
21496
+ });
20456
21497
  }
20457
21498
  async searchSingleDatasourceDocuments(datasourceId, query, options = {}) {
20458
21499
  const retrievalOptions = {
20459
21500
  ...this.activeRetrievalOptions,
20460
21501
  topK: options.topK,
20461
- scope: options.scope
21502
+ scope: options.scope,
21503
+ ...options.signal === undefined ? {} : { signal: options.signal }
20462
21504
  };
20463
21505
  const methods = this.methodRegistry.list().filter((method) => {
20464
21506
  const descriptor = method.describe();
@@ -20561,6 +21603,49 @@ ${result.content.replace(/\s+/gu, " ")}`);
20561
21603
  ${lines.join(`
20562
21604
  `)}`;
20563
21605
  }
21606
+ function skippedBySteer(toolName) {
21607
+ return {
21608
+ content: [
21609
+ {
21610
+ type: "text",
21611
+ text: `Skipped due to the user's steering message. ${toolName} was stopped before it returned; do not count it as completed work. Apply the steering message, then retry the search if it is still needed.`
21612
+ }
21613
+ ],
21614
+ details: { method: toolName, skipped: "steer" }
21615
+ };
21616
+ }
21617
+ function buildSteerMessage(text) {
21618
+ return `The user interrupted the running search with this correction:
21619
+
21620
+ ${text}
21621
+
21622
+ ` + `Treat it as direction for the search in progress, not as a new question: adjust which sources, queries, or claims you pursue, then continue. ` + `When finished, reply with the final answer to the original question, updated for this correction, and no tool call.`;
21623
+ }
21624
+ function frameSteers(messages, steerTexts) {
21625
+ if (steerTexts.size === 0)
21626
+ return messages;
21627
+ return messages.map((message) => {
21628
+ if (message.role !== "user" || typeof message.content === "string")
21629
+ return message;
21630
+ const [first, ...rest] = message.content;
21631
+ if (first?.type !== "text" || !steerTexts.has(first.text))
21632
+ return message;
21633
+ return { ...message, content: [{ ...first, text: buildSteerMessage(first.text) }, ...rest] };
21634
+ });
21635
+ }
21636
+ function formatSteerContext(steers) {
21637
+ if (steers.length === 0)
21638
+ return "";
21639
+ return `While the search ran, the user corrected it:
21640
+ ${steers.map((steer) => `- ${steer}`).join(`
21641
+ `)}
21642
+ ` + `The first answer predates these corrections; follow them over it.
21643
+
21644
+ `;
21645
+ }
21646
+ function buildSteerContinuationPrompt(steers) {
21647
+ return `${formatSteerContext(steers)}Apply the latest correction: search or verify further if it needs that, ` + `then reply with the complete final answer to the original question and no tool call.`;
21648
+ }
20564
21649
  function formatFirstAnswerContext(fastAnswer, delivered) {
20565
21650
  if (fastAnswer === undefined) {
20566
21651
  return "(the fast phase produced no answer — write the complete answer)";
@@ -20969,21 +22054,173 @@ ${this.description}`
20969
22054
  }
20970
22055
  }
20971
22056
 
20972
- // src/datasource/skills/clawgallery/client.ts
20973
- import { spawn as spawn9 } from "node:child_process";
20974
- import { existsSync as existsSync23 } from "node:fs";
20975
-
20976
- class ClawGalleryClient {
20977
- options;
20978
- constructor(options = {}) {
20979
- this.options = options;
20980
- }
20981
- async bootstrap(signal) {
20982
- if (this.options.path !== undefined && !existsSync23(this.options.path)) {
20983
- return {
20984
- ok: false,
20985
- reason: "nonzero-exit",
20986
- stdout: "",
22057
+ // src/datasource/connector.ts
22058
+ import { createHash as createHash12 } from "node:crypto";
22059
+ function connectorFailureToDiagnosticCode(reason) {
22060
+ switch (reason) {
22061
+ case "auth":
22062
+ return "datasource-auth-error";
22063
+ case "permission":
22064
+ return "datasource-permission-denied";
22065
+ case "rate-limited":
22066
+ return "datasource-rate-limited";
22067
+ case "unavailable":
22068
+ case "not-configured":
22069
+ return "datasource-unavailable";
22070
+ case "empty":
22071
+ return "datasource-empty";
22072
+ case "api-error":
22073
+ case "invalid-data":
22074
+ return "datasource-index-failed";
22075
+ }
22076
+ }
22077
+ function boundDiagnosticText(value) {
22078
+ const trimmed = value.trim();
22079
+ return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
22080
+ }
22081
+ function sanitizeIdSegment(value) {
22082
+ const normalized = value.replace(/[^\p{L}\p{N}._-]+/gu, "-");
22083
+ let start = 0;
22084
+ while (start < normalized.length && (normalized[start] === "-" || normalized[start] === "."))
22085
+ start += 1;
22086
+ let end = normalized.length;
22087
+ while (end > start && (normalized[end - 1] === "-" || normalized[end - 1] === "."))
22088
+ end -= 1;
22089
+ const cleaned = normalized.slice(start, end);
22090
+ if (cleaned === value && cleaned.length > 0 && cleaned.length <= 80)
22091
+ return cleaned;
22092
+ const hash = createHash12("sha256").update(value).digest("hex").slice(0, 8);
22093
+ const stem = cleaned.slice(0, 60);
22094
+ return stem.length > 0 ? `${stem}-${hash}` : hash;
22095
+ }
22096
+ var init_connector = () => {};
22097
+
22098
+ // src/datasource/sidecar/config.ts
22099
+ import { readFileSync as readFileSync19 } from "node:fs";
22100
+ import { homedir as homedir3 } from "node:os";
22101
+ import { dirname as dirname17, isAbsolute as isAbsolute8, join as join30, resolve as resolve14, sep as sep7 } from "node:path";
22102
+ import { parse as parseToml2 } from "smol-toml";
22103
+ function resolveSidecarOptions(input) {
22104
+ const { connector, workspaceRoot } = input;
22105
+ const setting = connector?.sidecar;
22106
+ if (setting === false || workspaceRoot === undefined)
22107
+ return;
22108
+ const object = typeof setting === "object" && setting !== null && !Array.isArray(setting) ? setting : {};
22109
+ if (object.enabled === false)
22110
+ return;
22111
+ const configured = [object.crawlerDbPath, connector?.databasePath].find((value) => typeof value === "string" && value.length > 0);
22112
+ const crawlerDbPath = expandHome(configured ?? archivePathFromCrawlerConfig(input) ?? input.defaultCrawlerDb);
22113
+ const workspaceId = typeof connector?.workspace === "string" && connector.workspace.length > 0 ? connector.workspace : undefined;
22114
+ return {
22115
+ descriptor: input.descriptor,
22116
+ crawlerDbPath,
22117
+ sidecarDbPath: join30(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(input.registrationName), sanitizeIdSegment(input.instanceId), "sidecar.sqlite"),
22118
+ ...workspaceId === undefined ? {} : { workspaceId },
22119
+ ...object.semantic === false ? {} : { embedder: createGatewayEmbedder() }
22120
+ };
22121
+ }
22122
+ function archivePathFromCrawlerConfig(input) {
22123
+ const configPath = input.connector?.configPath;
22124
+ if (input.descriptor.archivePathInConfig !== true || typeof configPath !== "string" || configPath.length === 0) {
22125
+ return;
22126
+ }
22127
+ const file = expandHome(configPath);
22128
+ try {
22129
+ const dbPath = parseToml2(readFileSync19(file, "utf8")).db_path;
22130
+ if (typeof dbPath !== "string" || dbPath.length === 0)
22131
+ return;
22132
+ const expanded = expandHome(dbPath);
22133
+ return isAbsolute8(expanded) ? expanded : resolve14(dirname17(file), expanded);
22134
+ } catch {
22135
+ return;
22136
+ }
22137
+ }
22138
+ function expandHome(path) {
22139
+ if (path === "~")
22140
+ return homedir3();
22141
+ const prefixed = path.startsWith("~/") || sep7 === "\\" && path.startsWith("~\\");
22142
+ return prefixed ? join30(homedir3(), path.slice(2)) : path;
22143
+ }
22144
+ var init_config2 = __esm(() => {
22145
+ init_connector();
22146
+ });
22147
+
22148
+ // src/datasource/sidecar/descriptors.ts
22149
+ function chatCrawler(id) {
22150
+ return {
22151
+ id,
22152
+ keyExpression: "m.event_id",
22153
+ chunkId: (key) => key,
22154
+ fromClause: "messages m",
22155
+ liveWhere: "(m.deleted_at is null or m.deleted_at = 0)",
22156
+ textExpression: "coalesce(m.text, '')",
22157
+ titleExpression: "m.chat_name",
22158
+ epochSecondsExpression: "m.ts",
22159
+ fts: { table: "messages_fts", joinOn: "messages_fts.rowid = m.rowid" },
22160
+ metadata: [
22161
+ { sql: "m.chat_jid", key: "chatId" },
22162
+ { sql: "m.chat_name", key: "chatName" },
22163
+ { sql: "m.sender_name", key: "senderName" },
22164
+ { sql: "m.ts", key: "timestamp" }
22165
+ ],
22166
+ channelIdExpression: "m.chat_jid",
22167
+ channelNameExpression: "m.chat_name",
22168
+ hierarchy: (metadata) => ["chats", str(metadata.chatName) ?? str(metadata.chatId) ?? "unknown"]
22169
+ };
22170
+ }
22171
+ var str = (value) => typeof value === "string" && value.length > 0 ? value : undefined, slack, CRAWLER_DESCRIPTORS;
22172
+ var init_descriptors = __esm(() => {
22173
+ slack = {
22174
+ id: "slack",
22175
+ archivePathInConfig: true,
22176
+ keyExpression: "m.channel_id || '|' || m.ts",
22177
+ chunkId: (_key, metadata) => `${metadata.channelId ?? ""}-${(metadata.timestamp ?? "").replaceAll(".", "-")}`,
22178
+ fromClause: "messages m left join channels c on c.id = m.channel_id and c.workspace_id = m.workspace_id " + "left join users u on u.id = m.user_id",
22179
+ liveWhere: "(m.deleted_ts is null or m.deleted_ts = '')",
22180
+ scopeExpression: "m.workspace_id",
22181
+ textExpression: "coalesce(nullif(m.normalized_text, ''), m.text)",
22182
+ titleExpression: "c.name",
22183
+ epochSecondsExpression: "cast(m.ts as real)",
22184
+ fts: { table: "message_fts", joinOn: "message_fts.message_key = m.channel_id || '|' || m.ts" },
22185
+ metadata: [
22186
+ { sql: "m.channel_id", key: "channelId" },
22187
+ { sql: "c.name", key: "channelName" },
22188
+ { sql: "m.workspace_id", key: "workspaceId" },
22189
+ { sql: "m.user_id", key: "userId" },
22190
+ { sql: "coalesce(nullif(u.real_name, ''), u.name)", key: "senderName" },
22191
+ { sql: "m.ts", key: "timestamp" }
22192
+ ],
22193
+ channelIdExpression: "m.channel_id",
22194
+ channelNameExpression: "c.name",
22195
+ hierarchy: (metadata) => [
22196
+ "workspaces",
22197
+ str(metadata.workspaceId) ?? "unknown",
22198
+ "channels",
22199
+ str(metadata.channelName) ?? str(metadata.channelId) ?? "unknown"
22200
+ ]
22201
+ };
22202
+ CRAWLER_DESCRIPTORS = {
22203
+ slack,
22204
+ telegram: chatCrawler("telegram"),
22205
+ whatsapp: chatCrawler("whatsapp")
22206
+ };
22207
+ });
22208
+
22209
+ // src/datasource/skills/clawgallery/client.ts
22210
+ import { spawn as spawn9 } from "node:child_process";
22211
+ import { existsSync as existsSync23 } from "node:fs";
22212
+
22213
+ class ClawGalleryClient {
22214
+ options;
22215
+ constructor(options = {}) {
22216
+ this.options = options;
22217
+ }
22218
+ async bootstrap(signal) {
22219
+ if (this.options.path !== undefined && !existsSync23(this.options.path)) {
22220
+ return {
22221
+ ok: false,
22222
+ reason: "nonzero-exit",
22223
+ stdout: "",
20987
22224
  stderr: "configured ClawGallery folder does not exist",
20988
22225
  code: null
20989
22226
  };
@@ -21084,7 +22321,7 @@ function spawnCli(binary, args, env, options, signal) {
21084
22321
  const timeout = setTimeout(() => {
21085
22322
  reason = "timeout";
21086
22323
  terminate2(child);
21087
- }, options.timeoutMs ?? DEFAULT_TIMEOUT_MS6);
22324
+ }, options.timeoutMs ?? DEFAULT_TIMEOUT_MS7);
21088
22325
  const abort = () => {
21089
22326
  reason = "aborted";
21090
22327
  terminate2(child);
@@ -21190,16 +22427,16 @@ function failure2(result, reason) {
21190
22427
  ok: false,
21191
22428
  reason: reason ?? result.reason ?? "nonzero-exit",
21192
22429
  stdout: result.stdout,
21193
- stderr: boundDiagnosticText(result.stderr),
22430
+ stderr: boundDiagnosticText2(result.stderr),
21194
22431
  code: result.code
21195
22432
  };
21196
22433
  }
21197
- function boundDiagnosticText(value) {
22434
+ function boundDiagnosticText2(value) {
21198
22435
  if (value.length === 0)
21199
22436
  return "";
21200
22437
  return value.trim().slice(0, 4000);
21201
22438
  }
21202
- var DEFAULT_BINARY2 = "clawgallery", DEFAULT_TIMEOUT_MS6 = 60000, DEFAULT_MAX_BUFFER_BYTES2, SAFE_ENV;
22439
+ var DEFAULT_BINARY2 = "clawgallery", DEFAULT_TIMEOUT_MS7 = 60000, DEFAULT_MAX_BUFFER_BYTES2, SAFE_ENV;
21203
22440
  var init_client5 = __esm(() => {
21204
22441
  init_portable_spawn();
21205
22442
  DEFAULT_MAX_BUFFER_BYTES2 = 4 * 1024 * 1024;
@@ -21493,7 +22730,7 @@ function statusToReason(status) {
21493
22730
  }
21494
22731
  async function requestText(url, options) {
21495
22732
  const fetchImpl = options.fetchImpl ?? globalThis.fetch;
21496
- const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS7;
22733
+ const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
21497
22734
  let response;
21498
22735
  try {
21499
22736
  response = await fetchImpl(url, {
@@ -21557,13 +22794,13 @@ function resolveToken(token, tokenEnv) {
21557
22794
  }
21558
22795
  return;
21559
22796
  }
21560
- var DEFAULT_TIMEOUT_MS7 = 15000;
22797
+ var DEFAULT_TIMEOUT_MS8 = 15000;
21561
22798
 
21562
22799
  // src/datasource/skills/cloud-drive/rclone-connector.ts
21563
22800
  import { spawn as spawn10 } from "node:child_process";
21564
- import { mkdirSync as mkdirSync18, readFileSync as readFileSync19, renameSync as renameSync10, rmSync as rmSync9, writeFileSync as writeFileSync17 } from "node:fs";
22801
+ import { mkdirSync as mkdirSync18, readFileSync as readFileSync20, renameSync as renameSync10, rmSync as rmSync9, writeFileSync as writeFileSync17 } from "node:fs";
21565
22802
  import { readFile as readFile4 } from "node:fs/promises";
21566
- import { dirname as dirname17, join as join30, normalize as normalize4 } from "node:path";
22803
+ import { dirname as dirname18, join as join31, normalize as normalize4 } from "node:path";
21567
22804
 
21568
22805
  class RcloneConnector {
21569
22806
  options;
@@ -21579,7 +22816,7 @@ class RcloneConnector {
21579
22816
  if (remote === undefined || remote.length === 0) {
21580
22817
  return { ok: false, reason: "not-configured", message: "rclone remote not configured" };
21581
22818
  }
21582
- const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
22819
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS9;
21583
22820
  const exportFormats = this.options.exportFormats ?? DEFAULT_EXPORT_FORMATS;
21584
22821
  let listResult;
21585
22822
  try {
@@ -21650,7 +22887,7 @@ class RcloneConnector {
21650
22887
  return;
21651
22888
  const destination = mirrorFilePath(paths.mirrorRoot, entry.path);
21652
22889
  const temp = `${destination}.autorag-tmp`;
21653
- mkdirSync18(dirname17(temp), { recursive: true });
22890
+ mkdirSync18(dirname18(temp), { recursive: true });
21654
22891
  let copied;
21655
22892
  try {
21656
22893
  copied = await this.runner(copyArgs(remote, entry.path, temp, exportFormats, this.options.concurrency, this.options.bandwidthLimit), timeoutMs);
@@ -21755,12 +22992,12 @@ function sameEntry(previous, current) {
21755
22992
  return previous?.path === current.path && previous.size === current.size && previous.modTime === current.modTime && previous.remoteId === current.remoteId && JSON.stringify(previous.hashes) === JSON.stringify(current.hashes);
21756
22993
  }
21757
22994
  function mirrorPaths(workspaceRoot, skillName, instanceId) {
21758
- const base = join30(workspaceRoot, ".autorag", "datasources", skillName, instanceId);
21759
- return { mirrorRoot: join30(base, "mirror"), manifestPath: join30(base, "manifest.json") };
22995
+ const base = join31(workspaceRoot, ".autorag", "datasources", skillName, instanceId);
22996
+ return { mirrorRoot: join31(base, "mirror"), manifestPath: join31(base, "manifest.json") };
21760
22997
  }
21761
22998
  function mirrorFilePath(root, path) {
21762
22999
  const safe = normalize4(path).replace(/^(\.\.(?:[/\\]|$))+|^[/\\]+/gu, "");
21763
- return join30(root, safe);
23000
+ return join31(root, safe);
21764
23001
  }
21765
23002
  function remoteTarget(remote, path) {
21766
23003
  return remote.endsWith(":") || remote.endsWith("/") ? `${remote}${path}` : `${remote}/${path}`;
@@ -21779,14 +23016,14 @@ function copyArgs(remote, path, destination, exportFormats, concurrency, bandwid
21779
23016
  }
21780
23017
  function loadManifest2(path) {
21781
23018
  try {
21782
- const parsed = JSON.parse(readFileSync19(path, "utf8"));
23019
+ const parsed = JSON.parse(readFileSync20(path, "utf8"));
21783
23020
  return parsed.version === 1 && Array.isArray(parsed.entries) ? parsed : undefined;
21784
23021
  } catch {
21785
23022
  return;
21786
23023
  }
21787
23024
  }
21788
23025
  function saveManifest(path, manifest) {
21789
- mkdirSync18(dirname17(path), { recursive: true });
23026
+ mkdirSync18(dirname18(path), { recursive: true });
21790
23027
  const temp = `${path}.autorag-tmp`;
21791
23028
  writeFileSync17(temp, `${JSON.stringify(manifest)}
21792
23029
  `, "utf8");
@@ -21888,7 +23125,7 @@ function runBinary(binary, args, timeoutMs, env, cwd) {
21888
23125
  });
21889
23126
  });
21890
23127
  }
21891
- var DEFAULT_BINARY3 = "rclone", DEFAULT_EXTENSIONS, DEFAULT_EXPORT_FORMATS = "txt,csv", DEFAULT_MAX_DOCUMENTS = 200, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_TIMEOUT_MS8 = 60000, MAX_CONTENT_CHARS2 = 1e5;
23128
+ var DEFAULT_BINARY3 = "rclone", DEFAULT_EXTENSIONS, DEFAULT_EXPORT_FORMATS = "txt,csv", DEFAULT_MAX_DOCUMENTS = 200, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_TIMEOUT_MS9 = 60000, MAX_CONTENT_CHARS2 = 1e5;
21892
23129
  var init_rclone_connector = __esm(() => {
21893
23130
  init_defaults();
21894
23131
  init_portable_spawn();
@@ -21896,52 +23133,11 @@ var init_rclone_connector = __esm(() => {
21896
23133
  DEFAULT_MAX_BYTES_PER_FILE = 2 * 1024 * 1024;
21897
23134
  });
21898
23135
 
21899
- // src/datasource/connector.ts
21900
- import { createHash as createHash12 } from "node:crypto";
21901
- function connectorFailureToDiagnosticCode(reason) {
21902
- switch (reason) {
21903
- case "auth":
21904
- return "datasource-auth-error";
21905
- case "permission":
21906
- return "datasource-permission-denied";
21907
- case "rate-limited":
21908
- return "datasource-rate-limited";
21909
- case "unavailable":
21910
- case "not-configured":
21911
- return "datasource-unavailable";
21912
- case "empty":
21913
- return "datasource-empty";
21914
- case "api-error":
21915
- case "invalid-data":
21916
- return "datasource-index-failed";
21917
- }
21918
- }
21919
- function boundDiagnosticText2(value) {
21920
- const trimmed = value.trim();
21921
- return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
21922
- }
21923
- function sanitizeIdSegment(value) {
21924
- const normalized = value.replace(/[^\p{L}\p{N}._-]+/gu, "-");
21925
- let start = 0;
21926
- while (start < normalized.length && (normalized[start] === "-" || normalized[start] === "."))
21927
- start += 1;
21928
- let end = normalized.length;
21929
- while (end > start && (normalized[end - 1] === "-" || normalized[end - 1] === "."))
21930
- end -= 1;
21931
- const cleaned = normalized.slice(start, end);
21932
- if (cleaned === value && cleaned.length > 0 && cleaned.length <= 80)
21933
- return cleaned;
21934
- const hash = createHash12("sha256").update(value).digest("hex").slice(0, 8);
21935
- const stem = cleaned.slice(0, 60);
21936
- return stem.length > 0 ? `${stem}-${hash}` : hash;
21937
- }
21938
- var init_connector = () => {};
21939
-
21940
23136
  // src/datasource/chunk-store.ts
21941
- import { existsSync as existsSync24, mkdirSync as mkdirSync19, readFileSync as readFileSync20, writeFileSync as writeFileSync18 } from "node:fs";
21942
- import { dirname as dirname18, join as join31 } from "node:path";
23137
+ import { existsSync as existsSync24, mkdirSync as mkdirSync19, readFileSync as readFileSync21, writeFileSync as writeFileSync18 } from "node:fs";
23138
+ import { dirname as dirname19, join as join32 } from "node:path";
21943
23139
  function datasourceChunkStorePath(workspaceRoot, skillName, instanceId) {
21944
- return join31(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId), "chunks.json");
23140
+ return join32(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId), "chunks.json");
21945
23141
  }
21946
23142
 
21947
23143
  class DatasourceChunkStore {
@@ -21980,7 +23176,7 @@ class DatasourceChunkStore {
21980
23176
  try {
21981
23177
  if (!existsSync24(this.storePath))
21982
23178
  return false;
21983
- const parsed = JSON.parse(readFileSync20(this.storePath, "utf8"));
23179
+ const parsed = JSON.parse(readFileSync21(this.storePath, "utf8"));
21984
23180
  if (parsed.version !== STORE_VERSION || !Array.isArray(parsed.chunks))
21985
23181
  return false;
21986
23182
  this.chunkList = parsed.chunks.filter((chunk) => typeof chunk?.chunkId === "string" && typeof chunk?.content === "string");
@@ -22026,7 +23222,7 @@ class DatasourceChunkStore {
22026
23222
  if (this.storePath === undefined)
22027
23223
  return;
22028
23224
  try {
22029
- mkdirSync19(dirname18(this.storePath), { recursive: true });
23225
+ mkdirSync19(dirname19(this.storePath), { recursive: true });
22030
23226
  const payload = {
22031
23227
  version: STORE_VERSION,
22032
23228
  skill: this.skillName,
@@ -22168,7 +23364,7 @@ class ConnectorDatasourceSkill {
22168
23364
  }
22169
23365
  if (!fetched.ok) {
22170
23366
  const code = connectorFailureToDiagnosticCode(fetched.reason);
22171
- const message = boundDiagnosticText2(fetched.message ?? fetched.reason);
23367
+ const message = boundDiagnosticText(fetched.message ?? fetched.reason);
22172
23368
  return this.fail(code, `${fetched.reason}: ${message}`);
22173
23369
  }
22174
23370
  const documents = this.applyDedupeWindow(fetched.documents);
@@ -22180,7 +23376,7 @@ class ConnectorDatasourceSkill {
22180
23376
  const diagnostics = (fetched.warnings ?? []).map((warning) => ({
22181
23377
  code: "datasource-index-failed",
22182
23378
  severity: "warning",
22183
- message: boundDiagnosticText2(warning),
23379
+ message: boundDiagnosticText(warning),
22184
23380
  instanceId: this.instanceId,
22185
23381
  source: this.definition.skillName
22186
23382
  }));
@@ -22297,7 +23493,7 @@ class ConnectorDatasourceSkill {
22297
23493
  return out;
22298
23494
  }
22299
23495
  fail(code, message) {
22300
- const sanitized = boundDiagnosticText2(message);
23496
+ const sanitized = boundDiagnosticText(message);
22301
23497
  this.lastError = sanitized;
22302
23498
  const diagnostic = {
22303
23499
  code,
@@ -22489,8 +23685,8 @@ var init_types4 = __esm(() => {
22489
23685
 
22490
23686
  // src/datasource/skills/discrawl/client.ts
22491
23687
  import { spawn as spawn12 } from "node:child_process";
22492
- import { mkdirSync as mkdirSync20, readFileSync as readFileSync21, writeFileSync as writeFileSync19 } from "node:fs";
22493
- import { dirname as dirname19 } from "node:path";
23688
+ import { mkdirSync as mkdirSync20, readFileSync as readFileSync22, writeFileSync as writeFileSync19 } from "node:fs";
23689
+ import { dirname as dirname20 } from "node:path";
22494
23690
 
22495
23691
  class DiscrawlClient {
22496
23692
  options;
@@ -22506,7 +23702,7 @@ class DiscrawlClient {
22506
23702
  const path = `${workspace}/.autorag/datasources/discrawl/config.toml`;
22507
23703
  let existing = "";
22508
23704
  try {
22509
- existing = readFileSync21(path, "utf8");
23705
+ existing = readFileSync22(path, "utf8");
22510
23706
  } catch {}
22511
23707
  const managed = existing.startsWith(`# AutoRAG managed discrawl embeddings v1
22512
23708
  `);
@@ -22514,7 +23710,7 @@ class DiscrawlClient {
22514
23710
  return { configured: false, rebuildRequired: false };
22515
23711
  const previous = managed ? parseManagedEmbedding(existing) : undefined;
22516
23712
  const config = managedDiscrawlConfig({ ...runtime, baseUrl: nativeGatewayBaseUrl(runtime.baseUrl) });
22517
- mkdirSync20(dirname19(path), { recursive: true });
23713
+ mkdirSync20(dirname20(path), { recursive: true });
22518
23714
  if (existing !== config)
22519
23715
  writeFileSync19(path, config, "utf8");
22520
23716
  return {
@@ -23539,8 +24735,8 @@ var init_github = __esm(() => {
23539
24735
 
23540
24736
  // src/datasource/skills/github-gist/connector.ts
23541
24737
  import { execFile as execFile2 } from "node:child_process";
23542
- import { existsSync as existsSync25, mkdirSync as mkdirSync21, readFileSync as readFileSync22, writeFileSync as writeFileSync20 } from "node:fs";
23543
- import { dirname as dirname20 } from "node:path";
24738
+ import { existsSync as existsSync25, mkdirSync as mkdirSync21, readFileSync as readFileSync23, writeFileSync as writeFileSync20 } from "node:fs";
24739
+ import { dirname as dirname21 } from "node:path";
23544
24740
  async function defaultGhTokenRunner() {
23545
24741
  return new Promise((resolve) => {
23546
24742
  execFile2("gh", ["auth", "token"], { timeout: GH_TOKEN_TIMEOUT_MS }, (error, stdout) => {
@@ -23662,7 +24858,7 @@ class GitHubGistConnector {
23662
24858
  if (statePath === undefined || !existsSync25(statePath))
23663
24859
  return { version: STATE_VERSION, gists: {} };
23664
24860
  try {
23665
- const parsed = JSON.parse(readFileSync22(statePath, "utf8"));
24861
+ const parsed = JSON.parse(readFileSync23(statePath, "utf8"));
23666
24862
  if (parsed.version !== STATE_VERSION || typeof parsed.gists !== "object" || parsed.gists === null) {
23667
24863
  return { version: STATE_VERSION, gists: {} };
23668
24864
  }
@@ -23676,7 +24872,7 @@ class GitHubGistConnector {
23676
24872
  if (statePath === undefined)
23677
24873
  return;
23678
24874
  try {
23679
- mkdirSync21(dirname20(statePath), { recursive: true });
24875
+ mkdirSync21(dirname21(statePath), { recursive: true });
23680
24876
  writeFileSync20(statePath, `${JSON.stringify(state)}
23681
24877
  `, "utf8");
23682
24878
  } catch {}
@@ -23722,8 +24918,8 @@ var init_connector3 = () => {};
23722
24918
 
23723
24919
  // src/datasource/skills/github-gist/semantic.ts
23724
24920
  import { createHash as createHash13 } from "node:crypto";
23725
- import { existsSync as existsSync26, mkdirSync as mkdirSync22, readFileSync as readFileSync23, writeFileSync as writeFileSync21 } from "node:fs";
23726
- import { dirname as dirname21 } from "node:path";
24921
+ import { existsSync as existsSync26, mkdirSync as mkdirSync22, readFileSync as readFileSync24, writeFileSync as writeFileSync21 } from "node:fs";
24922
+ import { dirname as dirname22 } from "node:path";
23727
24923
  function contentHash(chunk) {
23728
24924
  return createHash13("sha256").update(`${chunk.title ?? ""}
23729
24925
  ${chunk.content}`).digest("hex");
@@ -23816,7 +25012,7 @@ ${chunk.content}`));
23816
25012
  if (this.statePath === undefined || !existsSync26(this.statePath))
23817
25013
  return;
23818
25014
  try {
23819
- const parsed = JSON.parse(readFileSync23(this.statePath, "utf8"));
25015
+ const parsed = JSON.parse(readFileSync24(this.statePath, "utf8"));
23820
25016
  if (parsed.version !== VECTORS_VERSION || typeof parsed.entries !== "object" || parsed.entries === null)
23821
25017
  return;
23822
25018
  if (typeof parsed.identity?.dimension !== "number")
@@ -23831,7 +25027,7 @@ ${chunk.content}`));
23831
25027
  if (this.identity === undefined)
23832
25028
  return;
23833
25029
  try {
23834
- mkdirSync22(dirname21(this.statePath), { recursive: true });
25030
+ mkdirSync22(dirname22(this.statePath), { recursive: true });
23835
25031
  const payload = {
23836
25032
  version: VECTORS_VERSION,
23837
25033
  identity: this.identity,
@@ -23914,9 +25110,9 @@ var init_semantic = __esm(() => {
23914
25110
  });
23915
25111
 
23916
25112
  // src/datasource/skills/github-gist/skill.ts
23917
- import { join as join32 } from "node:path";
25113
+ import { join as join33 } from "node:path";
23918
25114
  function gistDatasourceDir(workspaceRoot, skillName, instanceId) {
23919
- return join32(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId));
25115
+ return join33(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId));
23920
25116
  }
23921
25117
  var GITHUB_GIST_SKILL_DEFINITION, DEFAULT_INSTANCE_ID3 = "default", GitHubGistSkill;
23922
25118
  var init_skill5 = __esm(() => {
@@ -23944,7 +25140,7 @@ var init_skill5 = __esm(() => {
23944
25140
  constructor(options = {}) {
23945
25141
  const { connector, connectorOptions, semantic, ...rest } = options;
23946
25142
  const instanceId = options.instanceId ?? DEFAULT_INSTANCE_ID3;
23947
- const statePath = connectorOptions?.statePath ?? (options.workspaceRoot === undefined ? undefined : join32(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "state.json"));
25143
+ const statePath = connectorOptions?.statePath ?? (options.workspaceRoot === undefined ? undefined : join33(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "state.json"));
23948
25144
  super(GITHUB_GIST_SKILL_DEFINITION, {
23949
25145
  ...rest,
23950
25146
  connector: connector ?? new GitHubGistConnector({
@@ -23954,7 +25150,7 @@ var init_skill5 = __esm(() => {
23954
25150
  });
23955
25151
  if (semantic?.enabled !== false) {
23956
25152
  this.semanticIndex = new GistSemanticIndex({
23957
- statePath: semantic?.statePath ?? (options.workspaceRoot === undefined ? undefined : join32(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "vectors.json"))
25153
+ statePath: semantic?.statePath ?? (options.workspaceRoot === undefined ? undefined : join33(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "vectors.json"))
23958
25154
  });
23959
25155
  this.semanticEmbedder = semantic?.embedder ?? createGatewayEmbedder(semantic?.embedderOptions ?? {});
23960
25156
  this.semanticMethod = new GitHubGistSemanticMethod({
@@ -23982,7 +25178,7 @@ var init_skill5 = __esm(() => {
23982
25178
  {
23983
25179
  code: "datasource-index-failed",
23984
25180
  severity: "warning",
23985
- message: boundDiagnosticText2(`semantic-unavailable: ${sync.message}`),
25181
+ message: boundDiagnosticText(`semantic-unavailable: ${sync.message}`),
23986
25182
  instanceId: this.instanceId,
23987
25183
  source: this.describe().name
23988
25184
  }
@@ -24677,7 +25873,7 @@ var init_types5 = __esm(() => {
24677
25873
  // src/datasource/skills/lazykatok/client.ts
24678
25874
  import { spawn as spawn14 } from "node:child_process";
24679
25875
  import { existsSync as existsSync27 } from "node:fs";
24680
- import { delimiter as delimiter4, join as join33 } from "node:path";
25876
+ import { delimiter as delimiter4, join as join34 } from "node:path";
24681
25877
 
24682
25878
  class LazykatokClient {
24683
25879
  options;
@@ -24865,7 +26061,7 @@ function lookupInPath2(executable, env) {
24865
26061
  for (const dir of pathEnv.split(delimiter4)) {
24866
26062
  if (dir.length === 0)
24867
26063
  continue;
24868
- const candidate = join33(dir, executable);
26064
+ const candidate = join34(dir, executable);
24869
26065
  if (existsSync27(candidate))
24870
26066
  return candidate;
24871
26067
  }
@@ -25355,7 +26551,7 @@ var init_lazykatok = __esm(() => {
25355
26551
 
25356
26552
  // src/datasource/skills/mail-export/connector.ts
25357
26553
  import { readdir as readdir3, readFile as readFile5, stat as stat6 } from "node:fs/promises";
25358
- import { basename as basename12, dirname as dirname22, extname as extname7, join as join34 } from "node:path";
26554
+ import { basename as basename12, dirname as dirname23, extname as extname7, join as join35 } from "node:path";
25359
26555
  import { simpleParser as simpleParser2 } from "mailparser";
25360
26556
 
25361
26557
  class MailExportConnector {
@@ -25381,7 +26577,7 @@ class MailExportConnector {
25381
26577
  continue;
25382
26578
  const extension = extname7(entry.name).toLowerCase();
25383
26579
  if (extension === ".eml" || extension === ".mbox") {
25384
- files.push(join34(entry.parentPath, entry.name));
26580
+ files.push(join35(entry.parentPath, entry.name));
25385
26581
  }
25386
26582
  }
25387
26583
  } else if (info.isFile()) {
@@ -25416,7 +26612,7 @@ class MailExportConnector {
25416
26612
  continue;
25417
26613
  }
25418
26614
  const stem = basename12(file, extname7(file));
25419
- const mailboxDir = basename12(dirname22(file));
26615
+ const mailboxDir = basename12(dirname23(file));
25420
26616
  const hierarchy = ["mailboxes", mailboxDir, stem];
25421
26617
  const extension = extname7(file).toLowerCase();
25422
26618
  const rawMessages = extension === ".mbox" ? splitMbox(bytes.toString("utf8")) : [bytes.toString("utf8")];
@@ -25537,7 +26733,7 @@ function ensurePrivateMailcrawlDataDir(path) {
25537
26733
  if (process.platform !== "win32")
25538
26734
  chmodSync(path, 448);
25539
26735
  }
25540
- var init_config2 = () => {};
26736
+ var init_config3 = () => {};
25541
26737
 
25542
26738
  // src/datasource/skills/mailcrawl/client.ts
25543
26739
  import { spawn as spawn15 } from "node:child_process";
@@ -25863,7 +27059,7 @@ function truncateUtf8(value, maxBytes) {
25863
27059
  var DEFAULT_BINARY4 = "mailcrawl", DEFAULT_SEARCH_TIMEOUT_MS3 = 60000, DEFAULT_INDEX_TIMEOUT_MS3 = 1800000, DEFAULT_MAX_BUFFER_BYTES4 = 1048576, LOOPBACK_HOSTS2, SAFE_ENV_KEYS2, SAFE_MAILCRAWL_ENV_KEYS, REMOTE_EMBEDDING_ENV_KEYS;
25864
27060
  var init_client9 = __esm(() => {
25865
27061
  init_portable_spawn();
25866
- init_config2();
27062
+ init_config3();
25867
27063
  LOOPBACK_HOSTS2 = new Set(["127.0.0.1", "localhost", "[::1]", "::1"]);
25868
27064
  SAFE_ENV_KEYS2 = new Set(["HOME", "LANG", "LC_ALL", "PATH", "TMPDIR", "TMP", "TEMP", "NO_COLOR"]);
25869
27065
  SAFE_MAILCRAWL_ENV_KEYS = new Set([
@@ -26110,7 +27306,7 @@ var TAGS;
26110
27306
  var init_skill9 = __esm(() => {
26111
27307
  init_scope2();
26112
27308
  init_client9();
26113
- init_config2();
27309
+ init_config3();
26114
27310
  init_methods4();
26115
27311
  TAGS = ["mailcrawl", "email", "pii"];
26116
27312
  });
@@ -26122,6 +27318,504 @@ var init_mailcrawl = __esm(() => {
26122
27318
  init_skill9();
26123
27319
  });
26124
27320
 
27321
+ // src/datasource/sidecar/fusion.ts
27322
+ function encodeVector(vector) {
27323
+ const floats = Float32Array.from(vector);
27324
+ return new Uint8Array(floats.buffer, floats.byteOffset, floats.byteLength);
27325
+ }
27326
+ function decodeVector(blob) {
27327
+ if (blob.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0)
27328
+ return;
27329
+ return new Float32Array(new Uint8Array(blob).buffer);
27330
+ }
27331
+ function cosine3(a, b) {
27332
+ if (a.length === 0 || a.length !== b.length)
27333
+ return 0;
27334
+ let dot = 0;
27335
+ let normA = 0;
27336
+ let normB = 0;
27337
+ for (let index = 0;index < a.length; index += 1) {
27338
+ const x = a[index] ?? 0;
27339
+ const y = b[index] ?? 0;
27340
+ dot += x * y;
27341
+ normA += x * x;
27342
+ normB += y * y;
27343
+ }
27344
+ if (normA === 0 || normB === 0)
27345
+ return 0;
27346
+ return dot / Math.sqrt(normA * normB);
27347
+ }
27348
+ function topKByCosine(query, rows, k) {
27349
+ if (k <= 0)
27350
+ return [];
27351
+ const scored = [];
27352
+ for (const row of rows) {
27353
+ const score = cosine3(query, row.vector);
27354
+ if (score > 0)
27355
+ scored.push({ key: row.key, score });
27356
+ }
27357
+ scored.sort((a, b) => b.score - a.score || a.key.localeCompare(b.key));
27358
+ return scored.slice(0, k);
27359
+ }
27360
+ function reciprocalRankFusion(lists) {
27361
+ const scores = new Map;
27362
+ for (const list of lists) {
27363
+ const seen = new Set;
27364
+ for (const [index, key] of list.entries()) {
27365
+ if (seen.has(key))
27366
+ continue;
27367
+ seen.add(key);
27368
+ scores.set(key, (scores.get(key) ?? 0) + 1 / (RRF_K2 + index + 1));
27369
+ }
27370
+ }
27371
+ return [...scores.entries()].map(([key, score]) => ({ key, score })).sort((a, b) => b.score - a.score);
27372
+ }
27373
+ var RRF_K2 = 60;
27374
+
27375
+ // src/datasource/sidecar/index.ts
27376
+ import { createHash as createHash14 } from "node:crypto";
27377
+ import { mkdirSync as mkdirSync24, statSync as statSync3 } from "node:fs";
27378
+ import { dirname as dirname24 } from "node:path";
27379
+ import { DatabaseSync as DatabaseSync2 } from "node:sqlite";
27380
+ import { pathToFileURL } from "node:url";
27381
+
27382
+ class CrawlerSidecar {
27383
+ descriptor;
27384
+ options;
27385
+ select;
27386
+ where;
27387
+ opened;
27388
+ attached;
27389
+ lastSemantic;
27390
+ constructor(options) {
27391
+ this.options = options;
27392
+ const { descriptor } = options;
27393
+ this.descriptor = descriptor;
27394
+ const metadataColumns = descriptor.metadata.map((column, index) => `${column.sql} as m${index}`);
27395
+ this.select = [
27396
+ `${descriptor.keyExpression} as k`,
27397
+ `${descriptor.textExpression} as content`,
27398
+ `${descriptor.titleExpression ?? "null"} as title`,
27399
+ `${descriptor.epochSecondsExpression ?? "null"} as epoch`,
27400
+ ...metadataColumns
27401
+ ].join(", ");
27402
+ const scope = descriptor.scopeExpression !== undefined && options.workspaceId !== undefined;
27403
+ const allowlist = [
27404
+ ...(options.channelIds?.length ?? 0) > 0 ? [`${descriptor.channelIdExpression} in (${options.channelIds?.map(() => "?").join(", ")})`] : [],
27405
+ ...(options.channelNames?.length ?? 0) > 0 ? [`${descriptor.channelNameExpression} in (${options.channelNames?.map(() => "?").join(", ")})`] : []
27406
+ ];
27407
+ this.where = [
27408
+ descriptor.liveWhere,
27409
+ ...scope ? [`${descriptor.scopeExpression} = ?`] : [],
27410
+ ...allowlist.length > 0 ? [`(${allowlist.join(" or ")})`] : []
27411
+ ].join(" and ");
27412
+ }
27413
+ get db() {
27414
+ if (this.opened !== undefined)
27415
+ return this.opened;
27416
+ mkdirSync24(dirname24(this.options.sidecarDbPath), { recursive: true });
27417
+ const db = new DatabaseSync2(this.options.sidecarDbPath);
27418
+ db.exec(`pragma busy_timeout = ${BUSY_TIMEOUT_MS}`);
27419
+ db.exec("pragma journal_mode = wal");
27420
+ db.exec(`
27421
+ create table if not exists sidecar_meta (k text primary key, v text not null);
27422
+ create table if not exists sidecar_vec (message_key text primary key, hash text not null, vec blob not null);
27423
+ `);
27424
+ this.opened = db;
27425
+ return db;
27426
+ }
27427
+ attachCrawler() {
27428
+ let file;
27429
+ try {
27430
+ const { dev, ino } = statSync3(this.options.crawlerDbPath);
27431
+ file = { dev, ino };
27432
+ } catch {
27433
+ this.detachCrawler();
27434
+ return false;
27435
+ }
27436
+ if (this.attached !== undefined) {
27437
+ if (this.attached.dev === file.dev && this.attached.ino === file.ino)
27438
+ return true;
27439
+ this.detachCrawler();
27440
+ }
27441
+ const crawlerUri = `${pathToFileURL(this.options.crawlerDbPath).href}?mode=ro`.replaceAll("'", "''");
27442
+ try {
27443
+ this.db.exec(`attach database '${crawlerUri}' as crawler`);
27444
+ } catch {
27445
+ return false;
27446
+ }
27447
+ const { descriptor } = this.options;
27448
+ try {
27449
+ this.db.prepare(`select ${this.select} from ${descriptor.fromClause} where ${this.where} limit 0`).all(...this.whereParams());
27450
+ this.db.prepare(`select 1 from ${descriptor.fts.table} limit 0`).all();
27451
+ } catch {
27452
+ this.db.exec("detach database crawler");
27453
+ return false;
27454
+ }
27455
+ this.attached = file;
27456
+ this.lastSemantic = undefined;
27457
+ return true;
27458
+ }
27459
+ detachCrawler() {
27460
+ if (this.attached === undefined)
27461
+ return;
27462
+ this.attached = undefined;
27463
+ this.lastSemantic = undefined;
27464
+ try {
27465
+ this.opened?.exec("detach database crawler");
27466
+ } catch {}
27467
+ }
27468
+ get semanticEnabled() {
27469
+ return this.options.embedder !== undefined;
27470
+ }
27471
+ close() {
27472
+ this.opened?.close();
27473
+ this.opened = undefined;
27474
+ this.attached = undefined;
27475
+ this.lastSemantic = undefined;
27476
+ }
27477
+ whereParams() {
27478
+ const { descriptor, workspaceId, channelIds = [], channelNames = [] } = this.options;
27479
+ return [
27480
+ ...descriptor.scopeExpression !== undefined && workspaceId !== undefined ? [workspaceId] : [],
27481
+ ...channelIds,
27482
+ ...channelNames
27483
+ ];
27484
+ }
27485
+ searchLexical(query, topK) {
27486
+ const match = toMatchQuery(query);
27487
+ if (match === undefined || topK <= 0 || !this.attachCrawler())
27488
+ return [];
27489
+ const { descriptor } = this.options;
27490
+ const rows = this.db.prepare(`select ${this.select}, bm25(${descriptor.fts.table}) as rank
27491
+ from ${descriptor.fromClause} join ${descriptor.fts.table} on ${descriptor.fts.joinOn}
27492
+ where ${descriptor.fts.table} match ? and ${this.where}
27493
+ order by rank limit ?`).all(match, ...this.whereParams(), topK);
27494
+ return rows.map((row) => this.toHit(this.toRow(row), -Number(row.rank)));
27495
+ }
27496
+ async searchSemantic(query, topK) {
27497
+ const trimmed = query.trim();
27498
+ if (this.options.embedder === undefined || trimmed.length === 0 || topK <= 0 || !this.attachCrawler())
27499
+ return [];
27500
+ const ranked = await this.rankSemantic(trimmed, topK * POOL_FACTOR);
27501
+ if (ranked.length === 0)
27502
+ return [];
27503
+ const live = this.hydrate(ranked.map((entry) => entry.key));
27504
+ const hits = [];
27505
+ for (const entry of ranked) {
27506
+ const row = live.get(entry.key);
27507
+ if (row !== undefined)
27508
+ hits.push(this.toHit(row, entry.score));
27509
+ if (hits.length >= topK)
27510
+ break;
27511
+ }
27512
+ return hits;
27513
+ }
27514
+ async rankSemantic(query, pool) {
27515
+ const now = Date.now();
27516
+ let cached = this.lastSemantic;
27517
+ if (cached === undefined || cached.query !== query || now - cached.at > SEMANTIC_REUSE_MS) {
27518
+ const embedder = this.options.embedder;
27519
+ if (embedder === undefined)
27520
+ return [];
27521
+ cached = {
27522
+ query,
27523
+ at: now,
27524
+ vector: embedder.embed([query]).then((vectors) => vectors[0], () => {
27525
+ return;
27526
+ }),
27527
+ ranked: undefined
27528
+ };
27529
+ this.lastSemantic = cached;
27530
+ }
27531
+ const vector = await cached.vector;
27532
+ if (vector === undefined) {
27533
+ if (this.lastSemantic === cached)
27534
+ this.lastSemantic = undefined;
27535
+ return [];
27536
+ }
27537
+ if (cached.ranked === undefined || cached.ranked.pool < pool) {
27538
+ const wide = pool * POOL_FACTOR;
27539
+ cached.ranked = { pool: wide, keys: topKByCosine(vector, this.storedVectors(), wide) };
27540
+ }
27541
+ return cached.ranked.keys.slice(0, pool);
27542
+ }
27543
+ async searchHybrid(query, topK) {
27544
+ if (topK <= 0)
27545
+ return [];
27546
+ const pool = topK * POOL_FACTOR;
27547
+ const lexical = this.searchLexical(query, pool);
27548
+ const semantic = await this.searchSemantic(query, pool);
27549
+ if (semantic.length === 0)
27550
+ return lexical.slice(0, topK);
27551
+ const byKey = new Map;
27552
+ for (const hit of [...semantic, ...lexical])
27553
+ byKey.set(hit.key, hit);
27554
+ const fused = [];
27555
+ for (const entry of reciprocalRankFusion([lexical.map((hit) => hit.key), semantic.map((hit) => hit.key)])) {
27556
+ const hit = byKey.get(entry.key);
27557
+ if (hit !== undefined)
27558
+ fused.push({ ...hit, score: entry.score });
27559
+ if (fused.length >= topK)
27560
+ break;
27561
+ }
27562
+ return fused;
27563
+ }
27564
+ async sync() {
27565
+ const { embedder } = this.options;
27566
+ if (embedder === undefined)
27567
+ return { ok: false, message: "no embedder configured" };
27568
+ if (!this.attachCrawler())
27569
+ return { ok: false, message: "crawler archive not available yet" };
27570
+ let identity;
27571
+ try {
27572
+ identity = await embedder.identity();
27573
+ } catch (error) {
27574
+ return { ok: false, message: errorMessage2(error) };
27575
+ }
27576
+ this.adoptIdentity(identity);
27577
+ const pruned = this.prune();
27578
+ const pending = this.pendingKeys();
27579
+ const batchSize = Math.max(1, this.options.batchSize ?? DEFAULT_BATCH_SIZE);
27580
+ const budget = Math.max(1, this.options.maxEmbedPerSync ?? DEFAULT_MAX_EMBED_PER_SYNC);
27581
+ const target = Math.min(pending.length, budget);
27582
+ const embed = async (rows) => {
27583
+ const vectors = await embedder.embed(rows.map((row) => row.input));
27584
+ if (vectors.length !== rows.length) {
27585
+ throw new Error(`embedder returned ${vectors.length} vectors for ${rows.length} messages`);
27586
+ }
27587
+ const wrong = vectors.find((vector) => vector.length !== identity.dimension);
27588
+ if (wrong !== undefined) {
27589
+ throw new Error(`embedder returned a ${wrong.length}-dimension vector, expected ${identity.dimension}`);
27590
+ }
27591
+ return vectors;
27592
+ };
27593
+ let embedded = 0;
27594
+ let skipped = 0;
27595
+ for (let offset = 0;offset < target; offset += batchSize) {
27596
+ const rows = [...this.hydrate(pending.slice(offset, Math.min(offset + batchSize, target))).values()];
27597
+ if (rows.length === 0)
27598
+ continue;
27599
+ try {
27600
+ this.store(rows, await embed(rows));
27601
+ embedded += rows.length;
27602
+ } catch (batchError) {
27603
+ const rejected = [];
27604
+ let lastError = batchError;
27605
+ for (const row of rows) {
27606
+ try {
27607
+ this.store([row], await embed([row]));
27608
+ embedded += 1;
27609
+ } catch (error) {
27610
+ rejected.push(row);
27611
+ lastError = error;
27612
+ }
27613
+ }
27614
+ if (rejected.length === 0)
27615
+ continue;
27616
+ if (embedded === 0)
27617
+ return { ok: false, message: errorMessage2(lastError) };
27618
+ this.store(rejected, undefined);
27619
+ skipped += rejected.length;
27620
+ }
27621
+ }
27622
+ return { ok: true, embedded, skipped, pruned, remaining: pending.length - embedded - skipped };
27623
+ }
27624
+ adoptIdentity(identity) {
27625
+ const serialized = JSON.stringify(identity);
27626
+ const current = this.db.prepare("select v from sidecar_meta where k = ?").get(IDENTITY_KEY);
27627
+ if (current?.v === serialized)
27628
+ return;
27629
+ this.transaction(() => {
27630
+ this.db.exec("delete from sidecar_vec");
27631
+ this.db.prepare("insert or replace into sidecar_meta (k, v) values (?, ?)").run(IDENTITY_KEY, serialized);
27632
+ });
27633
+ this.lastSemantic = undefined;
27634
+ }
27635
+ prune() {
27636
+ const { descriptor } = this.options;
27637
+ const result = this.db.prepare(`delete from sidecar_vec where message_key not in (
27638
+ select ${descriptor.keyExpression} from ${descriptor.fromClause} where ${this.where}
27639
+ )`).run(...this.whereParams());
27640
+ if (Number(result.changes) > 0)
27641
+ this.lastSemantic = undefined;
27642
+ return Number(result.changes);
27643
+ }
27644
+ pendingKeys() {
27645
+ const { descriptor } = this.options;
27646
+ const rows = this.db.prepare(`select ${this.select}, v.hash as stored_hash
27647
+ from ${descriptor.fromClause}
27648
+ left join sidecar_vec v on v.message_key = ${descriptor.keyExpression}
27649
+ where ${this.where}`).iterate(...this.whereParams());
27650
+ const pending = [];
27651
+ for (const row of rows) {
27652
+ if (row.stored_hash !== this.toRow(row).hash)
27653
+ pending.push(String(row.k));
27654
+ }
27655
+ return pending;
27656
+ }
27657
+ hydrate(keys) {
27658
+ const { descriptor } = this.options;
27659
+ const out = new Map;
27660
+ if (keys.length === 0)
27661
+ return out;
27662
+ const rows = this.db.prepare(`select ${this.select} from ${descriptor.fromClause}
27663
+ where ${descriptor.keyExpression} in (${keys.map(() => "?").join(", ")}) and ${this.where}`).all(...keys, ...this.whereParams());
27664
+ const byKey = new Map(rows.map((row) => [String(row.k), this.toRow(row)]));
27665
+ for (const key of keys) {
27666
+ const row = byKey.get(key);
27667
+ if (row !== undefined)
27668
+ out.set(key, row);
27669
+ }
27670
+ return out;
27671
+ }
27672
+ store(rows, vectors) {
27673
+ const upsert = this.db.prepare("insert or replace into sidecar_vec (message_key, hash, vec) values (?, ?, ?)");
27674
+ this.transaction(() => {
27675
+ for (const [index, row] of rows.entries()) {
27676
+ upsert.run(row.key, row.hash, encodeVector(vectors?.[index] ?? []));
27677
+ }
27678
+ });
27679
+ this.lastSemantic = undefined;
27680
+ }
27681
+ *storedVectors() {
27682
+ const rows = this.db.prepare("select message_key, vec from sidecar_vec where length(vec) > 0").iterate();
27683
+ for (const row of rows) {
27684
+ const vector = row.vec instanceof Uint8Array ? decodeVector(row.vec) : undefined;
27685
+ if (vector !== undefined)
27686
+ yield { key: String(row.message_key), vector };
27687
+ }
27688
+ }
27689
+ transaction(work) {
27690
+ this.db.exec("begin immediate");
27691
+ try {
27692
+ work();
27693
+ this.db.exec("commit");
27694
+ } catch (error) {
27695
+ this.db.exec("rollback");
27696
+ throw error;
27697
+ }
27698
+ }
27699
+ toRow(row) {
27700
+ const metadata = {};
27701
+ for (const [index, column] of this.options.descriptor.metadata.entries()) {
27702
+ const value = row[`m${index}`];
27703
+ if (typeof value === "string" && value.length > 0 || typeof value === "number")
27704
+ metadata[column.key] = String(value);
27705
+ }
27706
+ const title = typeof row.title === "string" && row.title.length > 0 ? row.title : undefined;
27707
+ const epoch = typeof row.epoch === "number" && Number.isFinite(row.epoch) ? row.epoch : undefined;
27708
+ const content = String(row.content ?? "");
27709
+ const input = `${title ?? ""}
27710
+ ${content}`.slice(0, MAX_EMBED_INPUT_CHARS);
27711
+ return {
27712
+ key: String(row.k),
27713
+ content,
27714
+ title,
27715
+ epoch,
27716
+ metadata,
27717
+ input,
27718
+ hash: createHash14("sha1").update(input).digest("hex")
27719
+ };
27720
+ }
27721
+ toHit(row, score) {
27722
+ return {
27723
+ key: row.key,
27724
+ content: row.content,
27725
+ score,
27726
+ ...row.title !== undefined ? { title: row.title } : {},
27727
+ ...row.epoch !== undefined ? { publishedAt: Math.round(row.epoch * 1000) } : {},
27728
+ metadata: row.metadata
27729
+ };
27730
+ }
27731
+ }
27732
+ function toMatchQuery(text) {
27733
+ const terms = [...new Set(text.match(/[\p{L}\p{N}_]+/gu) ?? [])];
27734
+ return terms.length === 0 ? undefined : terms.map((term) => `"${term}"`).join(" OR ");
27735
+ }
27736
+ function errorMessage2(error) {
27737
+ return error instanceof Error ? error.message : String(error);
27738
+ }
27739
+ var DEFAULT_BATCH_SIZE = 256, DEFAULT_MAX_EMBED_PER_SYNC = 2048, MAX_EMBED_INPUT_CHARS = 8192, BUSY_TIMEOUT_MS = 5000, POOL_FACTOR = 3, SEMANTIC_REUSE_MS = 1e4, IDENTITY_KEY = "identity";
27740
+ var init_sidecar = () => {};
27741
+
27742
+ // src/datasource/sidecar/methods.ts
27743
+ function sidecarMethodName(datasourceId, mode) {
27744
+ return mode === "bm25" ? `${datasourceId}-sidecar-bm25` : `${datasourceId}-${mode}`;
27745
+ }
27746
+
27747
+ class SidecarRetrievalMethod {
27748
+ options;
27749
+ constructor(options) {
27750
+ this.options = options;
27751
+ }
27752
+ describe() {
27753
+ const { datasourceId, skillType, mode, tags } = this.options;
27754
+ const { type, text, capabilities } = DESCRIPTIONS[mode];
27755
+ return {
27756
+ name: sidecarMethodName(datasourceId, mode),
27757
+ type,
27758
+ description: `${text} over ${skillType} via a read-only sidecar index`,
27759
+ status: "active",
27760
+ capabilities: [...capabilities, "scoped", "path-opaque-sources"],
27761
+ datasourceId,
27762
+ tags
27763
+ };
27764
+ }
27765
+ async retrieve(query, options) {
27766
+ const trimmed = query.trim();
27767
+ if (trimmed.length === 0)
27768
+ return [];
27769
+ const topK = options.topK ?? DEFAULT_TOP_K5;
27770
+ const { sidecar, mode } = this.options;
27771
+ const hits = mode === "bm25" ? sidecar.searchLexical(trimmed, topK) : mode === "semantic" ? await sidecar.searchSemantic(trimmed, topK) : await sidecar.searchHybrid(trimmed, topK);
27772
+ const results = [];
27773
+ for (const hit of hits) {
27774
+ const result = this.toResult(hit, options);
27775
+ if (result !== undefined)
27776
+ results.push(result);
27777
+ }
27778
+ return results;
27779
+ }
27780
+ toResult(hit, options) {
27781
+ const { descriptor, datasourceId, instanceId, mode } = this.options;
27782
+ const chunkId = descriptor.chunkId(hit.key, hit.metadata);
27783
+ const source = datasourceSourcePath(datasourceId, instanceId, chunkId);
27784
+ if (!matchesDatasourceScope(source, options.scope))
27785
+ return;
27786
+ const method = sidecarMethodName(datasourceId, mode);
27787
+ return {
27788
+ id: `${datasourceId}:${instanceId}:${chunkId}`,
27789
+ content: hit.content,
27790
+ source,
27791
+ score: hit.score,
27792
+ metadata: {
27793
+ ...hit.metadata,
27794
+ method,
27795
+ datasourceId,
27796
+ instanceId,
27797
+ backend: "sidecar",
27798
+ ...hit.title !== undefined ? { title: hit.title } : {},
27799
+ hierarchy: descriptor.hierarchy(hit.metadata).join("/"),
27800
+ ...hit.publishedAt !== undefined ? { publishedAt: hit.publishedAt } : {}
27801
+ }
27802
+ };
27803
+ }
27804
+ }
27805
+ var DEFAULT_TOP_K5 = 20, DESCRIPTIONS;
27806
+ var init_methods5 = __esm(() => {
27807
+ init_scope2();
27808
+ DESCRIPTIONS = {
27809
+ bm25: { type: "bm25", text: "BM25-ranked FTS5 retrieval", capabilities: ["lexical", "fts5", "bm25"] },
27810
+ semantic: { type: "vector", text: "Semantic (embedding) retrieval", capabilities: ["semantic"] },
27811
+ hybrid: {
27812
+ type: "hybrid",
27813
+ text: "Hybrid BM25 + semantic retrieval (reciprocal rank fusion)",
27814
+ capabilities: ["lexical", "semantic", "hybrid"]
27815
+ }
27816
+ };
27817
+ });
27818
+
26125
27819
  // src/datasource/crawler-skill.ts
26126
27820
  class CrawlerDatasourceSkill {
26127
27821
  definition;
@@ -26132,6 +27826,7 @@ class CrawlerDatasourceSkill {
26132
27826
  tags;
26133
27827
  channelIds;
26134
27828
  channelNames;
27829
+ sidecar;
26135
27830
  lastIndexedAt;
26136
27831
  constructor(definition, options) {
26137
27832
  this.definition = options.datasourceId === undefined ? definition : {
@@ -26147,6 +27842,11 @@ class CrawlerDatasourceSkill {
26147
27842
  this.lastIndexedAt = options.lastIndexedAt;
26148
27843
  this.channelIds = new Set(options.channelIds ?? []);
26149
27844
  this.channelNames = new Set(options.channelNames ?? []);
27845
+ this.sidecar = options.sidecar === undefined ? undefined : new CrawlerSidecar({
27846
+ ...options.sidecar,
27847
+ channelIds: options.channelIds ?? [],
27848
+ channelNames: options.channelNames ?? []
27849
+ });
26150
27850
  }
26151
27851
  describe() {
26152
27852
  return {
@@ -26182,26 +27882,89 @@ class CrawlerDatasourceSkill {
26182
27882
  skill: this.definition.datasourceId,
26183
27883
  chunkCount: result.count,
26184
27884
  indexedAt: this.lastIndexedAt,
26185
- diagnostics: []
27885
+ diagnostics: await this.syncSidecar()
26186
27886
  };
26187
27887
  } catch {
26188
27888
  return this.fail("datasource-unavailable", "spawn-error");
26189
27889
  }
26190
27890
  }
27891
+ async syncSidecar() {
27892
+ if (this.sidecar === undefined || !this.sidecar.semanticEnabled)
27893
+ return [];
27894
+ const base = { instanceId: this.instanceId, source: this.definition.datasourceId };
27895
+ let result;
27896
+ try {
27897
+ result = await this.sidecar.sync();
27898
+ } catch (error) {
27899
+ result = { ok: false, message: error instanceof Error ? error.message : String(error) };
27900
+ }
27901
+ if (!result.ok) {
27902
+ return [
27903
+ {
27904
+ code: "datasource-index-failed",
27905
+ severity: "warning",
27906
+ message: `semantic-unavailable: ${result.message}`.slice(0, 300),
27907
+ ...base
27908
+ }
27909
+ ];
27910
+ }
27911
+ const diagnostics = [];
27912
+ if (result.skipped > 0) {
27913
+ diagnostics.push({
27914
+ code: "datasource-index-partial",
27915
+ severity: "warning",
27916
+ message: `semantic index skipped ${result.skipped} message(s) the embedder rejected; they stay searchable by BM25 and are retried when edited`,
27917
+ ...base
27918
+ });
27919
+ }
27920
+ if (result.remaining > 0) {
27921
+ diagnostics.push({
27922
+ code: "datasource-index-partial",
27923
+ severity: "info",
27924
+ message: `semantic index catching up: ${result.embedded} embedded this refresh, ${result.remaining} remaining; hybrid search uses BM25 for the rest until a later refresh`,
27925
+ ...base
27926
+ });
27927
+ }
27928
+ return diagnostics;
27929
+ }
27930
+ close() {
27931
+ this.sidecar?.close();
27932
+ }
26191
27933
  retrievalMethods() {
27934
+ const cli = this.cliMethod();
27935
+ const { sidecar } = this;
27936
+ if (sidecar === undefined)
27937
+ return [cli];
27938
+ const shared = {
27939
+ sidecar,
27940
+ descriptor: sidecar.descriptor,
27941
+ datasourceId: this.definition.datasourceId,
27942
+ skillType: this.definition.skillType,
27943
+ instanceId: this.instanceId,
27944
+ tags: this.tags
27945
+ };
27946
+ const bm25 = new SidecarRetrievalMethod({ ...shared, mode: "bm25" });
27947
+ if (!sidecar.semanticEnabled)
27948
+ return [cli, bm25];
26192
27949
  return [
26193
- new CrawlerLexicalMethod({
26194
- client: this.client,
26195
- datasourceId: this.definition.datasourceId,
26196
- skillType: this.definition.skillType,
26197
- instanceId: this.instanceId,
26198
- tags: this.tags,
26199
- backendName: this.definition.backendName,
26200
- channelIds: this.channelIds,
26201
- channelNames: this.channelNames
26202
- })
27950
+ cli,
27951
+ bm25,
27952
+ new SidecarRetrievalMethod({ ...shared, mode: "semantic" }),
27953
+ new SidecarRetrievalMethod({ ...shared, mode: "hybrid" })
26203
27954
  ];
26204
27955
  }
27956
+ cliMethod() {
27957
+ return new CrawlerLexicalMethod({
27958
+ client: this.client,
27959
+ datasourceId: this.definition.datasourceId,
27960
+ skillType: this.definition.skillType,
27961
+ instanceId: this.instanceId,
27962
+ tags: this.tags,
27963
+ backendName: this.definition.backendName,
27964
+ channelIds: this.channelIds,
27965
+ channelNames: this.channelNames
27966
+ });
27967
+ }
26205
27968
  describeSources() {
26206
27969
  return this.instances.map((instanceId) => ({
26207
27970
  source: datasourceSourcePath(this.definition.datasourceId, instanceId),
@@ -26332,6 +28095,8 @@ var DEFAULT_INSTANCE_ID5 = "default", DEFAULT_POLLING_INTERVAL_MS4;
26332
28095
  var init_crawler_skill = __esm(() => {
26333
28096
  init_errors4();
26334
28097
  init_scope2();
28098
+ init_sidecar();
28099
+ init_methods5();
26335
28100
  DEFAULT_POLLING_INTERVAL_MS4 = 15 * 60 * 1000;
26336
28101
  });
26337
28102
 
@@ -26484,15 +28249,15 @@ var init_notion = __esm(() => {
26484
28249
  });
26485
28250
 
26486
28251
  // src/datasource/skills/obsidian/paths.ts
26487
- import { join as join35 } from "node:path";
28252
+ import { join as join36 } from "node:path";
26488
28253
  function obsidianDatasourceRoot(workspaceRoot, instanceId) {
26489
- return join35(workspaceRoot, AUTORAG_DIRNAME, DATASOURCES_DIRNAME, OBSIDIAN_DATASOURCE_DIRNAME, instanceId);
28254
+ return join36(workspaceRoot, AUTORAG_DIRNAME, DATASOURCES_DIRNAME, OBSIDIAN_DATASOURCE_DIRNAME, instanceId);
26490
28255
  }
26491
28256
  function obsidianQmdConfigDir(workspaceRoot, instanceId) {
26492
- return join35(obsidianDatasourceRoot(workspaceRoot, instanceId), "config");
28257
+ return join36(obsidianDatasourceRoot(workspaceRoot, instanceId), "config");
26493
28258
  }
26494
28259
  function obsidianQmdCacheDir(workspaceRoot, instanceId) {
26495
- return join35(obsidianDatasourceRoot(workspaceRoot, instanceId), "cache");
28260
+ return join36(obsidianDatasourceRoot(workspaceRoot, instanceId), "cache");
26496
28261
  }
26497
28262
  function stripEdgeDashes(value) {
26498
28263
  let start = 0;
@@ -26515,8 +28280,8 @@ var DEFAULT_QMD_BINARY = "qmd", DEFAULT_QMD_TIMEOUT_MS = 60000, DEFAULT_QMD_MAX_
26515
28280
 
26516
28281
  // src/datasource/skills/obsidian/client.ts
26517
28282
  import { spawn as spawn16 } from "node:child_process";
26518
- import { mkdirSync as mkdirSync24, writeFileSync as writeFileSync22 } from "node:fs";
26519
- import { isAbsolute as isAbsolute8, resolve as resolve14 } from "node:path";
28283
+ import { mkdirSync as mkdirSync25, writeFileSync as writeFileSync22 } from "node:fs";
28284
+ import { isAbsolute as isAbsolute9, resolve as resolve15 } from "node:path";
26520
28285
 
26521
28286
  class QmdClient {
26522
28287
  options;
@@ -26535,10 +28300,10 @@ class QmdClient {
26535
28300
  const cacheDir = obsidianQmdCacheDir(workspaceRoot, instanceId);
26536
28301
  try {
26537
28302
  if (this.options.configPath === undefined) {
26538
- mkdirSync24(configDir, { recursive: true });
26539
- writeFileSync22(resolve14(configDir, "index.yml"), renderIndexYaml(collectionName, vaultPath), "utf8");
28303
+ mkdirSync25(configDir, { recursive: true });
28304
+ writeFileSync22(resolve15(configDir, "index.yml"), renderIndexYaml(collectionName, vaultPath), "utf8");
26540
28305
  }
26541
- mkdirSync24(cacheDir, { recursive: true });
28306
+ mkdirSync25(cacheDir, { recursive: true });
26542
28307
  } catch (error) {
26543
28308
  return failure4("spawn-error", error instanceof Error ? error.message : "failed to write qmd config");
26544
28309
  }
@@ -26612,7 +28377,7 @@ function resolveVaultPath(vaultPath) {
26612
28377
  if (vaultPath === undefined || vaultPath.trim().length === 0)
26613
28378
  return;
26614
28379
  const trimmed = vaultPath.trim();
26615
- return isAbsolute8(trimmed) ? trimmed : resolve14(trimmed);
28380
+ return isAbsolute9(trimmed) ? trimmed : resolve15(trimmed);
26616
28381
  }
26617
28382
  function controlledEnv5(options) {
26618
28383
  const instanceId = options.instanceId ?? "default";
@@ -26931,7 +28696,7 @@ async function retrieveObsidian(client, mode, methodName, instanceId, query, opt
26931
28696
  const trimmed = query.trim();
26932
28697
  if (trimmed.length === 0)
26933
28698
  return [];
26934
- const topK = options.topK ?? DEFAULT_TOP_K5;
28699
+ const topK = options.topK ?? DEFAULT_TOP_K6;
26935
28700
  const searchOptions = { topK, signal: options.signal };
26936
28701
  const result = await client.search(mode, trimmed, searchOptions);
26937
28702
  if (!result.ok)
@@ -26966,8 +28731,8 @@ function toRetrievalResult3(hit, source, methodName, instanceId, mode) {
26966
28731
  }
26967
28732
  };
26968
28733
  }
26969
- var OBSIDIAN_DATASOURCE_ID = "obsidian", DEFAULT_OBSIDIAN_TAGS, DEFAULT_TOP_K5 = 20;
26970
- var init_methods5 = __esm(() => {
28734
+ var OBSIDIAN_DATASOURCE_ID = "obsidian", DEFAULT_OBSIDIAN_TAGS, DEFAULT_TOP_K6 = 20;
28735
+ var init_methods6 = __esm(() => {
26971
28736
  init_errors4();
26972
28737
  init_scope2();
26973
28738
  DEFAULT_OBSIDIAN_TAGS = ["obsidian", "notes"];
@@ -27157,7 +28922,7 @@ var OBSIDIAN_DATASOURCE_ID2 = "obsidian", OBSIDIAN_SKILL_TYPE = "obsidian-vault"
27157
28922
  var init_skill10 = __esm(() => {
27158
28923
  init_scope2();
27159
28924
  init_client10();
27160
- init_methods5();
28925
+ init_methods6();
27161
28926
  init_paths5();
27162
28927
  DEFAULT_POLLING_INTERVAL_MS5 = 15 * 60 * 1000;
27163
28928
  DEFAULT_OBSIDIAN_TAGS2 = ["obsidian", "notes"];
@@ -27166,7 +28931,7 @@ var init_skill10 = __esm(() => {
27166
28931
  // src/datasource/skills/obsidian/index.ts
27167
28932
  var init_obsidian = __esm(() => {
27168
28933
  init_client10();
27169
- init_methods5();
28934
+ init_methods6();
27170
28935
  init_paths5();
27171
28936
  init_skill10();
27172
28937
  });
@@ -27506,7 +29271,7 @@ var init_slack = __esm(() => {
27506
29271
 
27507
29272
  // src/datasource/skills/spotlight/connector.ts
27508
29273
  import { spawn as spawn17 } from "node:child_process";
27509
- import { createHash as createHash14 } from "node:crypto";
29274
+ import { createHash as createHash15 } from "node:crypto";
27510
29275
  import { readFile as fsReadFile, stat as stat7 } from "node:fs/promises";
27511
29276
  import { basename as basename13, extname as extname8 } from "node:path";
27512
29277
 
@@ -27529,7 +29294,7 @@ class SpotlightConnector {
27529
29294
  if (queries.length === 0) {
27530
29295
  return { ok: false, reason: "not-configured", message: "no spotlight queries configured" };
27531
29296
  }
27532
- const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS9;
29297
+ const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS10;
27533
29298
  const status = await this.run("mdutil", ["-s", "/"], timeoutMs);
27534
29299
  if (status.ok && /indexing disabled/iu.test(status.stdout)) {
27535
29300
  return {
@@ -27591,7 +29356,7 @@ class SpotlightConnector {
27591
29356
  if (content.length === 0)
27592
29357
  continue;
27593
29358
  documents.push({
27594
- docId: createHash14("sha256").update(path).digest("hex").slice(0, 16),
29359
+ docId: createHash15("sha256").update(path).digest("hex").slice(0, 16),
27595
29360
  hierarchy: ["files"],
27596
29361
  title: basename13(path),
27597
29362
  content,
@@ -27624,7 +29389,7 @@ class SpotlightConnector {
27624
29389
  return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
27625
29390
  }
27626
29391
  }
27627
- var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS5 = 300, DEFAULT_MAX_BYTES_PER_FILE3, DEFAULT_TIMEOUT_MS9 = 15000, MAX_CONTENT_CHARS4 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve) => {
29392
+ var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS5 = 300, DEFAULT_MAX_BYTES_PER_FILE3, DEFAULT_TIMEOUT_MS10 = 15000, MAX_CONTENT_CHARS4 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve) => {
27628
29393
  const child = spawn17(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
27629
29394
  let stdout = "";
27630
29395
  let stderr = "";
@@ -27741,10 +29506,12 @@ function parseHits3(stdout) {
27741
29506
  for (const [index, row] of rows.entries()) {
27742
29507
  if (!isRecord18(row))
27743
29508
  return;
27744
- const id = stringValue4(row, ["message_id", "event_id", "id"]);
27745
- const content = stringValue4(row, ["snippet", "text", "content"]);
27746
- if (id === undefined || content === undefined)
29509
+ const id = stringValue4(row, ["event_id", "message_id", "id"]);
29510
+ if (id === undefined)
27747
29511
  return;
29512
+ const content = stringValue4(row, ["snippet", "text", "content"]);
29513
+ if (content === undefined)
29514
+ continue;
27748
29515
  const chatId = stringValue4(row, ["chat_jid", "chat_id"]);
27749
29516
  const chatName = stringValue4(row, ["chat_name"]);
27750
29517
  const senderName = stringValue4(row, ["sender_name"]);
@@ -27861,10 +29628,12 @@ function parseHits4(stdout) {
27861
29628
  for (const [index, row] of rows.entries()) {
27862
29629
  if (!isRecord19(row))
27863
29630
  return;
27864
- const id = stringValue5(row, ["message_id", "event_id", "id"]);
27865
- const content = stringValue5(row, ["snippet", "text", "content"]);
27866
- if (id === undefined || content === undefined)
29631
+ const id = stringValue5(row, ["event_id", "message_id", "id"]);
29632
+ if (id === undefined)
27867
29633
  return;
29634
+ const content = stringValue5(row, ["snippet", "text", "content"]);
29635
+ if (content === undefined)
29636
+ continue;
27868
29637
  const chatId = stringValue5(row, ["chat_jid", "chat_id"]);
27869
29638
  const chatName = stringValue5(row, ["chat_name"]);
27870
29639
  const senderName = stringValue5(row, ["sender_name"]);
@@ -27954,6 +29723,31 @@ var init_wacrawl = __esm(() => {
27954
29723
  });
27955
29724
 
27956
29725
  // src/datasource/skills/factory.ts
29726
+ import { homedir as homedir4 } from "node:os";
29727
+ import { join as join37 } from "node:path";
29728
+ function chatCrawlerArgs(descriptor, defaultCrawlerDb, config, workspaceRoot, registrationName) {
29729
+ const sidecar = resolveSidecarOptions({
29730
+ descriptor,
29731
+ connector: config.connector,
29732
+ workspaceRoot,
29733
+ registrationName,
29734
+ instanceId: config.instanceId ?? "default",
29735
+ defaultCrawlerDb
29736
+ });
29737
+ return {
29738
+ datasourceId: registrationName,
29739
+ ...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
29740
+ ...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
29741
+ ...config.tags !== undefined ? { tags: config.tags } : {},
29742
+ channelIds: config.channels?.ids,
29743
+ channelNames: config.channels?.names,
29744
+ ...sidecar === undefined ? {} : { sidecar },
29745
+ connectorOptions: {
29746
+ ...config.connector,
29747
+ ...workspaceRoot === undefined ? {} : { workspacePath: workspaceRoot }
29748
+ }
29749
+ };
29750
+ }
27957
29751
  function larkConnectorOptions(connector) {
27958
29752
  if (connector === undefined)
27959
29753
  return {};
@@ -28014,6 +29808,8 @@ function buildDatasourceSkills(config, workspaceRoot, embeddingRuntime) {
28014
29808
  var BUILDERS, BUILTIN_DATASOURCE_SKILL_NAMES;
28015
29809
  var init_factory = __esm(() => {
28016
29810
  init_aliased_skill();
29811
+ init_config2();
29812
+ init_descriptors();
28017
29813
  init_clawgallery();
28018
29814
  init_cloud_drive();
28019
29815
  init_discrawl();
@@ -28031,42 +29827,9 @@ var init_factory = __esm(() => {
28031
29827
  init_telecrawl();
28032
29828
  init_wacrawl();
28033
29829
  BUILDERS = {
28034
- telegram: (config, _workspaceRoot, registrationName) => new TelecrawlSkill({
28035
- datasourceId: registrationName,
28036
- ...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
28037
- ...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
28038
- ...config.tags !== undefined ? { tags: config.tags } : {},
28039
- channelIds: config.channels?.ids,
28040
- channelNames: config.channels?.names,
28041
- connectorOptions: {
28042
- ...config.connector,
28043
- ..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
28044
- }
28045
- }),
28046
- whatsapp: (config, _workspaceRoot, registrationName) => new WacrawlSkill({
28047
- datasourceId: registrationName,
28048
- ...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
28049
- ...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
28050
- ...config.tags !== undefined ? { tags: config.tags } : {},
28051
- channelIds: config.channels?.ids,
28052
- channelNames: config.channels?.names,
28053
- connectorOptions: {
28054
- ...config.connector,
28055
- ..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
28056
- }
28057
- }),
28058
- slack: (config, _workspaceRoot, registrationName) => new SlackSkill({
28059
- datasourceId: registrationName,
28060
- ...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
28061
- ...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
28062
- ...config.tags !== undefined ? { tags: config.tags } : {},
28063
- channelIds: config.channels?.ids,
28064
- channelNames: config.channels?.names,
28065
- connectorOptions: {
28066
- ...config.connector,
28067
- ..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
28068
- }
28069
- }),
29830
+ telegram: (config, workspaceRoot, registrationName) => new TelecrawlSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.telegram, join37(homedir4(), ".telecrawl", "telecrawl.db"), config, workspaceRoot, registrationName)),
29831
+ whatsapp: (config, workspaceRoot, registrationName) => new WacrawlSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.whatsapp, join37(homedir4(), ".wacrawl", "wacrawl.db"), config, workspaceRoot, registrationName)),
29832
+ slack: (config, workspaceRoot, registrationName) => new SlackSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.slack, join37(homedir4(), ".slacrawl", "slacrawl.db"), config, workspaceRoot, registrationName)),
28070
29833
  discord: (config, workspaceRoot, registrationName, embeddingRuntime) => {
28071
29834
  const connector = config.connector ?? {};
28072
29835
  const clientOptions = {
@@ -28185,8 +29948,8 @@ var init_factory = __esm(() => {
28185
29948
 
28186
29949
  // src/cli/config.ts
28187
29950
  import { randomUUID as randomUUID9 } from "node:crypto";
28188
- import { existsSync as existsSync28, mkdirSync as mkdirSync25, readFileSync as readFileSync24, renameSync as renameSync11, unlinkSync as unlinkSync5, writeFileSync as writeFileSync23 } from "node:fs";
28189
- import { basename as basename14, dirname as dirname23, isAbsolute as isAbsolute9, join as join36, resolve as resolve15 } from "node:path";
29951
+ import { existsSync as existsSync28, mkdirSync as mkdirSync26, readFileSync as readFileSync25, renameSync as renameSync11, unlinkSync as unlinkSync5, writeFileSync as writeFileSync23 } from "node:fs";
29952
+ import { basename as basename14, dirname as dirname25, isAbsolute as isAbsolute10, join as join38, resolve as resolve16 } from "node:path";
28190
29953
  import { findEnvKeys, getEnvApiKey } from "@earendil-works/pi-ai/compat";
28191
29954
  import { getAgentDir as getAgentDir3, ModelRuntime as ModelRuntime2, SettingsManager as SettingsManager2 } from "@earendil-works/pi-coding-agent";
28192
29955
  function resolveConfigPath(input) {
@@ -28202,9 +29965,9 @@ function resolveConfigPath(input) {
28202
29965
  return { configPath: envConfig, explicit: true };
28203
29966
  }
28204
29967
  return {
28205
- configPath: join36(resolveAutoRAGHome(env), DEFAULT_CONFIG_FILENAME),
29968
+ configPath: join38(resolveAutoRAGHome(env), DEFAULT_CONFIG_FILENAME),
28206
29969
  explicit: false,
28207
- legacyPath: join36(cwd, LEGACY_CONFIG_FILENAME)
29970
+ legacyPath: join38(cwd, LEGACY_CONFIG_FILENAME)
28208
29971
  };
28209
29972
  }
28210
29973
  function readConfigFile(configPath, explicit) {
@@ -28222,7 +29985,7 @@ function readConfigFile(configPath, explicit) {
28222
29985
  }
28223
29986
  let text;
28224
29987
  try {
28225
- text = readFileSync24(configPath, "utf8");
29988
+ text = readFileSync25(configPath, "utf8");
28226
29989
  } catch (err) {
28227
29990
  throw new ConfigError(`Failed to read config file: ${err.message}`);
28228
29991
  }
@@ -28260,7 +30023,7 @@ function acquireConfigWriteLock(configPath) {
28260
30023
  });
28261
30024
  }
28262
30025
  function replaceFileAtomically(path, contents, assertCommitAllowed) {
28263
- const temporaryPath = join36(dirname23(path), `.${basename14(path)}.${process.pid}.${randomUUID9()}.tmp`);
30026
+ const temporaryPath = join38(dirname25(path), `.${basename14(path)}.${process.pid}.${randomUUID9()}.tmp`);
28264
30027
  try {
28265
30028
  writeFileSync23(temporaryPath, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
28266
30029
  assertCommitAllowed?.();
@@ -28273,12 +30036,12 @@ function migrateLegacyConfig(configPath, legacyPath) {
28273
30036
  if (existsSync28(configPath) || !existsSync28(legacyPath))
28274
30037
  return;
28275
30038
  const legacy = readConfigFile(legacyPath, true);
28276
- const legacyBytes = readFileSync24(legacyPath);
28277
- const migrated = normalizeLegacyConfigPaths(legacy ?? {}, dirname23(legacyPath));
30039
+ const legacyBytes = readFileSync25(legacyPath);
30040
+ const migrated = normalizeLegacyConfigPaths(legacy ?? {}, dirname25(legacyPath));
28278
30041
  const migratedPathsUnchanged = legacy?.workspacePath === migrated.workspacePath && legacy?.memoryPath === migrated.memoryPath && JSON.stringify(legacy?.searchPaths) === JSON.stringify(migrated.searchPaths);
28279
30042
  const migratedBytes = migratedPathsUnchanged ? legacyBytes : `${JSON.stringify(migrated, null, 2)}
28280
30043
  `;
28281
- mkdirSync25(dirname23(configPath), { recursive: true });
30044
+ mkdirSync26(dirname25(configPath), { recursive: true });
28282
30045
  const lock = acquireConfigWriteLock(configPath);
28283
30046
  try {
28284
30047
  const winner = readConfigFile(configPath, false);
@@ -28308,13 +30071,13 @@ function resolveConfigFileReadOnly(configPath, explicit, legacyPath) {
28308
30071
  const legacy = readConfigFile(legacyPath, false);
28309
30072
  if (legacy === undefined)
28310
30073
  return {};
28311
- return normalizeLegacyConfigPaths(legacy, dirname23(legacyPath));
30074
+ return normalizeLegacyConfigPaths(legacy, dirname25(legacyPath));
28312
30075
  }
28313
30076
  function resolveSearchPaths(searchPaths, origin) {
28314
30077
  return searchPaths.map((searchPath) => resolvePersistedPath(searchPath, origin));
28315
30078
  }
28316
30079
  function resolvePersistedPath(path, origin) {
28317
- return isAbsolute9(path) ? path : resolve15(origin, path);
30080
+ return isAbsolute10(path) ? path : resolve16(origin, path);
28318
30081
  }
28319
30082
  function normalizeLegacyConfigPaths(partial, origin) {
28320
30083
  const workspacePath = resolvePersistedPath(partial.workspacePath ?? ".", origin);
@@ -28501,7 +30264,7 @@ function normalizeEmbedder(raw, path) {
28501
30264
  out.passagePrefix = record.passagePrefix;
28502
30265
  }
28503
30266
  if (record.profile !== undefined) {
28504
- if (record.profile !== "qwen3-embedding-0.6b" && record.profile !== "embeddinggemma-300m") {
30267
+ if (record.profile !== "qwen3-embedding-0.6b") {
28505
30268
  throw new ConfigError(`${path}.profile must be a supported runtime profile`);
28506
30269
  }
28507
30270
  out.profile = record.profile;
@@ -28744,10 +30507,10 @@ function resolveConfig(input) {
28744
30507
  const file = readOnly ? resolveConfigFileReadOnly(configPath, explicit, legacyPath) : !explicit && legacyPath ? migrateLegacyConfig(configPath, legacyPath) ?? readConfigFile(configPath, explicit) ?? {} : readConfigFile(configPath, explicit) ?? {};
28745
30508
  const defaultSearchPaths = ["."];
28746
30509
  const defaultWorkspacePath = cwd;
28747
- const defaultMemoryPath = join36(resolveAutoRAGHome(env), "memory.json");
30510
+ const defaultMemoryPath = join38(resolveAutoRAGHome(env), "memory.json");
28748
30511
  const flagSearchPaths = parseCsv(flagString(flags, "search-paths"));
28749
30512
  const envSearchPaths = parseCsv(envString(env, "AUTORAG_SEARCH_PATHS"));
28750
- const configOrigin = dirname23(resolve15(configPath));
30513
+ const configOrigin = dirname25(resolve16(configPath));
28751
30514
  const fileWorkspacePath = typeof file.workspacePath === "string" ? resolvePersistedPath(file.workspacePath, configOrigin) : undefined;
28752
30515
  const fileSearchPaths = file.searchPaths ? resolveSearchPaths(file.searchPaths, fileWorkspacePath ?? configOrigin) : undefined;
28753
30516
  if (file.excludePaths !== undefined && (!Array.isArray(file.excludePaths) || file.excludePaths.some((path) => typeof path !== "string" || path.length === 0))) {
@@ -28770,7 +30533,7 @@ function resolveConfig(input) {
28770
30533
  const flagModelId = pickString(flags, env, "model-id", "AUTORAG_MODEL_ID", undefined);
28771
30534
  const model = applyRoleFlagOverrides(fileModel, flagModelProvider, flagModelId, "model");
28772
30535
  const config = {
28773
- configPath: resolve15(configPath),
30536
+ configPath: resolve16(configPath),
28774
30537
  searchPaths,
28775
30538
  workspacePath,
28776
30539
  memoryPath,
@@ -28946,36 +30709,38 @@ function buildWebSearchAgentOption(raw) {
28946
30709
  function normalizeRerankConfig(raw, path) {
28947
30710
  if (raw === false)
28948
30711
  return false;
28949
- const out = {
28950
- provider: DEFAULT_RERANK_PROVIDER,
28951
- model: DEFAULT_RERANK_MODEL,
28952
- apiKeyEnv: DEFAULT_RERANK_API_KEY_ENV,
28953
- topN: DEFAULT_RERANK_TOP_N
28954
- };
28955
- if (raw === undefined || raw === null)
28956
- return out;
28957
- if (typeof raw !== "object" || Array.isArray(raw)) {
28958
- throw new ConfigError(`${path} must be an object or false`);
28959
- }
28960
- const record = raw;
28961
- for (const key of Object.keys(record)) {
28962
- if (!RERANK_ALLOWLIST.has(key))
28963
- throw new ConfigError(`${path}.${key} is not a recognized field`);
28964
- }
28965
- if (record.enabled !== undefined) {
28966
- if (typeof record.enabled !== "boolean")
28967
- throw new ConfigError(`${path}.enabled must be a boolean`);
28968
- out.enabled = record.enabled;
30712
+ let record = {};
30713
+ if (raw !== undefined && raw !== null) {
30714
+ if (typeof raw !== "object" || Array.isArray(raw)) {
30715
+ throw new ConfigError(`${path} must be an object or false`);
30716
+ }
30717
+ record = raw;
30718
+ for (const key of Object.keys(record)) {
30719
+ if (!RERANK_ALLOWLIST.has(key))
30720
+ throw new ConfigError(`${path}.${key} is not a recognized field`);
30721
+ }
28969
30722
  }
30723
+ let provider = DEFAULT_RERANK_PROVIDER;
28970
30724
  if (record.provider !== undefined) {
28971
30725
  if (typeof record.provider !== "string" || record.provider.trim() === "") {
28972
30726
  throw new ConfigError(`${path}.provider must be a non-empty string`);
28973
30727
  }
28974
- out.provider = record.provider.trim();
30728
+ provider = record.provider.trim();
28975
30729
  }
28976
- if (!SUPPORTED_RERANK_PROVIDERS.includes(out.provider ?? "")) {
30730
+ if (!SUPPORTED_RERANK_PROVIDERS.includes(provider)) {
28977
30731
  throw new ConfigError(`${path}.provider must be one of: ${SUPPORTED_RERANK_PROVIDERS.join(", ")}`);
28978
30732
  }
30733
+ const out = {
30734
+ provider,
30735
+ model: defaultRerankModel(provider),
30736
+ apiKeyEnv: defaultRerankApiKeyEnv(provider),
30737
+ topN: DEFAULT_RERANK_TOP_N
30738
+ };
30739
+ if (record.enabled !== undefined) {
30740
+ if (typeof record.enabled !== "boolean")
30741
+ throw new ConfigError(`${path}.enabled must be a boolean`);
30742
+ out.enabled = record.enabled;
30743
+ }
28979
30744
  if (record.model !== undefined) {
28980
30745
  if (typeof record.model !== "string" || record.model.trim() === "") {
28981
30746
  throw new ConfigError(`${path}.model must be a non-empty string`);
@@ -29105,7 +30870,8 @@ function buildModelFromConfiguredEndpoint(reference) {
29105
30870
  input: reference.input ?? ["text"],
29106
30871
  cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
29107
30872
  contextWindow: reference.contextWindow ?? 128000,
29108
- maxTokens: reference.maxTokens ?? 16384
30873
+ maxTokens: reference.maxTokens ?? 16384,
30874
+ ...reference.provider === PROVIDER_ID ? { inputLimits: AUTORAG_INPUT_LIMITS } : {}
29109
30875
  };
29110
30876
  }
29111
30877
  function mergeCatalogModel(catalog, reference) {
@@ -29156,8 +30922,8 @@ function getModelRuntime(agentDir) {
29156
30922
  return cached;
29157
30923
  const created = (async () => {
29158
30924
  const runtime = await ModelRuntime2.create({
29159
- authPath: join36(agentDir, "auth.json"),
29160
- modelsPath: join36(agentDir, "models.json"),
30925
+ authPath: join38(agentDir, "auth.json"),
30926
+ modelsPath: join38(agentDir, "models.json"),
29161
30927
  allowModelNetwork: false
29162
30928
  });
29163
30929
  await registerAutoRAGProvider(runtime);
@@ -29416,7 +31182,7 @@ function writeConfigObject(path, config) {
29416
31182
  if (config === null || typeof config !== "object" || Array.isArray(config)) {
29417
31183
  throw new ConfigError("Config file must be a JSON object");
29418
31184
  }
29419
- mkdirSync25(dirname23(path), { recursive: true });
31185
+ mkdirSync26(dirname25(path), { recursive: true });
29420
31186
  const contents = `${JSON.stringify(config, null, 2)}
29421
31187
  `;
29422
31188
  const lock = acquireConfigWriteLock(path);
@@ -29427,9 +31193,9 @@ function writeConfigObject(path, config) {
29427
31193
  }
29428
31194
  }
29429
31195
  function writeDefaultConfig(path, partial, opts = {}) {
29430
- const cwd = resolve15(opts.cwd ?? process.cwd());
31196
+ const cwd = resolve16(opts.cwd ?? process.cwd());
29431
31197
  const workspacePath = resolvePersistedPath(partial.workspacePath ?? ".", cwd);
29432
- const memoryPath = partial.memoryPath === undefined ? join36(resolveAutoRAGHome(opts.env), "memory.json") : resolvePersistedPath(partial.memoryPath, workspacePath);
31198
+ const memoryPath = partial.memoryPath === undefined ? join38(resolveAutoRAGHome(opts.env), "memory.json") : resolvePersistedPath(partial.memoryPath, workspacePath);
29433
31199
  const full = {
29434
31200
  searchPaths: resolveSearchPaths(partial.searchPaths ?? ["."], cwd),
29435
31201
  workspacePath,
@@ -29463,7 +31229,7 @@ function writeDefaultConfig(path, partial, opts = {}) {
29463
31229
  full.p2p = { enabled: false };
29464
31230
  full.jev = partial.jev ?? { backend: DEFAULT_JEV_BACKEND };
29465
31231
  full.queryDecomposition = partial.queryDecomposition ?? { model: { ...DEFAULT_QUERY_DECOMPOSITION_MODEL } };
29466
- mkdirSync25(dirname23(path), { recursive: true });
31232
+ mkdirSync26(dirname25(path), { recursive: true });
29467
31233
  const contents = `${JSON.stringify(full, null, 2)}
29468
31234
  `;
29469
31235
  const lock = acquireConfigWriteLock(path);
@@ -29491,8 +31257,9 @@ function writeDefaultConfig(path, partial, opts = {}) {
29491
31257
  }
29492
31258
  }
29493
31259
  var DEFAULT_CONFIG_FILENAME = "config.json", LEGACY_CONFIG_FILENAME = "autorag.config.json", CONFIG_LOCK_RETRY_MS = 10, CONFIG_LOCK_TIMEOUT_MS = 1e4, CONFIG_LOCK_STALE_MS = 30000, ConfigError, MINSYNC_REQUIRED_MESSAGE = "MinSync is required and cannot be disabled; remove `minSync: false` / `minSync.enabled: false` from the config.", AGENT_MODEL_APIS, AGENT_MODEL_INPUTS, API_KEY_ENV_PATTERN2, POSITIVE_INT_FIELDS, EMBEDDER_ALLOWLIST, MINSYNC_ALLOWLIST, MINSYNC_LEGACY_IGNORED, P2P_ALLOWLIST, P2P_QUOTA_ALLOWLIST, LIMITS_ALLOWLIST, LIMITS_PREFETCH_ALLOWLIST, JEV_CONFIG_FIELDS, JEV_BACKEND_NAMES, DEFAULT_JEV_BACKEND = "openrouter", DEFAULT_QUERY_DECOMPOSITION_MODEL, RERANK_ALLOWLIST, UNKNOWN_MODEL_HINT = "Add baseUrl (and optional api/apiKeyEnv) for an OpenAI-compatible endpoint outside the pi-ai catalog, or use a pi-ai catalog model id.", modelRuntimeCache;
29494
- var init_config3 = __esm(() => {
31260
+ var init_config4 = __esm(() => {
29495
31261
  init_local_model();
31262
+ init_config();
29496
31263
  init_provider();
29497
31264
  init_home();
29498
31265
  init_factory();
@@ -29574,7 +31341,7 @@ var init_config3 = __esm(() => {
29574
31341
  model: true,
29575
31342
  confidenceThreshold: true
29576
31343
  };
29577
- JEV_BACKEND_NAMES = ["typesafe", "openrouter", "vercel"];
31344
+ JEV_BACKEND_NAMES = ["typesafe", "openrouter", "vercel", "autorag"];
29578
31345
  DEFAULT_QUERY_DECOMPOSITION_MODEL = { provider: "openrouter", id: "qwen/qwen3.7-flash" };
29579
31346
  RERANK_ALLOWLIST = new Set(["enabled", "provider", "model", "apiKeyEnv", "baseUrl", "topN", "timeoutMs"]);
29580
31347
  modelRuntimeCache = new Map;
@@ -29664,9 +31431,9 @@ function createAutoRAGLite(options = {}) {
29664
31431
  var init_core = __esm(() => {
29665
31432
  init_agent();
29666
31433
  init_search_documents();
29667
- init_config3();
31434
+ init_config4();
29668
31435
  init_memory();
29669
- init_config3();
31436
+ init_config4();
29670
31437
  });
29671
31438
  init_core();
29672
31439