@autorag/librarian 2.10.0 → 2.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +195 -21
- package/NOTICE +17 -21
- package/README.md +335 -161
- package/dist/agent/agent.d.ts +67 -2
- package/dist/agent/context-token-guard.d.ts +23 -0
- package/dist/agent/jev-extension.d.ts +5 -4
- package/dist/agent/jikji-find-tool.d.ts +1 -0
- package/dist/agent/pi-session.d.ts +35 -2
- package/dist/agent/request-limits.d.ts +20 -0
- package/dist/agent/search-all-tool.d.ts +1 -0
- package/dist/agent/search-single-datasource-tool.d.ts +1 -0
- package/dist/cli/config.d.ts +13 -11
- package/dist/cli/index.js +2573 -800
- package/dist/cloud/config.d.ts +18 -0
- package/dist/cloud/jev-backend.d.ts +90 -0
- package/dist/core.js +2185 -418
- package/dist/datasource/crawler-skill.d.ts +17 -0
- package/dist/datasource/sidecar/config.d.ts +26 -0
- package/dist/datasource/sidecar/descriptors.d.ts +67 -0
- package/dist/datasource/sidecar/fusion.d.ts +25 -0
- package/dist/datasource/sidecar/index.d.ts +131 -0
- package/dist/datasource/sidecar/methods.d.ts +27 -0
- package/dist/datasource/types.d.ts +3 -1
- package/dist/embedding-runtime/index.d.ts +6 -6
- package/dist/embedding-runtime/manifest.d.ts +0 -12
- package/dist/embedding-runtime/types.d.ts +1 -1
- package/dist/index.js +2210 -439
- package/dist/mcp/index.js +2205 -438
- package/dist/minsync/client.d.ts +3 -3
- package/dist/minsync/installer.d.ts +1 -1
- package/dist/minsync/process.d.ts +1 -0
- package/dist/minsync/types.d.ts +1 -1
- package/dist/parser/defaults.d.ts +6 -4
- package/dist/parser/ocr-engines.d.ts +6 -0
- package/dist/retrieval/index.d.ts +1 -1
- package/dist/retrieval/rerank.d.ts +20 -3
- package/licenses/embedding-assets.json +0 -35
- package/package.json +24 -27
- package/skills/autorag/SKILL.md +1 -1
- package/skills/autorag-doctor/SKILL.md +1 -1
- package/skills/autorag-lite-setup/SKILL.md +2 -2
- package/skills/autorag-setup/SKILL.md +2 -2
- package/licenses/gemma-notice.txt +0 -24
- package/licenses/gemma-section-3.2-flow-down.txt +0 -9
- package/licenses/gemma-terms-of-use.html +0 -4750
- package/licenses/gemma-terms-of-use.txt +0 -158
package/dist/core.js
CHANGED
|
@@ -61,6 +61,7 @@ function datasourceCodeToRetrieval(code) {
|
|
|
61
61
|
case "datasource-auth-error":
|
|
62
62
|
case "datasource-embedding-egress-rejected":
|
|
63
63
|
case "datasource-index-failed":
|
|
64
|
+
case "datasource-index-partial":
|
|
64
65
|
case "datasource-permission-denied":
|
|
65
66
|
case "datasource-remote-embedding-rejected":
|
|
66
67
|
return "retrieval-method-failed";
|
|
@@ -1112,19 +1113,6 @@ var init_manifest = __esm(() => {
|
|
|
1112
1113
|
artifactSha256: "06507c7b42688469c4e7298b0a1e16deff06caf291cf0a5b278c308249c3e439",
|
|
1113
1114
|
backend: "auto",
|
|
1114
1115
|
protocolCapabilities
|
|
1115
|
-
},
|
|
1116
|
-
"embeddinggemma-300m": {
|
|
1117
|
-
profileId: "embeddinggemma-300m",
|
|
1118
|
-
provider: "google",
|
|
1119
|
-
model: "embeddinggemma-300M-Q8_0.gguf",
|
|
1120
|
-
dimension: 768,
|
|
1121
|
-
queryPrefix: "task: search result | query: ",
|
|
1122
|
-
passagePrefix: "title: none | text: ",
|
|
1123
|
-
runtimeBuild: RUNTIME_BUILD,
|
|
1124
|
-
modelRevision: "0f741b5a6585bd53aeb15cd1372c56f2a0f65e12",
|
|
1125
|
-
artifactSha256: "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63",
|
|
1126
|
-
backend: "auto",
|
|
1127
|
-
protocolCapabilities
|
|
1128
1116
|
}
|
|
1129
1117
|
};
|
|
1130
1118
|
runtimeAssets = {
|
|
@@ -1170,17 +1158,6 @@ var init_manifest = __esm(() => {
|
|
|
1170
1158
|
unsuitableUse: "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256.",
|
|
1171
1159
|
upstreamCardUrl: "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B",
|
|
1172
1160
|
modelCard: "docs/model-cards/qwen3-embedding-0.6b.md"
|
|
1173
|
-
},
|
|
1174
|
-
embeddinggemma: {
|
|
1175
|
-
url: "https://huggingface.co/ggml-org/embeddinggemma-300M-GGUF/resolve/0f741b5a6585bd53aeb15cd1372c56f2a0f65e12/embeddinggemma-300M-Q8_0.gguf",
|
|
1176
|
-
revision: "0f741b5a6585bd53aeb15cd1372c56f2a0f65e12",
|
|
1177
|
-
sha256: "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63",
|
|
1178
|
-
license: "Gemma Terms of Use",
|
|
1179
|
-
noticeReference: "Google EmbeddingGemma model card and Gemma Terms of Use",
|
|
1180
|
-
intendedUse: "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.",
|
|
1181
|
-
unsuitableUse: "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256, uses restricted by the Gemma Prohibited Use Policy.",
|
|
1182
|
-
upstreamCardUrl: "https://ai.google.dev/gemma/docs/embeddinggemma/model_card",
|
|
1183
|
-
modelCard: "docs/model-cards/embeddinggemma-300m.md"
|
|
1184
1161
|
}
|
|
1185
1162
|
};
|
|
1186
1163
|
});
|
|
@@ -1618,7 +1595,7 @@ var init_supervisor = __esm(() => {
|
|
|
1618
1595
|
// src/embedding-runtime/index.ts
|
|
1619
1596
|
import { basename as basename4 } from "node:path";
|
|
1620
1597
|
function modelAsset(profile) {
|
|
1621
|
-
return
|
|
1598
|
+
return { id: profile.profileId, filename: basename4(profile.model), ...MODEL_ASSETS.qwen3 };
|
|
1622
1599
|
}
|
|
1623
1600
|
function runtimeAsset(platform, backend) {
|
|
1624
1601
|
const selected = platform ?? "darwin-arm64-metal";
|
|
@@ -5020,7 +4997,12 @@ var init_local_embedder = __esm(() => {
|
|
|
5020
4997
|
// src/minsync/process.ts
|
|
5021
4998
|
import { spawn as spawn7 } from "node:child_process";
|
|
5022
4999
|
function spawnProcess(command, args, cwd, options = {}) {
|
|
5023
|
-
return new Promise((resolve) => {
|
|
5000
|
+
return new Promise((resolve, reject) => {
|
|
5001
|
+
const { signal } = options;
|
|
5002
|
+
if (signal?.aborted) {
|
|
5003
|
+
reject(abortError(signal));
|
|
5004
|
+
return;
|
|
5005
|
+
}
|
|
5024
5006
|
const portable = portableSpawnCommand(command, args);
|
|
5025
5007
|
const child = spawn7(portable.command, [...portable.args], { cwd, stdio: ["ignore", "pipe", "pipe"] });
|
|
5026
5008
|
let stdout = "";
|
|
@@ -5038,15 +5020,26 @@ function spawnProcess(command, args, cwd, options = {}) {
|
|
|
5038
5020
|
timedOut = true;
|
|
5039
5021
|
child.kill("SIGTERM");
|
|
5040
5022
|
}, options.timeoutMs) : undefined;
|
|
5023
|
+
const onAbort = () => {
|
|
5024
|
+
child.kill("SIGTERM");
|
|
5025
|
+
};
|
|
5026
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
5027
|
+
const settle = () => {
|
|
5028
|
+
clearTimeout(timer);
|
|
5029
|
+
signal?.removeEventListener("abort", onAbort);
|
|
5030
|
+
};
|
|
5041
5031
|
child.on("error", (error) => {
|
|
5042
|
-
|
|
5043
|
-
|
|
5044
|
-
|
|
5032
|
+
settle();
|
|
5033
|
+
if (signal?.aborted)
|
|
5034
|
+
reject(abortError(signal));
|
|
5035
|
+
else
|
|
5036
|
+
resolve({ ok: false, stdout, stderr: error.message, code: null });
|
|
5045
5037
|
});
|
|
5046
5038
|
child.on("close", (code) => {
|
|
5047
|
-
|
|
5048
|
-
|
|
5049
|
-
|
|
5039
|
+
settle();
|
|
5040
|
+
if (signal?.aborted) {
|
|
5041
|
+
reject(abortError(signal));
|
|
5042
|
+
} else if (timedOut) {
|
|
5050
5043
|
resolve({ ok: false, stdout, stderr: "process timed out", code: null, timedOut: true });
|
|
5051
5044
|
} else {
|
|
5052
5045
|
resolve({ ok: code === 0, stdout, stderr, code });
|
|
@@ -5054,6 +5047,9 @@ function spawnProcess(command, args, cwd, options = {}) {
|
|
|
5054
5047
|
});
|
|
5055
5048
|
});
|
|
5056
5049
|
}
|
|
5050
|
+
function abortError(signal) {
|
|
5051
|
+
return signal.reason instanceof Error ? signal.reason : new DOMException("The operation was aborted", "AbortError");
|
|
5052
|
+
}
|
|
5057
5053
|
var init_process = __esm(() => {
|
|
5058
5054
|
init_portable_spawn();
|
|
5059
5055
|
});
|
|
@@ -5286,10 +5282,12 @@ class MinSyncClient {
|
|
|
5286
5282
|
} : {}
|
|
5287
5283
|
};
|
|
5288
5284
|
}
|
|
5289
|
-
async query(text, topK, mode = "vector") {
|
|
5285
|
+
async query(text, topK, mode = "vector", signal) {
|
|
5290
5286
|
if (!existsSync11(this.binaryPath))
|
|
5291
5287
|
throw minSyncBinaryMissingError();
|
|
5292
|
-
const result = await this.spawn(["query", "--format", "json", "--mode", mode, "-k", String(topK), text]
|
|
5288
|
+
const result = await this.spawn(["query", "--format", "json", "--mode", mode, "-k", String(topK), text], {
|
|
5289
|
+
...signal === undefined ? {} : { signal }
|
|
5290
|
+
});
|
|
5293
5291
|
if (!result.ok)
|
|
5294
5292
|
throw new MinSyncQueryError(result.code, result.stderr);
|
|
5295
5293
|
return parseQueryHits(result.stdout);
|
|
@@ -5362,7 +5360,7 @@ function isIdentity(value) {
|
|
|
5362
5360
|
function isRecord4(value) {
|
|
5363
5361
|
return typeof value === "object" && value !== null;
|
|
5364
5362
|
}
|
|
5365
|
-
var MINSYNC_OLLAMA_MIGRATION_MESSAGE = "This workspace uses the legacy 768-dimensional Ollama/TEI embedding path. Reindex explicitly, or pin an explicit
|
|
5363
|
+
var MINSYNC_OLLAMA_MIGRATION_MESSAGE = "This workspace uses the legacy 768-dimensional Ollama/TEI embedding path. Reindex explicitly, or pin an explicit Ollama/TEI embedder config before using the new default runtime.", API_KEY_ENV_PATTERN, MinSyncQueryError;
|
|
5366
5364
|
var init_client4 = __esm(() => {
|
|
5367
5365
|
init_embedder_config();
|
|
5368
5366
|
init_errors();
|
|
@@ -5606,7 +5604,7 @@ function isGitHubAsset(value) {
|
|
|
5606
5604
|
function isRecord5(value) {
|
|
5607
5605
|
return typeof value === "object" && value !== null;
|
|
5608
5606
|
}
|
|
5609
|
-
var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.4.
|
|
5607
|
+
var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.4.7", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
|
|
5610
5608
|
var init_installer2 = __esm(() => {
|
|
5611
5609
|
init_process();
|
|
5612
5610
|
SHA256_HEX_PATTERN = /^[a-f0-9]{64}$/;
|
|
@@ -5915,11 +5913,11 @@ class MinSyncVectorMethod {
|
|
|
5915
5913
|
});
|
|
5916
5914
|
let hits;
|
|
5917
5915
|
try {
|
|
5918
|
-
hits = await client.query(query, queryK, this.mode);
|
|
5916
|
+
hits = await client.query(query, queryK, this.mode, options.signal);
|
|
5919
5917
|
} catch (error) {
|
|
5920
|
-
if (this.mode !== "hybrid")
|
|
5918
|
+
if (this.mode !== "hybrid" || options.signal?.aborted)
|
|
5921
5919
|
throw error;
|
|
5922
|
-
hits = await client.query(query, queryK, "bm25");
|
|
5920
|
+
hits = await client.query(query, queryK, "bm25", options.signal);
|
|
5923
5921
|
}
|
|
5924
5922
|
const results = [];
|
|
5925
5923
|
for (const hit of hits) {
|
|
@@ -6242,10 +6240,16 @@ async function runTesseract(recognition, signal, cleanupResolve, cleanupReject)
|
|
|
6242
6240
|
};
|
|
6243
6241
|
signal.addEventListener("abort", abort, { once: true });
|
|
6244
6242
|
try {
|
|
6243
|
+
const tooSmall = ocrImageTooSmallMessage(recognition.bytes);
|
|
6244
|
+
if (tooSmall !== undefined)
|
|
6245
|
+
throw new Error(tooSmall);
|
|
6245
6246
|
const { cachePath } = recognition;
|
|
6246
6247
|
if (cachePath !== undefined)
|
|
6247
6248
|
await mkdir3(cachePath, { recursive: true });
|
|
6248
|
-
worker = await createWorker(recognition.language, undefined,
|
|
6249
|
+
worker = await createWorker(recognition.language, undefined, {
|
|
6250
|
+
...cachePath === undefined ? {} : { cachePath },
|
|
6251
|
+
errorHandler: containTesseractWorkerError
|
|
6252
|
+
});
|
|
6249
6253
|
if (signal.aborted)
|
|
6250
6254
|
throw new Error("OCR aborted before worker was ready");
|
|
6251
6255
|
const result = await worker.recognize(Buffer.from(recognition.bytes));
|
|
@@ -6255,6 +6259,129 @@ async function runTesseract(recognition, signal, cleanupResolve, cleanupReject)
|
|
|
6255
6259
|
await terminate().then(cleanupResolve, cleanupReject);
|
|
6256
6260
|
}
|
|
6257
6261
|
}
|
|
6262
|
+
function containTesseractWorkerError(_payload) {}
|
|
6263
|
+
function ocrImageTooSmallMessage(bytes) {
|
|
6264
|
+
const size = readImagePixelSize(bytes);
|
|
6265
|
+
if (size === undefined || size.width >= MIN_LEPTONICA_SIDE_PIXELS && size.height >= MIN_LEPTONICA_SIDE_PIXELS) {
|
|
6266
|
+
return;
|
|
6267
|
+
}
|
|
6268
|
+
return `Image is too small to scale (${size.width}x${size.height} vs minimum of ${MIN_LEPTONICA_SIDE_PIXELS} pixels).`;
|
|
6269
|
+
}
|
|
6270
|
+
function readImagePixelSize(bytes) {
|
|
6271
|
+
if (bytes.length >= 8 && bytes[0] === 137 && bytes[1] === 80)
|
|
6272
|
+
return readPngSize(bytes);
|
|
6273
|
+
if (bytes.length >= 2 && bytes[0] === 255 && bytes[1] === 216)
|
|
6274
|
+
return readJpegSize(bytes);
|
|
6275
|
+
if (bytes.length >= 12 && bytes[0] === 82 && bytes[1] === 73)
|
|
6276
|
+
return readWebpSize(bytes);
|
|
6277
|
+
return;
|
|
6278
|
+
}
|
|
6279
|
+
function readPngSize(bytes) {
|
|
6280
|
+
if (bytes.length < 24)
|
|
6281
|
+
return;
|
|
6282
|
+
if (bytes[0] !== 137 || bytes[1] !== 80 || bytes[2] !== 78 || bytes[3] !== 71 || bytes[4] !== 13 || bytes[5] !== 10 || bytes[6] !== 26 || bytes[7] !== 10) {
|
|
6283
|
+
return;
|
|
6284
|
+
}
|
|
6285
|
+
if (bytes[12] !== 73 || bytes[13] !== 72 || bytes[14] !== 68 || bytes[15] !== 82)
|
|
6286
|
+
return;
|
|
6287
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
6288
|
+
return { width: view.getUint32(16), height: view.getUint32(20) };
|
|
6289
|
+
}
|
|
6290
|
+
function readJpegSize(bytes) {
|
|
6291
|
+
if (bytes.length < 4 || bytes[0] !== 255 || bytes[1] !== 216)
|
|
6292
|
+
return;
|
|
6293
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
6294
|
+
let offset = 2;
|
|
6295
|
+
while (offset + 1 < bytes.length) {
|
|
6296
|
+
if (bytes[offset] !== 255)
|
|
6297
|
+
return;
|
|
6298
|
+
offset += 1;
|
|
6299
|
+
while (offset < bytes.length && bytes[offset] === 255)
|
|
6300
|
+
offset += 1;
|
|
6301
|
+
if (offset >= bytes.length)
|
|
6302
|
+
return;
|
|
6303
|
+
const marker = bytes[offset] ?? 0;
|
|
6304
|
+
offset += 1;
|
|
6305
|
+
if (marker === 0)
|
|
6306
|
+
return;
|
|
6307
|
+
if (marker === 1 || marker === 217 || marker >= 208 && marker <= 216)
|
|
6308
|
+
continue;
|
|
6309
|
+
if (offset + 2 > bytes.length)
|
|
6310
|
+
return;
|
|
6311
|
+
const segmentLength = view.getUint16(offset);
|
|
6312
|
+
if (segmentLength < 2)
|
|
6313
|
+
return;
|
|
6314
|
+
if (isJpegStartOfFrame(marker)) {
|
|
6315
|
+
if (segmentLength < 7 || offset + 7 > bytes.length)
|
|
6316
|
+
return;
|
|
6317
|
+
return { width: view.getUint16(offset + 5), height: view.getUint16(offset + 3) };
|
|
6318
|
+
}
|
|
6319
|
+
if (marker === 218)
|
|
6320
|
+
return;
|
|
6321
|
+
const next = offset + segmentLength;
|
|
6322
|
+
if (next <= offset || next > bytes.length)
|
|
6323
|
+
return;
|
|
6324
|
+
offset = next;
|
|
6325
|
+
}
|
|
6326
|
+
return;
|
|
6327
|
+
}
|
|
6328
|
+
function isJpegStartOfFrame(marker) {
|
|
6329
|
+
return marker >= 192 && marker <= 207 && marker !== 196 && marker !== 200 && marker !== 204;
|
|
6330
|
+
}
|
|
6331
|
+
function readWebpSize(bytes) {
|
|
6332
|
+
if (bytes.length < 20 || !asciiAt(bytes, 0, "RIFF") || !asciiAt(bytes, 8, "WEBP"))
|
|
6333
|
+
return;
|
|
6334
|
+
const chunkSize = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(16, true);
|
|
6335
|
+
if (asciiAt(bytes, 12, "VP8X"))
|
|
6336
|
+
return readVp8xSize(bytes, chunkSize);
|
|
6337
|
+
if (asciiAt(bytes, 12, "VP8L"))
|
|
6338
|
+
return readVp8lSize(bytes, chunkSize);
|
|
6339
|
+
if (asciiAt(bytes, 12, "VP8 "))
|
|
6340
|
+
return readVp8Size(bytes, chunkSize);
|
|
6341
|
+
return;
|
|
6342
|
+
}
|
|
6343
|
+
function readVp8xSize(bytes, chunkSize) {
|
|
6344
|
+
if (!webpPayloadFits(bytes, chunkSize, 10))
|
|
6345
|
+
return;
|
|
6346
|
+
const payload = 20;
|
|
6347
|
+
const width = 1 + readU24LE(bytes, payload + 4);
|
|
6348
|
+
const height = 1 + readU24LE(bytes, payload + 7);
|
|
6349
|
+
return { width, height };
|
|
6350
|
+
}
|
|
6351
|
+
function readVp8lSize(bytes, chunkSize) {
|
|
6352
|
+
if (!webpPayloadFits(bytes, chunkSize, 5) || bytes[20] !== 47)
|
|
6353
|
+
return;
|
|
6354
|
+
const b1 = bytes[21] ?? 0;
|
|
6355
|
+
const b2 = bytes[22] ?? 0;
|
|
6356
|
+
const b3 = bytes[23] ?? 0;
|
|
6357
|
+
const b4 = bytes[24] ?? 0;
|
|
6358
|
+
const width = 1 + (b1 | (b2 & 63) << 8);
|
|
6359
|
+
const height = 1 + (b2 >> 6 | b3 << 2 | (b4 & 15) << 10);
|
|
6360
|
+
return { width, height };
|
|
6361
|
+
}
|
|
6362
|
+
function readVp8Size(bytes, chunkSize) {
|
|
6363
|
+
if (!webpPayloadFits(bytes, chunkSize, 10) || (bytes[20] & 1) !== 0)
|
|
6364
|
+
return;
|
|
6365
|
+
if (bytes[23] !== 157 || bytes[24] !== 1 || bytes[25] !== 42)
|
|
6366
|
+
return;
|
|
6367
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
6368
|
+
return { width: view.getUint16(26, true) & 16383, height: view.getUint16(28, true) & 16383 };
|
|
6369
|
+
}
|
|
6370
|
+
function webpPayloadFits(bytes, chunkSize, needed) {
|
|
6371
|
+
return chunkSize >= needed && bytes.length >= 20 + needed;
|
|
6372
|
+
}
|
|
6373
|
+
function readU24LE(bytes, offset) {
|
|
6374
|
+
return (bytes[offset] ?? 0) | (bytes[offset + 1] ?? 0) << 8 | (bytes[offset + 2] ?? 0) << 16;
|
|
6375
|
+
}
|
|
6376
|
+
function asciiAt(bytes, offset, text) {
|
|
6377
|
+
if (offset + text.length > bytes.length)
|
|
6378
|
+
return false;
|
|
6379
|
+
for (let index = 0;index < text.length; index += 1) {
|
|
6380
|
+
if (bytes[offset + index] !== text.charCodeAt(index))
|
|
6381
|
+
return false;
|
|
6382
|
+
}
|
|
6383
|
+
return true;
|
|
6384
|
+
}
|
|
6258
6385
|
function withTimeout(operation, timeoutMs, onTimeout) {
|
|
6259
6386
|
return new Promise((resolve, reject) => {
|
|
6260
6387
|
const timeout = setTimeout(() => {
|
|
@@ -6270,7 +6397,7 @@ function withTimeout(operation, timeoutMs, onTimeout) {
|
|
|
6270
6397
|
});
|
|
6271
6398
|
});
|
|
6272
6399
|
}
|
|
6273
|
-
var TESSERACT_LANGUAGES, BUILTIN_OCR_LANGUAGES, DEFAULT_TESSERACT_TIMEOUT_MS = 30000;
|
|
6400
|
+
var TESSERACT_LANGUAGES, BUILTIN_OCR_LANGUAGES, DEFAULT_TESSERACT_TIMEOUT_MS = 30000, MIN_LEPTONICA_SIDE_PIXELS = 3;
|
|
6274
6401
|
var init_ocr_engines = __esm(() => {
|
|
6275
6402
|
TESSERACT_LANGUAGES = {
|
|
6276
6403
|
ko: "kor",
|
|
@@ -6479,8 +6606,8 @@ function createDefaultParserRegistry(options = {}) {
|
|
|
6479
6606
|
const kordocOptions = {
|
|
6480
6607
|
...options.kordoc,
|
|
6481
6608
|
images: ocrEnabled,
|
|
6482
|
-
...options.kordoc?.ocr === undefined
|
|
6483
|
-
ocr: builtinCovers ? true : createTesseractOcrProvider({
|
|
6609
|
+
...options.kordoc?.ocr === undefined ? {
|
|
6610
|
+
ocr: !ocrEnabled ? false : builtinCovers ? true : createTesseractOcrProvider({
|
|
6484
6611
|
languages,
|
|
6485
6612
|
timeoutMs: options.ocr?.timeoutMs,
|
|
6486
6613
|
cachePath: options.ocr?.cachePath
|
|
@@ -6618,6 +6745,23 @@ async function syncParsedMirrors(options) {
|
|
|
6618
6745
|
checkpoint();
|
|
6619
6746
|
continue;
|
|
6620
6747
|
}
|
|
6748
|
+
const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
|
|
6749
|
+
if (tooSmallImage !== undefined) {
|
|
6750
|
+
deleted += removePrevious(options.root, previous, entry.virtualPath);
|
|
6751
|
+
handledPrevious.add(entry.virtualPath);
|
|
6752
|
+
skipped += 1;
|
|
6753
|
+
recordSkip(entry, "parser-skipped");
|
|
6754
|
+
diagnostics.push({
|
|
6755
|
+
code: "parser-skipped",
|
|
6756
|
+
severity: "warning",
|
|
6757
|
+
message: tooSmallImage,
|
|
6758
|
+
source: entry.virtualPath
|
|
6759
|
+
});
|
|
6760
|
+
sinceCheckpoint += 1;
|
|
6761
|
+
if (sinceCheckpoint >= MIRROR_CHECKPOINT_EVERY)
|
|
6762
|
+
checkpoint();
|
|
6763
|
+
continue;
|
|
6764
|
+
}
|
|
6621
6765
|
let parsed;
|
|
6622
6766
|
try {
|
|
6623
6767
|
parsed = await parser.parse({ virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes });
|
|
@@ -6719,6 +6863,11 @@ async function detectMirrorStaleness(options) {
|
|
|
6719
6863
|
}
|
|
6720
6864
|
return diagnostics;
|
|
6721
6865
|
}
|
|
6866
|
+
function standaloneImageTooSmallMessage(virtualPath, bytes) {
|
|
6867
|
+
if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname4(virtualPath))))
|
|
6868
|
+
return;
|
|
6869
|
+
return ocrImageTooSmallMessage(bytes);
|
|
6870
|
+
}
|
|
6722
6871
|
function sourceReadFailure(error) {
|
|
6723
6872
|
if (!(error instanceof Error))
|
|
6724
6873
|
return;
|
|
@@ -6814,11 +6963,13 @@ function removeFile(path) {
|
|
|
6814
6963
|
if (existsSync15(path))
|
|
6815
6964
|
rmSync6(path, { force: true });
|
|
6816
6965
|
}
|
|
6817
|
-
var DEFAULT_MAX_SOURCE_BYTES, MIRROR_CHECKPOINT_EVERY = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.";
|
|
6966
|
+
var DEFAULT_MAX_SOURCE_BYTES, MIRROR_CHECKPOINT_EVERY = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", STANDALONE_OCR_IMAGE_EXTENSIONS;
|
|
6818
6967
|
var init_sync = __esm(() => {
|
|
6819
6968
|
init_source_paths();
|
|
6820
6969
|
init_defaults();
|
|
6821
6970
|
init_errors2();
|
|
6971
|
+
init_kordoc();
|
|
6972
|
+
init_ocr_engines();
|
|
6822
6973
|
init_text();
|
|
6823
6974
|
init_index_store();
|
|
6824
6975
|
init_paths();
|
|
@@ -6846,6 +6997,7 @@ var init_sync = __esm(() => {
|
|
|
6846
6997
|
"GrShaderCache",
|
|
6847
6998
|
"ShaderCache"
|
|
6848
6999
|
]);
|
|
7000
|
+
STANDALONE_OCR_IMAGE_EXTENSIONS = new Set(KORDOC_IMAGE_EXTENSIONS);
|
|
6849
7001
|
});
|
|
6850
7002
|
|
|
6851
7003
|
// src/observability/run-log.ts
|
|
@@ -7711,8 +7863,36 @@ class RetrievalMethodRegistry {
|
|
|
7711
7863
|
}
|
|
7712
7864
|
}
|
|
7713
7865
|
|
|
7866
|
+
// src/cloud/config.ts
|
|
7867
|
+
function normalizeBaseUrl(value) {
|
|
7868
|
+
const trimmed = value.trim();
|
|
7869
|
+
let end = trimmed.length;
|
|
7870
|
+
while (end > 0 && trimmed.charCodeAt(end - 1) === 47)
|
|
7871
|
+
end--;
|
|
7872
|
+
return trimmed.slice(0, end);
|
|
7873
|
+
}
|
|
7874
|
+
function resolveBaseUrl(env) {
|
|
7875
|
+
const configured = env[AUTORAG_BASE_URL_ENV];
|
|
7876
|
+
const value = typeof configured === "string" && configured.trim() !== "" ? configured : DEFAULT_BASE_URL;
|
|
7877
|
+
return normalizeBaseUrl(value);
|
|
7878
|
+
}
|
|
7879
|
+
var PROVIDER_ID = "autorag", PROVIDER_NAME = "AutoRAG", DEFAULT_BASE_URL = "https://api.dazziapp.com", AUTORAG_BASE_URL_ENV = "AUTORAG_BASE_URL", AUTORAG_API_KEY_ENV = "AUTORAG_API_KEY", API_PATH = "/v1", MODELS_PATH = "/v1/models", AUTHORIZE_PATH = "/desktop/authorize", TOKEN_PATH = "/api/desktop/token", AUTORAG_INPUT_LIMITS, OAUTH_CLIENT_ID = "agent", API_KEY_CONFIG_VALUE;
|
|
7880
|
+
var init_config = __esm(() => {
|
|
7881
|
+
AUTORAG_INPUT_LIMITS = {
|
|
7882
|
+
maxRequestBytes: 3500000,
|
|
7883
|
+
images: { resize: { maxBytes: 1e6 } }
|
|
7884
|
+
};
|
|
7885
|
+
API_KEY_CONFIG_VALUE = `$${AUTORAG_API_KEY_ENV}`;
|
|
7886
|
+
});
|
|
7887
|
+
|
|
7714
7888
|
// src/retrieval/rerank.ts
|
|
7715
7889
|
import { OpenRouter } from "@openrouter/sdk";
|
|
7890
|
+
function defaultRerankModel(provider) {
|
|
7891
|
+
return provider === PROVIDER_ID ? DEFAULT_AUTORAG_RERANK_MODEL : DEFAULT_RERANK_MODEL;
|
|
7892
|
+
}
|
|
7893
|
+
function defaultRerankApiKeyEnv(provider) {
|
|
7894
|
+
return provider === PROVIDER_ID ? DEFAULT_AUTORAG_RERANK_API_KEY_ENV : DEFAULT_RERANK_API_KEY_ENV;
|
|
7895
|
+
}
|
|
7716
7896
|
function createReranker(config) {
|
|
7717
7897
|
if (config === false || config === undefined)
|
|
7718
7898
|
return;
|
|
@@ -7720,30 +7900,36 @@ function createReranker(config) {
|
|
|
7720
7900
|
if (!SUPPORTED_RERANK_PROVIDERS.includes(provider)) {
|
|
7721
7901
|
throw new Error(`Unsupported rerank provider "${provider}"; supported providers: ${SUPPORTED_RERANK_PROVIDERS.join(", ")}`);
|
|
7722
7902
|
}
|
|
7723
|
-
return new OpenRouterReranker(config);
|
|
7903
|
+
return new OpenRouterReranker({ ...config, provider });
|
|
7904
|
+
}
|
|
7905
|
+
function autoragApiBaseUrl(env) {
|
|
7906
|
+
const root = resolveBaseUrl(env);
|
|
7907
|
+
return root.endsWith(API_PATH) ? root : `${root}${API_PATH}`;
|
|
7724
7908
|
}
|
|
7725
7909
|
|
|
7726
7910
|
class OpenRouterReranker {
|
|
7727
7911
|
model;
|
|
7728
|
-
provider
|
|
7912
|
+
provider;
|
|
7729
7913
|
client;
|
|
7730
7914
|
unavailableReason;
|
|
7731
7915
|
constructor(options = {}) {
|
|
7732
|
-
this.
|
|
7916
|
+
this.provider = options.provider ?? DEFAULT_RERANK_PROVIDER;
|
|
7917
|
+
this.model = options.model ?? defaultRerankModel(this.provider);
|
|
7733
7918
|
if (options.client !== undefined) {
|
|
7734
7919
|
this.client = options.client;
|
|
7735
7920
|
return;
|
|
7736
7921
|
}
|
|
7737
|
-
const apiKeyEnv = options.apiKeyEnv ?? DEFAULT_RERANK_API_KEY_ENV;
|
|
7738
7922
|
const env = options.env ?? process.env;
|
|
7923
|
+
const apiKeyEnv = options.apiKeyEnv ?? defaultRerankApiKeyEnv(this.provider);
|
|
7739
7924
|
const apiKey = options.apiKey ?? env[apiKeyEnv];
|
|
7740
7925
|
if (apiKey === undefined || apiKey.length === 0) {
|
|
7741
7926
|
this.unavailableReason = `no API key: set ${apiKeyEnv} or configure rerank.apiKeyEnv`;
|
|
7742
7927
|
return;
|
|
7743
7928
|
}
|
|
7929
|
+
const baseUrl = options.baseUrl ?? (this.provider === PROVIDER_ID ? autoragApiBaseUrl(env) : undefined);
|
|
7744
7930
|
const sdkOptions = { apiKey };
|
|
7745
|
-
if (
|
|
7746
|
-
sdkOptions.serverURL =
|
|
7931
|
+
if (baseUrl !== undefined)
|
|
7932
|
+
sdkOptions.serverURL = baseUrl;
|
|
7747
7933
|
if (options.timeoutMs !== undefined)
|
|
7748
7934
|
sdkOptions.timeoutMs = options.timeoutMs;
|
|
7749
7935
|
const sdk = new OpenRouter(sdkOptions);
|
|
@@ -7808,9 +7994,10 @@ class OpenRouterReranker {
|
|
|
7808
7994
|
return reranked;
|
|
7809
7995
|
}
|
|
7810
7996
|
}
|
|
7811
|
-
var DEFAULT_RERANK_MODEL = "voyageai/rerank-3-lite", DEFAULT_RERANK_PROVIDER = "openrouter", DEFAULT_RERANK_API_KEY_ENV = "OPENROUTER_API_KEY", DEFAULT_RERANK_TOP_N = 25, SUPPORTED_RERANK_PROVIDERS;
|
|
7997
|
+
var DEFAULT_RERANK_MODEL = "voyageai/rerank-3-lite", DEFAULT_RERANK_PROVIDER = "openrouter", DEFAULT_RERANK_API_KEY_ENV = "OPENROUTER_API_KEY", DEFAULT_AUTORAG_RERANK_MODEL = "qwen/qwen3-reranker-8b", DEFAULT_AUTORAG_RERANK_API_KEY_ENV = "AUTORAG_API_KEY", DEFAULT_RERANK_TOP_N = 25, SUPPORTED_RERANK_PROVIDERS;
|
|
7812
7998
|
var init_rerank = __esm(() => {
|
|
7813
|
-
|
|
7999
|
+
init_config();
|
|
8000
|
+
SUPPORTED_RERANK_PROVIDERS = [DEFAULT_RERANK_PROVIDER, PROVIDER_ID];
|
|
7814
8001
|
});
|
|
7815
8002
|
|
|
7816
8003
|
// src/retrieval/selection.ts
|
|
@@ -9029,22 +9216,219 @@ var init_fsearch_search_tool = __esm(() => {
|
|
|
9029
9216
|
});
|
|
9030
9217
|
});
|
|
9031
9218
|
|
|
9032
|
-
// src/cloud/
|
|
9033
|
-
|
|
9034
|
-
|
|
9035
|
-
|
|
9036
|
-
|
|
9037
|
-
|
|
9038
|
-
|
|
9219
|
+
// src/cloud/jev-backend.ts
|
|
9220
|
+
import {
|
|
9221
|
+
BackendError
|
|
9222
|
+
} from "jev-use";
|
|
9223
|
+
|
|
9224
|
+
class AutoragJevBackend {
|
|
9225
|
+
name = "autorag";
|
|
9226
|
+
apiKey;
|
|
9227
|
+
baseUrl;
|
|
9228
|
+
defaultModel;
|
|
9229
|
+
timeoutMs;
|
|
9230
|
+
maxRetries;
|
|
9231
|
+
fetchImpl;
|
|
9232
|
+
constructor(options = {}) {
|
|
9233
|
+
const env = options.env ?? process.env;
|
|
9234
|
+
const apiKey = options.apiKey ?? env[AUTORAG_API_KEY_ENV];
|
|
9235
|
+
if (apiKey === undefined || apiKey.trim() === "") {
|
|
9236
|
+
throw new Error(`jev.backend "autorag" needs ${AUTORAG_API_KEY_ENV}.`);
|
|
9237
|
+
}
|
|
9238
|
+
this.apiKey = apiKey;
|
|
9239
|
+
this.baseUrl = options.baseUrl ?? resolveBaseUrl(env);
|
|
9240
|
+
this.defaultModel = options.defaultModel ?? AUTORAG_DEFAULT_JEV_MODEL;
|
|
9241
|
+
this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS6;
|
|
9242
|
+
this.maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
|
|
9243
|
+
this.fetchImpl = options.fetchImpl;
|
|
9244
|
+
}
|
|
9245
|
+
async judge(request) {
|
|
9246
|
+
const model = request.model ?? this.defaultModel;
|
|
9247
|
+
const body = toNativeBody(request.state, request.questions, model);
|
|
9248
|
+
const started = Date.now();
|
|
9249
|
+
const response = await this.postJson(`${this.baseUrl}${JEV_PATH}`, body);
|
|
9250
|
+
return {
|
|
9251
|
+
answers: parseNativeAnswers(this.name, response.answers, request.questions),
|
|
9252
|
+
model: response.model ?? model,
|
|
9253
|
+
latencyMs: Date.now() - started,
|
|
9254
|
+
usage: {
|
|
9255
|
+
inputTokens: response.usage?.input_tokens,
|
|
9256
|
+
outputTokens: response.usage?.output_tokens
|
|
9257
|
+
}
|
|
9258
|
+
};
|
|
9259
|
+
}
|
|
9260
|
+
async postJson(url, body) {
|
|
9261
|
+
const doFetch = this.fetchImpl ?? fetch;
|
|
9262
|
+
let lastError;
|
|
9263
|
+
for (let attempt = 0;attempt <= this.maxRetries; attempt += 1) {
|
|
9264
|
+
if (attempt > 0)
|
|
9265
|
+
await sleep3(retryDelay(attempt, lastError));
|
|
9266
|
+
const controller = new AbortController;
|
|
9267
|
+
const timer = setTimeout(() => controller.abort(), this.timeoutMs);
|
|
9268
|
+
try {
|
|
9269
|
+
const response = await doFetch(url, {
|
|
9270
|
+
method: "POST",
|
|
9271
|
+
headers: {
|
|
9272
|
+
"Content-Type": "application/json",
|
|
9273
|
+
Accept: "application/json",
|
|
9274
|
+
Authorization: `Bearer ${this.apiKey}`
|
|
9275
|
+
},
|
|
9276
|
+
body: JSON.stringify(body),
|
|
9277
|
+
signal: controller.signal
|
|
9278
|
+
});
|
|
9279
|
+
if (response.ok)
|
|
9280
|
+
return await response.json();
|
|
9281
|
+
const message = await extractErrorMessage(response);
|
|
9282
|
+
lastError = new BackendError(this.name, `HTTP ${response.status}: ${message}`, response.status);
|
|
9283
|
+
lastError.retryAfterMs = parseRetryAfter(response);
|
|
9284
|
+
if (!isRetryable(response.status) || attempt === this.maxRetries)
|
|
9285
|
+
throw lastError;
|
|
9286
|
+
} catch (error) {
|
|
9287
|
+
if (error instanceof BackendError) {
|
|
9288
|
+
if (attempt === this.maxRetries || !isRetryable(error.status))
|
|
9289
|
+
throw error;
|
|
9290
|
+
continue;
|
|
9291
|
+
}
|
|
9292
|
+
const aborted = error?.name === "AbortError";
|
|
9293
|
+
lastError = new BackendError(this.name, aborted ? `request timed out after ${this.timeoutMs}ms` : `network error: ${String(error)}`, undefined, error);
|
|
9294
|
+
if (attempt === this.maxRetries)
|
|
9295
|
+
throw lastError;
|
|
9296
|
+
} finally {
|
|
9297
|
+
clearTimeout(timer);
|
|
9298
|
+
}
|
|
9299
|
+
}
|
|
9300
|
+
throw lastError ?? new BackendError(this.name, "unreachable");
|
|
9301
|
+
}
|
|
9039
9302
|
}
|
|
9040
|
-
function
|
|
9041
|
-
|
|
9042
|
-
const value = typeof configured === "string" && configured.trim() !== "" ? configured : DEFAULT_BASE_URL;
|
|
9043
|
-
return normalizeBaseUrl(value);
|
|
9303
|
+
function serializeState(state) {
|
|
9304
|
+
return typeof state === "string" ? state : JSON.stringify(state);
|
|
9044
9305
|
}
|
|
9045
|
-
|
|
9046
|
-
|
|
9047
|
-
|
|
9306
|
+
function toNativeQuestion(question) {
|
|
9307
|
+
switch (question.type) {
|
|
9308
|
+
case "noul":
|
|
9309
|
+
return {
|
|
9310
|
+
type: "noul",
|
|
9311
|
+
instructions: question.question,
|
|
9312
|
+
...question.criteria ? { criteria: question.criteria } : {}
|
|
9313
|
+
};
|
|
9314
|
+
case "choice":
|
|
9315
|
+
return {
|
|
9316
|
+
type: "choice",
|
|
9317
|
+
instructions: question.question,
|
|
9318
|
+
criteria: Object.fromEntries(optionEntries(question.options ?? []).map(([label, meaning]) => [label, meaning || label]))
|
|
9319
|
+
};
|
|
9320
|
+
case "score":
|
|
9321
|
+
return {
|
|
9322
|
+
type: "score",
|
|
9323
|
+
instructions: question.question,
|
|
9324
|
+
criteria: question.levels ?? []
|
|
9325
|
+
};
|
|
9326
|
+
}
|
|
9327
|
+
}
|
|
9328
|
+
function toNativeBody(state, questions, model) {
|
|
9329
|
+
return {
|
|
9330
|
+
model,
|
|
9331
|
+
state: serializeState(state),
|
|
9332
|
+
questions: Object.fromEntries(questions.map((question) => [question.id, toNativeQuestion(question)]))
|
|
9333
|
+
};
|
|
9334
|
+
}
|
|
9335
|
+
function parseNativeAnswers(backend, answers, questions) {
|
|
9336
|
+
if (!answers || typeof answers !== "object") {
|
|
9337
|
+
throw new BackendError(backend, "response has no answers object");
|
|
9338
|
+
}
|
|
9339
|
+
const byId = answers;
|
|
9340
|
+
return questions.map((question) => parseNativeAnswer(backend, question, byId[question.id]));
|
|
9341
|
+
}
|
|
9342
|
+
function parseNativeAnswer(backend, question, answer) {
|
|
9343
|
+
if (!answer) {
|
|
9344
|
+
throw new BackendError(backend, `response missing answer for question "${question.id}"`);
|
|
9345
|
+
}
|
|
9346
|
+
switch (question.type) {
|
|
9347
|
+
case "noul":
|
|
9348
|
+
return {
|
|
9349
|
+
answer: numberOrThrow(backend, question.id, answer.noul),
|
|
9350
|
+
confidence: answer.confidence
|
|
9351
|
+
};
|
|
9352
|
+
case "choice":
|
|
9353
|
+
return {
|
|
9354
|
+
answer: stringOrThrow(backend, question.id, answer.choice),
|
|
9355
|
+
confidence: answer.confidence,
|
|
9356
|
+
distribution: answer.probabilities
|
|
9357
|
+
};
|
|
9358
|
+
case "score":
|
|
9359
|
+
return {
|
|
9360
|
+
answer: numberOrThrow(backend, question.id, answer.score),
|
|
9361
|
+
confidence: answer.confidence,
|
|
9362
|
+
distribution: answer.probabilities,
|
|
9363
|
+
legend: answer.legend
|
|
9364
|
+
};
|
|
9365
|
+
}
|
|
9366
|
+
}
|
|
9367
|
+
function optionEntries(options) {
|
|
9368
|
+
return Array.isArray(options) ? options.map((label) => [label, ""]) : Object.entries(options);
|
|
9369
|
+
}
|
|
9370
|
+
function numberOrThrow(backend, id, value) {
|
|
9371
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
9372
|
+
throw new BackendError(backend, `answer "${id}" has no numeric value`);
|
|
9373
|
+
}
|
|
9374
|
+
return value;
|
|
9375
|
+
}
|
|
9376
|
+
function stringOrThrow(backend, id, value) {
|
|
9377
|
+
if (typeof value !== "string" || !value) {
|
|
9378
|
+
throw new BackendError(backend, `answer "${id}" has no choice value`);
|
|
9379
|
+
}
|
|
9380
|
+
return value;
|
|
9381
|
+
}
|
|
9382
|
+
function isRetryable(status) {
|
|
9383
|
+
if (status === undefined)
|
|
9384
|
+
return true;
|
|
9385
|
+
return status === 408 || status === 429 || status >= 500;
|
|
9386
|
+
}
|
|
9387
|
+
function retryDelay(attempt, lastError) {
|
|
9388
|
+
if (lastError?.retryAfterMs !== undefined)
|
|
9389
|
+
return Math.min(lastError.retryAfterMs, 60000);
|
|
9390
|
+
const base = Math.min(RETRY_BASE_MS * 2 ** (attempt - 1), RETRY_MAX_MS);
|
|
9391
|
+
return base * (1 + Math.random() * 0.25);
|
|
9392
|
+
}
|
|
9393
|
+
function parseRetryAfter(response) {
|
|
9394
|
+
const milliseconds = response.headers.get("retry-after-ms");
|
|
9395
|
+
if (milliseconds && Number.isFinite(Number(milliseconds)))
|
|
9396
|
+
return Number(milliseconds);
|
|
9397
|
+
const seconds = response.headers.get("retry-after");
|
|
9398
|
+
if (seconds && Number.isFinite(Number(seconds)))
|
|
9399
|
+
return Number(seconds) * 1000;
|
|
9400
|
+
return;
|
|
9401
|
+
}
|
|
9402
|
+
async function extractErrorMessage(response) {
|
|
9403
|
+
try {
|
|
9404
|
+
const data = await response.json();
|
|
9405
|
+
for (const key of ["error", "message", "detail"]) {
|
|
9406
|
+
const value = data[key];
|
|
9407
|
+
if (typeof value === "string")
|
|
9408
|
+
return value;
|
|
9409
|
+
if (value && typeof value === "object") {
|
|
9410
|
+
const message = value.message;
|
|
9411
|
+
if (typeof message === "string")
|
|
9412
|
+
return message;
|
|
9413
|
+
}
|
|
9414
|
+
if (Array.isArray(value) && value.length > 0) {
|
|
9415
|
+
const first = value[0];
|
|
9416
|
+
if (typeof first?.msg === "string")
|
|
9417
|
+
return first.msg;
|
|
9418
|
+
}
|
|
9419
|
+
}
|
|
9420
|
+
return JSON.stringify(data).slice(0, 300);
|
|
9421
|
+
} catch {
|
|
9422
|
+
return response.statusText || "request failed";
|
|
9423
|
+
}
|
|
9424
|
+
}
|
|
9425
|
+
function sleep3(ms) {
|
|
9426
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
9427
|
+
}
|
|
9428
|
+
var JEV_PATH, AUTORAG_DEFAULT_JEV_MODEL = "jev-1.13", DEFAULT_TIMEOUT_MS6 = 1e4, DEFAULT_MAX_RETRIES = 2, RETRY_BASE_MS = 500, RETRY_MAX_MS = 5000;
|
|
9429
|
+
var init_jev_backend = __esm(() => {
|
|
9430
|
+
init_config();
|
|
9431
|
+
JEV_PATH = `${API_PATH}/jev`;
|
|
9048
9432
|
});
|
|
9049
9433
|
|
|
9050
9434
|
// src/cloud/errors.ts
|
|
@@ -9175,7 +9559,8 @@ function mapServerModel(entry) {
|
|
|
9175
9559
|
cost: readPricing(entry.pricing),
|
|
9176
9560
|
reasoning: entry.reasoning === true,
|
|
9177
9561
|
contextWindow: readPositiveNumber(entry.context_window) ?? DEFAULT_CONTEXT_WINDOW,
|
|
9178
|
-
maxTokens: readPositiveNumber(entry.max_output_tokens) ?? DEFAULT_MAX_TOKENS
|
|
9562
|
+
maxTokens: readPositiveNumber(entry.max_output_tokens) ?? DEFAULT_MAX_TOKENS,
|
|
9563
|
+
inputLimits: AUTORAG_INPUT_LIMITS
|
|
9179
9564
|
};
|
|
9180
9565
|
}
|
|
9181
9566
|
function buildStoreEntry(models, providerId, baseUrl) {
|
|
@@ -9214,7 +9599,8 @@ function modelsFromStore(stored) {
|
|
|
9214
9599
|
cost: entry.cost,
|
|
9215
9600
|
reasoning: entry.reasoning,
|
|
9216
9601
|
contextWindow: entry.contextWindow,
|
|
9217
|
-
maxTokens: entry.maxTokens
|
|
9602
|
+
maxTokens: entry.maxTokens,
|
|
9603
|
+
inputLimits: AUTORAG_INPUT_LIMITS
|
|
9218
9604
|
});
|
|
9219
9605
|
}
|
|
9220
9606
|
return models;
|
|
@@ -9622,90 +10008,504 @@ var init_provider = __esm(() => {
|
|
|
9622
10008
|
init_oauth_login();
|
|
9623
10009
|
});
|
|
9624
10010
|
|
|
9625
|
-
// src/agent/
|
|
9626
|
-
import {
|
|
9627
|
-
|
|
9628
|
-
|
|
9629
|
-
|
|
9630
|
-
|
|
9631
|
-
|
|
9632
|
-
createAgentSessionServices,
|
|
9633
|
-
DefaultResourceLoader,
|
|
9634
|
-
getAgentDir,
|
|
9635
|
-
ModelRuntime,
|
|
9636
|
-
SessionManager,
|
|
9637
|
-
SettingsManager
|
|
9638
|
-
} from "@earendil-works/pi-coding-agent";
|
|
9639
|
-
function resolveInteractiveTuiMode(settingsManager) {
|
|
9640
|
-
const saved = settingsManager.getProjectSettings().tuiMode ?? settingsManager.getGlobalSettings().tuiMode;
|
|
9641
|
-
return saved ?? "regular";
|
|
9642
|
-
}
|
|
9643
|
-
function toToolDefinition(tool) {
|
|
9644
|
-
return {
|
|
9645
|
-
name: tool.name,
|
|
9646
|
-
label: tool.label,
|
|
9647
|
-
description: tool.description,
|
|
9648
|
-
parameters: tool.parameters,
|
|
9649
|
-
...tool.prepareArguments === undefined ? {} : { prepareArguments: tool.prepareArguments },
|
|
9650
|
-
execute: async (toolCallId, params, signal, onUpdate) => {
|
|
9651
|
-
return tool.execute(toolCallId, params, signal, onUpdate);
|
|
9652
|
-
}
|
|
10011
|
+
// src/agent/context-token-guard.ts
|
|
10012
|
+
import { calculateContextTokens, estimateTokens } from "@earendil-works/pi-coding-agent";
|
|
10013
|
+
function createContextTokenGuardedTransform(model, base, fixedOverheadTokens = () => 0) {
|
|
10014
|
+
return async (messages) => {
|
|
10015
|
+
const transformed = base === undefined ? messages : await base(messages);
|
|
10016
|
+
const currentModel = typeof model === "function" ? model() : model;
|
|
10017
|
+
return guardAgentContextMessages(currentModel, transformed, transformed === messages, fixedOverheadTokens());
|
|
9653
10018
|
};
|
|
9654
10019
|
}
|
|
9655
|
-
function
|
|
9656
|
-
|
|
9657
|
-
|
|
9658
|
-
|
|
9659
|
-
|
|
9660
|
-
|
|
9661
|
-
|
|
9662
|
-
|
|
9663
|
-
|
|
9664
|
-
|
|
9665
|
-
|
|
9666
|
-
|
|
9667
|
-
|
|
9668
|
-
|
|
9669
|
-
|
|
9670
|
-
|
|
9671
|
-
|
|
9672
|
-
|
|
9673
|
-
|
|
9674
|
-
|
|
9675
|
-
|
|
10020
|
+
function guardAgentContextMessages(model, messages, useReportedUsage = true, fixedOverheadTokens = 0) {
|
|
10021
|
+
if (model === undefined || !Number.isFinite(model.contextWindow) || model.contextWindow <= 0 || messages.length === 0) {
|
|
10022
|
+
return messages;
|
|
10023
|
+
}
|
|
10024
|
+
const usableContext = Math.floor(model.contextWindow / CONTEXT_HEADROOM);
|
|
10025
|
+
const outputReserve = Math.min(usableContext, Number.isFinite(model.maxTokens) ? Math.max(0, model.maxTokens) : 0);
|
|
10026
|
+
const inputBudget = Math.max(0, usableContext - outputReserve);
|
|
10027
|
+
const budget = Math.max(0, inputBudget - fixedOverheadTokens);
|
|
10028
|
+
const reported = useReportedUsage && findLastAssistantUsage(messages) !== undefined;
|
|
10029
|
+
if (estimateContextMessageTokens(messages, useReportedUsage) <= (reported ? inputBudget : budget)) {
|
|
10030
|
+
return messages;
|
|
10031
|
+
}
|
|
10032
|
+
const candidates = [];
|
|
10033
|
+
const candidateIndexes = [];
|
|
10034
|
+
const base = messages.slice();
|
|
10035
|
+
for (const [index, message] of base.entries()) {
|
|
10036
|
+
const text = textContent(message);
|
|
10037
|
+
if (text === undefined)
|
|
10038
|
+
continue;
|
|
10039
|
+
const section = parseCandidateSection(message, text);
|
|
10040
|
+
if (section === undefined)
|
|
10041
|
+
continue;
|
|
10042
|
+
candidates.push(section);
|
|
10043
|
+
candidateIndexes.push(index);
|
|
10044
|
+
base[index] = replaceTextContent(message, `${section.prefix}${section.suffix}`);
|
|
10045
|
+
}
|
|
10046
|
+
let guarded = messages;
|
|
10047
|
+
if (candidates.length > 0) {
|
|
10048
|
+
const fixedTokens = estimateContextMessageTokens(base, false);
|
|
10049
|
+
const maximumOmitted = candidates.reduce((total, section) => total + section.blocks.length, 0);
|
|
10050
|
+
const omissionMarker = (count) => `
|
|
10051
|
+
|
|
10052
|
+
${GUARD_MARKER} Retrieval candidates omitted: ${count}.
|
|
10053
|
+
`;
|
|
10054
|
+
let remaining = Math.max(0, budget - fixedTokens - Math.ceil(omissionMarker(maximumOmitted).length / 4));
|
|
10055
|
+
const kept = candidates.map(() => []);
|
|
10056
|
+
let omitted = 0;
|
|
10057
|
+
let firstOmitted = -1;
|
|
10058
|
+
for (const [sectionIndex, section] of candidates.entries()) {
|
|
10059
|
+
for (const block of section.blocks) {
|
|
10060
|
+
const tokens = Math.max(1, Math.ceil(block.length / 4));
|
|
10061
|
+
if (tokens <= remaining) {
|
|
10062
|
+
kept[sectionIndex]?.push(block);
|
|
10063
|
+
remaining -= tokens;
|
|
10064
|
+
continue;
|
|
10065
|
+
}
|
|
10066
|
+
omitted += section.blocks.length - (kept[sectionIndex]?.length ?? 0);
|
|
10067
|
+
for (const later of candidates.slice(sectionIndex + 1))
|
|
10068
|
+
omitted += later.blocks.length;
|
|
10069
|
+
firstOmitted = sectionIndex;
|
|
10070
|
+
break;
|
|
9676
10071
|
}
|
|
9677
|
-
|
|
9678
|
-
|
|
10072
|
+
if (firstOmitted !== -1)
|
|
10073
|
+
break;
|
|
10074
|
+
}
|
|
10075
|
+
if (omitted > 0) {
|
|
10076
|
+
const candidateByMessage = new Map;
|
|
10077
|
+
for (const [sectionIndex, messageIndex] of candidateIndexes.entries())
|
|
10078
|
+
candidateByMessage.set(messageIndex, sectionIndex);
|
|
10079
|
+
guarded = base.map((message, index) => {
|
|
10080
|
+
const sectionIndex = candidateByMessage.get(index);
|
|
10081
|
+
if (sectionIndex === undefined)
|
|
10082
|
+
return message;
|
|
10083
|
+
const section = candidates[sectionIndex];
|
|
10084
|
+
const marker = sectionIndex === firstOmitted ? omissionMarker(omitted) : "";
|
|
10085
|
+
return replaceTextContent(message, `${section.prefix}${kept[sectionIndex]?.join("") ?? ""}${marker}${section.suffix}`);
|
|
10086
|
+
});
|
|
10087
|
+
}
|
|
10088
|
+
}
|
|
10089
|
+
if (estimateContextMessageTokens(guarded, false) <= budget && guarded !== messages)
|
|
10090
|
+
return guarded;
|
|
10091
|
+
return trimFixedContext(guarded, budget);
|
|
9679
10092
|
}
|
|
9680
|
-
|
|
9681
|
-
if (
|
|
9682
|
-
const
|
|
9683
|
-
if (
|
|
9684
|
-
|
|
10093
|
+
function estimateContextMessageTokens(messages, useReportedUsage = false) {
|
|
10094
|
+
if (useReportedUsage) {
|
|
10095
|
+
const usage = findLastAssistantUsage(messages);
|
|
10096
|
+
if (usage !== undefined) {
|
|
10097
|
+
return calculateContextTokens(usage.usage) + messages.slice(usage.index + 1).reduce((total, message) => total + estimateMessageTokens(message), 0);
|
|
9685
10098
|
}
|
|
9686
10099
|
}
|
|
9687
|
-
|
|
9688
|
-
|
|
10100
|
+
return messages.reduce((total, message) => total + estimateMessageTokens(message), 0);
|
|
10101
|
+
}
|
|
10102
|
+
function trimFixedContext(messages, inputBudget) {
|
|
10103
|
+
const trimmed = messages.slice();
|
|
10104
|
+
const lastIndex = trimmed.length - 1;
|
|
10105
|
+
const shrinkable = trimmed.map((message, index) => ({ message, index })).filter(({ index, message }) => index !== lastIndex && message.role !== "system");
|
|
10106
|
+
for (const { message, index } of shrinkable) {
|
|
10107
|
+
const replacement = replaceMessageWithMarker(message);
|
|
10108
|
+
if (replacement === message || estimateMessageTokens(replacement) >= estimateMessageTokens(message))
|
|
10109
|
+
continue;
|
|
10110
|
+
trimmed[index] = replacement;
|
|
10111
|
+
if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
|
|
10112
|
+
return trimmed;
|
|
10113
|
+
}
|
|
10114
|
+
const lastMessage = trimmed[lastIndex];
|
|
10115
|
+
if (lastMessage !== undefined && lastMessage.role !== "system") {
|
|
10116
|
+
const replacement = replaceMessageWithMarker(lastMessage);
|
|
10117
|
+
if (replacement !== lastMessage) {
|
|
10118
|
+
trimmed[lastIndex] = replacement;
|
|
10119
|
+
if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
|
|
10120
|
+
return trimmed;
|
|
10121
|
+
}
|
|
10122
|
+
const fixedWithoutLast = trimmed.reduce((total, message, index) => index === lastIndex ? total : total + estimateMessageTokens(message), 0);
|
|
10123
|
+
if (fixedWithoutLast < inputBudget) {
|
|
10124
|
+
const fitted = fitMessageToTokens(lastMessage, inputBudget - fixedWithoutLast);
|
|
10125
|
+
if (fitted !== lastMessage) {
|
|
10126
|
+
trimmed[lastIndex] = fitted;
|
|
10127
|
+
if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
|
|
10128
|
+
return trimmed;
|
|
10129
|
+
}
|
|
10130
|
+
}
|
|
10131
|
+
}
|
|
10132
|
+
for (const [index, message] of trimmed.entries()) {
|
|
10133
|
+
if (message.role !== "system")
|
|
10134
|
+
continue;
|
|
10135
|
+
const replacement = replaceMessageWithMarker(message);
|
|
10136
|
+
if (replacement === message || estimateMessageTokens(replacement) >= estimateMessageTokens(message))
|
|
10137
|
+
continue;
|
|
10138
|
+
trimmed[index] = replacement;
|
|
10139
|
+
if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
|
|
10140
|
+
return trimmed;
|
|
10141
|
+
}
|
|
10142
|
+
return trimmed;
|
|
10143
|
+
}
|
|
10144
|
+
function fitMessageToTokens(message, maxTokens) {
|
|
10145
|
+
const original = textContent(message);
|
|
10146
|
+
if (original === undefined)
|
|
10147
|
+
return message;
|
|
10148
|
+
const marker = `
|
|
10149
|
+
|
|
10150
|
+
${GUARD_MARKER}
|
|
10151
|
+
`;
|
|
10152
|
+
let low = 0;
|
|
10153
|
+
let high = original.length;
|
|
10154
|
+
let best = replaceTextContent(message, marker);
|
|
10155
|
+
while (low <= high) {
|
|
10156
|
+
const middle = Math.floor((low + high) / 2);
|
|
10157
|
+
const candidate = replaceTextContent(message, excerpt(original, middle, marker));
|
|
10158
|
+
if (estimateMessageTokens(candidate) <= maxTokens) {
|
|
10159
|
+
best = candidate;
|
|
10160
|
+
low = middle + 1;
|
|
10161
|
+
} else {
|
|
10162
|
+
high = middle - 1;
|
|
10163
|
+
}
|
|
10164
|
+
}
|
|
10165
|
+
return best;
|
|
10166
|
+
}
|
|
10167
|
+
function excerpt(text, maxChars, marker) {
|
|
10168
|
+
if (text.length <= maxChars)
|
|
10169
|
+
return text;
|
|
10170
|
+
const available = Math.max(0, maxChars - marker.length);
|
|
10171
|
+
const head = Math.ceil(available * 0.7);
|
|
10172
|
+
const tail = Math.max(0, available - head);
|
|
10173
|
+
return `${text.slice(0, head)}${marker}${tail > 0 ? text.slice(-tail) : ""}`;
|
|
10174
|
+
}
|
|
10175
|
+
function replaceMessageWithMarker(message) {
|
|
10176
|
+
return replaceTextContent(message, `
|
|
10177
|
+
|
|
10178
|
+
${GUARD_MARKER}
|
|
10179
|
+
`);
|
|
10180
|
+
}
|
|
10181
|
+
function estimateMessageTokens(message) {
|
|
10182
|
+
if (message.role === "system")
|
|
10183
|
+
return estimateSystemMessageTokens(message);
|
|
10184
|
+
const estimate = estimateTokens(message);
|
|
10185
|
+
return message.role === "toolResult" ? estimate + Math.ceil(message.toolName.length / 4) : estimate;
|
|
10186
|
+
}
|
|
10187
|
+
function estimateSystemMessageTokens(message) {
|
|
10188
|
+
let chars = typeof message.content === "string" ? message.content.length : 0;
|
|
10189
|
+
if (Array.isArray(message.content)) {
|
|
10190
|
+
for (const block of message.content) {
|
|
10191
|
+
if (block.type === "text")
|
|
10192
|
+
chars += block.text.length;
|
|
10193
|
+
else if (block.type === "image")
|
|
10194
|
+
chars += 4800;
|
|
10195
|
+
}
|
|
10196
|
+
}
|
|
10197
|
+
chars += JSON.stringify(message.sections ?? {}).length;
|
|
10198
|
+
chars += JSON.stringify(message.toolsAdded ?? []).length;
|
|
10199
|
+
chars += JSON.stringify(message.toolsRemoved ?? []).length;
|
|
10200
|
+
return Math.ceil(chars / 4);
|
|
10201
|
+
}
|
|
10202
|
+
function findLastAssistantUsage(messages) {
|
|
10203
|
+
for (let index = messages.length - 1;index >= 0; index--) {
|
|
10204
|
+
const message = messages[index];
|
|
10205
|
+
if (message?.role !== "assistant" || message.stopReason === "aborted" || message.stopReason === "error")
|
|
10206
|
+
continue;
|
|
10207
|
+
if (calculateContextTokens(message.usage) > 0)
|
|
10208
|
+
return { index, usage: message.usage };
|
|
10209
|
+
}
|
|
10210
|
+
return;
|
|
10211
|
+
}
|
|
10212
|
+
function textContent(message) {
|
|
10213
|
+
if (message.role === "system") {
|
|
10214
|
+
if (typeof message.content === "string")
|
|
10215
|
+
return message.content;
|
|
10216
|
+
return message.content.filter((block) => block.type === "text").map((block) => block.text).join("");
|
|
10217
|
+
}
|
|
10218
|
+
if (message.role === "user" || message.role === "toolResult") {
|
|
10219
|
+
if (typeof message.content === "string")
|
|
10220
|
+
return message.content;
|
|
10221
|
+
let text = "";
|
|
10222
|
+
for (const block of message.content) {
|
|
10223
|
+
if (block.type !== "text")
|
|
10224
|
+
return;
|
|
10225
|
+
text += block.text;
|
|
10226
|
+
}
|
|
10227
|
+
return text;
|
|
10228
|
+
}
|
|
10229
|
+
if (message.role === "assistant") {
|
|
10230
|
+
let text = "";
|
|
10231
|
+
for (const block of message.content)
|
|
10232
|
+
if (block.type === "text")
|
|
10233
|
+
text += block.text;
|
|
10234
|
+
return text.length > 0 ? text : undefined;
|
|
10235
|
+
}
|
|
10236
|
+
return;
|
|
10237
|
+
}
|
|
10238
|
+
function replaceTextContent(message, text) {
|
|
10239
|
+
if (message.role === "system")
|
|
10240
|
+
return { ...message, content: text };
|
|
10241
|
+
if (message.role === "user") {
|
|
10242
|
+
if (typeof message.content === "string")
|
|
10243
|
+
return { ...message, content: text };
|
|
10244
|
+
return { ...message, content: [{ type: "text", text }] };
|
|
10245
|
+
}
|
|
10246
|
+
if (message.role === "toolResult")
|
|
10247
|
+
return { ...message, content: [{ type: "text", text }] };
|
|
10248
|
+
if (message.role === "assistant") {
|
|
10249
|
+
const content = [];
|
|
10250
|
+
let replaced = false;
|
|
10251
|
+
for (const block of message.content) {
|
|
10252
|
+
if (block.type !== "text") {
|
|
10253
|
+
content.push(block);
|
|
10254
|
+
continue;
|
|
10255
|
+
}
|
|
10256
|
+
if (replaced)
|
|
10257
|
+
continue;
|
|
10258
|
+
replaced = true;
|
|
10259
|
+
content.push({ ...block, text });
|
|
10260
|
+
}
|
|
10261
|
+
return replaced ? { ...message, content } : message;
|
|
10262
|
+
}
|
|
10263
|
+
return message;
|
|
10264
|
+
}
|
|
10265
|
+
function parseCandidateSection(message, text) {
|
|
10266
|
+
if (message.role === "user") {
|
|
10267
|
+
const markerMatch = /(?:Baseline retrieval (?:evidence \(already gathered for you\)|context):|initial candidates:)/u.exec(text);
|
|
10268
|
+
if (markerMatch === null)
|
|
10269
|
+
return;
|
|
10270
|
+
const sectionStart = (markerMatch.index ?? 0) + markerMatch[0].length;
|
|
10271
|
+
const sectionEnd = findBaselineSectionEnd(text, sectionStart);
|
|
10272
|
+
if (!/^\[e?\d+\] /mu.test(text.slice(sectionStart, sectionEnd)))
|
|
10273
|
+
return;
|
|
10274
|
+
return splitCandidates(text, sectionStart, sectionEnd);
|
|
10275
|
+
}
|
|
10276
|
+
if (message.role !== "toolResult")
|
|
10277
|
+
return;
|
|
10278
|
+
const isRetrieval = RETRIEVAL_TOOL_NAMES[message.toolName] === true || message.toolName.startsWith(RETRIEVAL_TOOL_PREFIX);
|
|
10279
|
+
if (!isRetrieval)
|
|
10280
|
+
return;
|
|
10281
|
+
if (message.toolName === "jikji_find") {
|
|
10282
|
+
const directiveStart = text.indexOf(`
|
|
10283
|
+
|
|
10284
|
+
directive:`);
|
|
10285
|
+
return splitCandidates(text, 0, directiveStart >= 0 ? directiveStart : text.length, /^- [^\n]*(?:\n|$)/gmu);
|
|
10286
|
+
}
|
|
10287
|
+
const diagnosticsStart = text.indexOf(`
|
|
10288
|
+
|
|
10289
|
+
Diagnostics:`);
|
|
10290
|
+
return splitCandidates(text, 0, diagnosticsStart >= 0 ? diagnosticsStart : text.length);
|
|
10291
|
+
}
|
|
10292
|
+
function findBaselineSectionEnd(text, startIndex) {
|
|
10293
|
+
const endMarkers = [
|
|
10294
|
+
`
|
|
10295
|
+
|
|
10296
|
+
Produce the best`,
|
|
10297
|
+
`
|
|
10298
|
+
|
|
10299
|
+
Treat candidates as unverified evidence`,
|
|
10300
|
+
`
|
|
10301
|
+
|
|
10302
|
+
Formatting and content rules`
|
|
10303
|
+
];
|
|
10304
|
+
const positions = endMarkers.map((marker) => text.indexOf(marker, startIndex)).filter((index) => index >= 0);
|
|
10305
|
+
return positions.length > 0 ? Math.min(...positions) : text.length;
|
|
10306
|
+
}
|
|
10307
|
+
function splitCandidates(text, sectionStart, sectionEnd, pattern = /^\[e?\d+\] [^\n]*(?:\n|$)/gmu) {
|
|
10308
|
+
const region = text.slice(sectionStart, sectionEnd);
|
|
10309
|
+
const matches = [...region.matchAll(pattern)];
|
|
10310
|
+
const firstStart = matches[0]?.index;
|
|
10311
|
+
if (firstStart === undefined)
|
|
10312
|
+
return;
|
|
10313
|
+
const blocks = [];
|
|
10314
|
+
for (const [index, match] of matches.entries()) {
|
|
10315
|
+
const start = match.index ?? 0;
|
|
10316
|
+
const end = index + 1 < matches.length ? matches[index + 1]?.index ?? region.length : region.length;
|
|
10317
|
+
blocks.push(region.slice(start, end));
|
|
10318
|
+
}
|
|
10319
|
+
return {
|
|
10320
|
+
prefix: text.slice(0, sectionStart) + region.slice(0, firstStart),
|
|
10321
|
+
suffix: text.slice(sectionEnd),
|
|
10322
|
+
blocks
|
|
10323
|
+
};
|
|
10324
|
+
}
|
|
10325
|
+
var CONTEXT_HEADROOM = 1.2, GUARD_MARKER = "[AutoRAG context guard: omitted context to stay within the model context budget.]", RETRIEVAL_TOOL_PREFIX = "search_datasource_", RETRIEVAL_TOOL_NAMES;
|
|
10326
|
+
var init_context_token_guard = __esm(() => {
|
|
10327
|
+
RETRIEVAL_TOOL_NAMES = {
|
|
10328
|
+
jikji_find: true,
|
|
10329
|
+
search_all_documents: true,
|
|
10330
|
+
semantic_search_local_docs: true,
|
|
10331
|
+
search_single_datasource_documents: true,
|
|
10332
|
+
everything_search: true,
|
|
10333
|
+
fsearch_search: true
|
|
10334
|
+
};
|
|
10335
|
+
});
|
|
10336
|
+
|
|
10337
|
+
// src/agent/request-limits.ts
|
|
10338
|
+
function fitRequestLimits(messages, limits) {
|
|
10339
|
+
const maxBytes = limits?.maxRequestBytes;
|
|
10340
|
+
const maxImages = limits?.images?.maxPerRequest;
|
|
10341
|
+
if (maxBytes === undefined && maxImages === undefined)
|
|
10342
|
+
return;
|
|
10343
|
+
const images = imageLocations(messages);
|
|
10344
|
+
if (images.length === 0)
|
|
10345
|
+
return;
|
|
10346
|
+
let keptImages = images.length;
|
|
10347
|
+
let bytes = maxBytes === undefined ? 0 : Buffer.byteLength(JSON.stringify(messages));
|
|
10348
|
+
const dropped = [];
|
|
10349
|
+
for (const image of images) {
|
|
10350
|
+
const overCount = maxImages !== undefined && keptImages > maxImages;
|
|
10351
|
+
const overBytes = maxBytes !== undefined && bytes > maxBytes;
|
|
10352
|
+
if (!overCount && !overBytes)
|
|
10353
|
+
break;
|
|
10354
|
+
dropped.push(image);
|
|
10355
|
+
keptImages--;
|
|
10356
|
+
bytes -= image.bytes - NOTE_BYTES;
|
|
10357
|
+
}
|
|
10358
|
+
if (dropped.length === 0)
|
|
10359
|
+
return;
|
|
10360
|
+
const byMessage = new Map;
|
|
10361
|
+
for (const { message, block } of dropped) {
|
|
10362
|
+
const blocks = byMessage.get(message) ?? new Set;
|
|
10363
|
+
blocks.add(block);
|
|
10364
|
+
byMessage.set(message, blocks);
|
|
10365
|
+
}
|
|
10366
|
+
return messages.map((message, index) => {
|
|
10367
|
+
const blocks = byMessage.get(index);
|
|
10368
|
+
if (blocks === undefined)
|
|
10369
|
+
return message;
|
|
10370
|
+
const content = message.content.map((part, block) => blocks.has(block) ? { type: "text", text: IMAGE_OMITTED_NOTE } : part);
|
|
10371
|
+
return { ...message, content };
|
|
10372
|
+
});
|
|
10373
|
+
}
|
|
10374
|
+
function imageLocations(messages) {
|
|
10375
|
+
const locations = [];
|
|
10376
|
+
messages.forEach((message, index) => {
|
|
10377
|
+
const content = message.content;
|
|
10378
|
+
if (!Array.isArray(content))
|
|
10379
|
+
return;
|
|
10380
|
+
content.forEach((part, block) => {
|
|
10381
|
+
if (!isImage(part))
|
|
10382
|
+
return;
|
|
10383
|
+
locations.push({ message: index, block, bytes: Buffer.byteLength(JSON.stringify(part)) });
|
|
10384
|
+
});
|
|
10385
|
+
});
|
|
10386
|
+
return locations;
|
|
10387
|
+
}
|
|
10388
|
+
function isImage(part) {
|
|
10389
|
+
return typeof part === "object" && part !== null && part.type === "image";
|
|
10390
|
+
}
|
|
10391
|
+
var IMAGE_OMITTED_NOTE = "[Image omitted from this request to stay within the model provider's request limits. Read the file again if you still need to see it.]", NOTE_BYTES;
|
|
10392
|
+
var init_request_limits = __esm(() => {
|
|
10393
|
+
NOTE_BYTES = Buffer.byteLength(JSON.stringify({ type: "text", text: IMAGE_OMITTED_NOTE }));
|
|
10394
|
+
});
|
|
10395
|
+
|
|
10396
|
+
// src/agent/pi-session.ts
|
|
10397
|
+
import { join as join21 } from "node:path";
|
|
10398
|
+
import { streamSimple as compatStreamSimple } from "@earendil-works/pi-ai/compat";
|
|
10399
|
+
import {
|
|
10400
|
+
createAgentSession,
|
|
10401
|
+
createAgentSessionFromServices,
|
|
10402
|
+
createAgentSessionRuntime,
|
|
10403
|
+
createAgentSessionServices,
|
|
10404
|
+
DefaultResourceLoader,
|
|
10405
|
+
getAgentDir,
|
|
10406
|
+
ModelRuntime,
|
|
10407
|
+
SessionManager,
|
|
10408
|
+
SettingsManager
|
|
10409
|
+
} from "@earendil-works/pi-coding-agent";
|
|
10410
|
+
import { getKeybindings, matchesKey } from "@earendil-works/pi-tui";
|
|
10411
|
+
function resolveInteractiveTuiMode(settingsManager) {
|
|
10412
|
+
const saved = settingsManager.getProjectSettings().tuiMode ?? settingsManager.getGlobalSettings().tuiMode;
|
|
10413
|
+
return saved ?? "regular";
|
|
10414
|
+
}
|
|
10415
|
+
function toToolDefinition(tool) {
|
|
10416
|
+
return {
|
|
10417
|
+
name: tool.name,
|
|
10418
|
+
label: tool.label,
|
|
10419
|
+
description: tool.description,
|
|
10420
|
+
parameters: tool.parameters,
|
|
10421
|
+
...tool.prepareArguments === undefined ? {} : { prepareArguments: tool.prepareArguments },
|
|
10422
|
+
execute: async (toolCallId, params, signal, onUpdate) => {
|
|
10423
|
+
return tool.execute(toolCallId, params, signal, onUpdate);
|
|
10424
|
+
}
|
|
10425
|
+
};
|
|
10426
|
+
}
|
|
10427
|
+
function modelConfig(model, configuredApiKey) {
|
|
10428
|
+
return {
|
|
10429
|
+
name: model.name,
|
|
10430
|
+
baseUrl: model.baseUrl,
|
|
10431
|
+
api: model.api,
|
|
10432
|
+
...configuredApiKey === undefined ? {} : { apiKey: configuredApiKey },
|
|
10433
|
+
authHeader: false,
|
|
10434
|
+
streamSimple: (requestModel, context, options) => compatStreamSimple(requestModel, context, {
|
|
10435
|
+
...options,
|
|
10436
|
+
...configuredApiKey === EMBEDDED_AUTH_MARKER ? { apiKey: undefined } : {}
|
|
10437
|
+
}),
|
|
10438
|
+
models: [
|
|
10439
|
+
{
|
|
10440
|
+
id: model.id,
|
|
10441
|
+
name: model.name,
|
|
10442
|
+
reasoning: model.reasoning,
|
|
10443
|
+
input: [...model.input],
|
|
10444
|
+
cost: model.cost,
|
|
10445
|
+
contextWindow: model.contextWindow,
|
|
10446
|
+
maxTokens: model.maxTokens,
|
|
10447
|
+
...model.inputLimits === undefined ? {} : { inputLimits: model.inputLimits },
|
|
10448
|
+
...model.thinkingLevelMap === undefined ? {} : { thinkingLevelMap: model.thinkingLevelMap }
|
|
10449
|
+
}
|
|
10450
|
+
]
|
|
10451
|
+
};
|
|
10452
|
+
}
|
|
10453
|
+
async function configureModelRuntime(modelRuntime, model, apiKey, providerApiKeys) {
|
|
10454
|
+
if (model !== undefined) {
|
|
10455
|
+
const configuredKey = apiKey ?? providerApiKeys?.[model.provider];
|
|
10456
|
+
if (modelRuntime.getModel(model.provider, model.id) === undefined) {
|
|
10457
|
+
modelRuntime.registerProvider(model.provider, modelConfig(model, configuredKey ?? EMBEDDED_AUTH_MARKER));
|
|
10458
|
+
}
|
|
10459
|
+
}
|
|
10460
|
+
const keys = new Map;
|
|
10461
|
+
if (model !== undefined && apiKey !== undefined)
|
|
9689
10462
|
keys.set(model.provider, apiKey);
|
|
9690
10463
|
for (const [provider, key] of Object.entries(providerApiKeys ?? {}))
|
|
9691
10464
|
keys.set(provider, key);
|
|
9692
10465
|
for (const [provider, key] of keys)
|
|
9693
10466
|
await modelRuntime.setRuntimeApiKey(provider, key);
|
|
9694
10467
|
}
|
|
10468
|
+
function enforceRequestLimits(pi) {
|
|
10469
|
+
pi.on("context", (event, ctx) => {
|
|
10470
|
+
const messages = fitRequestLimits(event.messages, ctx.model?.inputLimits);
|
|
10471
|
+
return messages === undefined ? undefined : { messages };
|
|
10472
|
+
});
|
|
10473
|
+
}
|
|
10474
|
+
function guardContext(pi, contextTransform) {
|
|
10475
|
+
pi.on("context", async (event, ctx) => {
|
|
10476
|
+
const activeTools = new Set(pi.getActiveTools());
|
|
10477
|
+
const toolDeclarations = pi.getAllTools().filter((tool) => activeTools.has(tool.name)).map(({ name, description, parameters }) => ({ name, description, parameters }));
|
|
10478
|
+
const fixedOverheadTokens = Math.ceil((ctx.getSystemPrompt().length + JSON.stringify(toolDeclarations).length) / 4);
|
|
10479
|
+
return {
|
|
10480
|
+
messages: await createContextTokenGuardedTransform(ctx.model, contextTransform, () => fixedOverheadTokens)(event.messages)
|
|
10481
|
+
};
|
|
10482
|
+
});
|
|
10483
|
+
}
|
|
9695
10484
|
function createAutoRAGExtension(getSystemPrompt, contextTransform) {
|
|
9696
10485
|
return (pi) => {
|
|
9697
10486
|
pi.on("before_agent_start", () => ({ systemPrompt: getSystemPrompt() }));
|
|
9698
|
-
|
|
9699
|
-
|
|
9700
|
-
}
|
|
10487
|
+
guardContext(pi, contextTransform);
|
|
10488
|
+
enforceRequestLimits(pi);
|
|
9701
10489
|
};
|
|
9702
10490
|
}
|
|
9703
|
-
function
|
|
10491
|
+
function interactiveKeyOf(data) {
|
|
10492
|
+
const bindings = getKeybindings();
|
|
10493
|
+
const keysOf = (action, fallback) => {
|
|
10494
|
+
const keys = bindings.getKeys(action);
|
|
10495
|
+
return keys.length > 0 ? keys : [fallback];
|
|
10496
|
+
};
|
|
10497
|
+
const windows = process.platform === "win32" || process.platform === "linux" && Boolean(process.env.WSL_DISTRO_NAME || process.env.WSL_INTEROP);
|
|
10498
|
+
if (keysOf("app.interrupt", "escape").some((key) => matchesKey(data, key)))
|
|
10499
|
+
return "interrupt";
|
|
10500
|
+
if (keysOf("app.message.followUp", windows ? "ctrl+q" : "alt+enter").some((key) => matchesKey(data, key)))
|
|
10501
|
+
return "followUp";
|
|
10502
|
+
return;
|
|
10503
|
+
}
|
|
10504
|
+
function createAutoRAGInteractiveExtension(getSystemPrompt, contextTransform, options, updateNotice) {
|
|
9704
10505
|
return (pi) => {
|
|
9705
10506
|
pi.on("before_agent_start", () => ({ systemPrompt: getSystemPrompt() }));
|
|
9706
|
-
|
|
9707
|
-
|
|
9708
|
-
}
|
|
10507
|
+
guardContext(pi, contextTransform);
|
|
10508
|
+
enforceRequestLimits(pi);
|
|
9709
10509
|
if (updateNotice !== undefined) {
|
|
9710
10510
|
pi.on("session_start", () => {
|
|
9711
10511
|
updateNotice().then((text) => {
|
|
@@ -9721,20 +10521,37 @@ function createAutoRAGInteractiveExtension(getSystemPrompt, contextTransform, on
|
|
|
9721
10521
|
});
|
|
9722
10522
|
});
|
|
9723
10523
|
}
|
|
9724
|
-
|
|
9725
|
-
|
|
9726
|
-
|
|
10524
|
+
const { onInput, isSearchRunning, onKey } = options;
|
|
10525
|
+
if (onKey !== undefined) {
|
|
10526
|
+
pi.on("session_start", (_event, ctx) => {
|
|
10527
|
+
ctx.ui.onTerminalInput((data) => {
|
|
10528
|
+
const key = interactiveKeyOf(data);
|
|
10529
|
+
return key !== undefined && onKey(key) ? { consume: true } : undefined;
|
|
10530
|
+
});
|
|
10531
|
+
});
|
|
10532
|
+
}
|
|
10533
|
+
pi.on("input", (event) => {
|
|
10534
|
+
const text = event.text.trim();
|
|
10535
|
+
if (event.source !== "interactive" || text.length === 0)
|
|
10536
|
+
return;
|
|
10537
|
+
if (text.startsWith("/") && isSearchRunning?.() !== true)
|
|
9727
10538
|
return;
|
|
10539
|
+
let disposition;
|
|
9728
10540
|
try {
|
|
9729
|
-
|
|
10541
|
+
disposition = onInput({ text, images: event.images, streamingBehavior: event.streamingBehavior }, pi);
|
|
9730
10542
|
} catch (error) {
|
|
9731
10543
|
pi.sendMessage({
|
|
9732
10544
|
customType: "autorag.error",
|
|
9733
10545
|
content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
|
|
9734
10546
|
display: true
|
|
9735
|
-
});
|
|
9736
|
-
|
|
9737
|
-
|
|
10547
|
+
}, { triggerTurn: false });
|
|
10548
|
+
return { action: "handled" };
|
|
10549
|
+
}
|
|
10550
|
+
return disposition.kind === "steer" ? {
|
|
10551
|
+
action: "transform",
|
|
10552
|
+
text: disposition.text,
|
|
10553
|
+
...event.images === undefined ? {} : { images: event.images }
|
|
10554
|
+
} : { action: "handled" };
|
|
9738
10555
|
});
|
|
9739
10556
|
};
|
|
9740
10557
|
}
|
|
@@ -9814,7 +10631,7 @@ async function createAutoRAGPiInteractiveRuntime(options) {
|
|
|
9814
10631
|
settingsManager,
|
|
9815
10632
|
resourceLoaderOptions: {
|
|
9816
10633
|
extensionFactories: [
|
|
9817
|
-
createAutoRAGInteractiveExtension(options.getSystemPrompt, options.contextTransform, options.
|
|
10634
|
+
createAutoRAGInteractiveExtension(options.getSystemPrompt, options.contextTransform, { onInput: options.onInput, isSearchRunning: options.isSearchRunning, onKey: options.onKey }, options.updateNotice),
|
|
9818
10635
|
...options.extensionFactories ?? []
|
|
9819
10636
|
],
|
|
9820
10637
|
systemPrompt: options.getSystemPrompt(),
|
|
@@ -9849,6 +10666,8 @@ async function createAutoRAGPiInteractiveRuntime(options) {
|
|
|
9849
10666
|
var PI_BUILTIN_TOOL_NAMES, EMBEDDED_AUTH_MARKER = "<autorag-embedded-model>";
|
|
9850
10667
|
var init_pi_session = __esm(() => {
|
|
9851
10668
|
init_provider();
|
|
10669
|
+
init_context_token_guard();
|
|
10670
|
+
init_request_limits();
|
|
9852
10671
|
PI_BUILTIN_TOOL_NAMES = ["read", "bash", "edit", "write", "grep", "find", "ls"];
|
|
9853
10672
|
});
|
|
9854
10673
|
|
|
@@ -9858,12 +10677,26 @@ import { Type as Type6 } from "typebox";
|
|
|
9858
10677
|
function createJevJudge(options = {}) {
|
|
9859
10678
|
let client;
|
|
9860
10679
|
return async (state, questions, callOptions = {}) => {
|
|
9861
|
-
client
|
|
9862
|
-
|
|
9863
|
-
|
|
9864
|
-
|
|
9865
|
-
|
|
9866
|
-
|
|
10680
|
+
if (client === undefined) {
|
|
10681
|
+
const env = options.env ?? process.env;
|
|
10682
|
+
const explicit = options.backend;
|
|
10683
|
+
let backend;
|
|
10684
|
+
if (typeof explicit === "string") {
|
|
10685
|
+
backend = explicit === "autorag" ? new AutoragJevBackend({ env }) : explicit;
|
|
10686
|
+
} else if (explicit !== undefined) {
|
|
10687
|
+
backend = explicit;
|
|
10688
|
+
} else if (env.JEV_BACKEND?.toLowerCase() === "autorag") {
|
|
10689
|
+
backend = new AutoragJevBackend({ env });
|
|
10690
|
+
} else {
|
|
10691
|
+
backend = undefined;
|
|
10692
|
+
}
|
|
10693
|
+
client = new Jev({
|
|
10694
|
+
...backend !== undefined ? { backend } : {},
|
|
10695
|
+
...options.env !== undefined ? { env: options.env } : {},
|
|
10696
|
+
...options.model !== undefined ? { model: options.model } : {},
|
|
10697
|
+
...options.confidenceThreshold !== undefined ? { confidenceThreshold: options.confidenceThreshold } : {}
|
|
10698
|
+
});
|
|
10699
|
+
}
|
|
9867
10700
|
const model = options.model ?? BACKEND_DEFAULT_MODEL[client.backend.name];
|
|
9868
10701
|
return client.judge(state, questions, {
|
|
9869
10702
|
...model !== undefined ? { model } : {},
|
|
@@ -9905,8 +10738,12 @@ function createJevExtension(judgeOrOptions = {}) {
|
|
|
9905
10738
|
}
|
|
9906
10739
|
var JEV_TOOL_NAME = "jev", OPENROUTER_DEFAULT_MODEL = "typesafe/jev-1.13", BACKEND_DEFAULT_MODEL, jevQuestionSchema, jevParameters;
|
|
9907
10740
|
var init_jev_extension = __esm(() => {
|
|
10741
|
+
init_jev_backend();
|
|
9908
10742
|
init_pi_session();
|
|
9909
|
-
BACKEND_DEFAULT_MODEL = {
|
|
10743
|
+
BACKEND_DEFAULT_MODEL = {
|
|
10744
|
+
openrouter: OPENROUTER_DEFAULT_MODEL,
|
|
10745
|
+
autorag: AUTORAG_DEFAULT_JEV_MODEL
|
|
10746
|
+
};
|
|
9910
10747
|
jevQuestionSchema = Type6.Object({
|
|
9911
10748
|
id: Type6.Optional(Type6.String({ description: "Caller-assigned id, echoed back in the verdict" })),
|
|
9912
10749
|
type: Type6.Union([Type6.Literal("noul"), Type6.Literal("choice"), Type6.Literal("score")], {
|
|
@@ -9948,7 +10785,7 @@ function createJikjiFindTool(provider, ledger = new EvidenceLedger) {
|
|
|
9948
10785
|
label: "Jikji Find",
|
|
9949
10786
|
description: "Discover local files via Jikji. Returns answer_paths with per-candidate next_read hints and a tool-call policy directive. Call this FIRST for local file discovery when Jikji is configured.",
|
|
9950
10787
|
parameters: jikjiFindSchema,
|
|
9951
|
-
async execute(_toolCallId, params) {
|
|
10788
|
+
async execute(_toolCallId, params, signal) {
|
|
9952
10789
|
const query = String(params.query ?? "").trim();
|
|
9953
10790
|
if (query.length === 0) {
|
|
9954
10791
|
return {
|
|
@@ -9971,7 +10808,8 @@ function createJikjiFindTool(provider, ledger = new EvidenceLedger) {
|
|
|
9971
10808
|
}
|
|
9972
10809
|
const { answerPack, policy, diagnostics, perRoot } = await provider.findJikji(query, {
|
|
9973
10810
|
topK: params.topK,
|
|
9974
|
-
first: params.first
|
|
10811
|
+
first: params.first,
|
|
10812
|
+
...signal === undefined ? {} : { signal }
|
|
9975
10813
|
});
|
|
9976
10814
|
if (answerPack === undefined) {
|
|
9977
10815
|
const diagMsg = diagnostics.length > 0 ? diagnostics.map((d) => d.message).join(" ") : "Jikji is unavailable or all roots failed.";
|
|
@@ -12285,7 +13123,7 @@ function createSearchAllDocumentsTool(provider, ledger = new EvidenceLedger) {
|
|
|
12285
13123
|
label: "Search All Documents",
|
|
12286
13124
|
description: "Search across all configured retrieval methods (posix, MinSync, datasources) and return every distinct chunk they found, with only pure duplicates removed. This is the exhaustive retrieval surface: one call can return many candidates, including several passages of the same document, so judge the evidence yourself rather than assuming it was pre-filtered. Authority is server-configured; tool arguments can only provide query, topK, and an optional narrowing scope.",
|
|
12287
13125
|
parameters: searchAllSchema,
|
|
12288
|
-
async execute(_toolCallId, params) {
|
|
13126
|
+
async execute(_toolCallId, params, signal) {
|
|
12289
13127
|
const query = params.query.trim();
|
|
12290
13128
|
if (query.length === 0) {
|
|
12291
13129
|
return {
|
|
@@ -12295,7 +13133,8 @@ function createSearchAllDocumentsTool(provider, ledger = new EvidenceLedger) {
|
|
|
12295
13133
|
}
|
|
12296
13134
|
const { results, diagnostics, perMethodCounts } = await provider.searchAllDocuments(query, {
|
|
12297
13135
|
topK: params.topK,
|
|
12298
|
-
scope: params.scope
|
|
13136
|
+
scope: params.scope,
|
|
13137
|
+
...signal === undefined ? {} : { signal }
|
|
12299
13138
|
});
|
|
12300
13139
|
const evidenceIds = results.map((result) => ledger.registerResult(SEARCH_ALL_DOCUMENTS_TOOL_NAME, result));
|
|
12301
13140
|
return {
|
|
@@ -12319,7 +13158,7 @@ Diagnostics: ${diagnostics.map((d) => `${d.source ?? "all"}:${d.code}`).join(",
|
|
|
12319
13158
|
if (results.length === 0)
|
|
12320
13159
|
return `No results.${diagnosticSummary}`;
|
|
12321
13160
|
const rows = results.map((result, index) => {
|
|
12322
|
-
const line = result.content.replace(/\s+/gu, " ")
|
|
13161
|
+
const line = result.content.replace(/\s+/gu, " ");
|
|
12323
13162
|
return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
|
|
12324
13163
|
${line}`;
|
|
12325
13164
|
});
|
|
@@ -12350,7 +13189,7 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
|
|
|
12350
13189
|
label: "Search MinSync Documents",
|
|
12351
13190
|
description: "Search parsed document mirrors with MinSync semantic vector retrieval. Use for conceptual and meaning-based search.",
|
|
12352
13191
|
parameters: searchMinSyncSchema,
|
|
12353
|
-
async execute(_toolCallId, params) {
|
|
13192
|
+
async execute(_toolCallId, params, signal) {
|
|
12354
13193
|
if (params.query.trim().length === 0) {
|
|
12355
13194
|
return {
|
|
12356
13195
|
content: [{ type: "text", text: "MinSync query was empty; no documents searched." }],
|
|
@@ -12360,7 +13199,8 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
|
|
|
12360
13199
|
const scope = resolveScope(params.scope);
|
|
12361
13200
|
const results = await getMethod().retrieve(params.query, {
|
|
12362
13201
|
topK: params.topK,
|
|
12363
|
-
scope
|
|
13202
|
+
scope,
|
|
13203
|
+
...signal === undefined ? {} : { signal }
|
|
12364
13204
|
});
|
|
12365
13205
|
const evidenceIds = results.map((result) => ledger.registerResult(SEARCH_MINSYNC_DOCUMENTS_TOOL_NAME, result));
|
|
12366
13206
|
return {
|
|
@@ -12379,7 +13219,7 @@ function formatResults2(results, evidenceIds) {
|
|
|
12379
13219
|
if (results.length === 0)
|
|
12380
13220
|
return "No MinSync results.";
|
|
12381
13221
|
const rows = results.map((result, index) => {
|
|
12382
|
-
const line = result.content.replace(/\s+/gu, " ")
|
|
13222
|
+
const line = result.content.replace(/\s+/gu, " ");
|
|
12383
13223
|
return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
|
|
12384
13224
|
${line}`;
|
|
12385
13225
|
});
|
|
@@ -12427,7 +13267,7 @@ function createTool(provider, spec, ledger) {
|
|
|
12427
13267
|
label: `Search ${spec.datasourceId}`,
|
|
12428
13268
|
description: `Search only the "${spec.datasourceId}" datasource connection: ${spec.description} ` + `This tool spawns only this connection's CLIs and returns only this connection's hits.${scopeLine} ` + `Authority is server-configured; tool arguments can only provide query, topK, and an optional narrowing scope.`,
|
|
12429
13269
|
parameters: searchSingleDatasourceSchema,
|
|
12430
|
-
async execute(_toolCallId, params) {
|
|
13270
|
+
async execute(_toolCallId, params, signal) {
|
|
12431
13271
|
const query = params.query.trim();
|
|
12432
13272
|
if (query.length === 0) {
|
|
12433
13273
|
return {
|
|
@@ -12443,7 +13283,8 @@ function createTool(provider, spec, ledger) {
|
|
|
12443
13283
|
}
|
|
12444
13284
|
const { results, diagnostics } = await provider.searchSingleDatasourceDocuments(spec.datasourceId, query, {
|
|
12445
13285
|
topK: params.topK,
|
|
12446
|
-
scope: params.scope
|
|
13286
|
+
scope: params.scope,
|
|
13287
|
+
...signal === undefined ? {} : { signal }
|
|
12447
13288
|
});
|
|
12448
13289
|
const evidenceIds = results.map((result) => ledger.registerResult(`search_datasource_${spec.datasourceId}`, result));
|
|
12449
13290
|
return {
|
|
@@ -12467,7 +13308,7 @@ Diagnostics: ${diagnostics.map((d) => `${d.source ?? "datasource"}:${d.code}`).j
|
|
|
12467
13308
|
if (results.length === 0)
|
|
12468
13309
|
return `No datasource results.${diagnosticSummary}`;
|
|
12469
13310
|
const rows = results.map((result, index) => {
|
|
12470
|
-
const line = result.content.replace(/\s+/gu, " ")
|
|
13311
|
+
const line = result.content.replace(/\s+/gu, " ");
|
|
12471
13312
|
return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
|
|
12472
13313
|
${line}`;
|
|
12473
13314
|
});
|
|
@@ -18030,7 +18871,7 @@ function mergeSources(merged, sources) {
|
|
|
18030
18871
|
existing.source.ageSeconds ??= source.ageSeconds;
|
|
18031
18872
|
}
|
|
18032
18873
|
}
|
|
18033
|
-
function
|
|
18874
|
+
function sleep4(ms, signal) {
|
|
18034
18875
|
if (ms <= 0)
|
|
18035
18876
|
return Promise.resolve();
|
|
18036
18877
|
let resolveFn;
|
|
@@ -18080,9 +18921,9 @@ async function searchPublicWeb(params, deadlines = {}) {
|
|
|
18080
18921
|
failures.push({ provider: { id, label: id }, error });
|
|
18081
18922
|
}
|
|
18082
18923
|
}));
|
|
18083
|
-
await Promise.race([all,
|
|
18924
|
+
await Promise.race([all, sleep4(softMs, signal)]);
|
|
18084
18925
|
if (!responses.some(isUseful) && failures.length < engineIds.length) {
|
|
18085
|
-
await Promise.race([all, firstSuccess,
|
|
18926
|
+
await Promise.race([all, firstSuccess, sleep4(Math.max(0, hardMs - softMs), signal)]);
|
|
18086
18927
|
}
|
|
18087
18928
|
straggler.abort();
|
|
18088
18929
|
const merged = new Map;
|
|
@@ -18498,6 +19339,20 @@ class AutoRAGAgent {
|
|
|
18498
19339
|
lastQuery;
|
|
18499
19340
|
lastSessionId;
|
|
18500
19341
|
activeRun = false;
|
|
19342
|
+
interactive = {
|
|
19343
|
+
run: undefined,
|
|
19344
|
+
followUps: [],
|
|
19345
|
+
steers: [],
|
|
19346
|
+
queuedSteers: [],
|
|
19347
|
+
preSessionSteers: [],
|
|
19348
|
+
acceptingSteers: false,
|
|
19349
|
+
harnessAbort: false,
|
|
19350
|
+
userAborted: false,
|
|
19351
|
+
steerInterrupt: new AbortController,
|
|
19352
|
+
nextSubmitIsFollowUp: false,
|
|
19353
|
+
steerTexts: new Set
|
|
19354
|
+
};
|
|
19355
|
+
activeRunAbort;
|
|
18501
19356
|
modelNativeSearchAuth;
|
|
18502
19357
|
retrievalTrace = [];
|
|
18503
19358
|
evidenceLedger = new EvidenceLedger;
|
|
@@ -18974,9 +19829,9 @@ ${context.text}
|
|
|
18974
19829
|
apiKey: resolved.apiKey,
|
|
18975
19830
|
providerApiKeys: resolved.providerApiKeys,
|
|
18976
19831
|
getSystemPrompt: () => systemPrompt,
|
|
18977
|
-
customTools: this.
|
|
19832
|
+
customTools: this.steerableTools(),
|
|
18978
19833
|
remoteSession: this.remoteSession,
|
|
18979
|
-
contextTransform: (messages) => this.withMemoryContext(messages),
|
|
19834
|
+
contextTransform: (messages) => this.withMemoryContext(frameSteers(messages, this.interactive.steerTexts)),
|
|
18980
19835
|
...this.jevExtension !== undefined ? { extensionFactories: [this.jevExtension], extensionToolNames: [JEV_TOOL_NAME] } : {}
|
|
18981
19836
|
});
|
|
18982
19837
|
const agent = piSession.session.agent;
|
|
@@ -18992,6 +19847,8 @@ ${context.text}
|
|
|
18992
19847
|
const unsubscribers = [...this.listeners].map((listener) => session.agent.subscribe(listener));
|
|
18993
19848
|
unsubscribers.push(session.agent.subscribe((event) => {
|
|
18994
19849
|
this.recordSearchToolEvent(event);
|
|
19850
|
+
if (event.type === "message_start" && event.message.role === "user")
|
|
19851
|
+
this.steersReachedModel();
|
|
18995
19852
|
}));
|
|
18996
19853
|
return unsubscribers;
|
|
18997
19854
|
}
|
|
@@ -19008,6 +19865,7 @@ ${context.text}
|
|
|
19008
19865
|
return;
|
|
19009
19866
|
this.searchToolCallCount += 1;
|
|
19010
19867
|
if (this.searchToolCallCount >= this.maxSearchToolCalls) {
|
|
19868
|
+
this.interactive.harnessAbort = true;
|
|
19011
19869
|
this.activeSession?.abort();
|
|
19012
19870
|
}
|
|
19013
19871
|
const details = event.result.details;
|
|
@@ -19054,9 +19912,11 @@ ${context.text}
|
|
|
19054
19912
|
getSystemPrompt: () => buildSystemPrompt(this.currentSystemPromptConfig({
|
|
19055
19913
|
modelId: this.boundPiRuntime?.session.model?.id ?? resolved?.model.id
|
|
19056
19914
|
})),
|
|
19057
|
-
contextTransform: (messages) => this.withMemoryContext(messages),
|
|
19058
|
-
customTools: this.
|
|
19059
|
-
|
|
19915
|
+
contextTransform: async (messages) => this.withMemoryContext(frameSteers(messages, this.interactive.steerTexts)),
|
|
19916
|
+
customTools: this.steerableTools(),
|
|
19917
|
+
onInput: (input, pi) => this.routeInteractiveInput(input, pi),
|
|
19918
|
+
isSearchRunning: () => this.interactive.run !== undefined,
|
|
19919
|
+
onKey: (key) => this.handleInteractiveKey(key),
|
|
19060
19920
|
...this.jevExtension !== undefined ? { extensionFactories: [this.jevExtension], extensionToolNames: [JEV_TOOL_NAME] } : {},
|
|
19061
19921
|
...this.updateNotice === undefined ? {} : { updateNotice: this.updateNotice }
|
|
19062
19922
|
});
|
|
@@ -19071,18 +19931,154 @@ ${context.text}
|
|
|
19071
19931
|
}
|
|
19072
19932
|
};
|
|
19073
19933
|
}
|
|
19934
|
+
steerableTools() {
|
|
19935
|
+
return this.tools.filter((tool) => !PI_BUILTIN_TOOL_NAMES.includes(tool.name)).map((tool) => this.searchToolNames.has(tool.name) || tool.name === WEB_SEARCH_TOOL_NAME || tool.name === WEB_FETCH_TOOL_NAME ? this.interruptibleBySteer(tool) : tool);
|
|
19936
|
+
}
|
|
19937
|
+
interruptibleBySteer(tool) {
|
|
19938
|
+
return {
|
|
19939
|
+
...tool,
|
|
19940
|
+
execute: async (toolCallId, params, signal, onUpdate) => {
|
|
19941
|
+
const steer = this.interactive.steerInterrupt.signal;
|
|
19942
|
+
if (steer.aborted)
|
|
19943
|
+
return skippedBySteer(tool.name);
|
|
19944
|
+
const combined = signal === undefined ? steer : AbortSignal.any([signal, steer]);
|
|
19945
|
+
let onSteer = () => {};
|
|
19946
|
+
const steered = new Promise((resolve) => {
|
|
19947
|
+
onSteer = () => resolve("steered");
|
|
19948
|
+
steer.addEventListener("abort", onSteer, { once: true });
|
|
19949
|
+
});
|
|
19950
|
+
try {
|
|
19951
|
+
const outcome = await Promise.race([tool.execute(toolCallId, params, combined, onUpdate), steered]);
|
|
19952
|
+
return outcome === "steered" ? skippedBySteer(tool.name) : outcome;
|
|
19953
|
+
} catch (error) {
|
|
19954
|
+
if (steer.aborted && signal?.aborted !== true)
|
|
19955
|
+
return skippedBySteer(tool.name);
|
|
19956
|
+
throw error;
|
|
19957
|
+
} finally {
|
|
19958
|
+
steer.removeEventListener("abort", onSteer);
|
|
19959
|
+
}
|
|
19960
|
+
}
|
|
19961
|
+
};
|
|
19962
|
+
}
|
|
19074
19963
|
deriveAnswer(text) {
|
|
19075
19964
|
const derived = deriveResultsFromAnswer(text, this.evidenceLedger, { allowLocalFiles: !this.remoteSession });
|
|
19076
19965
|
this.routingDiagnostics.push(...derived.diagnostics);
|
|
19077
19966
|
return derived.details;
|
|
19078
19967
|
}
|
|
19968
|
+
routeInteractiveInput(input, pi) {
|
|
19969
|
+
const state = this.interactive;
|
|
19970
|
+
const followUp = input.streamingBehavior === "followUp" || state.nextSubmitIsFollowUp;
|
|
19971
|
+
state.nextSubmitIsFollowUp = false;
|
|
19972
|
+
if (state.run === undefined) {
|
|
19973
|
+
this.startInteractiveQuery(input.text, pi);
|
|
19974
|
+
return { kind: "handled" };
|
|
19975
|
+
}
|
|
19976
|
+
if (followUp) {
|
|
19977
|
+
state.followUps.push(input.text);
|
|
19978
|
+
return { kind: "handled" };
|
|
19979
|
+
}
|
|
19980
|
+
if (input.streamingBehavior === "steer") {
|
|
19981
|
+
state.steers.push(input.text);
|
|
19982
|
+
state.steerTexts.add(input.text);
|
|
19983
|
+
state.steerInterrupt.abort();
|
|
19984
|
+
return { kind: "steer", text: input.text };
|
|
19985
|
+
}
|
|
19986
|
+
if (!this.steer(input.text, input.images))
|
|
19987
|
+
state.followUps.push(input.text);
|
|
19988
|
+
return { kind: "handled" };
|
|
19989
|
+
}
|
|
19990
|
+
handleInteractiveKey(key) {
|
|
19991
|
+
const state = this.interactive;
|
|
19992
|
+
if (state.run === undefined || this.boundPiRuntime?.session.isStreaming === true)
|
|
19993
|
+
return false;
|
|
19994
|
+
if (key === "followUp") {
|
|
19995
|
+
state.nextSubmitIsFollowUp = true;
|
|
19996
|
+
return false;
|
|
19997
|
+
}
|
|
19998
|
+
state.userAborted = true;
|
|
19999
|
+
state.steerInterrupt.abort();
|
|
20000
|
+
return true;
|
|
20001
|
+
}
|
|
20002
|
+
startInteractiveQuery(query, pi) {
|
|
20003
|
+
const state = this.interactive;
|
|
20004
|
+
const run = (async () => {
|
|
20005
|
+
let next = query;
|
|
20006
|
+
while (next !== undefined) {
|
|
20007
|
+
try {
|
|
20008
|
+
await this.runInteractivePiQuery(next, pi);
|
|
20009
|
+
} catch (error) {
|
|
20010
|
+
pi.sendMessage({
|
|
20011
|
+
customType: "autorag.error",
|
|
20012
|
+
content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
|
|
20013
|
+
display: true
|
|
20014
|
+
}, { triggerTurn: false });
|
|
20015
|
+
}
|
|
20016
|
+
next = state.followUps.shift();
|
|
20017
|
+
}
|
|
20018
|
+
})().finally(() => {
|
|
20019
|
+
state.run = undefined;
|
|
20020
|
+
});
|
|
20021
|
+
state.run = run;
|
|
20022
|
+
}
|
|
19079
20023
|
async runInteractivePiQuery(query, pi) {
|
|
19080
20024
|
pi.setSessionName(query.slice(0, 80));
|
|
19081
|
-
|
|
20025
|
+
const state = this.interactive;
|
|
20026
|
+
const session = this.boundPiRuntime?.session;
|
|
20027
|
+
const unsubscribe = session?.subscribe((event) => {
|
|
20028
|
+
if (event.type === "queue_update" && state.queuedSteers.length > 0 && !session.agent.hasQueuedMessages()) {
|
|
20029
|
+
state.queuedSteers.length = 0;
|
|
20030
|
+
}
|
|
20031
|
+
if (event.type === "agent_settled" && event.aborted && !state.harnessAbort)
|
|
20032
|
+
state.userAborted = true;
|
|
20033
|
+
});
|
|
20034
|
+
try {
|
|
20035
|
+
for await (const _event of this.searchDocumentsStream(query)) {}
|
|
20036
|
+
} finally {
|
|
20037
|
+
unsubscribe?.();
|
|
20038
|
+
if (state.queuedSteers.length > 0) {
|
|
20039
|
+
session?.agent.clearSteeringQueue();
|
|
20040
|
+
if (!state.userAborted)
|
|
20041
|
+
state.followUps.unshift(...state.queuedSteers);
|
|
20042
|
+
state.queuedSteers.length = 0;
|
|
20043
|
+
}
|
|
20044
|
+
}
|
|
20045
|
+
}
|
|
20046
|
+
async waitForIdle() {
|
|
20047
|
+
while (this.interactive.run !== undefined)
|
|
20048
|
+
await this.interactive.run;
|
|
20049
|
+
}
|
|
20050
|
+
steer(text, images = []) {
|
|
20051
|
+
const state = this.interactive;
|
|
20052
|
+
if (!this.activeRun || !state.acceptingSteers || state.userAborted || text.trim() === "")
|
|
20053
|
+
return false;
|
|
20054
|
+
state.steers.push(text);
|
|
20055
|
+
state.steerTexts.add(text);
|
|
20056
|
+
state.steerInterrupt.abort();
|
|
20057
|
+
const content = [{ type: "text", text }, ...images];
|
|
20058
|
+
const message = { role: "user", content, timestamp: Date.now() };
|
|
20059
|
+
const agent = this.activeSession?.agent;
|
|
20060
|
+
if (agent === undefined)
|
|
20061
|
+
state.preSessionSteers.push(message);
|
|
20062
|
+
else
|
|
20063
|
+
agent.steer(message);
|
|
20064
|
+
state.queuedSteers.push(text);
|
|
20065
|
+
return true;
|
|
19082
20066
|
}
|
|
19083
20067
|
abort() {
|
|
20068
|
+
if (!this.activeRun)
|
|
20069
|
+
return;
|
|
20070
|
+
const state = this.interactive;
|
|
20071
|
+
state.userAborted = true;
|
|
20072
|
+
state.steerInterrupt.abort();
|
|
20073
|
+
this.activeRunAbort?.abort();
|
|
19084
20074
|
this.activeSession?.abort();
|
|
19085
20075
|
}
|
|
20076
|
+
steersReachedModel() {
|
|
20077
|
+
const state = this.interactive;
|
|
20078
|
+
state.queuedSteers.length = 0;
|
|
20079
|
+
if (state.steerInterrupt.signal.aborted && !state.userAborted)
|
|
20080
|
+
state.steerInterrupt = new AbortController;
|
|
20081
|
+
}
|
|
19086
20082
|
startAutoRefresh(intervalMs, options = {}) {
|
|
19087
20083
|
this.stopAutoRefresh();
|
|
19088
20084
|
const tick = () => {
|
|
@@ -19142,6 +20138,15 @@ ${context.text}
|
|
|
19142
20138
|
const publishOnCapture = this.jevJudge === undefined;
|
|
19143
20139
|
let published = false;
|
|
19144
20140
|
const planAbort = new AbortController;
|
|
20141
|
+
this.activeRunAbort = planAbort;
|
|
20142
|
+
const steering = this.interactive;
|
|
20143
|
+
steering.steers.length = 0;
|
|
20144
|
+
steering.queuedSteers.length = 0;
|
|
20145
|
+
steering.preSessionSteers.length = 0;
|
|
20146
|
+
steering.harnessAbort = false;
|
|
20147
|
+
steering.userAborted = false;
|
|
20148
|
+
steering.steerInterrupt = new AbortController;
|
|
20149
|
+
steering.acceptingSteers = true;
|
|
19145
20150
|
const publishPreliminary = (details) => {
|
|
19146
20151
|
if (planAbort.signal.aborted || published)
|
|
19147
20152
|
return;
|
|
@@ -19177,6 +20182,9 @@ ${context.text}
|
|
|
19177
20182
|
session = await this.createSearchSession(resolved, buildSystemPrompt(this.currentSystemPromptConfig({ modelId: resolved.model.id })));
|
|
19178
20183
|
this.activeSession = session;
|
|
19179
20184
|
unsubscribers = this.configureSearchSession(session);
|
|
20185
|
+
for (const message of steering.preSessionSteers)
|
|
20186
|
+
session.agent.steer(message);
|
|
20187
|
+
steering.preSessionSteers.length = 0;
|
|
19180
20188
|
let timeout;
|
|
19181
20189
|
let timedOutAfterFastAnswer = false;
|
|
19182
20190
|
try {
|
|
@@ -19237,25 +20245,42 @@ Warning: the edited config did not validate, so the previous config was restored
|
|
|
19237
20245
|
}
|
|
19238
20246
|
return;
|
|
19239
20247
|
}
|
|
19240
|
-
|
|
20248
|
+
const steered = () => this.interactive.steers.length > 0;
|
|
20249
|
+
const stopped = () => this.interactive.userAborted;
|
|
20250
|
+
if (stopped())
|
|
20251
|
+
return;
|
|
20252
|
+
let directText;
|
|
20253
|
+
if (plan.route === "direct" && !steered()) {
|
|
19241
20254
|
setThinking(this.fastThinkingLevel);
|
|
19242
20255
|
await activeSession.prompt(this.buildDirectAnswerPrompt(trimmedQuery));
|
|
19243
|
-
|
|
19244
|
-
|
|
19245
|
-
|
|
20256
|
+
if (stopped())
|
|
20257
|
+
return;
|
|
20258
|
+
directText = finalAnswerText(transcript());
|
|
20259
|
+
if (!steered()) {
|
|
20260
|
+
if (directText !== undefined)
|
|
20261
|
+
captured = this.deriveAnswer(directText);
|
|
20262
|
+
return;
|
|
20263
|
+
}
|
|
20264
|
+
}
|
|
20265
|
+
const baseline = plan.route === "direct" ? "" : await this.prefetchInitialRetrievalContext(trimmedQuery, plan.queries, options, plan.datasources);
|
|
20266
|
+
if (stopped())
|
|
19246
20267
|
return;
|
|
20268
|
+
let fastText = directText;
|
|
20269
|
+
let preliminary;
|
|
20270
|
+
if (plan.route !== "direct") {
|
|
20271
|
+
setThinking(this.fastThinkingLevel);
|
|
20272
|
+
await activeSession.prompt(this.buildFastAnswerPrompt(trimmedQuery, options, baseline));
|
|
20273
|
+
if (stopped())
|
|
20274
|
+
return;
|
|
20275
|
+
fastText = finalAnswerText(transcript());
|
|
20276
|
+
preliminary = fastText === undefined ? undefined : this.deriveAnswer(fastText);
|
|
19247
20277
|
}
|
|
19248
|
-
const baseline = await this.prefetchInitialRetrievalContext(trimmedQuery, plan.queries, options, plan.datasources);
|
|
19249
|
-
setThinking(this.fastThinkingLevel);
|
|
19250
|
-
await activeSession.prompt(this.buildFastAnswerPrompt(trimmedQuery, options, baseline));
|
|
19251
|
-
const fastText = finalAnswerText(transcript());
|
|
19252
|
-
const preliminary = fastText === undefined ? undefined : this.deriveAnswer(fastText);
|
|
19253
20278
|
if (preliminary !== undefined) {
|
|
19254
20279
|
emitPreliminary(preliminary);
|
|
19255
|
-
const followUp = await this.shouldFollowUp(trimmedQuery, preliminary);
|
|
19256
|
-
if (planAbort.signal.aborted)
|
|
20280
|
+
const followUp = steered() || await this.shouldFollowUp(trimmedQuery, preliminary);
|
|
20281
|
+
if (planAbort.signal.aborted || stopped())
|
|
19257
20282
|
return;
|
|
19258
|
-
if (!followUp) {
|
|
20283
|
+
if (!followUp && !steered()) {
|
|
19259
20284
|
captured = preliminary;
|
|
19260
20285
|
return;
|
|
19261
20286
|
}
|
|
@@ -19263,14 +20288,23 @@ Warning: the edited config did not validate, so the previous config was restored
|
|
|
19263
20288
|
}
|
|
19264
20289
|
setThinking(this.finalThinkingLevel);
|
|
19265
20290
|
const fastAnswerDelivered = preliminary !== undefined && this.preliminaryCallback !== undefined;
|
|
19266
|
-
await activeSession.prompt(this.buildRefinementPrompt(trimmedQuery, options, fastText, fastAnswerDelivered));
|
|
20291
|
+
await activeSession.prompt(this.buildRefinementPrompt(trimmedQuery, options, fastText, fastAnswerDelivered, this.interactive.steers));
|
|
20292
|
+
while (!stopped() && !planAbort.signal.aborted && this.interactive.queuedSteers.length > 0) {
|
|
20293
|
+
await activeSession.prompt(buildSteerContinuationPrompt(this.interactive.steers));
|
|
20294
|
+
}
|
|
20295
|
+
steering.acceptingSteers = false;
|
|
20296
|
+
if (stopped())
|
|
20297
|
+
return;
|
|
19267
20298
|
const finalText = finalAnswerText(transcript());
|
|
19268
20299
|
if (finalText !== undefined)
|
|
19269
20300
|
captured = this.deriveAnswer(finalText);
|
|
19270
|
-
})()
|
|
20301
|
+
})().finally(() => {
|
|
20302
|
+
steering.acceptingSteers = false;
|
|
20303
|
+
}),
|
|
19271
20304
|
new Promise((resolve, reject) => {
|
|
19272
20305
|
timeout = setTimeout(() => {
|
|
19273
20306
|
planAbort.abort();
|
|
20307
|
+
this.interactive.harnessAbort = true;
|
|
19274
20308
|
Promise.resolve(session?.abort());
|
|
19275
20309
|
if (fastCaptured !== undefined && !this.remoteSession) {
|
|
19276
20310
|
timedOutAfterFastAnswer = true;
|
|
@@ -19717,12 +20751,13 @@ Warning: the edited config did not validate, so the previous config was restored
|
|
|
19717
20751
|
return selection.selected;
|
|
19718
20752
|
}
|
|
19719
20753
|
async prefetchInitialRetrievalContext(query, searchQueries, options, datasources = []) {
|
|
19720
|
-
const
|
|
20754
|
+
const signal = this.interactive.steerInterrupt.signal;
|
|
20755
|
+
const retrieveOptions = { topK: this.limits.prefetch.minSyncTopK, scope: options.scope, signal };
|
|
19721
20756
|
const formatSearchQueries = (list) => `Search queries (decomposed from the original question):
|
|
19722
20757
|
${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
|
|
19723
20758
|
`)}`;
|
|
19724
20759
|
const perQuery = await Promise.all(searchQueries.map((searchQuery) => Promise.all([
|
|
19725
|
-
this.jikjiClient === undefined ? Promise.resolve(undefined) : this.findJikji(searchQuery, { topK: this.limits.prefetch.jikjiTopK }).catch(() => {
|
|
20760
|
+
this.jikjiClient === undefined ? Promise.resolve(undefined) : this.findJikji(searchQuery, { topK: this.limits.prefetch.jikjiTopK, signal }).catch(() => {
|
|
19726
20761
|
return;
|
|
19727
20762
|
}),
|
|
19728
20763
|
this.minSyncMethod.retrieve(searchQuery, retrieveOptions).catch((error) => {
|
|
@@ -19732,7 +20767,8 @@ ${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
|
|
|
19732
20767
|
}),
|
|
19733
20768
|
Promise.all(datasources.map((datasourceId) => this.searchSingleDatasourceDocuments(datasourceId, searchQuery, {
|
|
19734
20769
|
topK: this.limits.singleDatasourceTopK,
|
|
19735
|
-
scope: options.scope
|
|
20770
|
+
scope: options.scope,
|
|
20771
|
+
signal
|
|
19736
20772
|
}).then((outcome) => outcome.results).catch(() => [])))
|
|
19737
20773
|
])));
|
|
19738
20774
|
for (const [index, [jikji, vector, perDatasource]] of perQuery.entries()) {
|
|
@@ -19782,7 +20818,7 @@ ${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
|
|
|
19782
20818
|
options.observedSources?.add(result.source);
|
|
19783
20819
|
for (const result of datasourceResults)
|
|
19784
20820
|
options.observedSources?.add(result.source);
|
|
19785
|
-
const reranked = await this.rerankPrefetchPool(query, jikjiPaths, minSyncResults, datasourceResults);
|
|
20821
|
+
const reranked = await this.rerankPrefetchPool(query, jikjiPaths, minSyncResults, datasourceResults, signal);
|
|
19786
20822
|
if (reranked !== undefined) {
|
|
19787
20823
|
const baseline = formatRerankedBaseline(reranked, reranked.map((result) => this.evidenceLedger.registerResult("baseline", result)));
|
|
19788
20824
|
return searchQueries.length > 1 ? `${formatSearchQueries(searchQueries)}
|
|
@@ -19814,7 +20850,7 @@ ${result.content.replace(/\s+/gu, " ")}`).join(`
|
|
|
19814
20850
|
|
|
19815
20851
|
`);
|
|
19816
20852
|
}
|
|
19817
|
-
async rerankPrefetchPool(query, answerPaths, minSyncResults, datasourceResults) {
|
|
20853
|
+
async rerankPrefetchPool(query, answerPaths, minSyncResults, datasourceResults, signal) {
|
|
19818
20854
|
const reranker = this.reranker;
|
|
19819
20855
|
if (reranker === undefined)
|
|
19820
20856
|
return;
|
|
@@ -19832,7 +20868,7 @@ ${result.content.replace(/\s+/gu, " ")}`).join(`
|
|
|
19832
20868
|
if (!reranker.describe().available)
|
|
19833
20869
|
return;
|
|
19834
20870
|
try {
|
|
19835
|
-
const reranked = await reranker.rerank(query, candidates, { topN: this.rerankTopN });
|
|
20871
|
+
const reranked = await reranker.rerank(query, candidates, { topN: this.rerankTopN, signal });
|
|
19836
20872
|
return reranked.length > 0 ? reranked : undefined;
|
|
19837
20873
|
} catch {
|
|
19838
20874
|
return;
|
|
@@ -19875,7 +20911,7 @@ ${baseline}
|
|
|
19875
20911
|
const fetch2 = names.has(WEB_FETCH_TOOL_NAME) ? ` (and ${WEB_FETCH_TOOL_NAME} to read a promising page)` : "";
|
|
19876
20912
|
return `When local evidence is missing, thin, or possibly outdated — or the question concerns current events, recent releases, prices, or other public facts — use ${WEB_SEARCH_TOOL_NAME}${fetch2} to confirm or correct claims and fill gaps; use URLs as result sources for web-derived evidence. `;
|
|
19877
20913
|
}
|
|
19878
|
-
buildRefinementPrompt(query, options, fastAnswer, fastAnswerDelivered) {
|
|
20914
|
+
buildRefinementPrompt(query, options, fastAnswer, fastAnswerDelivered, steers = []) {
|
|
19879
20915
|
const limit = typeof options.topK === "number" ? ` Cite at most ${options.topK} sources.` : "";
|
|
19880
20916
|
const scope = options.scope ? ` Restrict search to virtual path scope ${options.scope}.` : "";
|
|
19881
20917
|
const firstAnswer = formatFirstAnswerContext(fastAnswer, fastAnswerDelivered);
|
|
@@ -19898,7 +20934,7 @@ ${baseline}
|
|
|
19898
20934
|
|
|
19899
20935
|
` + `${firstAnswer}
|
|
19900
20936
|
|
|
19901
|
-
` + `Now verify it rigorously. ${this.discoveryHint((tools) => `Actively use ${tools} when discovering or exploring local files and folders. `)}Check important claims against source files with bash when needed, correct anything wrong or unsupported, fill gaps with retrieval tools, and resolve conflicts and freshness. ${this.webFallbackHint()}
|
|
20937
|
+
` + formatSteerContext(steers) + `Now verify it rigorously. ${this.discoveryHint((tools) => `Actively use ${tools} when discovering or exploring local files and folders. `)}Check important claims against source files with bash when needed, correct anything wrong or unsupported, fill gaps with retrieval tools, and resolve conflicts and freshness. ${this.webFallbackHint()}
|
|
19902
20938
|
|
|
19903
20939
|
` + `${answerRules}
|
|
19904
20940
|
|
|
@@ -20301,7 +21337,8 @@ ${initialRetrievalContext ?? "Pending; continue only with a brief progress state
|
|
|
20301
21337
|
const sourceRoots = planJikjiSourceRoots(this.searchPaths);
|
|
20302
21338
|
const findOpts = {
|
|
20303
21339
|
topK: opts?.topK,
|
|
20304
|
-
first: opts?.first
|
|
21340
|
+
first: opts?.first,
|
|
21341
|
+
...opts?.signal === undefined ? {} : { signal: opts.signal }
|
|
20305
21342
|
};
|
|
20306
21343
|
const diagnostics = [];
|
|
20307
21344
|
const okPacks = [];
|
|
@@ -20452,13 +21489,18 @@ ${initialRetrievalContext ?? "Pending; continue only with a brief progress state
|
|
|
20452
21489
|
return { results, diagnostics };
|
|
20453
21490
|
}
|
|
20454
21491
|
async searchAllDocuments(query, options = {}) {
|
|
20455
|
-
return this.retrieveWithDiagnostics(query, {
|
|
21492
|
+
return this.retrieveWithDiagnostics(query, {
|
|
21493
|
+
topK: options.topK,
|
|
21494
|
+
scope: options.scope,
|
|
21495
|
+
...options.signal === undefined ? {} : { signal: options.signal }
|
|
21496
|
+
});
|
|
20456
21497
|
}
|
|
20457
21498
|
async searchSingleDatasourceDocuments(datasourceId, query, options = {}) {
|
|
20458
21499
|
const retrievalOptions = {
|
|
20459
21500
|
...this.activeRetrievalOptions,
|
|
20460
21501
|
topK: options.topK,
|
|
20461
|
-
scope: options.scope
|
|
21502
|
+
scope: options.scope,
|
|
21503
|
+
...options.signal === undefined ? {} : { signal: options.signal }
|
|
20462
21504
|
};
|
|
20463
21505
|
const methods = this.methodRegistry.list().filter((method) => {
|
|
20464
21506
|
const descriptor = method.describe();
|
|
@@ -20561,6 +21603,49 @@ ${result.content.replace(/\s+/gu, " ")}`);
|
|
|
20561
21603
|
${lines.join(`
|
|
20562
21604
|
`)}`;
|
|
20563
21605
|
}
|
|
21606
|
+
function skippedBySteer(toolName) {
|
|
21607
|
+
return {
|
|
21608
|
+
content: [
|
|
21609
|
+
{
|
|
21610
|
+
type: "text",
|
|
21611
|
+
text: `Skipped due to the user's steering message. ${toolName} was stopped before it returned; do not count it as completed work. Apply the steering message, then retry the search if it is still needed.`
|
|
21612
|
+
}
|
|
21613
|
+
],
|
|
21614
|
+
details: { method: toolName, skipped: "steer" }
|
|
21615
|
+
};
|
|
21616
|
+
}
|
|
21617
|
+
function buildSteerMessage(text) {
|
|
21618
|
+
return `The user interrupted the running search with this correction:
|
|
21619
|
+
|
|
21620
|
+
${text}
|
|
21621
|
+
|
|
21622
|
+
` + `Treat it as direction for the search in progress, not as a new question: adjust which sources, queries, or claims you pursue, then continue. ` + `When finished, reply with the final answer to the original question, updated for this correction, and no tool call.`;
|
|
21623
|
+
}
|
|
21624
|
+
function frameSteers(messages, steerTexts) {
|
|
21625
|
+
if (steerTexts.size === 0)
|
|
21626
|
+
return messages;
|
|
21627
|
+
return messages.map((message) => {
|
|
21628
|
+
if (message.role !== "user" || typeof message.content === "string")
|
|
21629
|
+
return message;
|
|
21630
|
+
const [first, ...rest] = message.content;
|
|
21631
|
+
if (first?.type !== "text" || !steerTexts.has(first.text))
|
|
21632
|
+
return message;
|
|
21633
|
+
return { ...message, content: [{ ...first, text: buildSteerMessage(first.text) }, ...rest] };
|
|
21634
|
+
});
|
|
21635
|
+
}
|
|
21636
|
+
function formatSteerContext(steers) {
|
|
21637
|
+
if (steers.length === 0)
|
|
21638
|
+
return "";
|
|
21639
|
+
return `While the search ran, the user corrected it:
|
|
21640
|
+
${steers.map((steer) => `- ${steer}`).join(`
|
|
21641
|
+
`)}
|
|
21642
|
+
` + `The first answer predates these corrections; follow them over it.
|
|
21643
|
+
|
|
21644
|
+
`;
|
|
21645
|
+
}
|
|
21646
|
+
function buildSteerContinuationPrompt(steers) {
|
|
21647
|
+
return `${formatSteerContext(steers)}Apply the latest correction: search or verify further if it needs that, ` + `then reply with the complete final answer to the original question and no tool call.`;
|
|
21648
|
+
}
|
|
20564
21649
|
function formatFirstAnswerContext(fastAnswer, delivered) {
|
|
20565
21650
|
if (fastAnswer === undefined) {
|
|
20566
21651
|
return "(the fast phase produced no answer — write the complete answer)";
|
|
@@ -20969,21 +22054,173 @@ ${this.description}`
|
|
|
20969
22054
|
}
|
|
20970
22055
|
}
|
|
20971
22056
|
|
|
20972
|
-
// src/datasource/
|
|
20973
|
-
import {
|
|
20974
|
-
|
|
20975
|
-
|
|
20976
|
-
|
|
20977
|
-
|
|
20978
|
-
|
|
20979
|
-
|
|
20980
|
-
|
|
20981
|
-
|
|
20982
|
-
|
|
20983
|
-
|
|
20984
|
-
|
|
20985
|
-
|
|
20986
|
-
|
|
22057
|
+
// src/datasource/connector.ts
|
|
22058
|
+
import { createHash as createHash12 } from "node:crypto";
|
|
22059
|
+
function connectorFailureToDiagnosticCode(reason) {
|
|
22060
|
+
switch (reason) {
|
|
22061
|
+
case "auth":
|
|
22062
|
+
return "datasource-auth-error";
|
|
22063
|
+
case "permission":
|
|
22064
|
+
return "datasource-permission-denied";
|
|
22065
|
+
case "rate-limited":
|
|
22066
|
+
return "datasource-rate-limited";
|
|
22067
|
+
case "unavailable":
|
|
22068
|
+
case "not-configured":
|
|
22069
|
+
return "datasource-unavailable";
|
|
22070
|
+
case "empty":
|
|
22071
|
+
return "datasource-empty";
|
|
22072
|
+
case "api-error":
|
|
22073
|
+
case "invalid-data":
|
|
22074
|
+
return "datasource-index-failed";
|
|
22075
|
+
}
|
|
22076
|
+
}
|
|
22077
|
+
function boundDiagnosticText(value) {
|
|
22078
|
+
const trimmed = value.trim();
|
|
22079
|
+
return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
|
|
22080
|
+
}
|
|
22081
|
+
function sanitizeIdSegment(value) {
|
|
22082
|
+
const normalized = value.replace(/[^\p{L}\p{N}._-]+/gu, "-");
|
|
22083
|
+
let start = 0;
|
|
22084
|
+
while (start < normalized.length && (normalized[start] === "-" || normalized[start] === "."))
|
|
22085
|
+
start += 1;
|
|
22086
|
+
let end = normalized.length;
|
|
22087
|
+
while (end > start && (normalized[end - 1] === "-" || normalized[end - 1] === "."))
|
|
22088
|
+
end -= 1;
|
|
22089
|
+
const cleaned = normalized.slice(start, end);
|
|
22090
|
+
if (cleaned === value && cleaned.length > 0 && cleaned.length <= 80)
|
|
22091
|
+
return cleaned;
|
|
22092
|
+
const hash = createHash12("sha256").update(value).digest("hex").slice(0, 8);
|
|
22093
|
+
const stem = cleaned.slice(0, 60);
|
|
22094
|
+
return stem.length > 0 ? `${stem}-${hash}` : hash;
|
|
22095
|
+
}
|
|
22096
|
+
var init_connector = () => {};
|
|
22097
|
+
|
|
22098
|
+
// src/datasource/sidecar/config.ts
|
|
22099
|
+
import { readFileSync as readFileSync19 } from "node:fs";
|
|
22100
|
+
import { homedir as homedir3 } from "node:os";
|
|
22101
|
+
import { dirname as dirname17, isAbsolute as isAbsolute8, join as join30, resolve as resolve14, sep as sep7 } from "node:path";
|
|
22102
|
+
import { parse as parseToml2 } from "smol-toml";
|
|
22103
|
+
function resolveSidecarOptions(input) {
|
|
22104
|
+
const { connector, workspaceRoot } = input;
|
|
22105
|
+
const setting = connector?.sidecar;
|
|
22106
|
+
if (setting === false || workspaceRoot === undefined)
|
|
22107
|
+
return;
|
|
22108
|
+
const object = typeof setting === "object" && setting !== null && !Array.isArray(setting) ? setting : {};
|
|
22109
|
+
if (object.enabled === false)
|
|
22110
|
+
return;
|
|
22111
|
+
const configured = [object.crawlerDbPath, connector?.databasePath].find((value) => typeof value === "string" && value.length > 0);
|
|
22112
|
+
const crawlerDbPath = expandHome(configured ?? archivePathFromCrawlerConfig(input) ?? input.defaultCrawlerDb);
|
|
22113
|
+
const workspaceId = typeof connector?.workspace === "string" && connector.workspace.length > 0 ? connector.workspace : undefined;
|
|
22114
|
+
return {
|
|
22115
|
+
descriptor: input.descriptor,
|
|
22116
|
+
crawlerDbPath,
|
|
22117
|
+
sidecarDbPath: join30(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(input.registrationName), sanitizeIdSegment(input.instanceId), "sidecar.sqlite"),
|
|
22118
|
+
...workspaceId === undefined ? {} : { workspaceId },
|
|
22119
|
+
...object.semantic === false ? {} : { embedder: createGatewayEmbedder() }
|
|
22120
|
+
};
|
|
22121
|
+
}
|
|
22122
|
+
function archivePathFromCrawlerConfig(input) {
|
|
22123
|
+
const configPath = input.connector?.configPath;
|
|
22124
|
+
if (input.descriptor.archivePathInConfig !== true || typeof configPath !== "string" || configPath.length === 0) {
|
|
22125
|
+
return;
|
|
22126
|
+
}
|
|
22127
|
+
const file = expandHome(configPath);
|
|
22128
|
+
try {
|
|
22129
|
+
const dbPath = parseToml2(readFileSync19(file, "utf8")).db_path;
|
|
22130
|
+
if (typeof dbPath !== "string" || dbPath.length === 0)
|
|
22131
|
+
return;
|
|
22132
|
+
const expanded = expandHome(dbPath);
|
|
22133
|
+
return isAbsolute8(expanded) ? expanded : resolve14(dirname17(file), expanded);
|
|
22134
|
+
} catch {
|
|
22135
|
+
return;
|
|
22136
|
+
}
|
|
22137
|
+
}
|
|
22138
|
+
function expandHome(path) {
|
|
22139
|
+
if (path === "~")
|
|
22140
|
+
return homedir3();
|
|
22141
|
+
const prefixed = path.startsWith("~/") || sep7 === "\\" && path.startsWith("~\\");
|
|
22142
|
+
return prefixed ? join30(homedir3(), path.slice(2)) : path;
|
|
22143
|
+
}
|
|
22144
|
+
var init_config2 = __esm(() => {
|
|
22145
|
+
init_connector();
|
|
22146
|
+
});
|
|
22147
|
+
|
|
22148
|
+
// src/datasource/sidecar/descriptors.ts
|
|
22149
|
+
function chatCrawler(id) {
|
|
22150
|
+
return {
|
|
22151
|
+
id,
|
|
22152
|
+
keyExpression: "m.event_id",
|
|
22153
|
+
chunkId: (key) => key,
|
|
22154
|
+
fromClause: "messages m",
|
|
22155
|
+
liveWhere: "(m.deleted_at is null or m.deleted_at = 0)",
|
|
22156
|
+
textExpression: "coalesce(m.text, '')",
|
|
22157
|
+
titleExpression: "m.chat_name",
|
|
22158
|
+
epochSecondsExpression: "m.ts",
|
|
22159
|
+
fts: { table: "messages_fts", joinOn: "messages_fts.rowid = m.rowid" },
|
|
22160
|
+
metadata: [
|
|
22161
|
+
{ sql: "m.chat_jid", key: "chatId" },
|
|
22162
|
+
{ sql: "m.chat_name", key: "chatName" },
|
|
22163
|
+
{ sql: "m.sender_name", key: "senderName" },
|
|
22164
|
+
{ sql: "m.ts", key: "timestamp" }
|
|
22165
|
+
],
|
|
22166
|
+
channelIdExpression: "m.chat_jid",
|
|
22167
|
+
channelNameExpression: "m.chat_name",
|
|
22168
|
+
hierarchy: (metadata) => ["chats", str(metadata.chatName) ?? str(metadata.chatId) ?? "unknown"]
|
|
22169
|
+
};
|
|
22170
|
+
}
|
|
22171
|
+
var str = (value) => typeof value === "string" && value.length > 0 ? value : undefined, slack, CRAWLER_DESCRIPTORS;
|
|
22172
|
+
var init_descriptors = __esm(() => {
|
|
22173
|
+
slack = {
|
|
22174
|
+
id: "slack",
|
|
22175
|
+
archivePathInConfig: true,
|
|
22176
|
+
keyExpression: "m.channel_id || '|' || m.ts",
|
|
22177
|
+
chunkId: (_key, metadata) => `${metadata.channelId ?? ""}-${(metadata.timestamp ?? "").replaceAll(".", "-")}`,
|
|
22178
|
+
fromClause: "messages m left join channels c on c.id = m.channel_id and c.workspace_id = m.workspace_id " + "left join users u on u.id = m.user_id",
|
|
22179
|
+
liveWhere: "(m.deleted_ts is null or m.deleted_ts = '')",
|
|
22180
|
+
scopeExpression: "m.workspace_id",
|
|
22181
|
+
textExpression: "coalesce(nullif(m.normalized_text, ''), m.text)",
|
|
22182
|
+
titleExpression: "c.name",
|
|
22183
|
+
epochSecondsExpression: "cast(m.ts as real)",
|
|
22184
|
+
fts: { table: "message_fts", joinOn: "message_fts.message_key = m.channel_id || '|' || m.ts" },
|
|
22185
|
+
metadata: [
|
|
22186
|
+
{ sql: "m.channel_id", key: "channelId" },
|
|
22187
|
+
{ sql: "c.name", key: "channelName" },
|
|
22188
|
+
{ sql: "m.workspace_id", key: "workspaceId" },
|
|
22189
|
+
{ sql: "m.user_id", key: "userId" },
|
|
22190
|
+
{ sql: "coalesce(nullif(u.real_name, ''), u.name)", key: "senderName" },
|
|
22191
|
+
{ sql: "m.ts", key: "timestamp" }
|
|
22192
|
+
],
|
|
22193
|
+
channelIdExpression: "m.channel_id",
|
|
22194
|
+
channelNameExpression: "c.name",
|
|
22195
|
+
hierarchy: (metadata) => [
|
|
22196
|
+
"workspaces",
|
|
22197
|
+
str(metadata.workspaceId) ?? "unknown",
|
|
22198
|
+
"channels",
|
|
22199
|
+
str(metadata.channelName) ?? str(metadata.channelId) ?? "unknown"
|
|
22200
|
+
]
|
|
22201
|
+
};
|
|
22202
|
+
CRAWLER_DESCRIPTORS = {
|
|
22203
|
+
slack,
|
|
22204
|
+
telegram: chatCrawler("telegram"),
|
|
22205
|
+
whatsapp: chatCrawler("whatsapp")
|
|
22206
|
+
};
|
|
22207
|
+
});
|
|
22208
|
+
|
|
22209
|
+
// src/datasource/skills/clawgallery/client.ts
|
|
22210
|
+
import { spawn as spawn9 } from "node:child_process";
|
|
22211
|
+
import { existsSync as existsSync23 } from "node:fs";
|
|
22212
|
+
|
|
22213
|
+
class ClawGalleryClient {
|
|
22214
|
+
options;
|
|
22215
|
+
constructor(options = {}) {
|
|
22216
|
+
this.options = options;
|
|
22217
|
+
}
|
|
22218
|
+
async bootstrap(signal) {
|
|
22219
|
+
if (this.options.path !== undefined && !existsSync23(this.options.path)) {
|
|
22220
|
+
return {
|
|
22221
|
+
ok: false,
|
|
22222
|
+
reason: "nonzero-exit",
|
|
22223
|
+
stdout: "",
|
|
20987
22224
|
stderr: "configured ClawGallery folder does not exist",
|
|
20988
22225
|
code: null
|
|
20989
22226
|
};
|
|
@@ -21084,7 +22321,7 @@ function spawnCli(binary, args, env, options, signal) {
|
|
|
21084
22321
|
const timeout = setTimeout(() => {
|
|
21085
22322
|
reason = "timeout";
|
|
21086
22323
|
terminate2(child);
|
|
21087
|
-
}, options.timeoutMs ??
|
|
22324
|
+
}, options.timeoutMs ?? DEFAULT_TIMEOUT_MS7);
|
|
21088
22325
|
const abort = () => {
|
|
21089
22326
|
reason = "aborted";
|
|
21090
22327
|
terminate2(child);
|
|
@@ -21190,16 +22427,16 @@ function failure2(result, reason) {
|
|
|
21190
22427
|
ok: false,
|
|
21191
22428
|
reason: reason ?? result.reason ?? "nonzero-exit",
|
|
21192
22429
|
stdout: result.stdout,
|
|
21193
|
-
stderr:
|
|
22430
|
+
stderr: boundDiagnosticText2(result.stderr),
|
|
21194
22431
|
code: result.code
|
|
21195
22432
|
};
|
|
21196
22433
|
}
|
|
21197
|
-
function
|
|
22434
|
+
function boundDiagnosticText2(value) {
|
|
21198
22435
|
if (value.length === 0)
|
|
21199
22436
|
return "";
|
|
21200
22437
|
return value.trim().slice(0, 4000);
|
|
21201
22438
|
}
|
|
21202
|
-
var DEFAULT_BINARY2 = "clawgallery",
|
|
22439
|
+
var DEFAULT_BINARY2 = "clawgallery", DEFAULT_TIMEOUT_MS7 = 60000, DEFAULT_MAX_BUFFER_BYTES2, SAFE_ENV;
|
|
21203
22440
|
var init_client5 = __esm(() => {
|
|
21204
22441
|
init_portable_spawn();
|
|
21205
22442
|
DEFAULT_MAX_BUFFER_BYTES2 = 4 * 1024 * 1024;
|
|
@@ -21493,7 +22730,7 @@ function statusToReason(status) {
|
|
|
21493
22730
|
}
|
|
21494
22731
|
async function requestText(url, options) {
|
|
21495
22732
|
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
21496
|
-
const timeoutMs = options.timeoutMs ??
|
|
22733
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
21497
22734
|
let response;
|
|
21498
22735
|
try {
|
|
21499
22736
|
response = await fetchImpl(url, {
|
|
@@ -21557,13 +22794,13 @@ function resolveToken(token, tokenEnv) {
|
|
|
21557
22794
|
}
|
|
21558
22795
|
return;
|
|
21559
22796
|
}
|
|
21560
|
-
var
|
|
22797
|
+
var DEFAULT_TIMEOUT_MS8 = 15000;
|
|
21561
22798
|
|
|
21562
22799
|
// src/datasource/skills/cloud-drive/rclone-connector.ts
|
|
21563
22800
|
import { spawn as spawn10 } from "node:child_process";
|
|
21564
|
-
import { mkdirSync as mkdirSync18, readFileSync as
|
|
22801
|
+
import { mkdirSync as mkdirSync18, readFileSync as readFileSync20, renameSync as renameSync10, rmSync as rmSync9, writeFileSync as writeFileSync17 } from "node:fs";
|
|
21565
22802
|
import { readFile as readFile4 } from "node:fs/promises";
|
|
21566
|
-
import { dirname as
|
|
22803
|
+
import { dirname as dirname18, join as join31, normalize as normalize4 } from "node:path";
|
|
21567
22804
|
|
|
21568
22805
|
class RcloneConnector {
|
|
21569
22806
|
options;
|
|
@@ -21579,7 +22816,7 @@ class RcloneConnector {
|
|
|
21579
22816
|
if (remote === undefined || remote.length === 0) {
|
|
21580
22817
|
return { ok: false, reason: "not-configured", message: "rclone remote not configured" };
|
|
21581
22818
|
}
|
|
21582
|
-
const timeoutMs = this.options.timeoutMs ??
|
|
22819
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS9;
|
|
21583
22820
|
const exportFormats = this.options.exportFormats ?? DEFAULT_EXPORT_FORMATS;
|
|
21584
22821
|
let listResult;
|
|
21585
22822
|
try {
|
|
@@ -21650,7 +22887,7 @@ class RcloneConnector {
|
|
|
21650
22887
|
return;
|
|
21651
22888
|
const destination = mirrorFilePath(paths.mirrorRoot, entry.path);
|
|
21652
22889
|
const temp = `${destination}.autorag-tmp`;
|
|
21653
|
-
mkdirSync18(
|
|
22890
|
+
mkdirSync18(dirname18(temp), { recursive: true });
|
|
21654
22891
|
let copied;
|
|
21655
22892
|
try {
|
|
21656
22893
|
copied = await this.runner(copyArgs(remote, entry.path, temp, exportFormats, this.options.concurrency, this.options.bandwidthLimit), timeoutMs);
|
|
@@ -21755,12 +22992,12 @@ function sameEntry(previous, current) {
|
|
|
21755
22992
|
return previous?.path === current.path && previous.size === current.size && previous.modTime === current.modTime && previous.remoteId === current.remoteId && JSON.stringify(previous.hashes) === JSON.stringify(current.hashes);
|
|
21756
22993
|
}
|
|
21757
22994
|
function mirrorPaths(workspaceRoot, skillName, instanceId) {
|
|
21758
|
-
const base =
|
|
21759
|
-
return { mirrorRoot:
|
|
22995
|
+
const base = join31(workspaceRoot, ".autorag", "datasources", skillName, instanceId);
|
|
22996
|
+
return { mirrorRoot: join31(base, "mirror"), manifestPath: join31(base, "manifest.json") };
|
|
21760
22997
|
}
|
|
21761
22998
|
function mirrorFilePath(root, path) {
|
|
21762
22999
|
const safe = normalize4(path).replace(/^(\.\.(?:[/\\]|$))+|^[/\\]+/gu, "");
|
|
21763
|
-
return
|
|
23000
|
+
return join31(root, safe);
|
|
21764
23001
|
}
|
|
21765
23002
|
function remoteTarget(remote, path) {
|
|
21766
23003
|
return remote.endsWith(":") || remote.endsWith("/") ? `${remote}${path}` : `${remote}/${path}`;
|
|
@@ -21779,14 +23016,14 @@ function copyArgs(remote, path, destination, exportFormats, concurrency, bandwid
|
|
|
21779
23016
|
}
|
|
21780
23017
|
function loadManifest2(path) {
|
|
21781
23018
|
try {
|
|
21782
|
-
const parsed = JSON.parse(
|
|
23019
|
+
const parsed = JSON.parse(readFileSync20(path, "utf8"));
|
|
21783
23020
|
return parsed.version === 1 && Array.isArray(parsed.entries) ? parsed : undefined;
|
|
21784
23021
|
} catch {
|
|
21785
23022
|
return;
|
|
21786
23023
|
}
|
|
21787
23024
|
}
|
|
21788
23025
|
function saveManifest(path, manifest) {
|
|
21789
|
-
mkdirSync18(
|
|
23026
|
+
mkdirSync18(dirname18(path), { recursive: true });
|
|
21790
23027
|
const temp = `${path}.autorag-tmp`;
|
|
21791
23028
|
writeFileSync17(temp, `${JSON.stringify(manifest)}
|
|
21792
23029
|
`, "utf8");
|
|
@@ -21888,7 +23125,7 @@ function runBinary(binary, args, timeoutMs, env, cwd) {
|
|
|
21888
23125
|
});
|
|
21889
23126
|
});
|
|
21890
23127
|
}
|
|
21891
|
-
var DEFAULT_BINARY3 = "rclone", DEFAULT_EXTENSIONS, DEFAULT_EXPORT_FORMATS = "txt,csv", DEFAULT_MAX_DOCUMENTS = 200, DEFAULT_MAX_BYTES_PER_FILE,
|
|
23128
|
+
var DEFAULT_BINARY3 = "rclone", DEFAULT_EXTENSIONS, DEFAULT_EXPORT_FORMATS = "txt,csv", DEFAULT_MAX_DOCUMENTS = 200, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_TIMEOUT_MS9 = 60000, MAX_CONTENT_CHARS2 = 1e5;
|
|
21892
23129
|
var init_rclone_connector = __esm(() => {
|
|
21893
23130
|
init_defaults();
|
|
21894
23131
|
init_portable_spawn();
|
|
@@ -21896,52 +23133,11 @@ var init_rclone_connector = __esm(() => {
|
|
|
21896
23133
|
DEFAULT_MAX_BYTES_PER_FILE = 2 * 1024 * 1024;
|
|
21897
23134
|
});
|
|
21898
23135
|
|
|
21899
|
-
// src/datasource/connector.ts
|
|
21900
|
-
import { createHash as createHash12 } from "node:crypto";
|
|
21901
|
-
function connectorFailureToDiagnosticCode(reason) {
|
|
21902
|
-
switch (reason) {
|
|
21903
|
-
case "auth":
|
|
21904
|
-
return "datasource-auth-error";
|
|
21905
|
-
case "permission":
|
|
21906
|
-
return "datasource-permission-denied";
|
|
21907
|
-
case "rate-limited":
|
|
21908
|
-
return "datasource-rate-limited";
|
|
21909
|
-
case "unavailable":
|
|
21910
|
-
case "not-configured":
|
|
21911
|
-
return "datasource-unavailable";
|
|
21912
|
-
case "empty":
|
|
21913
|
-
return "datasource-empty";
|
|
21914
|
-
case "api-error":
|
|
21915
|
-
case "invalid-data":
|
|
21916
|
-
return "datasource-index-failed";
|
|
21917
|
-
}
|
|
21918
|
-
}
|
|
21919
|
-
function boundDiagnosticText2(value) {
|
|
21920
|
-
const trimmed = value.trim();
|
|
21921
|
-
return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
|
|
21922
|
-
}
|
|
21923
|
-
function sanitizeIdSegment(value) {
|
|
21924
|
-
const normalized = value.replace(/[^\p{L}\p{N}._-]+/gu, "-");
|
|
21925
|
-
let start = 0;
|
|
21926
|
-
while (start < normalized.length && (normalized[start] === "-" || normalized[start] === "."))
|
|
21927
|
-
start += 1;
|
|
21928
|
-
let end = normalized.length;
|
|
21929
|
-
while (end > start && (normalized[end - 1] === "-" || normalized[end - 1] === "."))
|
|
21930
|
-
end -= 1;
|
|
21931
|
-
const cleaned = normalized.slice(start, end);
|
|
21932
|
-
if (cleaned === value && cleaned.length > 0 && cleaned.length <= 80)
|
|
21933
|
-
return cleaned;
|
|
21934
|
-
const hash = createHash12("sha256").update(value).digest("hex").slice(0, 8);
|
|
21935
|
-
const stem = cleaned.slice(0, 60);
|
|
21936
|
-
return stem.length > 0 ? `${stem}-${hash}` : hash;
|
|
21937
|
-
}
|
|
21938
|
-
var init_connector = () => {};
|
|
21939
|
-
|
|
21940
23136
|
// src/datasource/chunk-store.ts
|
|
21941
|
-
import { existsSync as existsSync24, mkdirSync as mkdirSync19, readFileSync as
|
|
21942
|
-
import { dirname as
|
|
23137
|
+
import { existsSync as existsSync24, mkdirSync as mkdirSync19, readFileSync as readFileSync21, writeFileSync as writeFileSync18 } from "node:fs";
|
|
23138
|
+
import { dirname as dirname19, join as join32 } from "node:path";
|
|
21943
23139
|
function datasourceChunkStorePath(workspaceRoot, skillName, instanceId) {
|
|
21944
|
-
return
|
|
23140
|
+
return join32(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId), "chunks.json");
|
|
21945
23141
|
}
|
|
21946
23142
|
|
|
21947
23143
|
class DatasourceChunkStore {
|
|
@@ -21980,7 +23176,7 @@ class DatasourceChunkStore {
|
|
|
21980
23176
|
try {
|
|
21981
23177
|
if (!existsSync24(this.storePath))
|
|
21982
23178
|
return false;
|
|
21983
|
-
const parsed = JSON.parse(
|
|
23179
|
+
const parsed = JSON.parse(readFileSync21(this.storePath, "utf8"));
|
|
21984
23180
|
if (parsed.version !== STORE_VERSION || !Array.isArray(parsed.chunks))
|
|
21985
23181
|
return false;
|
|
21986
23182
|
this.chunkList = parsed.chunks.filter((chunk) => typeof chunk?.chunkId === "string" && typeof chunk?.content === "string");
|
|
@@ -22026,7 +23222,7 @@ class DatasourceChunkStore {
|
|
|
22026
23222
|
if (this.storePath === undefined)
|
|
22027
23223
|
return;
|
|
22028
23224
|
try {
|
|
22029
|
-
mkdirSync19(
|
|
23225
|
+
mkdirSync19(dirname19(this.storePath), { recursive: true });
|
|
22030
23226
|
const payload = {
|
|
22031
23227
|
version: STORE_VERSION,
|
|
22032
23228
|
skill: this.skillName,
|
|
@@ -22168,7 +23364,7 @@ class ConnectorDatasourceSkill {
|
|
|
22168
23364
|
}
|
|
22169
23365
|
if (!fetched.ok) {
|
|
22170
23366
|
const code = connectorFailureToDiagnosticCode(fetched.reason);
|
|
22171
|
-
const message =
|
|
23367
|
+
const message = boundDiagnosticText(fetched.message ?? fetched.reason);
|
|
22172
23368
|
return this.fail(code, `${fetched.reason}: ${message}`);
|
|
22173
23369
|
}
|
|
22174
23370
|
const documents = this.applyDedupeWindow(fetched.documents);
|
|
@@ -22180,7 +23376,7 @@ class ConnectorDatasourceSkill {
|
|
|
22180
23376
|
const diagnostics = (fetched.warnings ?? []).map((warning) => ({
|
|
22181
23377
|
code: "datasource-index-failed",
|
|
22182
23378
|
severity: "warning",
|
|
22183
|
-
message:
|
|
23379
|
+
message: boundDiagnosticText(warning),
|
|
22184
23380
|
instanceId: this.instanceId,
|
|
22185
23381
|
source: this.definition.skillName
|
|
22186
23382
|
}));
|
|
@@ -22297,7 +23493,7 @@ class ConnectorDatasourceSkill {
|
|
|
22297
23493
|
return out;
|
|
22298
23494
|
}
|
|
22299
23495
|
fail(code, message) {
|
|
22300
|
-
const sanitized =
|
|
23496
|
+
const sanitized = boundDiagnosticText(message);
|
|
22301
23497
|
this.lastError = sanitized;
|
|
22302
23498
|
const diagnostic = {
|
|
22303
23499
|
code,
|
|
@@ -22489,8 +23685,8 @@ var init_types4 = __esm(() => {
|
|
|
22489
23685
|
|
|
22490
23686
|
// src/datasource/skills/discrawl/client.ts
|
|
22491
23687
|
import { spawn as spawn12 } from "node:child_process";
|
|
22492
|
-
import { mkdirSync as mkdirSync20, readFileSync as
|
|
22493
|
-
import { dirname as
|
|
23688
|
+
import { mkdirSync as mkdirSync20, readFileSync as readFileSync22, writeFileSync as writeFileSync19 } from "node:fs";
|
|
23689
|
+
import { dirname as dirname20 } from "node:path";
|
|
22494
23690
|
|
|
22495
23691
|
class DiscrawlClient {
|
|
22496
23692
|
options;
|
|
@@ -22506,7 +23702,7 @@ class DiscrawlClient {
|
|
|
22506
23702
|
const path = `${workspace}/.autorag/datasources/discrawl/config.toml`;
|
|
22507
23703
|
let existing = "";
|
|
22508
23704
|
try {
|
|
22509
|
-
existing =
|
|
23705
|
+
existing = readFileSync22(path, "utf8");
|
|
22510
23706
|
} catch {}
|
|
22511
23707
|
const managed = existing.startsWith(`# AutoRAG managed discrawl embeddings v1
|
|
22512
23708
|
`);
|
|
@@ -22514,7 +23710,7 @@ class DiscrawlClient {
|
|
|
22514
23710
|
return { configured: false, rebuildRequired: false };
|
|
22515
23711
|
const previous = managed ? parseManagedEmbedding(existing) : undefined;
|
|
22516
23712
|
const config = managedDiscrawlConfig({ ...runtime, baseUrl: nativeGatewayBaseUrl(runtime.baseUrl) });
|
|
22517
|
-
mkdirSync20(
|
|
23713
|
+
mkdirSync20(dirname20(path), { recursive: true });
|
|
22518
23714
|
if (existing !== config)
|
|
22519
23715
|
writeFileSync19(path, config, "utf8");
|
|
22520
23716
|
return {
|
|
@@ -23539,8 +24735,8 @@ var init_github = __esm(() => {
|
|
|
23539
24735
|
|
|
23540
24736
|
// src/datasource/skills/github-gist/connector.ts
|
|
23541
24737
|
import { execFile as execFile2 } from "node:child_process";
|
|
23542
|
-
import { existsSync as existsSync25, mkdirSync as mkdirSync21, readFileSync as
|
|
23543
|
-
import { dirname as
|
|
24738
|
+
import { existsSync as existsSync25, mkdirSync as mkdirSync21, readFileSync as readFileSync23, writeFileSync as writeFileSync20 } from "node:fs";
|
|
24739
|
+
import { dirname as dirname21 } from "node:path";
|
|
23544
24740
|
async function defaultGhTokenRunner() {
|
|
23545
24741
|
return new Promise((resolve) => {
|
|
23546
24742
|
execFile2("gh", ["auth", "token"], { timeout: GH_TOKEN_TIMEOUT_MS }, (error, stdout) => {
|
|
@@ -23662,7 +24858,7 @@ class GitHubGistConnector {
|
|
|
23662
24858
|
if (statePath === undefined || !existsSync25(statePath))
|
|
23663
24859
|
return { version: STATE_VERSION, gists: {} };
|
|
23664
24860
|
try {
|
|
23665
|
-
const parsed = JSON.parse(
|
|
24861
|
+
const parsed = JSON.parse(readFileSync23(statePath, "utf8"));
|
|
23666
24862
|
if (parsed.version !== STATE_VERSION || typeof parsed.gists !== "object" || parsed.gists === null) {
|
|
23667
24863
|
return { version: STATE_VERSION, gists: {} };
|
|
23668
24864
|
}
|
|
@@ -23676,7 +24872,7 @@ class GitHubGistConnector {
|
|
|
23676
24872
|
if (statePath === undefined)
|
|
23677
24873
|
return;
|
|
23678
24874
|
try {
|
|
23679
|
-
mkdirSync21(
|
|
24875
|
+
mkdirSync21(dirname21(statePath), { recursive: true });
|
|
23680
24876
|
writeFileSync20(statePath, `${JSON.stringify(state)}
|
|
23681
24877
|
`, "utf8");
|
|
23682
24878
|
} catch {}
|
|
@@ -23722,8 +24918,8 @@ var init_connector3 = () => {};
|
|
|
23722
24918
|
|
|
23723
24919
|
// src/datasource/skills/github-gist/semantic.ts
|
|
23724
24920
|
import { createHash as createHash13 } from "node:crypto";
|
|
23725
|
-
import { existsSync as existsSync26, mkdirSync as mkdirSync22, readFileSync as
|
|
23726
|
-
import { dirname as
|
|
24921
|
+
import { existsSync as existsSync26, mkdirSync as mkdirSync22, readFileSync as readFileSync24, writeFileSync as writeFileSync21 } from "node:fs";
|
|
24922
|
+
import { dirname as dirname22 } from "node:path";
|
|
23727
24923
|
function contentHash(chunk) {
|
|
23728
24924
|
return createHash13("sha256").update(`${chunk.title ?? ""}
|
|
23729
24925
|
${chunk.content}`).digest("hex");
|
|
@@ -23816,7 +25012,7 @@ ${chunk.content}`));
|
|
|
23816
25012
|
if (this.statePath === undefined || !existsSync26(this.statePath))
|
|
23817
25013
|
return;
|
|
23818
25014
|
try {
|
|
23819
|
-
const parsed = JSON.parse(
|
|
25015
|
+
const parsed = JSON.parse(readFileSync24(this.statePath, "utf8"));
|
|
23820
25016
|
if (parsed.version !== VECTORS_VERSION || typeof parsed.entries !== "object" || parsed.entries === null)
|
|
23821
25017
|
return;
|
|
23822
25018
|
if (typeof parsed.identity?.dimension !== "number")
|
|
@@ -23831,7 +25027,7 @@ ${chunk.content}`));
|
|
|
23831
25027
|
if (this.identity === undefined)
|
|
23832
25028
|
return;
|
|
23833
25029
|
try {
|
|
23834
|
-
mkdirSync22(
|
|
25030
|
+
mkdirSync22(dirname22(this.statePath), { recursive: true });
|
|
23835
25031
|
const payload = {
|
|
23836
25032
|
version: VECTORS_VERSION,
|
|
23837
25033
|
identity: this.identity,
|
|
@@ -23914,9 +25110,9 @@ var init_semantic = __esm(() => {
|
|
|
23914
25110
|
});
|
|
23915
25111
|
|
|
23916
25112
|
// src/datasource/skills/github-gist/skill.ts
|
|
23917
|
-
import { join as
|
|
25113
|
+
import { join as join33 } from "node:path";
|
|
23918
25114
|
function gistDatasourceDir(workspaceRoot, skillName, instanceId) {
|
|
23919
|
-
return
|
|
25115
|
+
return join33(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId));
|
|
23920
25116
|
}
|
|
23921
25117
|
var GITHUB_GIST_SKILL_DEFINITION, DEFAULT_INSTANCE_ID3 = "default", GitHubGistSkill;
|
|
23922
25118
|
var init_skill5 = __esm(() => {
|
|
@@ -23944,7 +25140,7 @@ var init_skill5 = __esm(() => {
|
|
|
23944
25140
|
constructor(options = {}) {
|
|
23945
25141
|
const { connector, connectorOptions, semantic, ...rest } = options;
|
|
23946
25142
|
const instanceId = options.instanceId ?? DEFAULT_INSTANCE_ID3;
|
|
23947
|
-
const statePath = connectorOptions?.statePath ?? (options.workspaceRoot === undefined ? undefined :
|
|
25143
|
+
const statePath = connectorOptions?.statePath ?? (options.workspaceRoot === undefined ? undefined : join33(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "state.json"));
|
|
23948
25144
|
super(GITHUB_GIST_SKILL_DEFINITION, {
|
|
23949
25145
|
...rest,
|
|
23950
25146
|
connector: connector ?? new GitHubGistConnector({
|
|
@@ -23954,7 +25150,7 @@ var init_skill5 = __esm(() => {
|
|
|
23954
25150
|
});
|
|
23955
25151
|
if (semantic?.enabled !== false) {
|
|
23956
25152
|
this.semanticIndex = new GistSemanticIndex({
|
|
23957
|
-
statePath: semantic?.statePath ?? (options.workspaceRoot === undefined ? undefined :
|
|
25153
|
+
statePath: semantic?.statePath ?? (options.workspaceRoot === undefined ? undefined : join33(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "vectors.json"))
|
|
23958
25154
|
});
|
|
23959
25155
|
this.semanticEmbedder = semantic?.embedder ?? createGatewayEmbedder(semantic?.embedderOptions ?? {});
|
|
23960
25156
|
this.semanticMethod = new GitHubGistSemanticMethod({
|
|
@@ -23982,7 +25178,7 @@ var init_skill5 = __esm(() => {
|
|
|
23982
25178
|
{
|
|
23983
25179
|
code: "datasource-index-failed",
|
|
23984
25180
|
severity: "warning",
|
|
23985
|
-
message:
|
|
25181
|
+
message: boundDiagnosticText(`semantic-unavailable: ${sync.message}`),
|
|
23986
25182
|
instanceId: this.instanceId,
|
|
23987
25183
|
source: this.describe().name
|
|
23988
25184
|
}
|
|
@@ -24677,7 +25873,7 @@ var init_types5 = __esm(() => {
|
|
|
24677
25873
|
// src/datasource/skills/lazykatok/client.ts
|
|
24678
25874
|
import { spawn as spawn14 } from "node:child_process";
|
|
24679
25875
|
import { existsSync as existsSync27 } from "node:fs";
|
|
24680
|
-
import { delimiter as delimiter4, join as
|
|
25876
|
+
import { delimiter as delimiter4, join as join34 } from "node:path";
|
|
24681
25877
|
|
|
24682
25878
|
class LazykatokClient {
|
|
24683
25879
|
options;
|
|
@@ -24865,7 +26061,7 @@ function lookupInPath2(executable, env) {
|
|
|
24865
26061
|
for (const dir of pathEnv.split(delimiter4)) {
|
|
24866
26062
|
if (dir.length === 0)
|
|
24867
26063
|
continue;
|
|
24868
|
-
const candidate =
|
|
26064
|
+
const candidate = join34(dir, executable);
|
|
24869
26065
|
if (existsSync27(candidate))
|
|
24870
26066
|
return candidate;
|
|
24871
26067
|
}
|
|
@@ -25355,7 +26551,7 @@ var init_lazykatok = __esm(() => {
|
|
|
25355
26551
|
|
|
25356
26552
|
// src/datasource/skills/mail-export/connector.ts
|
|
25357
26553
|
import { readdir as readdir3, readFile as readFile5, stat as stat6 } from "node:fs/promises";
|
|
25358
|
-
import { basename as basename12, dirname as
|
|
26554
|
+
import { basename as basename12, dirname as dirname23, extname as extname7, join as join35 } from "node:path";
|
|
25359
26555
|
import { simpleParser as simpleParser2 } from "mailparser";
|
|
25360
26556
|
|
|
25361
26557
|
class MailExportConnector {
|
|
@@ -25381,7 +26577,7 @@ class MailExportConnector {
|
|
|
25381
26577
|
continue;
|
|
25382
26578
|
const extension = extname7(entry.name).toLowerCase();
|
|
25383
26579
|
if (extension === ".eml" || extension === ".mbox") {
|
|
25384
|
-
files.push(
|
|
26580
|
+
files.push(join35(entry.parentPath, entry.name));
|
|
25385
26581
|
}
|
|
25386
26582
|
}
|
|
25387
26583
|
} else if (info.isFile()) {
|
|
@@ -25416,7 +26612,7 @@ class MailExportConnector {
|
|
|
25416
26612
|
continue;
|
|
25417
26613
|
}
|
|
25418
26614
|
const stem = basename12(file, extname7(file));
|
|
25419
|
-
const mailboxDir = basename12(
|
|
26615
|
+
const mailboxDir = basename12(dirname23(file));
|
|
25420
26616
|
const hierarchy = ["mailboxes", mailboxDir, stem];
|
|
25421
26617
|
const extension = extname7(file).toLowerCase();
|
|
25422
26618
|
const rawMessages = extension === ".mbox" ? splitMbox(bytes.toString("utf8")) : [bytes.toString("utf8")];
|
|
@@ -25537,7 +26733,7 @@ function ensurePrivateMailcrawlDataDir(path) {
|
|
|
25537
26733
|
if (process.platform !== "win32")
|
|
25538
26734
|
chmodSync(path, 448);
|
|
25539
26735
|
}
|
|
25540
|
-
var
|
|
26736
|
+
var init_config3 = () => {};
|
|
25541
26737
|
|
|
25542
26738
|
// src/datasource/skills/mailcrawl/client.ts
|
|
25543
26739
|
import { spawn as spawn15 } from "node:child_process";
|
|
@@ -25863,7 +27059,7 @@ function truncateUtf8(value, maxBytes) {
|
|
|
25863
27059
|
var DEFAULT_BINARY4 = "mailcrawl", DEFAULT_SEARCH_TIMEOUT_MS3 = 60000, DEFAULT_INDEX_TIMEOUT_MS3 = 1800000, DEFAULT_MAX_BUFFER_BYTES4 = 1048576, LOOPBACK_HOSTS2, SAFE_ENV_KEYS2, SAFE_MAILCRAWL_ENV_KEYS, REMOTE_EMBEDDING_ENV_KEYS;
|
|
25864
27060
|
var init_client9 = __esm(() => {
|
|
25865
27061
|
init_portable_spawn();
|
|
25866
|
-
|
|
27062
|
+
init_config3();
|
|
25867
27063
|
LOOPBACK_HOSTS2 = new Set(["127.0.0.1", "localhost", "[::1]", "::1"]);
|
|
25868
27064
|
SAFE_ENV_KEYS2 = new Set(["HOME", "LANG", "LC_ALL", "PATH", "TMPDIR", "TMP", "TEMP", "NO_COLOR"]);
|
|
25869
27065
|
SAFE_MAILCRAWL_ENV_KEYS = new Set([
|
|
@@ -26110,7 +27306,7 @@ var TAGS;
|
|
|
26110
27306
|
var init_skill9 = __esm(() => {
|
|
26111
27307
|
init_scope2();
|
|
26112
27308
|
init_client9();
|
|
26113
|
-
|
|
27309
|
+
init_config3();
|
|
26114
27310
|
init_methods4();
|
|
26115
27311
|
TAGS = ["mailcrawl", "email", "pii"];
|
|
26116
27312
|
});
|
|
@@ -26122,6 +27318,504 @@ var init_mailcrawl = __esm(() => {
|
|
|
26122
27318
|
init_skill9();
|
|
26123
27319
|
});
|
|
26124
27320
|
|
|
27321
|
+
// src/datasource/sidecar/fusion.ts
|
|
27322
|
+
function encodeVector(vector) {
|
|
27323
|
+
const floats = Float32Array.from(vector);
|
|
27324
|
+
return new Uint8Array(floats.buffer, floats.byteOffset, floats.byteLength);
|
|
27325
|
+
}
|
|
27326
|
+
function decodeVector(blob) {
|
|
27327
|
+
if (blob.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0)
|
|
27328
|
+
return;
|
|
27329
|
+
return new Float32Array(new Uint8Array(blob).buffer);
|
|
27330
|
+
}
|
|
27331
|
+
function cosine3(a, b) {
|
|
27332
|
+
if (a.length === 0 || a.length !== b.length)
|
|
27333
|
+
return 0;
|
|
27334
|
+
let dot = 0;
|
|
27335
|
+
let normA = 0;
|
|
27336
|
+
let normB = 0;
|
|
27337
|
+
for (let index = 0;index < a.length; index += 1) {
|
|
27338
|
+
const x = a[index] ?? 0;
|
|
27339
|
+
const y = b[index] ?? 0;
|
|
27340
|
+
dot += x * y;
|
|
27341
|
+
normA += x * x;
|
|
27342
|
+
normB += y * y;
|
|
27343
|
+
}
|
|
27344
|
+
if (normA === 0 || normB === 0)
|
|
27345
|
+
return 0;
|
|
27346
|
+
return dot / Math.sqrt(normA * normB);
|
|
27347
|
+
}
|
|
27348
|
+
function topKByCosine(query, rows, k) {
|
|
27349
|
+
if (k <= 0)
|
|
27350
|
+
return [];
|
|
27351
|
+
const scored = [];
|
|
27352
|
+
for (const row of rows) {
|
|
27353
|
+
const score = cosine3(query, row.vector);
|
|
27354
|
+
if (score > 0)
|
|
27355
|
+
scored.push({ key: row.key, score });
|
|
27356
|
+
}
|
|
27357
|
+
scored.sort((a, b) => b.score - a.score || a.key.localeCompare(b.key));
|
|
27358
|
+
return scored.slice(0, k);
|
|
27359
|
+
}
|
|
27360
|
+
function reciprocalRankFusion(lists) {
|
|
27361
|
+
const scores = new Map;
|
|
27362
|
+
for (const list of lists) {
|
|
27363
|
+
const seen = new Set;
|
|
27364
|
+
for (const [index, key] of list.entries()) {
|
|
27365
|
+
if (seen.has(key))
|
|
27366
|
+
continue;
|
|
27367
|
+
seen.add(key);
|
|
27368
|
+
scores.set(key, (scores.get(key) ?? 0) + 1 / (RRF_K2 + index + 1));
|
|
27369
|
+
}
|
|
27370
|
+
}
|
|
27371
|
+
return [...scores.entries()].map(([key, score]) => ({ key, score })).sort((a, b) => b.score - a.score);
|
|
27372
|
+
}
|
|
27373
|
+
var RRF_K2 = 60;
|
|
27374
|
+
|
|
27375
|
+
// src/datasource/sidecar/index.ts
|
|
27376
|
+
import { createHash as createHash14 } from "node:crypto";
|
|
27377
|
+
import { mkdirSync as mkdirSync24, statSync as statSync3 } from "node:fs";
|
|
27378
|
+
import { dirname as dirname24 } from "node:path";
|
|
27379
|
+
import { DatabaseSync as DatabaseSync2 } from "node:sqlite";
|
|
27380
|
+
import { pathToFileURL } from "node:url";
|
|
27381
|
+
|
|
27382
|
+
class CrawlerSidecar {
|
|
27383
|
+
descriptor;
|
|
27384
|
+
options;
|
|
27385
|
+
select;
|
|
27386
|
+
where;
|
|
27387
|
+
opened;
|
|
27388
|
+
attached;
|
|
27389
|
+
lastSemantic;
|
|
27390
|
+
constructor(options) {
|
|
27391
|
+
this.options = options;
|
|
27392
|
+
const { descriptor } = options;
|
|
27393
|
+
this.descriptor = descriptor;
|
|
27394
|
+
const metadataColumns = descriptor.metadata.map((column, index) => `${column.sql} as m${index}`);
|
|
27395
|
+
this.select = [
|
|
27396
|
+
`${descriptor.keyExpression} as k`,
|
|
27397
|
+
`${descriptor.textExpression} as content`,
|
|
27398
|
+
`${descriptor.titleExpression ?? "null"} as title`,
|
|
27399
|
+
`${descriptor.epochSecondsExpression ?? "null"} as epoch`,
|
|
27400
|
+
...metadataColumns
|
|
27401
|
+
].join(", ");
|
|
27402
|
+
const scope = descriptor.scopeExpression !== undefined && options.workspaceId !== undefined;
|
|
27403
|
+
const allowlist = [
|
|
27404
|
+
...(options.channelIds?.length ?? 0) > 0 ? [`${descriptor.channelIdExpression} in (${options.channelIds?.map(() => "?").join(", ")})`] : [],
|
|
27405
|
+
...(options.channelNames?.length ?? 0) > 0 ? [`${descriptor.channelNameExpression} in (${options.channelNames?.map(() => "?").join(", ")})`] : []
|
|
27406
|
+
];
|
|
27407
|
+
this.where = [
|
|
27408
|
+
descriptor.liveWhere,
|
|
27409
|
+
...scope ? [`${descriptor.scopeExpression} = ?`] : [],
|
|
27410
|
+
...allowlist.length > 0 ? [`(${allowlist.join(" or ")})`] : []
|
|
27411
|
+
].join(" and ");
|
|
27412
|
+
}
|
|
27413
|
+
get db() {
|
|
27414
|
+
if (this.opened !== undefined)
|
|
27415
|
+
return this.opened;
|
|
27416
|
+
mkdirSync24(dirname24(this.options.sidecarDbPath), { recursive: true });
|
|
27417
|
+
const db = new DatabaseSync2(this.options.sidecarDbPath);
|
|
27418
|
+
db.exec(`pragma busy_timeout = ${BUSY_TIMEOUT_MS}`);
|
|
27419
|
+
db.exec("pragma journal_mode = wal");
|
|
27420
|
+
db.exec(`
|
|
27421
|
+
create table if not exists sidecar_meta (k text primary key, v text not null);
|
|
27422
|
+
create table if not exists sidecar_vec (message_key text primary key, hash text not null, vec blob not null);
|
|
27423
|
+
`);
|
|
27424
|
+
this.opened = db;
|
|
27425
|
+
return db;
|
|
27426
|
+
}
|
|
27427
|
+
attachCrawler() {
|
|
27428
|
+
let file;
|
|
27429
|
+
try {
|
|
27430
|
+
const { dev, ino } = statSync3(this.options.crawlerDbPath);
|
|
27431
|
+
file = { dev, ino };
|
|
27432
|
+
} catch {
|
|
27433
|
+
this.detachCrawler();
|
|
27434
|
+
return false;
|
|
27435
|
+
}
|
|
27436
|
+
if (this.attached !== undefined) {
|
|
27437
|
+
if (this.attached.dev === file.dev && this.attached.ino === file.ino)
|
|
27438
|
+
return true;
|
|
27439
|
+
this.detachCrawler();
|
|
27440
|
+
}
|
|
27441
|
+
const crawlerUri = `${pathToFileURL(this.options.crawlerDbPath).href}?mode=ro`.replaceAll("'", "''");
|
|
27442
|
+
try {
|
|
27443
|
+
this.db.exec(`attach database '${crawlerUri}' as crawler`);
|
|
27444
|
+
} catch {
|
|
27445
|
+
return false;
|
|
27446
|
+
}
|
|
27447
|
+
const { descriptor } = this.options;
|
|
27448
|
+
try {
|
|
27449
|
+
this.db.prepare(`select ${this.select} from ${descriptor.fromClause} where ${this.where} limit 0`).all(...this.whereParams());
|
|
27450
|
+
this.db.prepare(`select 1 from ${descriptor.fts.table} limit 0`).all();
|
|
27451
|
+
} catch {
|
|
27452
|
+
this.db.exec("detach database crawler");
|
|
27453
|
+
return false;
|
|
27454
|
+
}
|
|
27455
|
+
this.attached = file;
|
|
27456
|
+
this.lastSemantic = undefined;
|
|
27457
|
+
return true;
|
|
27458
|
+
}
|
|
27459
|
+
detachCrawler() {
|
|
27460
|
+
if (this.attached === undefined)
|
|
27461
|
+
return;
|
|
27462
|
+
this.attached = undefined;
|
|
27463
|
+
this.lastSemantic = undefined;
|
|
27464
|
+
try {
|
|
27465
|
+
this.opened?.exec("detach database crawler");
|
|
27466
|
+
} catch {}
|
|
27467
|
+
}
|
|
27468
|
+
get semanticEnabled() {
|
|
27469
|
+
return this.options.embedder !== undefined;
|
|
27470
|
+
}
|
|
27471
|
+
close() {
|
|
27472
|
+
this.opened?.close();
|
|
27473
|
+
this.opened = undefined;
|
|
27474
|
+
this.attached = undefined;
|
|
27475
|
+
this.lastSemantic = undefined;
|
|
27476
|
+
}
|
|
27477
|
+
whereParams() {
|
|
27478
|
+
const { descriptor, workspaceId, channelIds = [], channelNames = [] } = this.options;
|
|
27479
|
+
return [
|
|
27480
|
+
...descriptor.scopeExpression !== undefined && workspaceId !== undefined ? [workspaceId] : [],
|
|
27481
|
+
...channelIds,
|
|
27482
|
+
...channelNames
|
|
27483
|
+
];
|
|
27484
|
+
}
|
|
27485
|
+
searchLexical(query, topK) {
|
|
27486
|
+
const match = toMatchQuery(query);
|
|
27487
|
+
if (match === undefined || topK <= 0 || !this.attachCrawler())
|
|
27488
|
+
return [];
|
|
27489
|
+
const { descriptor } = this.options;
|
|
27490
|
+
const rows = this.db.prepare(`select ${this.select}, bm25(${descriptor.fts.table}) as rank
|
|
27491
|
+
from ${descriptor.fromClause} join ${descriptor.fts.table} on ${descriptor.fts.joinOn}
|
|
27492
|
+
where ${descriptor.fts.table} match ? and ${this.where}
|
|
27493
|
+
order by rank limit ?`).all(match, ...this.whereParams(), topK);
|
|
27494
|
+
return rows.map((row) => this.toHit(this.toRow(row), -Number(row.rank)));
|
|
27495
|
+
}
|
|
27496
|
+
async searchSemantic(query, topK) {
|
|
27497
|
+
const trimmed = query.trim();
|
|
27498
|
+
if (this.options.embedder === undefined || trimmed.length === 0 || topK <= 0 || !this.attachCrawler())
|
|
27499
|
+
return [];
|
|
27500
|
+
const ranked = await this.rankSemantic(trimmed, topK * POOL_FACTOR);
|
|
27501
|
+
if (ranked.length === 0)
|
|
27502
|
+
return [];
|
|
27503
|
+
const live = this.hydrate(ranked.map((entry) => entry.key));
|
|
27504
|
+
const hits = [];
|
|
27505
|
+
for (const entry of ranked) {
|
|
27506
|
+
const row = live.get(entry.key);
|
|
27507
|
+
if (row !== undefined)
|
|
27508
|
+
hits.push(this.toHit(row, entry.score));
|
|
27509
|
+
if (hits.length >= topK)
|
|
27510
|
+
break;
|
|
27511
|
+
}
|
|
27512
|
+
return hits;
|
|
27513
|
+
}
|
|
27514
|
+
async rankSemantic(query, pool) {
|
|
27515
|
+
const now = Date.now();
|
|
27516
|
+
let cached = this.lastSemantic;
|
|
27517
|
+
if (cached === undefined || cached.query !== query || now - cached.at > SEMANTIC_REUSE_MS) {
|
|
27518
|
+
const embedder = this.options.embedder;
|
|
27519
|
+
if (embedder === undefined)
|
|
27520
|
+
return [];
|
|
27521
|
+
cached = {
|
|
27522
|
+
query,
|
|
27523
|
+
at: now,
|
|
27524
|
+
vector: embedder.embed([query]).then((vectors) => vectors[0], () => {
|
|
27525
|
+
return;
|
|
27526
|
+
}),
|
|
27527
|
+
ranked: undefined
|
|
27528
|
+
};
|
|
27529
|
+
this.lastSemantic = cached;
|
|
27530
|
+
}
|
|
27531
|
+
const vector = await cached.vector;
|
|
27532
|
+
if (vector === undefined) {
|
|
27533
|
+
if (this.lastSemantic === cached)
|
|
27534
|
+
this.lastSemantic = undefined;
|
|
27535
|
+
return [];
|
|
27536
|
+
}
|
|
27537
|
+
if (cached.ranked === undefined || cached.ranked.pool < pool) {
|
|
27538
|
+
const wide = pool * POOL_FACTOR;
|
|
27539
|
+
cached.ranked = { pool: wide, keys: topKByCosine(vector, this.storedVectors(), wide) };
|
|
27540
|
+
}
|
|
27541
|
+
return cached.ranked.keys.slice(0, pool);
|
|
27542
|
+
}
|
|
27543
|
+
async searchHybrid(query, topK) {
|
|
27544
|
+
if (topK <= 0)
|
|
27545
|
+
return [];
|
|
27546
|
+
const pool = topK * POOL_FACTOR;
|
|
27547
|
+
const lexical = this.searchLexical(query, pool);
|
|
27548
|
+
const semantic = await this.searchSemantic(query, pool);
|
|
27549
|
+
if (semantic.length === 0)
|
|
27550
|
+
return lexical.slice(0, topK);
|
|
27551
|
+
const byKey = new Map;
|
|
27552
|
+
for (const hit of [...semantic, ...lexical])
|
|
27553
|
+
byKey.set(hit.key, hit);
|
|
27554
|
+
const fused = [];
|
|
27555
|
+
for (const entry of reciprocalRankFusion([lexical.map((hit) => hit.key), semantic.map((hit) => hit.key)])) {
|
|
27556
|
+
const hit = byKey.get(entry.key);
|
|
27557
|
+
if (hit !== undefined)
|
|
27558
|
+
fused.push({ ...hit, score: entry.score });
|
|
27559
|
+
if (fused.length >= topK)
|
|
27560
|
+
break;
|
|
27561
|
+
}
|
|
27562
|
+
return fused;
|
|
27563
|
+
}
|
|
27564
|
+
async sync() {
|
|
27565
|
+
const { embedder } = this.options;
|
|
27566
|
+
if (embedder === undefined)
|
|
27567
|
+
return { ok: false, message: "no embedder configured" };
|
|
27568
|
+
if (!this.attachCrawler())
|
|
27569
|
+
return { ok: false, message: "crawler archive not available yet" };
|
|
27570
|
+
let identity;
|
|
27571
|
+
try {
|
|
27572
|
+
identity = await embedder.identity();
|
|
27573
|
+
} catch (error) {
|
|
27574
|
+
return { ok: false, message: errorMessage2(error) };
|
|
27575
|
+
}
|
|
27576
|
+
this.adoptIdentity(identity);
|
|
27577
|
+
const pruned = this.prune();
|
|
27578
|
+
const pending = this.pendingKeys();
|
|
27579
|
+
const batchSize = Math.max(1, this.options.batchSize ?? DEFAULT_BATCH_SIZE);
|
|
27580
|
+
const budget = Math.max(1, this.options.maxEmbedPerSync ?? DEFAULT_MAX_EMBED_PER_SYNC);
|
|
27581
|
+
const target = Math.min(pending.length, budget);
|
|
27582
|
+
const embed = async (rows) => {
|
|
27583
|
+
const vectors = await embedder.embed(rows.map((row) => row.input));
|
|
27584
|
+
if (vectors.length !== rows.length) {
|
|
27585
|
+
throw new Error(`embedder returned ${vectors.length} vectors for ${rows.length} messages`);
|
|
27586
|
+
}
|
|
27587
|
+
const wrong = vectors.find((vector) => vector.length !== identity.dimension);
|
|
27588
|
+
if (wrong !== undefined) {
|
|
27589
|
+
throw new Error(`embedder returned a ${wrong.length}-dimension vector, expected ${identity.dimension}`);
|
|
27590
|
+
}
|
|
27591
|
+
return vectors;
|
|
27592
|
+
};
|
|
27593
|
+
let embedded = 0;
|
|
27594
|
+
let skipped = 0;
|
|
27595
|
+
for (let offset = 0;offset < target; offset += batchSize) {
|
|
27596
|
+
const rows = [...this.hydrate(pending.slice(offset, Math.min(offset + batchSize, target))).values()];
|
|
27597
|
+
if (rows.length === 0)
|
|
27598
|
+
continue;
|
|
27599
|
+
try {
|
|
27600
|
+
this.store(rows, await embed(rows));
|
|
27601
|
+
embedded += rows.length;
|
|
27602
|
+
} catch (batchError) {
|
|
27603
|
+
const rejected = [];
|
|
27604
|
+
let lastError = batchError;
|
|
27605
|
+
for (const row of rows) {
|
|
27606
|
+
try {
|
|
27607
|
+
this.store([row], await embed([row]));
|
|
27608
|
+
embedded += 1;
|
|
27609
|
+
} catch (error) {
|
|
27610
|
+
rejected.push(row);
|
|
27611
|
+
lastError = error;
|
|
27612
|
+
}
|
|
27613
|
+
}
|
|
27614
|
+
if (rejected.length === 0)
|
|
27615
|
+
continue;
|
|
27616
|
+
if (embedded === 0)
|
|
27617
|
+
return { ok: false, message: errorMessage2(lastError) };
|
|
27618
|
+
this.store(rejected, undefined);
|
|
27619
|
+
skipped += rejected.length;
|
|
27620
|
+
}
|
|
27621
|
+
}
|
|
27622
|
+
return { ok: true, embedded, skipped, pruned, remaining: pending.length - embedded - skipped };
|
|
27623
|
+
}
|
|
27624
|
+
adoptIdentity(identity) {
|
|
27625
|
+
const serialized = JSON.stringify(identity);
|
|
27626
|
+
const current = this.db.prepare("select v from sidecar_meta where k = ?").get(IDENTITY_KEY);
|
|
27627
|
+
if (current?.v === serialized)
|
|
27628
|
+
return;
|
|
27629
|
+
this.transaction(() => {
|
|
27630
|
+
this.db.exec("delete from sidecar_vec");
|
|
27631
|
+
this.db.prepare("insert or replace into sidecar_meta (k, v) values (?, ?)").run(IDENTITY_KEY, serialized);
|
|
27632
|
+
});
|
|
27633
|
+
this.lastSemantic = undefined;
|
|
27634
|
+
}
|
|
27635
|
+
prune() {
|
|
27636
|
+
const { descriptor } = this.options;
|
|
27637
|
+
const result = this.db.prepare(`delete from sidecar_vec where message_key not in (
|
|
27638
|
+
select ${descriptor.keyExpression} from ${descriptor.fromClause} where ${this.where}
|
|
27639
|
+
)`).run(...this.whereParams());
|
|
27640
|
+
if (Number(result.changes) > 0)
|
|
27641
|
+
this.lastSemantic = undefined;
|
|
27642
|
+
return Number(result.changes);
|
|
27643
|
+
}
|
|
27644
|
+
pendingKeys() {
|
|
27645
|
+
const { descriptor } = this.options;
|
|
27646
|
+
const rows = this.db.prepare(`select ${this.select}, v.hash as stored_hash
|
|
27647
|
+
from ${descriptor.fromClause}
|
|
27648
|
+
left join sidecar_vec v on v.message_key = ${descriptor.keyExpression}
|
|
27649
|
+
where ${this.where}`).iterate(...this.whereParams());
|
|
27650
|
+
const pending = [];
|
|
27651
|
+
for (const row of rows) {
|
|
27652
|
+
if (row.stored_hash !== this.toRow(row).hash)
|
|
27653
|
+
pending.push(String(row.k));
|
|
27654
|
+
}
|
|
27655
|
+
return pending;
|
|
27656
|
+
}
|
|
27657
|
+
hydrate(keys) {
|
|
27658
|
+
const { descriptor } = this.options;
|
|
27659
|
+
const out = new Map;
|
|
27660
|
+
if (keys.length === 0)
|
|
27661
|
+
return out;
|
|
27662
|
+
const rows = this.db.prepare(`select ${this.select} from ${descriptor.fromClause}
|
|
27663
|
+
where ${descriptor.keyExpression} in (${keys.map(() => "?").join(", ")}) and ${this.where}`).all(...keys, ...this.whereParams());
|
|
27664
|
+
const byKey = new Map(rows.map((row) => [String(row.k), this.toRow(row)]));
|
|
27665
|
+
for (const key of keys) {
|
|
27666
|
+
const row = byKey.get(key);
|
|
27667
|
+
if (row !== undefined)
|
|
27668
|
+
out.set(key, row);
|
|
27669
|
+
}
|
|
27670
|
+
return out;
|
|
27671
|
+
}
|
|
27672
|
+
store(rows, vectors) {
|
|
27673
|
+
const upsert = this.db.prepare("insert or replace into sidecar_vec (message_key, hash, vec) values (?, ?, ?)");
|
|
27674
|
+
this.transaction(() => {
|
|
27675
|
+
for (const [index, row] of rows.entries()) {
|
|
27676
|
+
upsert.run(row.key, row.hash, encodeVector(vectors?.[index] ?? []));
|
|
27677
|
+
}
|
|
27678
|
+
});
|
|
27679
|
+
this.lastSemantic = undefined;
|
|
27680
|
+
}
|
|
27681
|
+
*storedVectors() {
|
|
27682
|
+
const rows = this.db.prepare("select message_key, vec from sidecar_vec where length(vec) > 0").iterate();
|
|
27683
|
+
for (const row of rows) {
|
|
27684
|
+
const vector = row.vec instanceof Uint8Array ? decodeVector(row.vec) : undefined;
|
|
27685
|
+
if (vector !== undefined)
|
|
27686
|
+
yield { key: String(row.message_key), vector };
|
|
27687
|
+
}
|
|
27688
|
+
}
|
|
27689
|
+
transaction(work) {
|
|
27690
|
+
this.db.exec("begin immediate");
|
|
27691
|
+
try {
|
|
27692
|
+
work();
|
|
27693
|
+
this.db.exec("commit");
|
|
27694
|
+
} catch (error) {
|
|
27695
|
+
this.db.exec("rollback");
|
|
27696
|
+
throw error;
|
|
27697
|
+
}
|
|
27698
|
+
}
|
|
27699
|
+
toRow(row) {
|
|
27700
|
+
const metadata = {};
|
|
27701
|
+
for (const [index, column] of this.options.descriptor.metadata.entries()) {
|
|
27702
|
+
const value = row[`m${index}`];
|
|
27703
|
+
if (typeof value === "string" && value.length > 0 || typeof value === "number")
|
|
27704
|
+
metadata[column.key] = String(value);
|
|
27705
|
+
}
|
|
27706
|
+
const title = typeof row.title === "string" && row.title.length > 0 ? row.title : undefined;
|
|
27707
|
+
const epoch = typeof row.epoch === "number" && Number.isFinite(row.epoch) ? row.epoch : undefined;
|
|
27708
|
+
const content = String(row.content ?? "");
|
|
27709
|
+
const input = `${title ?? ""}
|
|
27710
|
+
${content}`.slice(0, MAX_EMBED_INPUT_CHARS);
|
|
27711
|
+
return {
|
|
27712
|
+
key: String(row.k),
|
|
27713
|
+
content,
|
|
27714
|
+
title,
|
|
27715
|
+
epoch,
|
|
27716
|
+
metadata,
|
|
27717
|
+
input,
|
|
27718
|
+
hash: createHash14("sha1").update(input).digest("hex")
|
|
27719
|
+
};
|
|
27720
|
+
}
|
|
27721
|
+
toHit(row, score) {
|
|
27722
|
+
return {
|
|
27723
|
+
key: row.key,
|
|
27724
|
+
content: row.content,
|
|
27725
|
+
score,
|
|
27726
|
+
...row.title !== undefined ? { title: row.title } : {},
|
|
27727
|
+
...row.epoch !== undefined ? { publishedAt: Math.round(row.epoch * 1000) } : {},
|
|
27728
|
+
metadata: row.metadata
|
|
27729
|
+
};
|
|
27730
|
+
}
|
|
27731
|
+
}
|
|
27732
|
+
function toMatchQuery(text) {
|
|
27733
|
+
const terms = [...new Set(text.match(/[\p{L}\p{N}_]+/gu) ?? [])];
|
|
27734
|
+
return terms.length === 0 ? undefined : terms.map((term) => `"${term}"`).join(" OR ");
|
|
27735
|
+
}
|
|
27736
|
+
function errorMessage2(error) {
|
|
27737
|
+
return error instanceof Error ? error.message : String(error);
|
|
27738
|
+
}
|
|
27739
|
+
var DEFAULT_BATCH_SIZE = 256, DEFAULT_MAX_EMBED_PER_SYNC = 2048, MAX_EMBED_INPUT_CHARS = 8192, BUSY_TIMEOUT_MS = 5000, POOL_FACTOR = 3, SEMANTIC_REUSE_MS = 1e4, IDENTITY_KEY = "identity";
|
|
27740
|
+
var init_sidecar = () => {};
|
|
27741
|
+
|
|
27742
|
+
// src/datasource/sidecar/methods.ts
|
|
27743
|
+
function sidecarMethodName(datasourceId, mode) {
|
|
27744
|
+
return mode === "bm25" ? `${datasourceId}-sidecar-bm25` : `${datasourceId}-${mode}`;
|
|
27745
|
+
}
|
|
27746
|
+
|
|
27747
|
+
class SidecarRetrievalMethod {
|
|
27748
|
+
options;
|
|
27749
|
+
constructor(options) {
|
|
27750
|
+
this.options = options;
|
|
27751
|
+
}
|
|
27752
|
+
describe() {
|
|
27753
|
+
const { datasourceId, skillType, mode, tags } = this.options;
|
|
27754
|
+
const { type, text, capabilities } = DESCRIPTIONS[mode];
|
|
27755
|
+
return {
|
|
27756
|
+
name: sidecarMethodName(datasourceId, mode),
|
|
27757
|
+
type,
|
|
27758
|
+
description: `${text} over ${skillType} via a read-only sidecar index`,
|
|
27759
|
+
status: "active",
|
|
27760
|
+
capabilities: [...capabilities, "scoped", "path-opaque-sources"],
|
|
27761
|
+
datasourceId,
|
|
27762
|
+
tags
|
|
27763
|
+
};
|
|
27764
|
+
}
|
|
27765
|
+
async retrieve(query, options) {
|
|
27766
|
+
const trimmed = query.trim();
|
|
27767
|
+
if (trimmed.length === 0)
|
|
27768
|
+
return [];
|
|
27769
|
+
const topK = options.topK ?? DEFAULT_TOP_K5;
|
|
27770
|
+
const { sidecar, mode } = this.options;
|
|
27771
|
+
const hits = mode === "bm25" ? sidecar.searchLexical(trimmed, topK) : mode === "semantic" ? await sidecar.searchSemantic(trimmed, topK) : await sidecar.searchHybrid(trimmed, topK);
|
|
27772
|
+
const results = [];
|
|
27773
|
+
for (const hit of hits) {
|
|
27774
|
+
const result = this.toResult(hit, options);
|
|
27775
|
+
if (result !== undefined)
|
|
27776
|
+
results.push(result);
|
|
27777
|
+
}
|
|
27778
|
+
return results;
|
|
27779
|
+
}
|
|
27780
|
+
toResult(hit, options) {
|
|
27781
|
+
const { descriptor, datasourceId, instanceId, mode } = this.options;
|
|
27782
|
+
const chunkId = descriptor.chunkId(hit.key, hit.metadata);
|
|
27783
|
+
const source = datasourceSourcePath(datasourceId, instanceId, chunkId);
|
|
27784
|
+
if (!matchesDatasourceScope(source, options.scope))
|
|
27785
|
+
return;
|
|
27786
|
+
const method = sidecarMethodName(datasourceId, mode);
|
|
27787
|
+
return {
|
|
27788
|
+
id: `${datasourceId}:${instanceId}:${chunkId}`,
|
|
27789
|
+
content: hit.content,
|
|
27790
|
+
source,
|
|
27791
|
+
score: hit.score,
|
|
27792
|
+
metadata: {
|
|
27793
|
+
...hit.metadata,
|
|
27794
|
+
method,
|
|
27795
|
+
datasourceId,
|
|
27796
|
+
instanceId,
|
|
27797
|
+
backend: "sidecar",
|
|
27798
|
+
...hit.title !== undefined ? { title: hit.title } : {},
|
|
27799
|
+
hierarchy: descriptor.hierarchy(hit.metadata).join("/"),
|
|
27800
|
+
...hit.publishedAt !== undefined ? { publishedAt: hit.publishedAt } : {}
|
|
27801
|
+
}
|
|
27802
|
+
};
|
|
27803
|
+
}
|
|
27804
|
+
}
|
|
27805
|
+
var DEFAULT_TOP_K5 = 20, DESCRIPTIONS;
|
|
27806
|
+
var init_methods5 = __esm(() => {
|
|
27807
|
+
init_scope2();
|
|
27808
|
+
DESCRIPTIONS = {
|
|
27809
|
+
bm25: { type: "bm25", text: "BM25-ranked FTS5 retrieval", capabilities: ["lexical", "fts5", "bm25"] },
|
|
27810
|
+
semantic: { type: "vector", text: "Semantic (embedding) retrieval", capabilities: ["semantic"] },
|
|
27811
|
+
hybrid: {
|
|
27812
|
+
type: "hybrid",
|
|
27813
|
+
text: "Hybrid BM25 + semantic retrieval (reciprocal rank fusion)",
|
|
27814
|
+
capabilities: ["lexical", "semantic", "hybrid"]
|
|
27815
|
+
}
|
|
27816
|
+
};
|
|
27817
|
+
});
|
|
27818
|
+
|
|
26125
27819
|
// src/datasource/crawler-skill.ts
|
|
26126
27820
|
class CrawlerDatasourceSkill {
|
|
26127
27821
|
definition;
|
|
@@ -26132,6 +27826,7 @@ class CrawlerDatasourceSkill {
|
|
|
26132
27826
|
tags;
|
|
26133
27827
|
channelIds;
|
|
26134
27828
|
channelNames;
|
|
27829
|
+
sidecar;
|
|
26135
27830
|
lastIndexedAt;
|
|
26136
27831
|
constructor(definition, options) {
|
|
26137
27832
|
this.definition = options.datasourceId === undefined ? definition : {
|
|
@@ -26147,6 +27842,11 @@ class CrawlerDatasourceSkill {
|
|
|
26147
27842
|
this.lastIndexedAt = options.lastIndexedAt;
|
|
26148
27843
|
this.channelIds = new Set(options.channelIds ?? []);
|
|
26149
27844
|
this.channelNames = new Set(options.channelNames ?? []);
|
|
27845
|
+
this.sidecar = options.sidecar === undefined ? undefined : new CrawlerSidecar({
|
|
27846
|
+
...options.sidecar,
|
|
27847
|
+
channelIds: options.channelIds ?? [],
|
|
27848
|
+
channelNames: options.channelNames ?? []
|
|
27849
|
+
});
|
|
26150
27850
|
}
|
|
26151
27851
|
describe() {
|
|
26152
27852
|
return {
|
|
@@ -26182,26 +27882,89 @@ class CrawlerDatasourceSkill {
|
|
|
26182
27882
|
skill: this.definition.datasourceId,
|
|
26183
27883
|
chunkCount: result.count,
|
|
26184
27884
|
indexedAt: this.lastIndexedAt,
|
|
26185
|
-
diagnostics:
|
|
27885
|
+
diagnostics: await this.syncSidecar()
|
|
26186
27886
|
};
|
|
26187
27887
|
} catch {
|
|
26188
27888
|
return this.fail("datasource-unavailable", "spawn-error");
|
|
26189
27889
|
}
|
|
26190
27890
|
}
|
|
27891
|
+
async syncSidecar() {
|
|
27892
|
+
if (this.sidecar === undefined || !this.sidecar.semanticEnabled)
|
|
27893
|
+
return [];
|
|
27894
|
+
const base = { instanceId: this.instanceId, source: this.definition.datasourceId };
|
|
27895
|
+
let result;
|
|
27896
|
+
try {
|
|
27897
|
+
result = await this.sidecar.sync();
|
|
27898
|
+
} catch (error) {
|
|
27899
|
+
result = { ok: false, message: error instanceof Error ? error.message : String(error) };
|
|
27900
|
+
}
|
|
27901
|
+
if (!result.ok) {
|
|
27902
|
+
return [
|
|
27903
|
+
{
|
|
27904
|
+
code: "datasource-index-failed",
|
|
27905
|
+
severity: "warning",
|
|
27906
|
+
message: `semantic-unavailable: ${result.message}`.slice(0, 300),
|
|
27907
|
+
...base
|
|
27908
|
+
}
|
|
27909
|
+
];
|
|
27910
|
+
}
|
|
27911
|
+
const diagnostics = [];
|
|
27912
|
+
if (result.skipped > 0) {
|
|
27913
|
+
diagnostics.push({
|
|
27914
|
+
code: "datasource-index-partial",
|
|
27915
|
+
severity: "warning",
|
|
27916
|
+
message: `semantic index skipped ${result.skipped} message(s) the embedder rejected; they stay searchable by BM25 and are retried when edited`,
|
|
27917
|
+
...base
|
|
27918
|
+
});
|
|
27919
|
+
}
|
|
27920
|
+
if (result.remaining > 0) {
|
|
27921
|
+
diagnostics.push({
|
|
27922
|
+
code: "datasource-index-partial",
|
|
27923
|
+
severity: "info",
|
|
27924
|
+
message: `semantic index catching up: ${result.embedded} embedded this refresh, ${result.remaining} remaining; hybrid search uses BM25 for the rest until a later refresh`,
|
|
27925
|
+
...base
|
|
27926
|
+
});
|
|
27927
|
+
}
|
|
27928
|
+
return diagnostics;
|
|
27929
|
+
}
|
|
27930
|
+
close() {
|
|
27931
|
+
this.sidecar?.close();
|
|
27932
|
+
}
|
|
26191
27933
|
retrievalMethods() {
|
|
27934
|
+
const cli = this.cliMethod();
|
|
27935
|
+
const { sidecar } = this;
|
|
27936
|
+
if (sidecar === undefined)
|
|
27937
|
+
return [cli];
|
|
27938
|
+
const shared = {
|
|
27939
|
+
sidecar,
|
|
27940
|
+
descriptor: sidecar.descriptor,
|
|
27941
|
+
datasourceId: this.definition.datasourceId,
|
|
27942
|
+
skillType: this.definition.skillType,
|
|
27943
|
+
instanceId: this.instanceId,
|
|
27944
|
+
tags: this.tags
|
|
27945
|
+
};
|
|
27946
|
+
const bm25 = new SidecarRetrievalMethod({ ...shared, mode: "bm25" });
|
|
27947
|
+
if (!sidecar.semanticEnabled)
|
|
27948
|
+
return [cli, bm25];
|
|
26192
27949
|
return [
|
|
26193
|
-
|
|
26194
|
-
|
|
26195
|
-
|
|
26196
|
-
|
|
26197
|
-
instanceId: this.instanceId,
|
|
26198
|
-
tags: this.tags,
|
|
26199
|
-
backendName: this.definition.backendName,
|
|
26200
|
-
channelIds: this.channelIds,
|
|
26201
|
-
channelNames: this.channelNames
|
|
26202
|
-
})
|
|
27950
|
+
cli,
|
|
27951
|
+
bm25,
|
|
27952
|
+
new SidecarRetrievalMethod({ ...shared, mode: "semantic" }),
|
|
27953
|
+
new SidecarRetrievalMethod({ ...shared, mode: "hybrid" })
|
|
26203
27954
|
];
|
|
26204
27955
|
}
|
|
27956
|
+
cliMethod() {
|
|
27957
|
+
return new CrawlerLexicalMethod({
|
|
27958
|
+
client: this.client,
|
|
27959
|
+
datasourceId: this.definition.datasourceId,
|
|
27960
|
+
skillType: this.definition.skillType,
|
|
27961
|
+
instanceId: this.instanceId,
|
|
27962
|
+
tags: this.tags,
|
|
27963
|
+
backendName: this.definition.backendName,
|
|
27964
|
+
channelIds: this.channelIds,
|
|
27965
|
+
channelNames: this.channelNames
|
|
27966
|
+
});
|
|
27967
|
+
}
|
|
26205
27968
|
describeSources() {
|
|
26206
27969
|
return this.instances.map((instanceId) => ({
|
|
26207
27970
|
source: datasourceSourcePath(this.definition.datasourceId, instanceId),
|
|
@@ -26332,6 +28095,8 @@ var DEFAULT_INSTANCE_ID5 = "default", DEFAULT_POLLING_INTERVAL_MS4;
|
|
|
26332
28095
|
var init_crawler_skill = __esm(() => {
|
|
26333
28096
|
init_errors4();
|
|
26334
28097
|
init_scope2();
|
|
28098
|
+
init_sidecar();
|
|
28099
|
+
init_methods5();
|
|
26335
28100
|
DEFAULT_POLLING_INTERVAL_MS4 = 15 * 60 * 1000;
|
|
26336
28101
|
});
|
|
26337
28102
|
|
|
@@ -26484,15 +28249,15 @@ var init_notion = __esm(() => {
|
|
|
26484
28249
|
});
|
|
26485
28250
|
|
|
26486
28251
|
// src/datasource/skills/obsidian/paths.ts
|
|
26487
|
-
import { join as
|
|
28252
|
+
import { join as join36 } from "node:path";
|
|
26488
28253
|
function obsidianDatasourceRoot(workspaceRoot, instanceId) {
|
|
26489
|
-
return
|
|
28254
|
+
return join36(workspaceRoot, AUTORAG_DIRNAME, DATASOURCES_DIRNAME, OBSIDIAN_DATASOURCE_DIRNAME, instanceId);
|
|
26490
28255
|
}
|
|
26491
28256
|
function obsidianQmdConfigDir(workspaceRoot, instanceId) {
|
|
26492
|
-
return
|
|
28257
|
+
return join36(obsidianDatasourceRoot(workspaceRoot, instanceId), "config");
|
|
26493
28258
|
}
|
|
26494
28259
|
function obsidianQmdCacheDir(workspaceRoot, instanceId) {
|
|
26495
|
-
return
|
|
28260
|
+
return join36(obsidianDatasourceRoot(workspaceRoot, instanceId), "cache");
|
|
26496
28261
|
}
|
|
26497
28262
|
function stripEdgeDashes(value) {
|
|
26498
28263
|
let start = 0;
|
|
@@ -26515,8 +28280,8 @@ var DEFAULT_QMD_BINARY = "qmd", DEFAULT_QMD_TIMEOUT_MS = 60000, DEFAULT_QMD_MAX_
|
|
|
26515
28280
|
|
|
26516
28281
|
// src/datasource/skills/obsidian/client.ts
|
|
26517
28282
|
import { spawn as spawn16 } from "node:child_process";
|
|
26518
|
-
import { mkdirSync as
|
|
26519
|
-
import { isAbsolute as
|
|
28283
|
+
import { mkdirSync as mkdirSync25, writeFileSync as writeFileSync22 } from "node:fs";
|
|
28284
|
+
import { isAbsolute as isAbsolute9, resolve as resolve15 } from "node:path";
|
|
26520
28285
|
|
|
26521
28286
|
class QmdClient {
|
|
26522
28287
|
options;
|
|
@@ -26535,10 +28300,10 @@ class QmdClient {
|
|
|
26535
28300
|
const cacheDir = obsidianQmdCacheDir(workspaceRoot, instanceId);
|
|
26536
28301
|
try {
|
|
26537
28302
|
if (this.options.configPath === undefined) {
|
|
26538
|
-
|
|
26539
|
-
writeFileSync22(
|
|
28303
|
+
mkdirSync25(configDir, { recursive: true });
|
|
28304
|
+
writeFileSync22(resolve15(configDir, "index.yml"), renderIndexYaml(collectionName, vaultPath), "utf8");
|
|
26540
28305
|
}
|
|
26541
|
-
|
|
28306
|
+
mkdirSync25(cacheDir, { recursive: true });
|
|
26542
28307
|
} catch (error) {
|
|
26543
28308
|
return failure4("spawn-error", error instanceof Error ? error.message : "failed to write qmd config");
|
|
26544
28309
|
}
|
|
@@ -26612,7 +28377,7 @@ function resolveVaultPath(vaultPath) {
|
|
|
26612
28377
|
if (vaultPath === undefined || vaultPath.trim().length === 0)
|
|
26613
28378
|
return;
|
|
26614
28379
|
const trimmed = vaultPath.trim();
|
|
26615
|
-
return
|
|
28380
|
+
return isAbsolute9(trimmed) ? trimmed : resolve15(trimmed);
|
|
26616
28381
|
}
|
|
26617
28382
|
function controlledEnv5(options) {
|
|
26618
28383
|
const instanceId = options.instanceId ?? "default";
|
|
@@ -26931,7 +28696,7 @@ async function retrieveObsidian(client, mode, methodName, instanceId, query, opt
|
|
|
26931
28696
|
const trimmed = query.trim();
|
|
26932
28697
|
if (trimmed.length === 0)
|
|
26933
28698
|
return [];
|
|
26934
|
-
const topK = options.topK ??
|
|
28699
|
+
const topK = options.topK ?? DEFAULT_TOP_K6;
|
|
26935
28700
|
const searchOptions = { topK, signal: options.signal };
|
|
26936
28701
|
const result = await client.search(mode, trimmed, searchOptions);
|
|
26937
28702
|
if (!result.ok)
|
|
@@ -26966,8 +28731,8 @@ function toRetrievalResult3(hit, source, methodName, instanceId, mode) {
|
|
|
26966
28731
|
}
|
|
26967
28732
|
};
|
|
26968
28733
|
}
|
|
26969
|
-
var OBSIDIAN_DATASOURCE_ID = "obsidian", DEFAULT_OBSIDIAN_TAGS,
|
|
26970
|
-
var
|
|
28734
|
+
var OBSIDIAN_DATASOURCE_ID = "obsidian", DEFAULT_OBSIDIAN_TAGS, DEFAULT_TOP_K6 = 20;
|
|
28735
|
+
var init_methods6 = __esm(() => {
|
|
26971
28736
|
init_errors4();
|
|
26972
28737
|
init_scope2();
|
|
26973
28738
|
DEFAULT_OBSIDIAN_TAGS = ["obsidian", "notes"];
|
|
@@ -27157,7 +28922,7 @@ var OBSIDIAN_DATASOURCE_ID2 = "obsidian", OBSIDIAN_SKILL_TYPE = "obsidian-vault"
|
|
|
27157
28922
|
var init_skill10 = __esm(() => {
|
|
27158
28923
|
init_scope2();
|
|
27159
28924
|
init_client10();
|
|
27160
|
-
|
|
28925
|
+
init_methods6();
|
|
27161
28926
|
init_paths5();
|
|
27162
28927
|
DEFAULT_POLLING_INTERVAL_MS5 = 15 * 60 * 1000;
|
|
27163
28928
|
DEFAULT_OBSIDIAN_TAGS2 = ["obsidian", "notes"];
|
|
@@ -27166,7 +28931,7 @@ var init_skill10 = __esm(() => {
|
|
|
27166
28931
|
// src/datasource/skills/obsidian/index.ts
|
|
27167
28932
|
var init_obsidian = __esm(() => {
|
|
27168
28933
|
init_client10();
|
|
27169
|
-
|
|
28934
|
+
init_methods6();
|
|
27170
28935
|
init_paths5();
|
|
27171
28936
|
init_skill10();
|
|
27172
28937
|
});
|
|
@@ -27506,7 +29271,7 @@ var init_slack = __esm(() => {
|
|
|
27506
29271
|
|
|
27507
29272
|
// src/datasource/skills/spotlight/connector.ts
|
|
27508
29273
|
import { spawn as spawn17 } from "node:child_process";
|
|
27509
|
-
import { createHash as
|
|
29274
|
+
import { createHash as createHash15 } from "node:crypto";
|
|
27510
29275
|
import { readFile as fsReadFile, stat as stat7 } from "node:fs/promises";
|
|
27511
29276
|
import { basename as basename13, extname as extname8 } from "node:path";
|
|
27512
29277
|
|
|
@@ -27529,7 +29294,7 @@ class SpotlightConnector {
|
|
|
27529
29294
|
if (queries.length === 0) {
|
|
27530
29295
|
return { ok: false, reason: "not-configured", message: "no spotlight queries configured" };
|
|
27531
29296
|
}
|
|
27532
|
-
const timeoutMs = this.options.timeoutMs ??
|
|
29297
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS10;
|
|
27533
29298
|
const status = await this.run("mdutil", ["-s", "/"], timeoutMs);
|
|
27534
29299
|
if (status.ok && /indexing disabled/iu.test(status.stdout)) {
|
|
27535
29300
|
return {
|
|
@@ -27591,7 +29356,7 @@ class SpotlightConnector {
|
|
|
27591
29356
|
if (content.length === 0)
|
|
27592
29357
|
continue;
|
|
27593
29358
|
documents.push({
|
|
27594
|
-
docId:
|
|
29359
|
+
docId: createHash15("sha256").update(path).digest("hex").slice(0, 16),
|
|
27595
29360
|
hierarchy: ["files"],
|
|
27596
29361
|
title: basename13(path),
|
|
27597
29362
|
content,
|
|
@@ -27624,7 +29389,7 @@ class SpotlightConnector {
|
|
|
27624
29389
|
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
27625
29390
|
}
|
|
27626
29391
|
}
|
|
27627
|
-
var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS5 = 300, DEFAULT_MAX_BYTES_PER_FILE3,
|
|
29392
|
+
var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS5 = 300, DEFAULT_MAX_BYTES_PER_FILE3, DEFAULT_TIMEOUT_MS10 = 15000, MAX_CONTENT_CHARS4 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve) => {
|
|
27628
29393
|
const child = spawn17(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
|
|
27629
29394
|
let stdout = "";
|
|
27630
29395
|
let stderr = "";
|
|
@@ -27741,10 +29506,12 @@ function parseHits3(stdout) {
|
|
|
27741
29506
|
for (const [index, row] of rows.entries()) {
|
|
27742
29507
|
if (!isRecord18(row))
|
|
27743
29508
|
return;
|
|
27744
|
-
const id = stringValue4(row, ["
|
|
27745
|
-
|
|
27746
|
-
if (id === undefined || content === undefined)
|
|
29509
|
+
const id = stringValue4(row, ["event_id", "message_id", "id"]);
|
|
29510
|
+
if (id === undefined)
|
|
27747
29511
|
return;
|
|
29512
|
+
const content = stringValue4(row, ["snippet", "text", "content"]);
|
|
29513
|
+
if (content === undefined)
|
|
29514
|
+
continue;
|
|
27748
29515
|
const chatId = stringValue4(row, ["chat_jid", "chat_id"]);
|
|
27749
29516
|
const chatName = stringValue4(row, ["chat_name"]);
|
|
27750
29517
|
const senderName = stringValue4(row, ["sender_name"]);
|
|
@@ -27861,10 +29628,12 @@ function parseHits4(stdout) {
|
|
|
27861
29628
|
for (const [index, row] of rows.entries()) {
|
|
27862
29629
|
if (!isRecord19(row))
|
|
27863
29630
|
return;
|
|
27864
|
-
const id = stringValue5(row, ["
|
|
27865
|
-
|
|
27866
|
-
if (id === undefined || content === undefined)
|
|
29631
|
+
const id = stringValue5(row, ["event_id", "message_id", "id"]);
|
|
29632
|
+
if (id === undefined)
|
|
27867
29633
|
return;
|
|
29634
|
+
const content = stringValue5(row, ["snippet", "text", "content"]);
|
|
29635
|
+
if (content === undefined)
|
|
29636
|
+
continue;
|
|
27868
29637
|
const chatId = stringValue5(row, ["chat_jid", "chat_id"]);
|
|
27869
29638
|
const chatName = stringValue5(row, ["chat_name"]);
|
|
27870
29639
|
const senderName = stringValue5(row, ["sender_name"]);
|
|
@@ -27954,6 +29723,31 @@ var init_wacrawl = __esm(() => {
|
|
|
27954
29723
|
});
|
|
27955
29724
|
|
|
27956
29725
|
// src/datasource/skills/factory.ts
|
|
29726
|
+
import { homedir as homedir4 } from "node:os";
|
|
29727
|
+
import { join as join37 } from "node:path";
|
|
29728
|
+
function chatCrawlerArgs(descriptor, defaultCrawlerDb, config, workspaceRoot, registrationName) {
|
|
29729
|
+
const sidecar = resolveSidecarOptions({
|
|
29730
|
+
descriptor,
|
|
29731
|
+
connector: config.connector,
|
|
29732
|
+
workspaceRoot,
|
|
29733
|
+
registrationName,
|
|
29734
|
+
instanceId: config.instanceId ?? "default",
|
|
29735
|
+
defaultCrawlerDb
|
|
29736
|
+
});
|
|
29737
|
+
return {
|
|
29738
|
+
datasourceId: registrationName,
|
|
29739
|
+
...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
|
|
29740
|
+
...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
|
|
29741
|
+
...config.tags !== undefined ? { tags: config.tags } : {},
|
|
29742
|
+
channelIds: config.channels?.ids,
|
|
29743
|
+
channelNames: config.channels?.names,
|
|
29744
|
+
...sidecar === undefined ? {} : { sidecar },
|
|
29745
|
+
connectorOptions: {
|
|
29746
|
+
...config.connector,
|
|
29747
|
+
...workspaceRoot === undefined ? {} : { workspacePath: workspaceRoot }
|
|
29748
|
+
}
|
|
29749
|
+
};
|
|
29750
|
+
}
|
|
27957
29751
|
function larkConnectorOptions(connector) {
|
|
27958
29752
|
if (connector === undefined)
|
|
27959
29753
|
return {};
|
|
@@ -28014,6 +29808,8 @@ function buildDatasourceSkills(config, workspaceRoot, embeddingRuntime) {
|
|
|
28014
29808
|
var BUILDERS, BUILTIN_DATASOURCE_SKILL_NAMES;
|
|
28015
29809
|
var init_factory = __esm(() => {
|
|
28016
29810
|
init_aliased_skill();
|
|
29811
|
+
init_config2();
|
|
29812
|
+
init_descriptors();
|
|
28017
29813
|
init_clawgallery();
|
|
28018
29814
|
init_cloud_drive();
|
|
28019
29815
|
init_discrawl();
|
|
@@ -28031,42 +29827,9 @@ var init_factory = __esm(() => {
|
|
|
28031
29827
|
init_telecrawl();
|
|
28032
29828
|
init_wacrawl();
|
|
28033
29829
|
BUILDERS = {
|
|
28034
|
-
telegram: (config,
|
|
28035
|
-
|
|
28036
|
-
|
|
28037
|
-
...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
|
|
28038
|
-
...config.tags !== undefined ? { tags: config.tags } : {},
|
|
28039
|
-
channelIds: config.channels?.ids,
|
|
28040
|
-
channelNames: config.channels?.names,
|
|
28041
|
-
connectorOptions: {
|
|
28042
|
-
...config.connector,
|
|
28043
|
-
..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
|
|
28044
|
-
}
|
|
28045
|
-
}),
|
|
28046
|
-
whatsapp: (config, _workspaceRoot, registrationName) => new WacrawlSkill({
|
|
28047
|
-
datasourceId: registrationName,
|
|
28048
|
-
...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
|
|
28049
|
-
...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
|
|
28050
|
-
...config.tags !== undefined ? { tags: config.tags } : {},
|
|
28051
|
-
channelIds: config.channels?.ids,
|
|
28052
|
-
channelNames: config.channels?.names,
|
|
28053
|
-
connectorOptions: {
|
|
28054
|
-
...config.connector,
|
|
28055
|
-
..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
|
|
28056
|
-
}
|
|
28057
|
-
}),
|
|
28058
|
-
slack: (config, _workspaceRoot, registrationName) => new SlackSkill({
|
|
28059
|
-
datasourceId: registrationName,
|
|
28060
|
-
...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
|
|
28061
|
-
...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
|
|
28062
|
-
...config.tags !== undefined ? { tags: config.tags } : {},
|
|
28063
|
-
channelIds: config.channels?.ids,
|
|
28064
|
-
channelNames: config.channels?.names,
|
|
28065
|
-
connectorOptions: {
|
|
28066
|
-
...config.connector,
|
|
28067
|
-
..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
|
|
28068
|
-
}
|
|
28069
|
-
}),
|
|
29830
|
+
telegram: (config, workspaceRoot, registrationName) => new TelecrawlSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.telegram, join37(homedir4(), ".telecrawl", "telecrawl.db"), config, workspaceRoot, registrationName)),
|
|
29831
|
+
whatsapp: (config, workspaceRoot, registrationName) => new WacrawlSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.whatsapp, join37(homedir4(), ".wacrawl", "wacrawl.db"), config, workspaceRoot, registrationName)),
|
|
29832
|
+
slack: (config, workspaceRoot, registrationName) => new SlackSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.slack, join37(homedir4(), ".slacrawl", "slacrawl.db"), config, workspaceRoot, registrationName)),
|
|
28070
29833
|
discord: (config, workspaceRoot, registrationName, embeddingRuntime) => {
|
|
28071
29834
|
const connector = config.connector ?? {};
|
|
28072
29835
|
const clientOptions = {
|
|
@@ -28185,8 +29948,8 @@ var init_factory = __esm(() => {
|
|
|
28185
29948
|
|
|
28186
29949
|
// src/cli/config.ts
|
|
28187
29950
|
import { randomUUID as randomUUID9 } from "node:crypto";
|
|
28188
|
-
import { existsSync as existsSync28, mkdirSync as
|
|
28189
|
-
import { basename as basename14, dirname as
|
|
29951
|
+
import { existsSync as existsSync28, mkdirSync as mkdirSync26, readFileSync as readFileSync25, renameSync as renameSync11, unlinkSync as unlinkSync5, writeFileSync as writeFileSync23 } from "node:fs";
|
|
29952
|
+
import { basename as basename14, dirname as dirname25, isAbsolute as isAbsolute10, join as join38, resolve as resolve16 } from "node:path";
|
|
28190
29953
|
import { findEnvKeys, getEnvApiKey } from "@earendil-works/pi-ai/compat";
|
|
28191
29954
|
import { getAgentDir as getAgentDir3, ModelRuntime as ModelRuntime2, SettingsManager as SettingsManager2 } from "@earendil-works/pi-coding-agent";
|
|
28192
29955
|
function resolveConfigPath(input) {
|
|
@@ -28202,9 +29965,9 @@ function resolveConfigPath(input) {
|
|
|
28202
29965
|
return { configPath: envConfig, explicit: true };
|
|
28203
29966
|
}
|
|
28204
29967
|
return {
|
|
28205
|
-
configPath:
|
|
29968
|
+
configPath: join38(resolveAutoRAGHome(env), DEFAULT_CONFIG_FILENAME),
|
|
28206
29969
|
explicit: false,
|
|
28207
|
-
legacyPath:
|
|
29970
|
+
legacyPath: join38(cwd, LEGACY_CONFIG_FILENAME)
|
|
28208
29971
|
};
|
|
28209
29972
|
}
|
|
28210
29973
|
function readConfigFile(configPath, explicit) {
|
|
@@ -28222,7 +29985,7 @@ function readConfigFile(configPath, explicit) {
|
|
|
28222
29985
|
}
|
|
28223
29986
|
let text;
|
|
28224
29987
|
try {
|
|
28225
|
-
text =
|
|
29988
|
+
text = readFileSync25(configPath, "utf8");
|
|
28226
29989
|
} catch (err) {
|
|
28227
29990
|
throw new ConfigError(`Failed to read config file: ${err.message}`);
|
|
28228
29991
|
}
|
|
@@ -28260,7 +30023,7 @@ function acquireConfigWriteLock(configPath) {
|
|
|
28260
30023
|
});
|
|
28261
30024
|
}
|
|
28262
30025
|
function replaceFileAtomically(path, contents, assertCommitAllowed) {
|
|
28263
|
-
const temporaryPath =
|
|
30026
|
+
const temporaryPath = join38(dirname25(path), `.${basename14(path)}.${process.pid}.${randomUUID9()}.tmp`);
|
|
28264
30027
|
try {
|
|
28265
30028
|
writeFileSync23(temporaryPath, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
|
|
28266
30029
|
assertCommitAllowed?.();
|
|
@@ -28273,12 +30036,12 @@ function migrateLegacyConfig(configPath, legacyPath) {
|
|
|
28273
30036
|
if (existsSync28(configPath) || !existsSync28(legacyPath))
|
|
28274
30037
|
return;
|
|
28275
30038
|
const legacy = readConfigFile(legacyPath, true);
|
|
28276
|
-
const legacyBytes =
|
|
28277
|
-
const migrated = normalizeLegacyConfigPaths(legacy ?? {},
|
|
30039
|
+
const legacyBytes = readFileSync25(legacyPath);
|
|
30040
|
+
const migrated = normalizeLegacyConfigPaths(legacy ?? {}, dirname25(legacyPath));
|
|
28278
30041
|
const migratedPathsUnchanged = legacy?.workspacePath === migrated.workspacePath && legacy?.memoryPath === migrated.memoryPath && JSON.stringify(legacy?.searchPaths) === JSON.stringify(migrated.searchPaths);
|
|
28279
30042
|
const migratedBytes = migratedPathsUnchanged ? legacyBytes : `${JSON.stringify(migrated, null, 2)}
|
|
28280
30043
|
`;
|
|
28281
|
-
|
|
30044
|
+
mkdirSync26(dirname25(configPath), { recursive: true });
|
|
28282
30045
|
const lock = acquireConfigWriteLock(configPath);
|
|
28283
30046
|
try {
|
|
28284
30047
|
const winner = readConfigFile(configPath, false);
|
|
@@ -28308,13 +30071,13 @@ function resolveConfigFileReadOnly(configPath, explicit, legacyPath) {
|
|
|
28308
30071
|
const legacy = readConfigFile(legacyPath, false);
|
|
28309
30072
|
if (legacy === undefined)
|
|
28310
30073
|
return {};
|
|
28311
|
-
return normalizeLegacyConfigPaths(legacy,
|
|
30074
|
+
return normalizeLegacyConfigPaths(legacy, dirname25(legacyPath));
|
|
28312
30075
|
}
|
|
28313
30076
|
function resolveSearchPaths(searchPaths, origin) {
|
|
28314
30077
|
return searchPaths.map((searchPath) => resolvePersistedPath(searchPath, origin));
|
|
28315
30078
|
}
|
|
28316
30079
|
function resolvePersistedPath(path, origin) {
|
|
28317
|
-
return
|
|
30080
|
+
return isAbsolute10(path) ? path : resolve16(origin, path);
|
|
28318
30081
|
}
|
|
28319
30082
|
function normalizeLegacyConfigPaths(partial, origin) {
|
|
28320
30083
|
const workspacePath = resolvePersistedPath(partial.workspacePath ?? ".", origin);
|
|
@@ -28501,7 +30264,7 @@ function normalizeEmbedder(raw, path) {
|
|
|
28501
30264
|
out.passagePrefix = record.passagePrefix;
|
|
28502
30265
|
}
|
|
28503
30266
|
if (record.profile !== undefined) {
|
|
28504
|
-
if (record.profile !== "qwen3-embedding-0.6b"
|
|
30267
|
+
if (record.profile !== "qwen3-embedding-0.6b") {
|
|
28505
30268
|
throw new ConfigError(`${path}.profile must be a supported runtime profile`);
|
|
28506
30269
|
}
|
|
28507
30270
|
out.profile = record.profile;
|
|
@@ -28744,10 +30507,10 @@ function resolveConfig(input) {
|
|
|
28744
30507
|
const file = readOnly ? resolveConfigFileReadOnly(configPath, explicit, legacyPath) : !explicit && legacyPath ? migrateLegacyConfig(configPath, legacyPath) ?? readConfigFile(configPath, explicit) ?? {} : readConfigFile(configPath, explicit) ?? {};
|
|
28745
30508
|
const defaultSearchPaths = ["."];
|
|
28746
30509
|
const defaultWorkspacePath = cwd;
|
|
28747
|
-
const defaultMemoryPath =
|
|
30510
|
+
const defaultMemoryPath = join38(resolveAutoRAGHome(env), "memory.json");
|
|
28748
30511
|
const flagSearchPaths = parseCsv(flagString(flags, "search-paths"));
|
|
28749
30512
|
const envSearchPaths = parseCsv(envString(env, "AUTORAG_SEARCH_PATHS"));
|
|
28750
|
-
const configOrigin =
|
|
30513
|
+
const configOrigin = dirname25(resolve16(configPath));
|
|
28751
30514
|
const fileWorkspacePath = typeof file.workspacePath === "string" ? resolvePersistedPath(file.workspacePath, configOrigin) : undefined;
|
|
28752
30515
|
const fileSearchPaths = file.searchPaths ? resolveSearchPaths(file.searchPaths, fileWorkspacePath ?? configOrigin) : undefined;
|
|
28753
30516
|
if (file.excludePaths !== undefined && (!Array.isArray(file.excludePaths) || file.excludePaths.some((path) => typeof path !== "string" || path.length === 0))) {
|
|
@@ -28770,7 +30533,7 @@ function resolveConfig(input) {
|
|
|
28770
30533
|
const flagModelId = pickString(flags, env, "model-id", "AUTORAG_MODEL_ID", undefined);
|
|
28771
30534
|
const model = applyRoleFlagOverrides(fileModel, flagModelProvider, flagModelId, "model");
|
|
28772
30535
|
const config = {
|
|
28773
|
-
configPath:
|
|
30536
|
+
configPath: resolve16(configPath),
|
|
28774
30537
|
searchPaths,
|
|
28775
30538
|
workspacePath,
|
|
28776
30539
|
memoryPath,
|
|
@@ -28946,36 +30709,38 @@ function buildWebSearchAgentOption(raw) {
|
|
|
28946
30709
|
function normalizeRerankConfig(raw, path) {
|
|
28947
30710
|
if (raw === false)
|
|
28948
30711
|
return false;
|
|
28949
|
-
|
|
28950
|
-
|
|
28951
|
-
|
|
28952
|
-
|
|
28953
|
-
|
|
28954
|
-
|
|
28955
|
-
|
|
28956
|
-
|
|
28957
|
-
|
|
28958
|
-
|
|
28959
|
-
}
|
|
28960
|
-
const record = raw;
|
|
28961
|
-
for (const key of Object.keys(record)) {
|
|
28962
|
-
if (!RERANK_ALLOWLIST.has(key))
|
|
28963
|
-
throw new ConfigError(`${path}.${key} is not a recognized field`);
|
|
28964
|
-
}
|
|
28965
|
-
if (record.enabled !== undefined) {
|
|
28966
|
-
if (typeof record.enabled !== "boolean")
|
|
28967
|
-
throw new ConfigError(`${path}.enabled must be a boolean`);
|
|
28968
|
-
out.enabled = record.enabled;
|
|
30712
|
+
let record = {};
|
|
30713
|
+
if (raw !== undefined && raw !== null) {
|
|
30714
|
+
if (typeof raw !== "object" || Array.isArray(raw)) {
|
|
30715
|
+
throw new ConfigError(`${path} must be an object or false`);
|
|
30716
|
+
}
|
|
30717
|
+
record = raw;
|
|
30718
|
+
for (const key of Object.keys(record)) {
|
|
30719
|
+
if (!RERANK_ALLOWLIST.has(key))
|
|
30720
|
+
throw new ConfigError(`${path}.${key} is not a recognized field`);
|
|
30721
|
+
}
|
|
28969
30722
|
}
|
|
30723
|
+
let provider = DEFAULT_RERANK_PROVIDER;
|
|
28970
30724
|
if (record.provider !== undefined) {
|
|
28971
30725
|
if (typeof record.provider !== "string" || record.provider.trim() === "") {
|
|
28972
30726
|
throw new ConfigError(`${path}.provider must be a non-empty string`);
|
|
28973
30727
|
}
|
|
28974
|
-
|
|
30728
|
+
provider = record.provider.trim();
|
|
28975
30729
|
}
|
|
28976
|
-
if (!SUPPORTED_RERANK_PROVIDERS.includes(
|
|
30730
|
+
if (!SUPPORTED_RERANK_PROVIDERS.includes(provider)) {
|
|
28977
30731
|
throw new ConfigError(`${path}.provider must be one of: ${SUPPORTED_RERANK_PROVIDERS.join(", ")}`);
|
|
28978
30732
|
}
|
|
30733
|
+
const out = {
|
|
30734
|
+
provider,
|
|
30735
|
+
model: defaultRerankModel(provider),
|
|
30736
|
+
apiKeyEnv: defaultRerankApiKeyEnv(provider),
|
|
30737
|
+
topN: DEFAULT_RERANK_TOP_N
|
|
30738
|
+
};
|
|
30739
|
+
if (record.enabled !== undefined) {
|
|
30740
|
+
if (typeof record.enabled !== "boolean")
|
|
30741
|
+
throw new ConfigError(`${path}.enabled must be a boolean`);
|
|
30742
|
+
out.enabled = record.enabled;
|
|
30743
|
+
}
|
|
28979
30744
|
if (record.model !== undefined) {
|
|
28980
30745
|
if (typeof record.model !== "string" || record.model.trim() === "") {
|
|
28981
30746
|
throw new ConfigError(`${path}.model must be a non-empty string`);
|
|
@@ -29105,7 +30870,8 @@ function buildModelFromConfiguredEndpoint(reference) {
|
|
|
29105
30870
|
input: reference.input ?? ["text"],
|
|
29106
30871
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
29107
30872
|
contextWindow: reference.contextWindow ?? 128000,
|
|
29108
|
-
maxTokens: reference.maxTokens ?? 16384
|
|
30873
|
+
maxTokens: reference.maxTokens ?? 16384,
|
|
30874
|
+
...reference.provider === PROVIDER_ID ? { inputLimits: AUTORAG_INPUT_LIMITS } : {}
|
|
29109
30875
|
};
|
|
29110
30876
|
}
|
|
29111
30877
|
function mergeCatalogModel(catalog, reference) {
|
|
@@ -29156,8 +30922,8 @@ function getModelRuntime(agentDir) {
|
|
|
29156
30922
|
return cached;
|
|
29157
30923
|
const created = (async () => {
|
|
29158
30924
|
const runtime = await ModelRuntime2.create({
|
|
29159
|
-
authPath:
|
|
29160
|
-
modelsPath:
|
|
30925
|
+
authPath: join38(agentDir, "auth.json"),
|
|
30926
|
+
modelsPath: join38(agentDir, "models.json"),
|
|
29161
30927
|
allowModelNetwork: false
|
|
29162
30928
|
});
|
|
29163
30929
|
await registerAutoRAGProvider(runtime);
|
|
@@ -29416,7 +31182,7 @@ function writeConfigObject(path, config) {
|
|
|
29416
31182
|
if (config === null || typeof config !== "object" || Array.isArray(config)) {
|
|
29417
31183
|
throw new ConfigError("Config file must be a JSON object");
|
|
29418
31184
|
}
|
|
29419
|
-
|
|
31185
|
+
mkdirSync26(dirname25(path), { recursive: true });
|
|
29420
31186
|
const contents = `${JSON.stringify(config, null, 2)}
|
|
29421
31187
|
`;
|
|
29422
31188
|
const lock = acquireConfigWriteLock(path);
|
|
@@ -29427,9 +31193,9 @@ function writeConfigObject(path, config) {
|
|
|
29427
31193
|
}
|
|
29428
31194
|
}
|
|
29429
31195
|
function writeDefaultConfig(path, partial, opts = {}) {
|
|
29430
|
-
const cwd =
|
|
31196
|
+
const cwd = resolve16(opts.cwd ?? process.cwd());
|
|
29431
31197
|
const workspacePath = resolvePersistedPath(partial.workspacePath ?? ".", cwd);
|
|
29432
|
-
const memoryPath = partial.memoryPath === undefined ?
|
|
31198
|
+
const memoryPath = partial.memoryPath === undefined ? join38(resolveAutoRAGHome(opts.env), "memory.json") : resolvePersistedPath(partial.memoryPath, workspacePath);
|
|
29433
31199
|
const full = {
|
|
29434
31200
|
searchPaths: resolveSearchPaths(partial.searchPaths ?? ["."], cwd),
|
|
29435
31201
|
workspacePath,
|
|
@@ -29463,7 +31229,7 @@ function writeDefaultConfig(path, partial, opts = {}) {
|
|
|
29463
31229
|
full.p2p = { enabled: false };
|
|
29464
31230
|
full.jev = partial.jev ?? { backend: DEFAULT_JEV_BACKEND };
|
|
29465
31231
|
full.queryDecomposition = partial.queryDecomposition ?? { model: { ...DEFAULT_QUERY_DECOMPOSITION_MODEL } };
|
|
29466
|
-
|
|
31232
|
+
mkdirSync26(dirname25(path), { recursive: true });
|
|
29467
31233
|
const contents = `${JSON.stringify(full, null, 2)}
|
|
29468
31234
|
`;
|
|
29469
31235
|
const lock = acquireConfigWriteLock(path);
|
|
@@ -29491,8 +31257,9 @@ function writeDefaultConfig(path, partial, opts = {}) {
|
|
|
29491
31257
|
}
|
|
29492
31258
|
}
|
|
29493
31259
|
var DEFAULT_CONFIG_FILENAME = "config.json", LEGACY_CONFIG_FILENAME = "autorag.config.json", CONFIG_LOCK_RETRY_MS = 10, CONFIG_LOCK_TIMEOUT_MS = 1e4, CONFIG_LOCK_STALE_MS = 30000, ConfigError, MINSYNC_REQUIRED_MESSAGE = "MinSync is required and cannot be disabled; remove `minSync: false` / `minSync.enabled: false` from the config.", AGENT_MODEL_APIS, AGENT_MODEL_INPUTS, API_KEY_ENV_PATTERN2, POSITIVE_INT_FIELDS, EMBEDDER_ALLOWLIST, MINSYNC_ALLOWLIST, MINSYNC_LEGACY_IGNORED, P2P_ALLOWLIST, P2P_QUOTA_ALLOWLIST, LIMITS_ALLOWLIST, LIMITS_PREFETCH_ALLOWLIST, JEV_CONFIG_FIELDS, JEV_BACKEND_NAMES, DEFAULT_JEV_BACKEND = "openrouter", DEFAULT_QUERY_DECOMPOSITION_MODEL, RERANK_ALLOWLIST, UNKNOWN_MODEL_HINT = "Add baseUrl (and optional api/apiKeyEnv) for an OpenAI-compatible endpoint outside the pi-ai catalog, or use a pi-ai catalog model id.", modelRuntimeCache;
|
|
29494
|
-
var
|
|
31260
|
+
var init_config4 = __esm(() => {
|
|
29495
31261
|
init_local_model();
|
|
31262
|
+
init_config();
|
|
29496
31263
|
init_provider();
|
|
29497
31264
|
init_home();
|
|
29498
31265
|
init_factory();
|
|
@@ -29574,7 +31341,7 @@ var init_config3 = __esm(() => {
|
|
|
29574
31341
|
model: true,
|
|
29575
31342
|
confidenceThreshold: true
|
|
29576
31343
|
};
|
|
29577
|
-
JEV_BACKEND_NAMES = ["typesafe", "openrouter", "vercel"];
|
|
31344
|
+
JEV_BACKEND_NAMES = ["typesafe", "openrouter", "vercel", "autorag"];
|
|
29578
31345
|
DEFAULT_QUERY_DECOMPOSITION_MODEL = { provider: "openrouter", id: "qwen/qwen3.7-flash" };
|
|
29579
31346
|
RERANK_ALLOWLIST = new Set(["enabled", "provider", "model", "apiKeyEnv", "baseUrl", "topN", "timeoutMs"]);
|
|
29580
31347
|
modelRuntimeCache = new Map;
|
|
@@ -29664,9 +31431,9 @@ function createAutoRAGLite(options = {}) {
|
|
|
29664
31431
|
var init_core = __esm(() => {
|
|
29665
31432
|
init_agent();
|
|
29666
31433
|
init_search_documents();
|
|
29667
|
-
|
|
31434
|
+
init_config4();
|
|
29668
31435
|
init_memory();
|
|
29669
|
-
|
|
31436
|
+
init_config4();
|
|
29670
31437
|
});
|
|
29671
31438
|
init_core();
|
|
29672
31439
|
|