@autorag/librarian 2.10.0 → 2.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +195 -21
- package/NOTICE +17 -21
- package/README.md +335 -161
- package/dist/agent/agent.d.ts +67 -2
- package/dist/agent/context-token-guard.d.ts +23 -0
- package/dist/agent/jev-extension.d.ts +5 -4
- package/dist/agent/jikji-find-tool.d.ts +1 -0
- package/dist/agent/pi-session.d.ts +35 -2
- package/dist/agent/request-limits.d.ts +20 -0
- package/dist/agent/search-all-tool.d.ts +1 -0
- package/dist/agent/search-single-datasource-tool.d.ts +1 -0
- package/dist/cli/config.d.ts +13 -11
- package/dist/cli/index.js +2573 -800
- package/dist/cloud/config.d.ts +18 -0
- package/dist/cloud/jev-backend.d.ts +90 -0
- package/dist/core.js +2185 -418
- package/dist/datasource/crawler-skill.d.ts +17 -0
- package/dist/datasource/sidecar/config.d.ts +26 -0
- package/dist/datasource/sidecar/descriptors.d.ts +67 -0
- package/dist/datasource/sidecar/fusion.d.ts +25 -0
- package/dist/datasource/sidecar/index.d.ts +131 -0
- package/dist/datasource/sidecar/methods.d.ts +27 -0
- package/dist/datasource/types.d.ts +3 -1
- package/dist/embedding-runtime/index.d.ts +6 -6
- package/dist/embedding-runtime/manifest.d.ts +0 -12
- package/dist/embedding-runtime/types.d.ts +1 -1
- package/dist/index.js +2210 -439
- package/dist/mcp/index.js +2205 -438
- package/dist/minsync/client.d.ts +3 -3
- package/dist/minsync/installer.d.ts +1 -1
- package/dist/minsync/process.d.ts +1 -0
- package/dist/minsync/types.d.ts +1 -1
- package/dist/parser/defaults.d.ts +6 -4
- package/dist/parser/ocr-engines.d.ts +6 -0
- package/dist/retrieval/index.d.ts +1 -1
- package/dist/retrieval/rerank.d.ts +20 -3
- package/licenses/embedding-assets.json +0 -35
- package/package.json +24 -27
- package/skills/autorag/SKILL.md +1 -1
- package/skills/autorag-doctor/SKILL.md +1 -1
- package/skills/autorag-lite-setup/SKILL.md +2 -2
- package/skills/autorag-setup/SKILL.md +2 -2
- package/licenses/gemma-notice.txt +0 -24
- package/licenses/gemma-section-3.2-flow-down.txt +0 -9
- package/licenses/gemma-terms-of-use.html +0 -4750
- package/licenses/gemma-terms-of-use.txt +0 -158
package/dist/mcp/index.js
CHANGED
|
@@ -62,6 +62,7 @@ function datasourceCodeToRetrieval(code) {
|
|
|
62
62
|
case "datasource-auth-error":
|
|
63
63
|
case "datasource-embedding-egress-rejected":
|
|
64
64
|
case "datasource-index-failed":
|
|
65
|
+
case "datasource-index-partial":
|
|
65
66
|
case "datasource-permission-denied":
|
|
66
67
|
case "datasource-remote-embedding-rejected":
|
|
67
68
|
return "retrieval-method-failed";
|
|
@@ -1113,19 +1114,6 @@ var init_manifest = __esm(() => {
|
|
|
1113
1114
|
artifactSha256: "06507c7b42688469c4e7298b0a1e16deff06caf291cf0a5b278c308249c3e439",
|
|
1114
1115
|
backend: "auto",
|
|
1115
1116
|
protocolCapabilities
|
|
1116
|
-
},
|
|
1117
|
-
"embeddinggemma-300m": {
|
|
1118
|
-
profileId: "embeddinggemma-300m",
|
|
1119
|
-
provider: "google",
|
|
1120
|
-
model: "embeddinggemma-300M-Q8_0.gguf",
|
|
1121
|
-
dimension: 768,
|
|
1122
|
-
queryPrefix: "task: search result | query: ",
|
|
1123
|
-
passagePrefix: "title: none | text: ",
|
|
1124
|
-
runtimeBuild: RUNTIME_BUILD,
|
|
1125
|
-
modelRevision: "0f741b5a6585bd53aeb15cd1372c56f2a0f65e12",
|
|
1126
|
-
artifactSha256: "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63",
|
|
1127
|
-
backend: "auto",
|
|
1128
|
-
protocolCapabilities
|
|
1129
1117
|
}
|
|
1130
1118
|
};
|
|
1131
1119
|
runtimeAssets = {
|
|
@@ -1171,17 +1159,6 @@ var init_manifest = __esm(() => {
|
|
|
1171
1159
|
unsuitableUse: "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256.",
|
|
1172
1160
|
upstreamCardUrl: "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B",
|
|
1173
1161
|
modelCard: "docs/model-cards/qwen3-embedding-0.6b.md"
|
|
1174
|
-
},
|
|
1175
|
-
embeddinggemma: {
|
|
1176
|
-
url: "https://huggingface.co/ggml-org/embeddinggemma-300M-GGUF/resolve/0f741b5a6585bd53aeb15cd1372c56f2a0f65e12/embeddinggemma-300M-Q8_0.gguf",
|
|
1177
|
-
revision: "0f741b5a6585bd53aeb15cd1372c56f2a0f65e12",
|
|
1178
|
-
sha256: "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63",
|
|
1179
|
-
license: "Gemma Terms of Use",
|
|
1180
|
-
noticeReference: "Google EmbeddingGemma model card and Gemma Terms of Use",
|
|
1181
|
-
intendedUse: "Local loopback embedding for AutoRAG MinSync and native-datasource semantic retrieval. Not a generative model.",
|
|
1182
|
-
unsuitableUse: "Chat or generation, remote embedding of corpus text, shipping weights in the npm package, any GGUF other than the pinned SHA-256, uses restricted by the Gemma Prohibited Use Policy.",
|
|
1183
|
-
upstreamCardUrl: "https://ai.google.dev/gemma/docs/embeddinggemma/model_card",
|
|
1184
|
-
modelCard: "docs/model-cards/embeddinggemma-300m.md"
|
|
1185
1162
|
}
|
|
1186
1163
|
};
|
|
1187
1164
|
});
|
|
@@ -1619,7 +1596,7 @@ var init_supervisor = __esm(() => {
|
|
|
1619
1596
|
// src/embedding-runtime/index.ts
|
|
1620
1597
|
import { basename as basename4 } from "node:path";
|
|
1621
1598
|
function modelAsset(profile) {
|
|
1622
|
-
return
|
|
1599
|
+
return { id: profile.profileId, filename: basename4(profile.model), ...MODEL_ASSETS.qwen3 };
|
|
1623
1600
|
}
|
|
1624
1601
|
function runtimeAsset(platform, backend) {
|
|
1625
1602
|
const selected = platform ?? "darwin-arm64-metal";
|
|
@@ -5021,7 +4998,12 @@ var init_local_embedder = __esm(() => {
|
|
|
5021
4998
|
// src/minsync/process.ts
|
|
5022
4999
|
import { spawn as spawn7 } from "node:child_process";
|
|
5023
5000
|
function spawnProcess(command, args, cwd, options = {}) {
|
|
5024
|
-
return new Promise((resolve) => {
|
|
5001
|
+
return new Promise((resolve, reject) => {
|
|
5002
|
+
const { signal } = options;
|
|
5003
|
+
if (signal?.aborted) {
|
|
5004
|
+
reject(abortError(signal));
|
|
5005
|
+
return;
|
|
5006
|
+
}
|
|
5025
5007
|
const portable = portableSpawnCommand(command, args);
|
|
5026
5008
|
const child = spawn7(portable.command, [...portable.args], { cwd, stdio: ["ignore", "pipe", "pipe"] });
|
|
5027
5009
|
let stdout = "";
|
|
@@ -5039,15 +5021,26 @@ function spawnProcess(command, args, cwd, options = {}) {
|
|
|
5039
5021
|
timedOut = true;
|
|
5040
5022
|
child.kill("SIGTERM");
|
|
5041
5023
|
}, options.timeoutMs) : undefined;
|
|
5024
|
+
const onAbort = () => {
|
|
5025
|
+
child.kill("SIGTERM");
|
|
5026
|
+
};
|
|
5027
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
5028
|
+
const settle = () => {
|
|
5029
|
+
clearTimeout(timer);
|
|
5030
|
+
signal?.removeEventListener("abort", onAbort);
|
|
5031
|
+
};
|
|
5042
5032
|
child.on("error", (error) => {
|
|
5043
|
-
|
|
5044
|
-
|
|
5045
|
-
|
|
5033
|
+
settle();
|
|
5034
|
+
if (signal?.aborted)
|
|
5035
|
+
reject(abortError(signal));
|
|
5036
|
+
else
|
|
5037
|
+
resolve({ ok: false, stdout, stderr: error.message, code: null });
|
|
5046
5038
|
});
|
|
5047
5039
|
child.on("close", (code) => {
|
|
5048
|
-
|
|
5049
|
-
|
|
5050
|
-
|
|
5040
|
+
settle();
|
|
5041
|
+
if (signal?.aborted) {
|
|
5042
|
+
reject(abortError(signal));
|
|
5043
|
+
} else if (timedOut) {
|
|
5051
5044
|
resolve({ ok: false, stdout, stderr: "process timed out", code: null, timedOut: true });
|
|
5052
5045
|
} else {
|
|
5053
5046
|
resolve({ ok: code === 0, stdout, stderr, code });
|
|
@@ -5055,6 +5048,9 @@ function spawnProcess(command, args, cwd, options = {}) {
|
|
|
5055
5048
|
});
|
|
5056
5049
|
});
|
|
5057
5050
|
}
|
|
5051
|
+
function abortError(signal) {
|
|
5052
|
+
return signal.reason instanceof Error ? signal.reason : new DOMException("The operation was aborted", "AbortError");
|
|
5053
|
+
}
|
|
5058
5054
|
var init_process = __esm(() => {
|
|
5059
5055
|
init_portable_spawn();
|
|
5060
5056
|
});
|
|
@@ -5287,10 +5283,12 @@ class MinSyncClient {
|
|
|
5287
5283
|
} : {}
|
|
5288
5284
|
};
|
|
5289
5285
|
}
|
|
5290
|
-
async query(text, topK, mode = "vector") {
|
|
5286
|
+
async query(text, topK, mode = "vector", signal) {
|
|
5291
5287
|
if (!existsSync11(this.binaryPath))
|
|
5292
5288
|
throw minSyncBinaryMissingError();
|
|
5293
|
-
const result = await this.spawn(["query", "--format", "json", "--mode", mode, "-k", String(topK), text]
|
|
5289
|
+
const result = await this.spawn(["query", "--format", "json", "--mode", mode, "-k", String(topK), text], {
|
|
5290
|
+
...signal === undefined ? {} : { signal }
|
|
5291
|
+
});
|
|
5294
5292
|
if (!result.ok)
|
|
5295
5293
|
throw new MinSyncQueryError(result.code, result.stderr);
|
|
5296
5294
|
return parseQueryHits(result.stdout);
|
|
@@ -5363,7 +5361,7 @@ function isIdentity(value) {
|
|
|
5363
5361
|
function isRecord4(value) {
|
|
5364
5362
|
return typeof value === "object" && value !== null;
|
|
5365
5363
|
}
|
|
5366
|
-
var MINSYNC_OLLAMA_MIGRATION_MESSAGE = "This workspace uses the legacy 768-dimensional Ollama/TEI embedding path. Reindex explicitly, or pin an explicit
|
|
5364
|
+
var MINSYNC_OLLAMA_MIGRATION_MESSAGE = "This workspace uses the legacy 768-dimensional Ollama/TEI embedding path. Reindex explicitly, or pin an explicit Ollama/TEI embedder config before using the new default runtime.", API_KEY_ENV_PATTERN, MinSyncQueryError;
|
|
5367
5365
|
var init_client4 = __esm(() => {
|
|
5368
5366
|
init_embedder_config();
|
|
5369
5367
|
init_errors();
|
|
@@ -5607,7 +5605,7 @@ function isGitHubAsset(value) {
|
|
|
5607
5605
|
function isRecord5(value) {
|
|
5608
5606
|
return typeof value === "object" && value !== null;
|
|
5609
5607
|
}
|
|
5610
|
-
var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.4.
|
|
5608
|
+
var LATEST_RELEASE_URL = "https://api.github.com/repos/NomaDamas/MinSync/releases/latest", SHA256_HEX_PATTERN, MINSYNC_VERSION = "0.4.7", MINSYNC_CARGO_INSTALL_TIMEOUT_MS, CARGO_PROBE_TIMEOUT_MS = 5000, MinSyncReleaseError;
|
|
5611
5609
|
var init_installer2 = __esm(() => {
|
|
5612
5610
|
init_process();
|
|
5613
5611
|
SHA256_HEX_PATTERN = /^[a-f0-9]{64}$/;
|
|
@@ -5916,11 +5914,11 @@ class MinSyncVectorMethod {
|
|
|
5916
5914
|
});
|
|
5917
5915
|
let hits;
|
|
5918
5916
|
try {
|
|
5919
|
-
hits = await client.query(query, queryK, this.mode);
|
|
5917
|
+
hits = await client.query(query, queryK, this.mode, options.signal);
|
|
5920
5918
|
} catch (error) {
|
|
5921
|
-
if (this.mode !== "hybrid")
|
|
5919
|
+
if (this.mode !== "hybrid" || options.signal?.aborted)
|
|
5922
5920
|
throw error;
|
|
5923
|
-
hits = await client.query(query, queryK, "bm25");
|
|
5921
|
+
hits = await client.query(query, queryK, "bm25", options.signal);
|
|
5924
5922
|
}
|
|
5925
5923
|
const results = [];
|
|
5926
5924
|
for (const hit of hits) {
|
|
@@ -6243,10 +6241,16 @@ async function runTesseract(recognition, signal, cleanupResolve, cleanupReject)
|
|
|
6243
6241
|
};
|
|
6244
6242
|
signal.addEventListener("abort", abort, { once: true });
|
|
6245
6243
|
try {
|
|
6244
|
+
const tooSmall = ocrImageTooSmallMessage(recognition.bytes);
|
|
6245
|
+
if (tooSmall !== undefined)
|
|
6246
|
+
throw new Error(tooSmall);
|
|
6246
6247
|
const { cachePath } = recognition;
|
|
6247
6248
|
if (cachePath !== undefined)
|
|
6248
6249
|
await mkdir3(cachePath, { recursive: true });
|
|
6249
|
-
worker = await createWorker(recognition.language, undefined,
|
|
6250
|
+
worker = await createWorker(recognition.language, undefined, {
|
|
6251
|
+
...cachePath === undefined ? {} : { cachePath },
|
|
6252
|
+
errorHandler: containTesseractWorkerError
|
|
6253
|
+
});
|
|
6250
6254
|
if (signal.aborted)
|
|
6251
6255
|
throw new Error("OCR aborted before worker was ready");
|
|
6252
6256
|
const result = await worker.recognize(Buffer.from(recognition.bytes));
|
|
@@ -6256,6 +6260,129 @@ async function runTesseract(recognition, signal, cleanupResolve, cleanupReject)
|
|
|
6256
6260
|
await terminate().then(cleanupResolve, cleanupReject);
|
|
6257
6261
|
}
|
|
6258
6262
|
}
|
|
6263
|
+
function containTesseractWorkerError(_payload) {}
|
|
6264
|
+
function ocrImageTooSmallMessage(bytes) {
|
|
6265
|
+
const size = readImagePixelSize(bytes);
|
|
6266
|
+
if (size === undefined || size.width >= MIN_LEPTONICA_SIDE_PIXELS && size.height >= MIN_LEPTONICA_SIDE_PIXELS) {
|
|
6267
|
+
return;
|
|
6268
|
+
}
|
|
6269
|
+
return `Image is too small to scale (${size.width}x${size.height} vs minimum of ${MIN_LEPTONICA_SIDE_PIXELS} pixels).`;
|
|
6270
|
+
}
|
|
6271
|
+
function readImagePixelSize(bytes) {
|
|
6272
|
+
if (bytes.length >= 8 && bytes[0] === 137 && bytes[1] === 80)
|
|
6273
|
+
return readPngSize(bytes);
|
|
6274
|
+
if (bytes.length >= 2 && bytes[0] === 255 && bytes[1] === 216)
|
|
6275
|
+
return readJpegSize(bytes);
|
|
6276
|
+
if (bytes.length >= 12 && bytes[0] === 82 && bytes[1] === 73)
|
|
6277
|
+
return readWebpSize(bytes);
|
|
6278
|
+
return;
|
|
6279
|
+
}
|
|
6280
|
+
function readPngSize(bytes) {
|
|
6281
|
+
if (bytes.length < 24)
|
|
6282
|
+
return;
|
|
6283
|
+
if (bytes[0] !== 137 || bytes[1] !== 80 || bytes[2] !== 78 || bytes[3] !== 71 || bytes[4] !== 13 || bytes[5] !== 10 || bytes[6] !== 26 || bytes[7] !== 10) {
|
|
6284
|
+
return;
|
|
6285
|
+
}
|
|
6286
|
+
if (bytes[12] !== 73 || bytes[13] !== 72 || bytes[14] !== 68 || bytes[15] !== 82)
|
|
6287
|
+
return;
|
|
6288
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
6289
|
+
return { width: view.getUint32(16), height: view.getUint32(20) };
|
|
6290
|
+
}
|
|
6291
|
+
function readJpegSize(bytes) {
|
|
6292
|
+
if (bytes.length < 4 || bytes[0] !== 255 || bytes[1] !== 216)
|
|
6293
|
+
return;
|
|
6294
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
6295
|
+
let offset = 2;
|
|
6296
|
+
while (offset + 1 < bytes.length) {
|
|
6297
|
+
if (bytes[offset] !== 255)
|
|
6298
|
+
return;
|
|
6299
|
+
offset += 1;
|
|
6300
|
+
while (offset < bytes.length && bytes[offset] === 255)
|
|
6301
|
+
offset += 1;
|
|
6302
|
+
if (offset >= bytes.length)
|
|
6303
|
+
return;
|
|
6304
|
+
const marker = bytes[offset] ?? 0;
|
|
6305
|
+
offset += 1;
|
|
6306
|
+
if (marker === 0)
|
|
6307
|
+
return;
|
|
6308
|
+
if (marker === 1 || marker === 217 || marker >= 208 && marker <= 216)
|
|
6309
|
+
continue;
|
|
6310
|
+
if (offset + 2 > bytes.length)
|
|
6311
|
+
return;
|
|
6312
|
+
const segmentLength = view.getUint16(offset);
|
|
6313
|
+
if (segmentLength < 2)
|
|
6314
|
+
return;
|
|
6315
|
+
if (isJpegStartOfFrame(marker)) {
|
|
6316
|
+
if (segmentLength < 7 || offset + 7 > bytes.length)
|
|
6317
|
+
return;
|
|
6318
|
+
return { width: view.getUint16(offset + 5), height: view.getUint16(offset + 3) };
|
|
6319
|
+
}
|
|
6320
|
+
if (marker === 218)
|
|
6321
|
+
return;
|
|
6322
|
+
const next = offset + segmentLength;
|
|
6323
|
+
if (next <= offset || next > bytes.length)
|
|
6324
|
+
return;
|
|
6325
|
+
offset = next;
|
|
6326
|
+
}
|
|
6327
|
+
return;
|
|
6328
|
+
}
|
|
6329
|
+
function isJpegStartOfFrame(marker) {
|
|
6330
|
+
return marker >= 192 && marker <= 207 && marker !== 196 && marker !== 200 && marker !== 204;
|
|
6331
|
+
}
|
|
6332
|
+
function readWebpSize(bytes) {
|
|
6333
|
+
if (bytes.length < 20 || !asciiAt(bytes, 0, "RIFF") || !asciiAt(bytes, 8, "WEBP"))
|
|
6334
|
+
return;
|
|
6335
|
+
const chunkSize = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength).getUint32(16, true);
|
|
6336
|
+
if (asciiAt(bytes, 12, "VP8X"))
|
|
6337
|
+
return readVp8xSize(bytes, chunkSize);
|
|
6338
|
+
if (asciiAt(bytes, 12, "VP8L"))
|
|
6339
|
+
return readVp8lSize(bytes, chunkSize);
|
|
6340
|
+
if (asciiAt(bytes, 12, "VP8 "))
|
|
6341
|
+
return readVp8Size(bytes, chunkSize);
|
|
6342
|
+
return;
|
|
6343
|
+
}
|
|
6344
|
+
function readVp8xSize(bytes, chunkSize) {
|
|
6345
|
+
if (!webpPayloadFits(bytes, chunkSize, 10))
|
|
6346
|
+
return;
|
|
6347
|
+
const payload = 20;
|
|
6348
|
+
const width = 1 + readU24LE(bytes, payload + 4);
|
|
6349
|
+
const height = 1 + readU24LE(bytes, payload + 7);
|
|
6350
|
+
return { width, height };
|
|
6351
|
+
}
|
|
6352
|
+
function readVp8lSize(bytes, chunkSize) {
|
|
6353
|
+
if (!webpPayloadFits(bytes, chunkSize, 5) || bytes[20] !== 47)
|
|
6354
|
+
return;
|
|
6355
|
+
const b1 = bytes[21] ?? 0;
|
|
6356
|
+
const b2 = bytes[22] ?? 0;
|
|
6357
|
+
const b3 = bytes[23] ?? 0;
|
|
6358
|
+
const b4 = bytes[24] ?? 0;
|
|
6359
|
+
const width = 1 + (b1 | (b2 & 63) << 8);
|
|
6360
|
+
const height = 1 + (b2 >> 6 | b3 << 2 | (b4 & 15) << 10);
|
|
6361
|
+
return { width, height };
|
|
6362
|
+
}
|
|
6363
|
+
function readVp8Size(bytes, chunkSize) {
|
|
6364
|
+
if (!webpPayloadFits(bytes, chunkSize, 10) || (bytes[20] & 1) !== 0)
|
|
6365
|
+
return;
|
|
6366
|
+
if (bytes[23] !== 157 || bytes[24] !== 1 || bytes[25] !== 42)
|
|
6367
|
+
return;
|
|
6368
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
6369
|
+
return { width: view.getUint16(26, true) & 16383, height: view.getUint16(28, true) & 16383 };
|
|
6370
|
+
}
|
|
6371
|
+
function webpPayloadFits(bytes, chunkSize, needed) {
|
|
6372
|
+
return chunkSize >= needed && bytes.length >= 20 + needed;
|
|
6373
|
+
}
|
|
6374
|
+
function readU24LE(bytes, offset) {
|
|
6375
|
+
return (bytes[offset] ?? 0) | (bytes[offset + 1] ?? 0) << 8 | (bytes[offset + 2] ?? 0) << 16;
|
|
6376
|
+
}
|
|
6377
|
+
function asciiAt(bytes, offset, text) {
|
|
6378
|
+
if (offset + text.length > bytes.length)
|
|
6379
|
+
return false;
|
|
6380
|
+
for (let index = 0;index < text.length; index += 1) {
|
|
6381
|
+
if (bytes[offset + index] !== text.charCodeAt(index))
|
|
6382
|
+
return false;
|
|
6383
|
+
}
|
|
6384
|
+
return true;
|
|
6385
|
+
}
|
|
6259
6386
|
function withTimeout(operation, timeoutMs, onTimeout) {
|
|
6260
6387
|
return new Promise((resolve, reject) => {
|
|
6261
6388
|
const timeout = setTimeout(() => {
|
|
@@ -6271,7 +6398,7 @@ function withTimeout(operation, timeoutMs, onTimeout) {
|
|
|
6271
6398
|
});
|
|
6272
6399
|
});
|
|
6273
6400
|
}
|
|
6274
|
-
var TESSERACT_LANGUAGES, BUILTIN_OCR_LANGUAGES, DEFAULT_TESSERACT_TIMEOUT_MS = 30000;
|
|
6401
|
+
var TESSERACT_LANGUAGES, BUILTIN_OCR_LANGUAGES, DEFAULT_TESSERACT_TIMEOUT_MS = 30000, MIN_LEPTONICA_SIDE_PIXELS = 3;
|
|
6275
6402
|
var init_ocr_engines = __esm(() => {
|
|
6276
6403
|
TESSERACT_LANGUAGES = {
|
|
6277
6404
|
ko: "kor",
|
|
@@ -6480,8 +6607,8 @@ function createDefaultParserRegistry(options = {}) {
|
|
|
6480
6607
|
const kordocOptions = {
|
|
6481
6608
|
...options.kordoc,
|
|
6482
6609
|
images: ocrEnabled,
|
|
6483
|
-
...options.kordoc?.ocr === undefined
|
|
6484
|
-
ocr: builtinCovers ? true : createTesseractOcrProvider({
|
|
6610
|
+
...options.kordoc?.ocr === undefined ? {
|
|
6611
|
+
ocr: !ocrEnabled ? false : builtinCovers ? true : createTesseractOcrProvider({
|
|
6485
6612
|
languages,
|
|
6486
6613
|
timeoutMs: options.ocr?.timeoutMs,
|
|
6487
6614
|
cachePath: options.ocr?.cachePath
|
|
@@ -6619,6 +6746,23 @@ async function syncParsedMirrors(options) {
|
|
|
6619
6746
|
checkpoint();
|
|
6620
6747
|
continue;
|
|
6621
6748
|
}
|
|
6749
|
+
const tooSmallImage = standaloneImageTooSmallMessage(entry.virtualPath, bytes);
|
|
6750
|
+
if (tooSmallImage !== undefined) {
|
|
6751
|
+
deleted += removePrevious(options.root, previous, entry.virtualPath);
|
|
6752
|
+
handledPrevious.add(entry.virtualPath);
|
|
6753
|
+
skipped += 1;
|
|
6754
|
+
recordSkip(entry, "parser-skipped");
|
|
6755
|
+
diagnostics.push({
|
|
6756
|
+
code: "parser-skipped",
|
|
6757
|
+
severity: "warning",
|
|
6758
|
+
message: tooSmallImage,
|
|
6759
|
+
source: entry.virtualPath
|
|
6760
|
+
});
|
|
6761
|
+
sinceCheckpoint += 1;
|
|
6762
|
+
if (sinceCheckpoint >= MIRROR_CHECKPOINT_EVERY)
|
|
6763
|
+
checkpoint();
|
|
6764
|
+
continue;
|
|
6765
|
+
}
|
|
6622
6766
|
let parsed;
|
|
6623
6767
|
try {
|
|
6624
6768
|
parsed = await parser.parse({ virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes });
|
|
@@ -6720,6 +6864,11 @@ async function detectMirrorStaleness(options) {
|
|
|
6720
6864
|
}
|
|
6721
6865
|
return diagnostics;
|
|
6722
6866
|
}
|
|
6867
|
+
function standaloneImageTooSmallMessage(virtualPath, bytes) {
|
|
6868
|
+
if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname4(virtualPath))))
|
|
6869
|
+
return;
|
|
6870
|
+
return ocrImageTooSmallMessage(bytes);
|
|
6871
|
+
}
|
|
6723
6872
|
function sourceReadFailure(error) {
|
|
6724
6873
|
if (!(error instanceof Error))
|
|
6725
6874
|
return;
|
|
@@ -6815,11 +6964,13 @@ function removeFile(path) {
|
|
|
6815
6964
|
if (existsSync15(path))
|
|
6816
6965
|
rmSync6(path, { force: true });
|
|
6817
6966
|
}
|
|
6818
|
-
var DEFAULT_MAX_SOURCE_BYTES, MIRROR_CHECKPOINT_EVERY = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.";
|
|
6967
|
+
var DEFAULT_MAX_SOURCE_BYTES, MIRROR_CHECKPOINT_EVERY = 25, PRODUCT_ARTIFACT_FILE_NAMES, SKIP_DIR_NAMES, PARSER_FAILURE_MESSAGE = "The registered parser failed on this file; it was skipped during indexing.", STANDALONE_OCR_IMAGE_EXTENSIONS;
|
|
6819
6968
|
var init_sync = __esm(() => {
|
|
6820
6969
|
init_source_paths();
|
|
6821
6970
|
init_defaults();
|
|
6822
6971
|
init_errors2();
|
|
6972
|
+
init_kordoc();
|
|
6973
|
+
init_ocr_engines();
|
|
6823
6974
|
init_text();
|
|
6824
6975
|
init_index_store();
|
|
6825
6976
|
init_paths();
|
|
@@ -6847,6 +6998,7 @@ var init_sync = __esm(() => {
|
|
|
6847
6998
|
"GrShaderCache",
|
|
6848
6999
|
"ShaderCache"
|
|
6849
7000
|
]);
|
|
7001
|
+
STANDALONE_OCR_IMAGE_EXTENSIONS = new Set(KORDOC_IMAGE_EXTENSIONS);
|
|
6850
7002
|
});
|
|
6851
7003
|
|
|
6852
7004
|
// src/observability/run-log.ts
|
|
@@ -7712,8 +7864,36 @@ class RetrievalMethodRegistry {
|
|
|
7712
7864
|
}
|
|
7713
7865
|
}
|
|
7714
7866
|
|
|
7867
|
+
// src/cloud/config.ts
|
|
7868
|
+
function normalizeBaseUrl(value) {
|
|
7869
|
+
const trimmed = value.trim();
|
|
7870
|
+
let end = trimmed.length;
|
|
7871
|
+
while (end > 0 && trimmed.charCodeAt(end - 1) === 47)
|
|
7872
|
+
end--;
|
|
7873
|
+
return trimmed.slice(0, end);
|
|
7874
|
+
}
|
|
7875
|
+
function resolveBaseUrl(env) {
|
|
7876
|
+
const configured = env[AUTORAG_BASE_URL_ENV];
|
|
7877
|
+
const value = typeof configured === "string" && configured.trim() !== "" ? configured : DEFAULT_BASE_URL;
|
|
7878
|
+
return normalizeBaseUrl(value);
|
|
7879
|
+
}
|
|
7880
|
+
var PROVIDER_ID = "autorag", PROVIDER_NAME = "AutoRAG", DEFAULT_BASE_URL = "https://api.dazziapp.com", AUTORAG_BASE_URL_ENV = "AUTORAG_BASE_URL", AUTORAG_API_KEY_ENV = "AUTORAG_API_KEY", API_PATH = "/v1", MODELS_PATH = "/v1/models", AUTHORIZE_PATH = "/desktop/authorize", TOKEN_PATH = "/api/desktop/token", AUTORAG_INPUT_LIMITS, OAUTH_CLIENT_ID = "agent", API_KEY_CONFIG_VALUE;
|
|
7881
|
+
var init_config = __esm(() => {
|
|
7882
|
+
AUTORAG_INPUT_LIMITS = {
|
|
7883
|
+
maxRequestBytes: 3500000,
|
|
7884
|
+
images: { resize: { maxBytes: 1e6 } }
|
|
7885
|
+
};
|
|
7886
|
+
API_KEY_CONFIG_VALUE = `$${AUTORAG_API_KEY_ENV}`;
|
|
7887
|
+
});
|
|
7888
|
+
|
|
7715
7889
|
// src/retrieval/rerank.ts
|
|
7716
7890
|
import { OpenRouter } from "@openrouter/sdk";
|
|
7891
|
+
function defaultRerankModel(provider) {
|
|
7892
|
+
return provider === PROVIDER_ID ? DEFAULT_AUTORAG_RERANK_MODEL : DEFAULT_RERANK_MODEL;
|
|
7893
|
+
}
|
|
7894
|
+
function defaultRerankApiKeyEnv(provider) {
|
|
7895
|
+
return provider === PROVIDER_ID ? DEFAULT_AUTORAG_RERANK_API_KEY_ENV : DEFAULT_RERANK_API_KEY_ENV;
|
|
7896
|
+
}
|
|
7717
7897
|
function createReranker(config) {
|
|
7718
7898
|
if (config === false || config === undefined)
|
|
7719
7899
|
return;
|
|
@@ -7721,30 +7901,36 @@ function createReranker(config) {
|
|
|
7721
7901
|
if (!SUPPORTED_RERANK_PROVIDERS.includes(provider)) {
|
|
7722
7902
|
throw new Error(`Unsupported rerank provider "${provider}"; supported providers: ${SUPPORTED_RERANK_PROVIDERS.join(", ")}`);
|
|
7723
7903
|
}
|
|
7724
|
-
return new OpenRouterReranker(config);
|
|
7904
|
+
return new OpenRouterReranker({ ...config, provider });
|
|
7905
|
+
}
|
|
7906
|
+
function autoragApiBaseUrl(env) {
|
|
7907
|
+
const root = resolveBaseUrl(env);
|
|
7908
|
+
return root.endsWith(API_PATH) ? root : `${root}${API_PATH}`;
|
|
7725
7909
|
}
|
|
7726
7910
|
|
|
7727
7911
|
class OpenRouterReranker {
|
|
7728
7912
|
model;
|
|
7729
|
-
provider
|
|
7913
|
+
provider;
|
|
7730
7914
|
client;
|
|
7731
7915
|
unavailableReason;
|
|
7732
7916
|
constructor(options = {}) {
|
|
7733
|
-
this.
|
|
7917
|
+
this.provider = options.provider ?? DEFAULT_RERANK_PROVIDER;
|
|
7918
|
+
this.model = options.model ?? defaultRerankModel(this.provider);
|
|
7734
7919
|
if (options.client !== undefined) {
|
|
7735
7920
|
this.client = options.client;
|
|
7736
7921
|
return;
|
|
7737
7922
|
}
|
|
7738
|
-
const apiKeyEnv = options.apiKeyEnv ?? DEFAULT_RERANK_API_KEY_ENV;
|
|
7739
7923
|
const env = options.env ?? process.env;
|
|
7924
|
+
const apiKeyEnv = options.apiKeyEnv ?? defaultRerankApiKeyEnv(this.provider);
|
|
7740
7925
|
const apiKey = options.apiKey ?? env[apiKeyEnv];
|
|
7741
7926
|
if (apiKey === undefined || apiKey.length === 0) {
|
|
7742
7927
|
this.unavailableReason = `no API key: set ${apiKeyEnv} or configure rerank.apiKeyEnv`;
|
|
7743
7928
|
return;
|
|
7744
7929
|
}
|
|
7930
|
+
const baseUrl = options.baseUrl ?? (this.provider === PROVIDER_ID ? autoragApiBaseUrl(env) : undefined);
|
|
7745
7931
|
const sdkOptions = { apiKey };
|
|
7746
|
-
if (
|
|
7747
|
-
sdkOptions.serverURL =
|
|
7932
|
+
if (baseUrl !== undefined)
|
|
7933
|
+
sdkOptions.serverURL = baseUrl;
|
|
7748
7934
|
if (options.timeoutMs !== undefined)
|
|
7749
7935
|
sdkOptions.timeoutMs = options.timeoutMs;
|
|
7750
7936
|
const sdk = new OpenRouter(sdkOptions);
|
|
@@ -7809,9 +7995,10 @@ class OpenRouterReranker {
|
|
|
7809
7995
|
return reranked;
|
|
7810
7996
|
}
|
|
7811
7997
|
}
|
|
7812
|
-
var DEFAULT_RERANK_MODEL = "voyageai/rerank-3-lite", DEFAULT_RERANK_PROVIDER = "openrouter", DEFAULT_RERANK_API_KEY_ENV = "OPENROUTER_API_KEY", DEFAULT_RERANK_TOP_N = 25, SUPPORTED_RERANK_PROVIDERS;
|
|
7998
|
+
var DEFAULT_RERANK_MODEL = "voyageai/rerank-3-lite", DEFAULT_RERANK_PROVIDER = "openrouter", DEFAULT_RERANK_API_KEY_ENV = "OPENROUTER_API_KEY", DEFAULT_AUTORAG_RERANK_MODEL = "qwen/qwen3-reranker-8b", DEFAULT_AUTORAG_RERANK_API_KEY_ENV = "AUTORAG_API_KEY", DEFAULT_RERANK_TOP_N = 25, SUPPORTED_RERANK_PROVIDERS;
|
|
7813
7999
|
var init_rerank = __esm(() => {
|
|
7814
|
-
|
|
8000
|
+
init_config();
|
|
8001
|
+
SUPPORTED_RERANK_PROVIDERS = [DEFAULT_RERANK_PROVIDER, PROVIDER_ID];
|
|
7815
8002
|
});
|
|
7816
8003
|
|
|
7817
8004
|
// src/retrieval/selection.ts
|
|
@@ -9030,22 +9217,219 @@ var init_fsearch_search_tool = __esm(() => {
|
|
|
9030
9217
|
});
|
|
9031
9218
|
});
|
|
9032
9219
|
|
|
9033
|
-
// src/cloud/
|
|
9034
|
-
|
|
9035
|
-
|
|
9036
|
-
|
|
9037
|
-
|
|
9038
|
-
|
|
9039
|
-
|
|
9220
|
+
// src/cloud/jev-backend.ts
|
|
9221
|
+
import {
|
|
9222
|
+
BackendError
|
|
9223
|
+
} from "jev-use";
|
|
9224
|
+
|
|
9225
|
+
class AutoragJevBackend {
|
|
9226
|
+
name = "autorag";
|
|
9227
|
+
apiKey;
|
|
9228
|
+
baseUrl;
|
|
9229
|
+
defaultModel;
|
|
9230
|
+
timeoutMs;
|
|
9231
|
+
maxRetries;
|
|
9232
|
+
fetchImpl;
|
|
9233
|
+
constructor(options = {}) {
|
|
9234
|
+
const env = options.env ?? process.env;
|
|
9235
|
+
const apiKey = options.apiKey ?? env[AUTORAG_API_KEY_ENV];
|
|
9236
|
+
if (apiKey === undefined || apiKey.trim() === "") {
|
|
9237
|
+
throw new Error(`jev.backend "autorag" needs ${AUTORAG_API_KEY_ENV}.`);
|
|
9238
|
+
}
|
|
9239
|
+
this.apiKey = apiKey;
|
|
9240
|
+
this.baseUrl = options.baseUrl ?? resolveBaseUrl(env);
|
|
9241
|
+
this.defaultModel = options.defaultModel ?? AUTORAG_DEFAULT_JEV_MODEL;
|
|
9242
|
+
this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS6;
|
|
9243
|
+
this.maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
|
|
9244
|
+
this.fetchImpl = options.fetchImpl;
|
|
9245
|
+
}
|
|
9246
|
+
async judge(request) {
|
|
9247
|
+
const model = request.model ?? this.defaultModel;
|
|
9248
|
+
const body = toNativeBody(request.state, request.questions, model);
|
|
9249
|
+
const started = Date.now();
|
|
9250
|
+
const response = await this.postJson(`${this.baseUrl}${JEV_PATH}`, body);
|
|
9251
|
+
return {
|
|
9252
|
+
answers: parseNativeAnswers(this.name, response.answers, request.questions),
|
|
9253
|
+
model: response.model ?? model,
|
|
9254
|
+
latencyMs: Date.now() - started,
|
|
9255
|
+
usage: {
|
|
9256
|
+
inputTokens: response.usage?.input_tokens,
|
|
9257
|
+
outputTokens: response.usage?.output_tokens
|
|
9258
|
+
}
|
|
9259
|
+
};
|
|
9260
|
+
}
|
|
9261
|
+
async postJson(url, body) {
|
|
9262
|
+
const doFetch = this.fetchImpl ?? fetch;
|
|
9263
|
+
let lastError;
|
|
9264
|
+
for (let attempt = 0;attempt <= this.maxRetries; attempt += 1) {
|
|
9265
|
+
if (attempt > 0)
|
|
9266
|
+
await sleep3(retryDelay(attempt, lastError));
|
|
9267
|
+
const controller = new AbortController;
|
|
9268
|
+
const timer = setTimeout(() => controller.abort(), this.timeoutMs);
|
|
9269
|
+
try {
|
|
9270
|
+
const response = await doFetch(url, {
|
|
9271
|
+
method: "POST",
|
|
9272
|
+
headers: {
|
|
9273
|
+
"Content-Type": "application/json",
|
|
9274
|
+
Accept: "application/json",
|
|
9275
|
+
Authorization: `Bearer ${this.apiKey}`
|
|
9276
|
+
},
|
|
9277
|
+
body: JSON.stringify(body),
|
|
9278
|
+
signal: controller.signal
|
|
9279
|
+
});
|
|
9280
|
+
if (response.ok)
|
|
9281
|
+
return await response.json();
|
|
9282
|
+
const message = await extractErrorMessage(response);
|
|
9283
|
+
lastError = new BackendError(this.name, `HTTP ${response.status}: ${message}`, response.status);
|
|
9284
|
+
lastError.retryAfterMs = parseRetryAfter(response);
|
|
9285
|
+
if (!isRetryable(response.status) || attempt === this.maxRetries)
|
|
9286
|
+
throw lastError;
|
|
9287
|
+
} catch (error) {
|
|
9288
|
+
if (error instanceof BackendError) {
|
|
9289
|
+
if (attempt === this.maxRetries || !isRetryable(error.status))
|
|
9290
|
+
throw error;
|
|
9291
|
+
continue;
|
|
9292
|
+
}
|
|
9293
|
+
const aborted = error?.name === "AbortError";
|
|
9294
|
+
lastError = new BackendError(this.name, aborted ? `request timed out after ${this.timeoutMs}ms` : `network error: ${String(error)}`, undefined, error);
|
|
9295
|
+
if (attempt === this.maxRetries)
|
|
9296
|
+
throw lastError;
|
|
9297
|
+
} finally {
|
|
9298
|
+
clearTimeout(timer);
|
|
9299
|
+
}
|
|
9300
|
+
}
|
|
9301
|
+
throw lastError ?? new BackendError(this.name, "unreachable");
|
|
9302
|
+
}
|
|
9040
9303
|
}
|
|
9041
|
-
function
|
|
9042
|
-
|
|
9043
|
-
const value = typeof configured === "string" && configured.trim() !== "" ? configured : DEFAULT_BASE_URL;
|
|
9044
|
-
return normalizeBaseUrl(value);
|
|
9304
|
+
function serializeState(state) {
|
|
9305
|
+
return typeof state === "string" ? state : JSON.stringify(state);
|
|
9045
9306
|
}
|
|
9046
|
-
|
|
9047
|
-
|
|
9048
|
-
|
|
9307
|
+
function toNativeQuestion(question) {
|
|
9308
|
+
switch (question.type) {
|
|
9309
|
+
case "noul":
|
|
9310
|
+
return {
|
|
9311
|
+
type: "noul",
|
|
9312
|
+
instructions: question.question,
|
|
9313
|
+
...question.criteria ? { criteria: question.criteria } : {}
|
|
9314
|
+
};
|
|
9315
|
+
case "choice":
|
|
9316
|
+
return {
|
|
9317
|
+
type: "choice",
|
|
9318
|
+
instructions: question.question,
|
|
9319
|
+
criteria: Object.fromEntries(optionEntries(question.options ?? []).map(([label, meaning]) => [label, meaning || label]))
|
|
9320
|
+
};
|
|
9321
|
+
case "score":
|
|
9322
|
+
return {
|
|
9323
|
+
type: "score",
|
|
9324
|
+
instructions: question.question,
|
|
9325
|
+
criteria: question.levels ?? []
|
|
9326
|
+
};
|
|
9327
|
+
}
|
|
9328
|
+
}
|
|
9329
|
+
function toNativeBody(state, questions, model) {
|
|
9330
|
+
return {
|
|
9331
|
+
model,
|
|
9332
|
+
state: serializeState(state),
|
|
9333
|
+
questions: Object.fromEntries(questions.map((question) => [question.id, toNativeQuestion(question)]))
|
|
9334
|
+
};
|
|
9335
|
+
}
|
|
9336
|
+
function parseNativeAnswers(backend, answers, questions) {
|
|
9337
|
+
if (!answers || typeof answers !== "object") {
|
|
9338
|
+
throw new BackendError(backend, "response has no answers object");
|
|
9339
|
+
}
|
|
9340
|
+
const byId = answers;
|
|
9341
|
+
return questions.map((question) => parseNativeAnswer(backend, question, byId[question.id]));
|
|
9342
|
+
}
|
|
9343
|
+
function parseNativeAnswer(backend, question, answer) {
|
|
9344
|
+
if (!answer) {
|
|
9345
|
+
throw new BackendError(backend, `response missing answer for question "${question.id}"`);
|
|
9346
|
+
}
|
|
9347
|
+
switch (question.type) {
|
|
9348
|
+
case "noul":
|
|
9349
|
+
return {
|
|
9350
|
+
answer: numberOrThrow(backend, question.id, answer.noul),
|
|
9351
|
+
confidence: answer.confidence
|
|
9352
|
+
};
|
|
9353
|
+
case "choice":
|
|
9354
|
+
return {
|
|
9355
|
+
answer: stringOrThrow(backend, question.id, answer.choice),
|
|
9356
|
+
confidence: answer.confidence,
|
|
9357
|
+
distribution: answer.probabilities
|
|
9358
|
+
};
|
|
9359
|
+
case "score":
|
|
9360
|
+
return {
|
|
9361
|
+
answer: numberOrThrow(backend, question.id, answer.score),
|
|
9362
|
+
confidence: answer.confidence,
|
|
9363
|
+
distribution: answer.probabilities,
|
|
9364
|
+
legend: answer.legend
|
|
9365
|
+
};
|
|
9366
|
+
}
|
|
9367
|
+
}
|
|
9368
|
+
function optionEntries(options) {
|
|
9369
|
+
return Array.isArray(options) ? options.map((label) => [label, ""]) : Object.entries(options);
|
|
9370
|
+
}
|
|
9371
|
+
function numberOrThrow(backend, id, value) {
|
|
9372
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
9373
|
+
throw new BackendError(backend, `answer "${id}" has no numeric value`);
|
|
9374
|
+
}
|
|
9375
|
+
return value;
|
|
9376
|
+
}
|
|
9377
|
+
function stringOrThrow(backend, id, value) {
|
|
9378
|
+
if (typeof value !== "string" || !value) {
|
|
9379
|
+
throw new BackendError(backend, `answer "${id}" has no choice value`);
|
|
9380
|
+
}
|
|
9381
|
+
return value;
|
|
9382
|
+
}
|
|
9383
|
+
function isRetryable(status) {
|
|
9384
|
+
if (status === undefined)
|
|
9385
|
+
return true;
|
|
9386
|
+
return status === 408 || status === 429 || status >= 500;
|
|
9387
|
+
}
|
|
9388
|
+
function retryDelay(attempt, lastError) {
|
|
9389
|
+
if (lastError?.retryAfterMs !== undefined)
|
|
9390
|
+
return Math.min(lastError.retryAfterMs, 60000);
|
|
9391
|
+
const base = Math.min(RETRY_BASE_MS * 2 ** (attempt - 1), RETRY_MAX_MS);
|
|
9392
|
+
return base * (1 + Math.random() * 0.25);
|
|
9393
|
+
}
|
|
9394
|
+
function parseRetryAfter(response) {
|
|
9395
|
+
const milliseconds = response.headers.get("retry-after-ms");
|
|
9396
|
+
if (milliseconds && Number.isFinite(Number(milliseconds)))
|
|
9397
|
+
return Number(milliseconds);
|
|
9398
|
+
const seconds = response.headers.get("retry-after");
|
|
9399
|
+
if (seconds && Number.isFinite(Number(seconds)))
|
|
9400
|
+
return Number(seconds) * 1000;
|
|
9401
|
+
return;
|
|
9402
|
+
}
|
|
9403
|
+
async function extractErrorMessage(response) {
|
|
9404
|
+
try {
|
|
9405
|
+
const data = await response.json();
|
|
9406
|
+
for (const key of ["error", "message", "detail"]) {
|
|
9407
|
+
const value = data[key];
|
|
9408
|
+
if (typeof value === "string")
|
|
9409
|
+
return value;
|
|
9410
|
+
if (value && typeof value === "object") {
|
|
9411
|
+
const message = value.message;
|
|
9412
|
+
if (typeof message === "string")
|
|
9413
|
+
return message;
|
|
9414
|
+
}
|
|
9415
|
+
if (Array.isArray(value) && value.length > 0) {
|
|
9416
|
+
const first = value[0];
|
|
9417
|
+
if (typeof first?.msg === "string")
|
|
9418
|
+
return first.msg;
|
|
9419
|
+
}
|
|
9420
|
+
}
|
|
9421
|
+
return JSON.stringify(data).slice(0, 300);
|
|
9422
|
+
} catch {
|
|
9423
|
+
return response.statusText || "request failed";
|
|
9424
|
+
}
|
|
9425
|
+
}
|
|
9426
|
+
function sleep3(ms) {
|
|
9427
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
9428
|
+
}
|
|
9429
|
+
var JEV_PATH, AUTORAG_DEFAULT_JEV_MODEL = "jev-1.13", DEFAULT_TIMEOUT_MS6 = 1e4, DEFAULT_MAX_RETRIES = 2, RETRY_BASE_MS = 500, RETRY_MAX_MS = 5000;
|
|
9430
|
+
var init_jev_backend = __esm(() => {
|
|
9431
|
+
init_config();
|
|
9432
|
+
JEV_PATH = `${API_PATH}/jev`;
|
|
9049
9433
|
});
|
|
9050
9434
|
|
|
9051
9435
|
// src/cloud/errors.ts
|
|
@@ -9176,7 +9560,8 @@ function mapServerModel(entry) {
|
|
|
9176
9560
|
cost: readPricing(entry.pricing),
|
|
9177
9561
|
reasoning: entry.reasoning === true,
|
|
9178
9562
|
contextWindow: readPositiveNumber(entry.context_window) ?? DEFAULT_CONTEXT_WINDOW,
|
|
9179
|
-
maxTokens: readPositiveNumber(entry.max_output_tokens) ?? DEFAULT_MAX_TOKENS
|
|
9563
|
+
maxTokens: readPositiveNumber(entry.max_output_tokens) ?? DEFAULT_MAX_TOKENS,
|
|
9564
|
+
inputLimits: AUTORAG_INPUT_LIMITS
|
|
9180
9565
|
};
|
|
9181
9566
|
}
|
|
9182
9567
|
function buildStoreEntry(models, providerId, baseUrl) {
|
|
@@ -9215,7 +9600,8 @@ function modelsFromStore(stored) {
|
|
|
9215
9600
|
cost: entry.cost,
|
|
9216
9601
|
reasoning: entry.reasoning,
|
|
9217
9602
|
contextWindow: entry.contextWindow,
|
|
9218
|
-
maxTokens: entry.maxTokens
|
|
9603
|
+
maxTokens: entry.maxTokens,
|
|
9604
|
+
inputLimits: AUTORAG_INPUT_LIMITS
|
|
9219
9605
|
});
|
|
9220
9606
|
}
|
|
9221
9607
|
return models;
|
|
@@ -9623,69 +10009,456 @@ var init_provider = __esm(() => {
|
|
|
9623
10009
|
init_oauth_login();
|
|
9624
10010
|
});
|
|
9625
10011
|
|
|
9626
|
-
// src/agent/
|
|
9627
|
-
import {
|
|
9628
|
-
|
|
9629
|
-
|
|
9630
|
-
|
|
9631
|
-
|
|
9632
|
-
|
|
9633
|
-
createAgentSessionServices,
|
|
9634
|
-
DefaultResourceLoader,
|
|
9635
|
-
getAgentDir,
|
|
9636
|
-
ModelRuntime,
|
|
9637
|
-
SessionManager,
|
|
9638
|
-
SettingsManager
|
|
9639
|
-
} from "@earendil-works/pi-coding-agent";
|
|
9640
|
-
function resolveInteractiveTuiMode(settingsManager) {
|
|
9641
|
-
const saved = settingsManager.getProjectSettings().tuiMode ?? settingsManager.getGlobalSettings().tuiMode;
|
|
9642
|
-
return saved ?? "regular";
|
|
9643
|
-
}
|
|
9644
|
-
function toToolDefinition(tool) {
|
|
9645
|
-
return {
|
|
9646
|
-
name: tool.name,
|
|
9647
|
-
label: tool.label,
|
|
9648
|
-
description: tool.description,
|
|
9649
|
-
parameters: tool.parameters,
|
|
9650
|
-
...tool.prepareArguments === undefined ? {} : { prepareArguments: tool.prepareArguments },
|
|
9651
|
-
execute: async (toolCallId, params, signal, onUpdate) => {
|
|
9652
|
-
return tool.execute(toolCallId, params, signal, onUpdate);
|
|
9653
|
-
}
|
|
10012
|
+
// src/agent/context-token-guard.ts
|
|
10013
|
+
import { calculateContextTokens, estimateTokens } from "@earendil-works/pi-coding-agent";
|
|
10014
|
+
function createContextTokenGuardedTransform(model, base, fixedOverheadTokens = () => 0) {
|
|
10015
|
+
return async (messages) => {
|
|
10016
|
+
const transformed = base === undefined ? messages : await base(messages);
|
|
10017
|
+
const currentModel = typeof model === "function" ? model() : model;
|
|
10018
|
+
return guardAgentContextMessages(currentModel, transformed, transformed === messages, fixedOverheadTokens());
|
|
9654
10019
|
};
|
|
9655
10020
|
}
|
|
9656
|
-
function
|
|
9657
|
-
|
|
9658
|
-
|
|
9659
|
-
|
|
9660
|
-
|
|
9661
|
-
|
|
9662
|
-
|
|
9663
|
-
|
|
9664
|
-
|
|
9665
|
-
|
|
9666
|
-
|
|
9667
|
-
|
|
9668
|
-
|
|
9669
|
-
|
|
9670
|
-
|
|
9671
|
-
|
|
9672
|
-
|
|
9673
|
-
|
|
9674
|
-
|
|
9675
|
-
|
|
9676
|
-
|
|
10021
|
+
function guardAgentContextMessages(model, messages, useReportedUsage = true, fixedOverheadTokens = 0) {
|
|
10022
|
+
if (model === undefined || !Number.isFinite(model.contextWindow) || model.contextWindow <= 0 || messages.length === 0) {
|
|
10023
|
+
return messages;
|
|
10024
|
+
}
|
|
10025
|
+
const usableContext = Math.floor(model.contextWindow / CONTEXT_HEADROOM);
|
|
10026
|
+
const outputReserve = Math.min(usableContext, Number.isFinite(model.maxTokens) ? Math.max(0, model.maxTokens) : 0);
|
|
10027
|
+
const inputBudget = Math.max(0, usableContext - outputReserve);
|
|
10028
|
+
const budget = Math.max(0, inputBudget - fixedOverheadTokens);
|
|
10029
|
+
const reported = useReportedUsage && findLastAssistantUsage(messages) !== undefined;
|
|
10030
|
+
if (estimateContextMessageTokens(messages, useReportedUsage) <= (reported ? inputBudget : budget)) {
|
|
10031
|
+
return messages;
|
|
10032
|
+
}
|
|
10033
|
+
const candidates = [];
|
|
10034
|
+
const candidateIndexes = [];
|
|
10035
|
+
const base = messages.slice();
|
|
10036
|
+
for (const [index, message] of base.entries()) {
|
|
10037
|
+
const text = textContent(message);
|
|
10038
|
+
if (text === undefined)
|
|
10039
|
+
continue;
|
|
10040
|
+
const section = parseCandidateSection(message, text);
|
|
10041
|
+
if (section === undefined)
|
|
10042
|
+
continue;
|
|
10043
|
+
candidates.push(section);
|
|
10044
|
+
candidateIndexes.push(index);
|
|
10045
|
+
base[index] = replaceTextContent(message, `${section.prefix}${section.suffix}`);
|
|
10046
|
+
}
|
|
10047
|
+
let guarded = messages;
|
|
10048
|
+
if (candidates.length > 0) {
|
|
10049
|
+
const fixedTokens = estimateContextMessageTokens(base, false);
|
|
10050
|
+
const maximumOmitted = candidates.reduce((total, section) => total + section.blocks.length, 0);
|
|
10051
|
+
const omissionMarker = (count) => `
|
|
10052
|
+
|
|
10053
|
+
${GUARD_MARKER} Retrieval candidates omitted: ${count}.
|
|
10054
|
+
`;
|
|
10055
|
+
let remaining = Math.max(0, budget - fixedTokens - Math.ceil(omissionMarker(maximumOmitted).length / 4));
|
|
10056
|
+
const kept = candidates.map(() => []);
|
|
10057
|
+
let omitted = 0;
|
|
10058
|
+
let firstOmitted = -1;
|
|
10059
|
+
for (const [sectionIndex, section] of candidates.entries()) {
|
|
10060
|
+
for (const block of section.blocks) {
|
|
10061
|
+
const tokens = Math.max(1, Math.ceil(block.length / 4));
|
|
10062
|
+
if (tokens <= remaining) {
|
|
10063
|
+
kept[sectionIndex]?.push(block);
|
|
10064
|
+
remaining -= tokens;
|
|
10065
|
+
continue;
|
|
10066
|
+
}
|
|
10067
|
+
omitted += section.blocks.length - (kept[sectionIndex]?.length ?? 0);
|
|
10068
|
+
for (const later of candidates.slice(sectionIndex + 1))
|
|
10069
|
+
omitted += later.blocks.length;
|
|
10070
|
+
firstOmitted = sectionIndex;
|
|
10071
|
+
break;
|
|
9677
10072
|
}
|
|
9678
|
-
|
|
9679
|
-
|
|
10073
|
+
if (firstOmitted !== -1)
|
|
10074
|
+
break;
|
|
10075
|
+
}
|
|
10076
|
+
if (omitted > 0) {
|
|
10077
|
+
const candidateByMessage = new Map;
|
|
10078
|
+
for (const [sectionIndex, messageIndex] of candidateIndexes.entries())
|
|
10079
|
+
candidateByMessage.set(messageIndex, sectionIndex);
|
|
10080
|
+
guarded = base.map((message, index) => {
|
|
10081
|
+
const sectionIndex = candidateByMessage.get(index);
|
|
10082
|
+
if (sectionIndex === undefined)
|
|
10083
|
+
return message;
|
|
10084
|
+
const section = candidates[sectionIndex];
|
|
10085
|
+
const marker = sectionIndex === firstOmitted ? omissionMarker(omitted) : "";
|
|
10086
|
+
return replaceTextContent(message, `${section.prefix}${kept[sectionIndex]?.join("") ?? ""}${marker}${section.suffix}`);
|
|
10087
|
+
});
|
|
10088
|
+
}
|
|
10089
|
+
}
|
|
10090
|
+
if (estimateContextMessageTokens(guarded, false) <= budget && guarded !== messages)
|
|
10091
|
+
return guarded;
|
|
10092
|
+
return trimFixedContext(guarded, budget);
|
|
9680
10093
|
}
|
|
9681
|
-
|
|
9682
|
-
if (
|
|
9683
|
-
const
|
|
9684
|
-
if (
|
|
9685
|
-
|
|
10094
|
+
function estimateContextMessageTokens(messages, useReportedUsage = false) {
|
|
10095
|
+
if (useReportedUsage) {
|
|
10096
|
+
const usage = findLastAssistantUsage(messages);
|
|
10097
|
+
if (usage !== undefined) {
|
|
10098
|
+
return calculateContextTokens(usage.usage) + messages.slice(usage.index + 1).reduce((total, message) => total + estimateMessageTokens(message), 0);
|
|
9686
10099
|
}
|
|
9687
10100
|
}
|
|
9688
|
-
|
|
10101
|
+
return messages.reduce((total, message) => total + estimateMessageTokens(message), 0);
|
|
10102
|
+
}
|
|
10103
|
+
function trimFixedContext(messages, inputBudget) {
|
|
10104
|
+
const trimmed = messages.slice();
|
|
10105
|
+
const lastIndex = trimmed.length - 1;
|
|
10106
|
+
const shrinkable = trimmed.map((message, index) => ({ message, index })).filter(({ index, message }) => index !== lastIndex && message.role !== "system");
|
|
10107
|
+
for (const { message, index } of shrinkable) {
|
|
10108
|
+
const replacement = replaceMessageWithMarker(message);
|
|
10109
|
+
if (replacement === message || estimateMessageTokens(replacement) >= estimateMessageTokens(message))
|
|
10110
|
+
continue;
|
|
10111
|
+
trimmed[index] = replacement;
|
|
10112
|
+
if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
|
|
10113
|
+
return trimmed;
|
|
10114
|
+
}
|
|
10115
|
+
const lastMessage = trimmed[lastIndex];
|
|
10116
|
+
if (lastMessage !== undefined && lastMessage.role !== "system") {
|
|
10117
|
+
const replacement = replaceMessageWithMarker(lastMessage);
|
|
10118
|
+
if (replacement !== lastMessage) {
|
|
10119
|
+
trimmed[lastIndex] = replacement;
|
|
10120
|
+
if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
|
|
10121
|
+
return trimmed;
|
|
10122
|
+
}
|
|
10123
|
+
const fixedWithoutLast = trimmed.reduce((total, message, index) => index === lastIndex ? total : total + estimateMessageTokens(message), 0);
|
|
10124
|
+
if (fixedWithoutLast < inputBudget) {
|
|
10125
|
+
const fitted = fitMessageToTokens(lastMessage, inputBudget - fixedWithoutLast);
|
|
10126
|
+
if (fitted !== lastMessage) {
|
|
10127
|
+
trimmed[lastIndex] = fitted;
|
|
10128
|
+
if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
|
|
10129
|
+
return trimmed;
|
|
10130
|
+
}
|
|
10131
|
+
}
|
|
10132
|
+
}
|
|
10133
|
+
for (const [index, message] of trimmed.entries()) {
|
|
10134
|
+
if (message.role !== "system")
|
|
10135
|
+
continue;
|
|
10136
|
+
const replacement = replaceMessageWithMarker(message);
|
|
10137
|
+
if (replacement === message || estimateMessageTokens(replacement) >= estimateMessageTokens(message))
|
|
10138
|
+
continue;
|
|
10139
|
+
trimmed[index] = replacement;
|
|
10140
|
+
if (estimateContextMessageTokens(trimmed, false) <= inputBudget)
|
|
10141
|
+
return trimmed;
|
|
10142
|
+
}
|
|
10143
|
+
return trimmed;
|
|
10144
|
+
}
|
|
10145
|
+
function fitMessageToTokens(message, maxTokens) {
|
|
10146
|
+
const original = textContent(message);
|
|
10147
|
+
if (original === undefined)
|
|
10148
|
+
return message;
|
|
10149
|
+
const marker = `
|
|
10150
|
+
|
|
10151
|
+
${GUARD_MARKER}
|
|
10152
|
+
`;
|
|
10153
|
+
let low = 0;
|
|
10154
|
+
let high = original.length;
|
|
10155
|
+
let best = replaceTextContent(message, marker);
|
|
10156
|
+
while (low <= high) {
|
|
10157
|
+
const middle = Math.floor((low + high) / 2);
|
|
10158
|
+
const candidate = replaceTextContent(message, excerpt(original, middle, marker));
|
|
10159
|
+
if (estimateMessageTokens(candidate) <= maxTokens) {
|
|
10160
|
+
best = candidate;
|
|
10161
|
+
low = middle + 1;
|
|
10162
|
+
} else {
|
|
10163
|
+
high = middle - 1;
|
|
10164
|
+
}
|
|
10165
|
+
}
|
|
10166
|
+
return best;
|
|
10167
|
+
}
|
|
10168
|
+
function excerpt(text, maxChars, marker) {
|
|
10169
|
+
if (text.length <= maxChars)
|
|
10170
|
+
return text;
|
|
10171
|
+
const available = Math.max(0, maxChars - marker.length);
|
|
10172
|
+
const head = Math.ceil(available * 0.7);
|
|
10173
|
+
const tail = Math.max(0, available - head);
|
|
10174
|
+
return `${text.slice(0, head)}${marker}${tail > 0 ? text.slice(-tail) : ""}`;
|
|
10175
|
+
}
|
|
10176
|
+
function replaceMessageWithMarker(message) {
|
|
10177
|
+
return replaceTextContent(message, `
|
|
10178
|
+
|
|
10179
|
+
${GUARD_MARKER}
|
|
10180
|
+
`);
|
|
10181
|
+
}
|
|
10182
|
+
function estimateMessageTokens(message) {
|
|
10183
|
+
if (message.role === "system")
|
|
10184
|
+
return estimateSystemMessageTokens(message);
|
|
10185
|
+
const estimate = estimateTokens(message);
|
|
10186
|
+
return message.role === "toolResult" ? estimate + Math.ceil(message.toolName.length / 4) : estimate;
|
|
10187
|
+
}
|
|
10188
|
+
function estimateSystemMessageTokens(message) {
|
|
10189
|
+
let chars = typeof message.content === "string" ? message.content.length : 0;
|
|
10190
|
+
if (Array.isArray(message.content)) {
|
|
10191
|
+
for (const block of message.content) {
|
|
10192
|
+
if (block.type === "text")
|
|
10193
|
+
chars += block.text.length;
|
|
10194
|
+
else if (block.type === "image")
|
|
10195
|
+
chars += 4800;
|
|
10196
|
+
}
|
|
10197
|
+
}
|
|
10198
|
+
chars += JSON.stringify(message.sections ?? {}).length;
|
|
10199
|
+
chars += JSON.stringify(message.toolsAdded ?? []).length;
|
|
10200
|
+
chars += JSON.stringify(message.toolsRemoved ?? []).length;
|
|
10201
|
+
return Math.ceil(chars / 4);
|
|
10202
|
+
}
|
|
10203
|
+
function findLastAssistantUsage(messages) {
|
|
10204
|
+
for (let index = messages.length - 1;index >= 0; index--) {
|
|
10205
|
+
const message = messages[index];
|
|
10206
|
+
if (message?.role !== "assistant" || message.stopReason === "aborted" || message.stopReason === "error")
|
|
10207
|
+
continue;
|
|
10208
|
+
if (calculateContextTokens(message.usage) > 0)
|
|
10209
|
+
return { index, usage: message.usage };
|
|
10210
|
+
}
|
|
10211
|
+
return;
|
|
10212
|
+
}
|
|
10213
|
+
function textContent(message) {
|
|
10214
|
+
if (message.role === "system") {
|
|
10215
|
+
if (typeof message.content === "string")
|
|
10216
|
+
return message.content;
|
|
10217
|
+
return message.content.filter((block) => block.type === "text").map((block) => block.text).join("");
|
|
10218
|
+
}
|
|
10219
|
+
if (message.role === "user" || message.role === "toolResult") {
|
|
10220
|
+
if (typeof message.content === "string")
|
|
10221
|
+
return message.content;
|
|
10222
|
+
let text = "";
|
|
10223
|
+
for (const block of message.content) {
|
|
10224
|
+
if (block.type !== "text")
|
|
10225
|
+
return;
|
|
10226
|
+
text += block.text;
|
|
10227
|
+
}
|
|
10228
|
+
return text;
|
|
10229
|
+
}
|
|
10230
|
+
if (message.role === "assistant") {
|
|
10231
|
+
let text = "";
|
|
10232
|
+
for (const block of message.content)
|
|
10233
|
+
if (block.type === "text")
|
|
10234
|
+
text += block.text;
|
|
10235
|
+
return text.length > 0 ? text : undefined;
|
|
10236
|
+
}
|
|
10237
|
+
return;
|
|
10238
|
+
}
|
|
10239
|
+
function replaceTextContent(message, text) {
|
|
10240
|
+
if (message.role === "system")
|
|
10241
|
+
return { ...message, content: text };
|
|
10242
|
+
if (message.role === "user") {
|
|
10243
|
+
if (typeof message.content === "string")
|
|
10244
|
+
return { ...message, content: text };
|
|
10245
|
+
return { ...message, content: [{ type: "text", text }] };
|
|
10246
|
+
}
|
|
10247
|
+
if (message.role === "toolResult")
|
|
10248
|
+
return { ...message, content: [{ type: "text", text }] };
|
|
10249
|
+
if (message.role === "assistant") {
|
|
10250
|
+
const content = [];
|
|
10251
|
+
let replaced = false;
|
|
10252
|
+
for (const block of message.content) {
|
|
10253
|
+
if (block.type !== "text") {
|
|
10254
|
+
content.push(block);
|
|
10255
|
+
continue;
|
|
10256
|
+
}
|
|
10257
|
+
if (replaced)
|
|
10258
|
+
continue;
|
|
10259
|
+
replaced = true;
|
|
10260
|
+
content.push({ ...block, text });
|
|
10261
|
+
}
|
|
10262
|
+
return replaced ? { ...message, content } : message;
|
|
10263
|
+
}
|
|
10264
|
+
return message;
|
|
10265
|
+
}
|
|
10266
|
+
function parseCandidateSection(message, text) {
|
|
10267
|
+
if (message.role === "user") {
|
|
10268
|
+
const markerMatch = /(?:Baseline retrieval (?:evidence \(already gathered for you\)|context):|initial candidates:)/u.exec(text);
|
|
10269
|
+
if (markerMatch === null)
|
|
10270
|
+
return;
|
|
10271
|
+
const sectionStart = (markerMatch.index ?? 0) + markerMatch[0].length;
|
|
10272
|
+
const sectionEnd = findBaselineSectionEnd(text, sectionStart);
|
|
10273
|
+
if (!/^\[e?\d+\] /mu.test(text.slice(sectionStart, sectionEnd)))
|
|
10274
|
+
return;
|
|
10275
|
+
return splitCandidates(text, sectionStart, sectionEnd);
|
|
10276
|
+
}
|
|
10277
|
+
if (message.role !== "toolResult")
|
|
10278
|
+
return;
|
|
10279
|
+
const isRetrieval = RETRIEVAL_TOOL_NAMES[message.toolName] === true || message.toolName.startsWith(RETRIEVAL_TOOL_PREFIX);
|
|
10280
|
+
if (!isRetrieval)
|
|
10281
|
+
return;
|
|
10282
|
+
if (message.toolName === "jikji_find") {
|
|
10283
|
+
const directiveStart = text.indexOf(`
|
|
10284
|
+
|
|
10285
|
+
directive:`);
|
|
10286
|
+
return splitCandidates(text, 0, directiveStart >= 0 ? directiveStart : text.length, /^- [^\n]*(?:\n|$)/gmu);
|
|
10287
|
+
}
|
|
10288
|
+
const diagnosticsStart = text.indexOf(`
|
|
10289
|
+
|
|
10290
|
+
Diagnostics:`);
|
|
10291
|
+
return splitCandidates(text, 0, diagnosticsStart >= 0 ? diagnosticsStart : text.length);
|
|
10292
|
+
}
|
|
10293
|
+
function findBaselineSectionEnd(text, startIndex) {
|
|
10294
|
+
const endMarkers = [
|
|
10295
|
+
`
|
|
10296
|
+
|
|
10297
|
+
Produce the best`,
|
|
10298
|
+
`
|
|
10299
|
+
|
|
10300
|
+
Treat candidates as unverified evidence`,
|
|
10301
|
+
`
|
|
10302
|
+
|
|
10303
|
+
Formatting and content rules`
|
|
10304
|
+
];
|
|
10305
|
+
const positions = endMarkers.map((marker) => text.indexOf(marker, startIndex)).filter((index) => index >= 0);
|
|
10306
|
+
return positions.length > 0 ? Math.min(...positions) : text.length;
|
|
10307
|
+
}
|
|
10308
|
+
function splitCandidates(text, sectionStart, sectionEnd, pattern = /^\[e?\d+\] [^\n]*(?:\n|$)/gmu) {
|
|
10309
|
+
const region = text.slice(sectionStart, sectionEnd);
|
|
10310
|
+
const matches = [...region.matchAll(pattern)];
|
|
10311
|
+
const firstStart = matches[0]?.index;
|
|
10312
|
+
if (firstStart === undefined)
|
|
10313
|
+
return;
|
|
10314
|
+
const blocks = [];
|
|
10315
|
+
for (const [index, match] of matches.entries()) {
|
|
10316
|
+
const start = match.index ?? 0;
|
|
10317
|
+
const end = index + 1 < matches.length ? matches[index + 1]?.index ?? region.length : region.length;
|
|
10318
|
+
blocks.push(region.slice(start, end));
|
|
10319
|
+
}
|
|
10320
|
+
return {
|
|
10321
|
+
prefix: text.slice(0, sectionStart) + region.slice(0, firstStart),
|
|
10322
|
+
suffix: text.slice(sectionEnd),
|
|
10323
|
+
blocks
|
|
10324
|
+
};
|
|
10325
|
+
}
|
|
10326
|
+
var CONTEXT_HEADROOM = 1.2, GUARD_MARKER = "[AutoRAG context guard: omitted context to stay within the model context budget.]", RETRIEVAL_TOOL_PREFIX = "search_datasource_", RETRIEVAL_TOOL_NAMES;
|
|
10327
|
+
var init_context_token_guard = __esm(() => {
|
|
10328
|
+
RETRIEVAL_TOOL_NAMES = {
|
|
10329
|
+
jikji_find: true,
|
|
10330
|
+
search_all_documents: true,
|
|
10331
|
+
semantic_search_local_docs: true,
|
|
10332
|
+
search_single_datasource_documents: true,
|
|
10333
|
+
everything_search: true,
|
|
10334
|
+
fsearch_search: true
|
|
10335
|
+
};
|
|
10336
|
+
});
|
|
10337
|
+
|
|
10338
|
+
// src/agent/request-limits.ts
|
|
10339
|
+
function fitRequestLimits(messages, limits) {
|
|
10340
|
+
const maxBytes = limits?.maxRequestBytes;
|
|
10341
|
+
const maxImages = limits?.images?.maxPerRequest;
|
|
10342
|
+
if (maxBytes === undefined && maxImages === undefined)
|
|
10343
|
+
return;
|
|
10344
|
+
const images = imageLocations(messages);
|
|
10345
|
+
if (images.length === 0)
|
|
10346
|
+
return;
|
|
10347
|
+
let keptImages = images.length;
|
|
10348
|
+
let bytes = maxBytes === undefined ? 0 : Buffer.byteLength(JSON.stringify(messages));
|
|
10349
|
+
const dropped = [];
|
|
10350
|
+
for (const image of images) {
|
|
10351
|
+
const overCount = maxImages !== undefined && keptImages > maxImages;
|
|
10352
|
+
const overBytes = maxBytes !== undefined && bytes > maxBytes;
|
|
10353
|
+
if (!overCount && !overBytes)
|
|
10354
|
+
break;
|
|
10355
|
+
dropped.push(image);
|
|
10356
|
+
keptImages--;
|
|
10357
|
+
bytes -= image.bytes - NOTE_BYTES;
|
|
10358
|
+
}
|
|
10359
|
+
if (dropped.length === 0)
|
|
10360
|
+
return;
|
|
10361
|
+
const byMessage = new Map;
|
|
10362
|
+
for (const { message, block } of dropped) {
|
|
10363
|
+
const blocks = byMessage.get(message) ?? new Set;
|
|
10364
|
+
blocks.add(block);
|
|
10365
|
+
byMessage.set(message, blocks);
|
|
10366
|
+
}
|
|
10367
|
+
return messages.map((message, index) => {
|
|
10368
|
+
const blocks = byMessage.get(index);
|
|
10369
|
+
if (blocks === undefined)
|
|
10370
|
+
return message;
|
|
10371
|
+
const content = message.content.map((part, block) => blocks.has(block) ? { type: "text", text: IMAGE_OMITTED_NOTE } : part);
|
|
10372
|
+
return { ...message, content };
|
|
10373
|
+
});
|
|
10374
|
+
}
|
|
10375
|
+
function imageLocations(messages) {
|
|
10376
|
+
const locations = [];
|
|
10377
|
+
messages.forEach((message, index) => {
|
|
10378
|
+
const content = message.content;
|
|
10379
|
+
if (!Array.isArray(content))
|
|
10380
|
+
return;
|
|
10381
|
+
content.forEach((part, block) => {
|
|
10382
|
+
if (!isImage(part))
|
|
10383
|
+
return;
|
|
10384
|
+
locations.push({ message: index, block, bytes: Buffer.byteLength(JSON.stringify(part)) });
|
|
10385
|
+
});
|
|
10386
|
+
});
|
|
10387
|
+
return locations;
|
|
10388
|
+
}
|
|
10389
|
+
function isImage(part) {
|
|
10390
|
+
return typeof part === "object" && part !== null && part.type === "image";
|
|
10391
|
+
}
|
|
10392
|
+
var IMAGE_OMITTED_NOTE = "[Image omitted from this request to stay within the model provider's request limits. Read the file again if you still need to see it.]", NOTE_BYTES;
|
|
10393
|
+
var init_request_limits = __esm(() => {
|
|
10394
|
+
NOTE_BYTES = Buffer.byteLength(JSON.stringify({ type: "text", text: IMAGE_OMITTED_NOTE }));
|
|
10395
|
+
});
|
|
10396
|
+
|
|
10397
|
+
// src/agent/pi-session.ts
|
|
10398
|
+
import { join as join21 } from "node:path";
|
|
10399
|
+
import { streamSimple as compatStreamSimple } from "@earendil-works/pi-ai/compat";
|
|
10400
|
+
import {
|
|
10401
|
+
createAgentSession,
|
|
10402
|
+
createAgentSessionFromServices,
|
|
10403
|
+
createAgentSessionRuntime,
|
|
10404
|
+
createAgentSessionServices,
|
|
10405
|
+
DefaultResourceLoader,
|
|
10406
|
+
getAgentDir,
|
|
10407
|
+
ModelRuntime,
|
|
10408
|
+
SessionManager,
|
|
10409
|
+
SettingsManager
|
|
10410
|
+
} from "@earendil-works/pi-coding-agent";
|
|
10411
|
+
import { getKeybindings, matchesKey } from "@earendil-works/pi-tui";
|
|
10412
|
+
function resolveInteractiveTuiMode(settingsManager) {
|
|
10413
|
+
const saved = settingsManager.getProjectSettings().tuiMode ?? settingsManager.getGlobalSettings().tuiMode;
|
|
10414
|
+
return saved ?? "regular";
|
|
10415
|
+
}
|
|
10416
|
+
function toToolDefinition(tool) {
|
|
10417
|
+
return {
|
|
10418
|
+
name: tool.name,
|
|
10419
|
+
label: tool.label,
|
|
10420
|
+
description: tool.description,
|
|
10421
|
+
parameters: tool.parameters,
|
|
10422
|
+
...tool.prepareArguments === undefined ? {} : { prepareArguments: tool.prepareArguments },
|
|
10423
|
+
execute: async (toolCallId, params, signal, onUpdate) => {
|
|
10424
|
+
return tool.execute(toolCallId, params, signal, onUpdate);
|
|
10425
|
+
}
|
|
10426
|
+
};
|
|
10427
|
+
}
|
|
10428
|
+
function modelConfig(model, configuredApiKey) {
|
|
10429
|
+
return {
|
|
10430
|
+
name: model.name,
|
|
10431
|
+
baseUrl: model.baseUrl,
|
|
10432
|
+
api: model.api,
|
|
10433
|
+
...configuredApiKey === undefined ? {} : { apiKey: configuredApiKey },
|
|
10434
|
+
authHeader: false,
|
|
10435
|
+
streamSimple: (requestModel, context, options) => compatStreamSimple(requestModel, context, {
|
|
10436
|
+
...options,
|
|
10437
|
+
...configuredApiKey === EMBEDDED_AUTH_MARKER ? { apiKey: undefined } : {}
|
|
10438
|
+
}),
|
|
10439
|
+
models: [
|
|
10440
|
+
{
|
|
10441
|
+
id: model.id,
|
|
10442
|
+
name: model.name,
|
|
10443
|
+
reasoning: model.reasoning,
|
|
10444
|
+
input: [...model.input],
|
|
10445
|
+
cost: model.cost,
|
|
10446
|
+
contextWindow: model.contextWindow,
|
|
10447
|
+
maxTokens: model.maxTokens,
|
|
10448
|
+
...model.inputLimits === undefined ? {} : { inputLimits: model.inputLimits },
|
|
10449
|
+
...model.thinkingLevelMap === undefined ? {} : { thinkingLevelMap: model.thinkingLevelMap }
|
|
10450
|
+
}
|
|
10451
|
+
]
|
|
10452
|
+
};
|
|
10453
|
+
}
|
|
10454
|
+
async function configureModelRuntime(modelRuntime, model, apiKey, providerApiKeys) {
|
|
10455
|
+
if (model !== undefined) {
|
|
10456
|
+
const configuredKey = apiKey ?? providerApiKeys?.[model.provider];
|
|
10457
|
+
if (modelRuntime.getModel(model.provider, model.id) === undefined) {
|
|
10458
|
+
modelRuntime.registerProvider(model.provider, modelConfig(model, configuredKey ?? EMBEDDED_AUTH_MARKER));
|
|
10459
|
+
}
|
|
10460
|
+
}
|
|
10461
|
+
const keys = new Map;
|
|
9689
10462
|
if (model !== undefined && apiKey !== undefined)
|
|
9690
10463
|
keys.set(model.provider, apiKey);
|
|
9691
10464
|
for (const [provider, key] of Object.entries(providerApiKeys ?? {}))
|
|
@@ -9693,20 +10466,47 @@ async function configureModelRuntime(modelRuntime, model, apiKey, providerApiKey
|
|
|
9693
10466
|
for (const [provider, key] of keys)
|
|
9694
10467
|
await modelRuntime.setRuntimeApiKey(provider, key);
|
|
9695
10468
|
}
|
|
10469
|
+
function enforceRequestLimits(pi) {
|
|
10470
|
+
pi.on("context", (event, ctx) => {
|
|
10471
|
+
const messages = fitRequestLimits(event.messages, ctx.model?.inputLimits);
|
|
10472
|
+
return messages === undefined ? undefined : { messages };
|
|
10473
|
+
});
|
|
10474
|
+
}
|
|
10475
|
+
function guardContext(pi, contextTransform) {
|
|
10476
|
+
pi.on("context", async (event, ctx) => {
|
|
10477
|
+
const activeTools = new Set(pi.getActiveTools());
|
|
10478
|
+
const toolDeclarations = pi.getAllTools().filter((tool) => activeTools.has(tool.name)).map(({ name, description, parameters }) => ({ name, description, parameters }));
|
|
10479
|
+
const fixedOverheadTokens = Math.ceil((ctx.getSystemPrompt().length + JSON.stringify(toolDeclarations).length) / 4);
|
|
10480
|
+
return {
|
|
10481
|
+
messages: await createContextTokenGuardedTransform(ctx.model, contextTransform, () => fixedOverheadTokens)(event.messages)
|
|
10482
|
+
};
|
|
10483
|
+
});
|
|
10484
|
+
}
|
|
9696
10485
|
function createAutoRAGExtension(getSystemPrompt, contextTransform) {
|
|
9697
10486
|
return (pi) => {
|
|
9698
10487
|
pi.on("before_agent_start", () => ({ systemPrompt: getSystemPrompt() }));
|
|
9699
|
-
|
|
9700
|
-
|
|
9701
|
-
|
|
10488
|
+
guardContext(pi, contextTransform);
|
|
10489
|
+
enforceRequestLimits(pi);
|
|
10490
|
+
};
|
|
10491
|
+
}
|
|
10492
|
+
function interactiveKeyOf(data) {
|
|
10493
|
+
const bindings = getKeybindings();
|
|
10494
|
+
const keysOf = (action, fallback) => {
|
|
10495
|
+
const keys = bindings.getKeys(action);
|
|
10496
|
+
return keys.length > 0 ? keys : [fallback];
|
|
9702
10497
|
};
|
|
10498
|
+
const windows = process.platform === "win32" || process.platform === "linux" && Boolean(process.env.WSL_DISTRO_NAME || process.env.WSL_INTEROP);
|
|
10499
|
+
if (keysOf("app.interrupt", "escape").some((key) => matchesKey(data, key)))
|
|
10500
|
+
return "interrupt";
|
|
10501
|
+
if (keysOf("app.message.followUp", windows ? "ctrl+q" : "alt+enter").some((key) => matchesKey(data, key)))
|
|
10502
|
+
return "followUp";
|
|
10503
|
+
return;
|
|
9703
10504
|
}
|
|
9704
|
-
function createAutoRAGInteractiveExtension(getSystemPrompt, contextTransform,
|
|
10505
|
+
function createAutoRAGInteractiveExtension(getSystemPrompt, contextTransform, options, updateNotice) {
|
|
9705
10506
|
return (pi) => {
|
|
9706
10507
|
pi.on("before_agent_start", () => ({ systemPrompt: getSystemPrompt() }));
|
|
9707
|
-
|
|
9708
|
-
|
|
9709
|
-
}
|
|
10508
|
+
guardContext(pi, contextTransform);
|
|
10509
|
+
enforceRequestLimits(pi);
|
|
9710
10510
|
if (updateNotice !== undefined) {
|
|
9711
10511
|
pi.on("session_start", () => {
|
|
9712
10512
|
updateNotice().then((text) => {
|
|
@@ -9722,20 +10522,37 @@ function createAutoRAGInteractiveExtension(getSystemPrompt, contextTransform, on
|
|
|
9722
10522
|
});
|
|
9723
10523
|
});
|
|
9724
10524
|
}
|
|
9725
|
-
|
|
9726
|
-
|
|
9727
|
-
|
|
10525
|
+
const { onInput, isSearchRunning, onKey } = options;
|
|
10526
|
+
if (onKey !== undefined) {
|
|
10527
|
+
pi.on("session_start", (_event, ctx) => {
|
|
10528
|
+
ctx.ui.onTerminalInput((data) => {
|
|
10529
|
+
const key = interactiveKeyOf(data);
|
|
10530
|
+
return key !== undefined && onKey(key) ? { consume: true } : undefined;
|
|
10531
|
+
});
|
|
10532
|
+
});
|
|
10533
|
+
}
|
|
10534
|
+
pi.on("input", (event) => {
|
|
10535
|
+
const text = event.text.trim();
|
|
10536
|
+
if (event.source !== "interactive" || text.length === 0)
|
|
10537
|
+
return;
|
|
10538
|
+
if (text.startsWith("/") && isSearchRunning?.() !== true)
|
|
9728
10539
|
return;
|
|
10540
|
+
let disposition;
|
|
9729
10541
|
try {
|
|
9730
|
-
|
|
10542
|
+
disposition = onInput({ text, images: event.images, streamingBehavior: event.streamingBehavior }, pi);
|
|
9731
10543
|
} catch (error) {
|
|
9732
10544
|
pi.sendMessage({
|
|
9733
10545
|
customType: "autorag.error",
|
|
9734
10546
|
content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
|
|
9735
10547
|
display: true
|
|
9736
|
-
});
|
|
9737
|
-
|
|
9738
|
-
|
|
10548
|
+
}, { triggerTurn: false });
|
|
10549
|
+
return { action: "handled" };
|
|
10550
|
+
}
|
|
10551
|
+
return disposition.kind === "steer" ? {
|
|
10552
|
+
action: "transform",
|
|
10553
|
+
text: disposition.text,
|
|
10554
|
+
...event.images === undefined ? {} : { images: event.images }
|
|
10555
|
+
} : { action: "handled" };
|
|
9739
10556
|
});
|
|
9740
10557
|
};
|
|
9741
10558
|
}
|
|
@@ -9815,7 +10632,7 @@ async function createAutoRAGPiInteractiveRuntime(options) {
|
|
|
9815
10632
|
settingsManager,
|
|
9816
10633
|
resourceLoaderOptions: {
|
|
9817
10634
|
extensionFactories: [
|
|
9818
|
-
createAutoRAGInteractiveExtension(options.getSystemPrompt, options.contextTransform, options.
|
|
10635
|
+
createAutoRAGInteractiveExtension(options.getSystemPrompt, options.contextTransform, { onInput: options.onInput, isSearchRunning: options.isSearchRunning, onKey: options.onKey }, options.updateNotice),
|
|
9819
10636
|
...options.extensionFactories ?? []
|
|
9820
10637
|
],
|
|
9821
10638
|
systemPrompt: options.getSystemPrompt(),
|
|
@@ -9850,6 +10667,8 @@ async function createAutoRAGPiInteractiveRuntime(options) {
|
|
|
9850
10667
|
var PI_BUILTIN_TOOL_NAMES, EMBEDDED_AUTH_MARKER = "<autorag-embedded-model>";
|
|
9851
10668
|
var init_pi_session = __esm(() => {
|
|
9852
10669
|
init_provider();
|
|
10670
|
+
init_context_token_guard();
|
|
10671
|
+
init_request_limits();
|
|
9853
10672
|
PI_BUILTIN_TOOL_NAMES = ["read", "bash", "edit", "write", "grep", "find", "ls"];
|
|
9854
10673
|
});
|
|
9855
10674
|
|
|
@@ -9859,12 +10678,26 @@ import { Type as Type6 } from "typebox";
|
|
|
9859
10678
|
function createJevJudge(options = {}) {
|
|
9860
10679
|
let client;
|
|
9861
10680
|
return async (state, questions, callOptions = {}) => {
|
|
9862
|
-
client
|
|
9863
|
-
|
|
9864
|
-
|
|
9865
|
-
|
|
9866
|
-
|
|
9867
|
-
|
|
10681
|
+
if (client === undefined) {
|
|
10682
|
+
const env = options.env ?? process.env;
|
|
10683
|
+
const explicit = options.backend;
|
|
10684
|
+
let backend;
|
|
10685
|
+
if (typeof explicit === "string") {
|
|
10686
|
+
backend = explicit === "autorag" ? new AutoragJevBackend({ env }) : explicit;
|
|
10687
|
+
} else if (explicit !== undefined) {
|
|
10688
|
+
backend = explicit;
|
|
10689
|
+
} else if (env.JEV_BACKEND?.toLowerCase() === "autorag") {
|
|
10690
|
+
backend = new AutoragJevBackend({ env });
|
|
10691
|
+
} else {
|
|
10692
|
+
backend = undefined;
|
|
10693
|
+
}
|
|
10694
|
+
client = new Jev({
|
|
10695
|
+
...backend !== undefined ? { backend } : {},
|
|
10696
|
+
...options.env !== undefined ? { env: options.env } : {},
|
|
10697
|
+
...options.model !== undefined ? { model: options.model } : {},
|
|
10698
|
+
...options.confidenceThreshold !== undefined ? { confidenceThreshold: options.confidenceThreshold } : {}
|
|
10699
|
+
});
|
|
10700
|
+
}
|
|
9868
10701
|
const model = options.model ?? BACKEND_DEFAULT_MODEL[client.backend.name];
|
|
9869
10702
|
return client.judge(state, questions, {
|
|
9870
10703
|
...model !== undefined ? { model } : {},
|
|
@@ -9906,8 +10739,12 @@ function createJevExtension(judgeOrOptions = {}) {
|
|
|
9906
10739
|
}
|
|
9907
10740
|
var JEV_TOOL_NAME = "jev", OPENROUTER_DEFAULT_MODEL = "typesafe/jev-1.13", BACKEND_DEFAULT_MODEL, jevQuestionSchema, jevParameters;
|
|
9908
10741
|
var init_jev_extension = __esm(() => {
|
|
10742
|
+
init_jev_backend();
|
|
9909
10743
|
init_pi_session();
|
|
9910
|
-
BACKEND_DEFAULT_MODEL = {
|
|
10744
|
+
BACKEND_DEFAULT_MODEL = {
|
|
10745
|
+
openrouter: OPENROUTER_DEFAULT_MODEL,
|
|
10746
|
+
autorag: AUTORAG_DEFAULT_JEV_MODEL
|
|
10747
|
+
};
|
|
9911
10748
|
jevQuestionSchema = Type6.Object({
|
|
9912
10749
|
id: Type6.Optional(Type6.String({ description: "Caller-assigned id, echoed back in the verdict" })),
|
|
9913
10750
|
type: Type6.Union([Type6.Literal("noul"), Type6.Literal("choice"), Type6.Literal("score")], {
|
|
@@ -9949,7 +10786,7 @@ function createJikjiFindTool(provider, ledger = new EvidenceLedger) {
|
|
|
9949
10786
|
label: "Jikji Find",
|
|
9950
10787
|
description: "Discover local files via Jikji. Returns answer_paths with per-candidate next_read hints and a tool-call policy directive. Call this FIRST for local file discovery when Jikji is configured.",
|
|
9951
10788
|
parameters: jikjiFindSchema,
|
|
9952
|
-
async execute(_toolCallId, params) {
|
|
10789
|
+
async execute(_toolCallId, params, signal) {
|
|
9953
10790
|
const query = String(params.query ?? "").trim();
|
|
9954
10791
|
if (query.length === 0) {
|
|
9955
10792
|
return {
|
|
@@ -9972,7 +10809,8 @@ function createJikjiFindTool(provider, ledger = new EvidenceLedger) {
|
|
|
9972
10809
|
}
|
|
9973
10810
|
const { answerPack, policy, diagnostics, perRoot } = await provider.findJikji(query, {
|
|
9974
10811
|
topK: params.topK,
|
|
9975
|
-
first: params.first
|
|
10812
|
+
first: params.first,
|
|
10813
|
+
...signal === undefined ? {} : { signal }
|
|
9976
10814
|
});
|
|
9977
10815
|
if (answerPack === undefined) {
|
|
9978
10816
|
const diagMsg = diagnostics.length > 0 ? diagnostics.map((d) => d.message).join(" ") : "Jikji is unavailable or all roots failed.";
|
|
@@ -12286,7 +13124,7 @@ function createSearchAllDocumentsTool(provider, ledger = new EvidenceLedger) {
|
|
|
12286
13124
|
label: "Search All Documents",
|
|
12287
13125
|
description: "Search across all configured retrieval methods (posix, MinSync, datasources) and return every distinct chunk they found, with only pure duplicates removed. This is the exhaustive retrieval surface: one call can return many candidates, including several passages of the same document, so judge the evidence yourself rather than assuming it was pre-filtered. Authority is server-configured; tool arguments can only provide query, topK, and an optional narrowing scope.",
|
|
12288
13126
|
parameters: searchAllSchema,
|
|
12289
|
-
async execute(_toolCallId, params) {
|
|
13127
|
+
async execute(_toolCallId, params, signal) {
|
|
12290
13128
|
const query = params.query.trim();
|
|
12291
13129
|
if (query.length === 0) {
|
|
12292
13130
|
return {
|
|
@@ -12296,7 +13134,8 @@ function createSearchAllDocumentsTool(provider, ledger = new EvidenceLedger) {
|
|
|
12296
13134
|
}
|
|
12297
13135
|
const { results, diagnostics, perMethodCounts } = await provider.searchAllDocuments(query, {
|
|
12298
13136
|
topK: params.topK,
|
|
12299
|
-
scope: params.scope
|
|
13137
|
+
scope: params.scope,
|
|
13138
|
+
...signal === undefined ? {} : { signal }
|
|
12300
13139
|
});
|
|
12301
13140
|
const evidenceIds = results.map((result) => ledger.registerResult(SEARCH_ALL_DOCUMENTS_TOOL_NAME, result));
|
|
12302
13141
|
return {
|
|
@@ -12320,7 +13159,7 @@ Diagnostics: ${diagnostics.map((d) => `${d.source ?? "all"}:${d.code}`).join(",
|
|
|
12320
13159
|
if (results.length === 0)
|
|
12321
13160
|
return `No results.${diagnosticSummary}`;
|
|
12322
13161
|
const rows = results.map((result, index) => {
|
|
12323
|
-
const line = result.content.replace(/\s+/gu, " ")
|
|
13162
|
+
const line = result.content.replace(/\s+/gu, " ");
|
|
12324
13163
|
return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
|
|
12325
13164
|
${line}`;
|
|
12326
13165
|
});
|
|
@@ -12351,7 +13190,7 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
|
|
|
12351
13190
|
label: "Search MinSync Documents",
|
|
12352
13191
|
description: "Search parsed document mirrors with MinSync semantic vector retrieval. Use for conceptual and meaning-based search.",
|
|
12353
13192
|
parameters: searchMinSyncSchema,
|
|
12354
|
-
async execute(_toolCallId, params) {
|
|
13193
|
+
async execute(_toolCallId, params, signal) {
|
|
12355
13194
|
if (params.query.trim().length === 0) {
|
|
12356
13195
|
return {
|
|
12357
13196
|
content: [{ type: "text", text: "MinSync query was empty; no documents searched." }],
|
|
@@ -12361,7 +13200,8 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
|
|
|
12361
13200
|
const scope = resolveScope(params.scope);
|
|
12362
13201
|
const results = await getMethod().retrieve(params.query, {
|
|
12363
13202
|
topK: params.topK,
|
|
12364
|
-
scope
|
|
13203
|
+
scope,
|
|
13204
|
+
...signal === undefined ? {} : { signal }
|
|
12365
13205
|
});
|
|
12366
13206
|
const evidenceIds = results.map((result) => ledger.registerResult(SEARCH_MINSYNC_DOCUMENTS_TOOL_NAME, result));
|
|
12367
13207
|
return {
|
|
@@ -12380,7 +13220,7 @@ function formatResults2(results, evidenceIds) {
|
|
|
12380
13220
|
if (results.length === 0)
|
|
12381
13221
|
return "No MinSync results.";
|
|
12382
13222
|
const rows = results.map((result, index) => {
|
|
12383
|
-
const line = result.content.replace(/\s+/gu, " ")
|
|
13223
|
+
const line = result.content.replace(/\s+/gu, " ");
|
|
12384
13224
|
return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
|
|
12385
13225
|
${line}`;
|
|
12386
13226
|
});
|
|
@@ -12428,7 +13268,7 @@ function createTool(provider, spec, ledger) {
|
|
|
12428
13268
|
label: `Search ${spec.datasourceId}`,
|
|
12429
13269
|
description: `Search only the "${spec.datasourceId}" datasource connection: ${spec.description} ` + `This tool spawns only this connection's CLIs and returns only this connection's hits.${scopeLine} ` + `Authority is server-configured; tool arguments can only provide query, topK, and an optional narrowing scope.`,
|
|
12430
13270
|
parameters: searchSingleDatasourceSchema,
|
|
12431
|
-
async execute(_toolCallId, params) {
|
|
13271
|
+
async execute(_toolCallId, params, signal) {
|
|
12432
13272
|
const query = params.query.trim();
|
|
12433
13273
|
if (query.length === 0) {
|
|
12434
13274
|
return {
|
|
@@ -12444,7 +13284,8 @@ function createTool(provider, spec, ledger) {
|
|
|
12444
13284
|
}
|
|
12445
13285
|
const { results, diagnostics } = await provider.searchSingleDatasourceDocuments(spec.datasourceId, query, {
|
|
12446
13286
|
topK: params.topK,
|
|
12447
|
-
scope: params.scope
|
|
13287
|
+
scope: params.scope,
|
|
13288
|
+
...signal === undefined ? {} : { signal }
|
|
12448
13289
|
});
|
|
12449
13290
|
const evidenceIds = results.map((result) => ledger.registerResult(`search_datasource_${spec.datasourceId}`, result));
|
|
12450
13291
|
return {
|
|
@@ -12468,7 +13309,7 @@ Diagnostics: ${diagnostics.map((d) => `${d.source ?? "datasource"}:${d.code}`).j
|
|
|
12468
13309
|
if (results.length === 0)
|
|
12469
13310
|
return `No datasource results.${diagnosticSummary}`;
|
|
12470
13311
|
const rows = results.map((result, index) => {
|
|
12471
|
-
const line = result.content.replace(/\s+/gu, " ")
|
|
13312
|
+
const line = result.content.replace(/\s+/gu, " ");
|
|
12472
13313
|
return `[${evidenceIds[index]}] ${result.source} score=${result.score.toFixed(4)}
|
|
12473
13314
|
${line}`;
|
|
12474
13315
|
});
|
|
@@ -18031,7 +18872,7 @@ function mergeSources(merged, sources) {
|
|
|
18031
18872
|
existing.source.ageSeconds ??= source.ageSeconds;
|
|
18032
18873
|
}
|
|
18033
18874
|
}
|
|
18034
|
-
function
|
|
18875
|
+
function sleep4(ms, signal) {
|
|
18035
18876
|
if (ms <= 0)
|
|
18036
18877
|
return Promise.resolve();
|
|
18037
18878
|
let resolveFn;
|
|
@@ -18081,9 +18922,9 @@ async function searchPublicWeb(params, deadlines = {}) {
|
|
|
18081
18922
|
failures.push({ provider: { id, label: id }, error });
|
|
18082
18923
|
}
|
|
18083
18924
|
}));
|
|
18084
|
-
await Promise.race([all,
|
|
18925
|
+
await Promise.race([all, sleep4(softMs, signal)]);
|
|
18085
18926
|
if (!responses.some(isUseful) && failures.length < engineIds.length) {
|
|
18086
|
-
await Promise.race([all, firstSuccess,
|
|
18927
|
+
await Promise.race([all, firstSuccess, sleep4(Math.max(0, hardMs - softMs), signal)]);
|
|
18087
18928
|
}
|
|
18088
18929
|
straggler.abort();
|
|
18089
18930
|
const merged = new Map;
|
|
@@ -18499,6 +19340,20 @@ class AutoRAGAgent {
|
|
|
18499
19340
|
lastQuery;
|
|
18500
19341
|
lastSessionId;
|
|
18501
19342
|
activeRun = false;
|
|
19343
|
+
interactive = {
|
|
19344
|
+
run: undefined,
|
|
19345
|
+
followUps: [],
|
|
19346
|
+
steers: [],
|
|
19347
|
+
queuedSteers: [],
|
|
19348
|
+
preSessionSteers: [],
|
|
19349
|
+
acceptingSteers: false,
|
|
19350
|
+
harnessAbort: false,
|
|
19351
|
+
userAborted: false,
|
|
19352
|
+
steerInterrupt: new AbortController,
|
|
19353
|
+
nextSubmitIsFollowUp: false,
|
|
19354
|
+
steerTexts: new Set
|
|
19355
|
+
};
|
|
19356
|
+
activeRunAbort;
|
|
18502
19357
|
modelNativeSearchAuth;
|
|
18503
19358
|
retrievalTrace = [];
|
|
18504
19359
|
evidenceLedger = new EvidenceLedger;
|
|
@@ -18975,9 +19830,9 @@ ${context.text}
|
|
|
18975
19830
|
apiKey: resolved.apiKey,
|
|
18976
19831
|
providerApiKeys: resolved.providerApiKeys,
|
|
18977
19832
|
getSystemPrompt: () => systemPrompt,
|
|
18978
|
-
customTools: this.
|
|
19833
|
+
customTools: this.steerableTools(),
|
|
18979
19834
|
remoteSession: this.remoteSession,
|
|
18980
|
-
contextTransform: (messages) => this.withMemoryContext(messages),
|
|
19835
|
+
contextTransform: (messages) => this.withMemoryContext(frameSteers(messages, this.interactive.steerTexts)),
|
|
18981
19836
|
...this.jevExtension !== undefined ? { extensionFactories: [this.jevExtension], extensionToolNames: [JEV_TOOL_NAME] } : {}
|
|
18982
19837
|
});
|
|
18983
19838
|
const agent = piSession.session.agent;
|
|
@@ -18993,6 +19848,8 @@ ${context.text}
|
|
|
18993
19848
|
const unsubscribers = [...this.listeners].map((listener) => session.agent.subscribe(listener));
|
|
18994
19849
|
unsubscribers.push(session.agent.subscribe((event) => {
|
|
18995
19850
|
this.recordSearchToolEvent(event);
|
|
19851
|
+
if (event.type === "message_start" && event.message.role === "user")
|
|
19852
|
+
this.steersReachedModel();
|
|
18996
19853
|
}));
|
|
18997
19854
|
return unsubscribers;
|
|
18998
19855
|
}
|
|
@@ -19009,6 +19866,7 @@ ${context.text}
|
|
|
19009
19866
|
return;
|
|
19010
19867
|
this.searchToolCallCount += 1;
|
|
19011
19868
|
if (this.searchToolCallCount >= this.maxSearchToolCalls) {
|
|
19869
|
+
this.interactive.harnessAbort = true;
|
|
19012
19870
|
this.activeSession?.abort();
|
|
19013
19871
|
}
|
|
19014
19872
|
const details = event.result.details;
|
|
@@ -19055,9 +19913,11 @@ ${context.text}
|
|
|
19055
19913
|
getSystemPrompt: () => buildSystemPrompt(this.currentSystemPromptConfig({
|
|
19056
19914
|
modelId: this.boundPiRuntime?.session.model?.id ?? resolved?.model.id
|
|
19057
19915
|
})),
|
|
19058
|
-
contextTransform: (messages) => this.withMemoryContext(messages),
|
|
19059
|
-
customTools: this.
|
|
19060
|
-
|
|
19916
|
+
contextTransform: async (messages) => this.withMemoryContext(frameSteers(messages, this.interactive.steerTexts)),
|
|
19917
|
+
customTools: this.steerableTools(),
|
|
19918
|
+
onInput: (input, pi) => this.routeInteractiveInput(input, pi),
|
|
19919
|
+
isSearchRunning: () => this.interactive.run !== undefined,
|
|
19920
|
+
onKey: (key) => this.handleInteractiveKey(key),
|
|
19061
19921
|
...this.jevExtension !== undefined ? { extensionFactories: [this.jevExtension], extensionToolNames: [JEV_TOOL_NAME] } : {},
|
|
19062
19922
|
...this.updateNotice === undefined ? {} : { updateNotice: this.updateNotice }
|
|
19063
19923
|
});
|
|
@@ -19072,18 +19932,154 @@ ${context.text}
|
|
|
19072
19932
|
}
|
|
19073
19933
|
};
|
|
19074
19934
|
}
|
|
19935
|
+
steerableTools() {
|
|
19936
|
+
return this.tools.filter((tool) => !PI_BUILTIN_TOOL_NAMES.includes(tool.name)).map((tool) => this.searchToolNames.has(tool.name) || tool.name === WEB_SEARCH_TOOL_NAME || tool.name === WEB_FETCH_TOOL_NAME ? this.interruptibleBySteer(tool) : tool);
|
|
19937
|
+
}
|
|
19938
|
+
interruptibleBySteer(tool) {
|
|
19939
|
+
return {
|
|
19940
|
+
...tool,
|
|
19941
|
+
execute: async (toolCallId, params, signal, onUpdate) => {
|
|
19942
|
+
const steer = this.interactive.steerInterrupt.signal;
|
|
19943
|
+
if (steer.aborted)
|
|
19944
|
+
return skippedBySteer(tool.name);
|
|
19945
|
+
const combined = signal === undefined ? steer : AbortSignal.any([signal, steer]);
|
|
19946
|
+
let onSteer = () => {};
|
|
19947
|
+
const steered = new Promise((resolve) => {
|
|
19948
|
+
onSteer = () => resolve("steered");
|
|
19949
|
+
steer.addEventListener("abort", onSteer, { once: true });
|
|
19950
|
+
});
|
|
19951
|
+
try {
|
|
19952
|
+
const outcome = await Promise.race([tool.execute(toolCallId, params, combined, onUpdate), steered]);
|
|
19953
|
+
return outcome === "steered" ? skippedBySteer(tool.name) : outcome;
|
|
19954
|
+
} catch (error) {
|
|
19955
|
+
if (steer.aborted && signal?.aborted !== true)
|
|
19956
|
+
return skippedBySteer(tool.name);
|
|
19957
|
+
throw error;
|
|
19958
|
+
} finally {
|
|
19959
|
+
steer.removeEventListener("abort", onSteer);
|
|
19960
|
+
}
|
|
19961
|
+
}
|
|
19962
|
+
};
|
|
19963
|
+
}
|
|
19075
19964
|
deriveAnswer(text) {
|
|
19076
19965
|
const derived = deriveResultsFromAnswer(text, this.evidenceLedger, { allowLocalFiles: !this.remoteSession });
|
|
19077
19966
|
this.routingDiagnostics.push(...derived.diagnostics);
|
|
19078
19967
|
return derived.details;
|
|
19079
19968
|
}
|
|
19969
|
+
routeInteractiveInput(input, pi) {
|
|
19970
|
+
const state = this.interactive;
|
|
19971
|
+
const followUp = input.streamingBehavior === "followUp" || state.nextSubmitIsFollowUp;
|
|
19972
|
+
state.nextSubmitIsFollowUp = false;
|
|
19973
|
+
if (state.run === undefined) {
|
|
19974
|
+
this.startInteractiveQuery(input.text, pi);
|
|
19975
|
+
return { kind: "handled" };
|
|
19976
|
+
}
|
|
19977
|
+
if (followUp) {
|
|
19978
|
+
state.followUps.push(input.text);
|
|
19979
|
+
return { kind: "handled" };
|
|
19980
|
+
}
|
|
19981
|
+
if (input.streamingBehavior === "steer") {
|
|
19982
|
+
state.steers.push(input.text);
|
|
19983
|
+
state.steerTexts.add(input.text);
|
|
19984
|
+
state.steerInterrupt.abort();
|
|
19985
|
+
return { kind: "steer", text: input.text };
|
|
19986
|
+
}
|
|
19987
|
+
if (!this.steer(input.text, input.images))
|
|
19988
|
+
state.followUps.push(input.text);
|
|
19989
|
+
return { kind: "handled" };
|
|
19990
|
+
}
|
|
19991
|
+
handleInteractiveKey(key) {
|
|
19992
|
+
const state = this.interactive;
|
|
19993
|
+
if (state.run === undefined || this.boundPiRuntime?.session.isStreaming === true)
|
|
19994
|
+
return false;
|
|
19995
|
+
if (key === "followUp") {
|
|
19996
|
+
state.nextSubmitIsFollowUp = true;
|
|
19997
|
+
return false;
|
|
19998
|
+
}
|
|
19999
|
+
state.userAborted = true;
|
|
20000
|
+
state.steerInterrupt.abort();
|
|
20001
|
+
return true;
|
|
20002
|
+
}
|
|
20003
|
+
startInteractiveQuery(query, pi) {
|
|
20004
|
+
const state = this.interactive;
|
|
20005
|
+
const run = (async () => {
|
|
20006
|
+
let next = query;
|
|
20007
|
+
while (next !== undefined) {
|
|
20008
|
+
try {
|
|
20009
|
+
await this.runInteractivePiQuery(next, pi);
|
|
20010
|
+
} catch (error) {
|
|
20011
|
+
pi.sendMessage({
|
|
20012
|
+
customType: "autorag.error",
|
|
20013
|
+
content: [{ type: "text", text: error instanceof Error ? error.message : String(error) }],
|
|
20014
|
+
display: true
|
|
20015
|
+
}, { triggerTurn: false });
|
|
20016
|
+
}
|
|
20017
|
+
next = state.followUps.shift();
|
|
20018
|
+
}
|
|
20019
|
+
})().finally(() => {
|
|
20020
|
+
state.run = undefined;
|
|
20021
|
+
});
|
|
20022
|
+
state.run = run;
|
|
20023
|
+
}
|
|
19080
20024
|
async runInteractivePiQuery(query, pi) {
|
|
19081
20025
|
pi.setSessionName(query.slice(0, 80));
|
|
19082
|
-
|
|
20026
|
+
const state = this.interactive;
|
|
20027
|
+
const session = this.boundPiRuntime?.session;
|
|
20028
|
+
const unsubscribe = session?.subscribe((event) => {
|
|
20029
|
+
if (event.type === "queue_update" && state.queuedSteers.length > 0 && !session.agent.hasQueuedMessages()) {
|
|
20030
|
+
state.queuedSteers.length = 0;
|
|
20031
|
+
}
|
|
20032
|
+
if (event.type === "agent_settled" && event.aborted && !state.harnessAbort)
|
|
20033
|
+
state.userAborted = true;
|
|
20034
|
+
});
|
|
20035
|
+
try {
|
|
20036
|
+
for await (const _event of this.searchDocumentsStream(query)) {}
|
|
20037
|
+
} finally {
|
|
20038
|
+
unsubscribe?.();
|
|
20039
|
+
if (state.queuedSteers.length > 0) {
|
|
20040
|
+
session?.agent.clearSteeringQueue();
|
|
20041
|
+
if (!state.userAborted)
|
|
20042
|
+
state.followUps.unshift(...state.queuedSteers);
|
|
20043
|
+
state.queuedSteers.length = 0;
|
|
20044
|
+
}
|
|
20045
|
+
}
|
|
20046
|
+
}
|
|
20047
|
+
async waitForIdle() {
|
|
20048
|
+
while (this.interactive.run !== undefined)
|
|
20049
|
+
await this.interactive.run;
|
|
20050
|
+
}
|
|
20051
|
+
steer(text, images = []) {
|
|
20052
|
+
const state = this.interactive;
|
|
20053
|
+
if (!this.activeRun || !state.acceptingSteers || state.userAborted || text.trim() === "")
|
|
20054
|
+
return false;
|
|
20055
|
+
state.steers.push(text);
|
|
20056
|
+
state.steerTexts.add(text);
|
|
20057
|
+
state.steerInterrupt.abort();
|
|
20058
|
+
const content = [{ type: "text", text }, ...images];
|
|
20059
|
+
const message = { role: "user", content, timestamp: Date.now() };
|
|
20060
|
+
const agent = this.activeSession?.agent;
|
|
20061
|
+
if (agent === undefined)
|
|
20062
|
+
state.preSessionSteers.push(message);
|
|
20063
|
+
else
|
|
20064
|
+
agent.steer(message);
|
|
20065
|
+
state.queuedSteers.push(text);
|
|
20066
|
+
return true;
|
|
19083
20067
|
}
|
|
19084
20068
|
abort() {
|
|
20069
|
+
if (!this.activeRun)
|
|
20070
|
+
return;
|
|
20071
|
+
const state = this.interactive;
|
|
20072
|
+
state.userAborted = true;
|
|
20073
|
+
state.steerInterrupt.abort();
|
|
20074
|
+
this.activeRunAbort?.abort();
|
|
19085
20075
|
this.activeSession?.abort();
|
|
19086
20076
|
}
|
|
20077
|
+
steersReachedModel() {
|
|
20078
|
+
const state = this.interactive;
|
|
20079
|
+
state.queuedSteers.length = 0;
|
|
20080
|
+
if (state.steerInterrupt.signal.aborted && !state.userAborted)
|
|
20081
|
+
state.steerInterrupt = new AbortController;
|
|
20082
|
+
}
|
|
19087
20083
|
startAutoRefresh(intervalMs, options = {}) {
|
|
19088
20084
|
this.stopAutoRefresh();
|
|
19089
20085
|
const tick = () => {
|
|
@@ -19143,6 +20139,15 @@ ${context.text}
|
|
|
19143
20139
|
const publishOnCapture = this.jevJudge === undefined;
|
|
19144
20140
|
let published = false;
|
|
19145
20141
|
const planAbort = new AbortController;
|
|
20142
|
+
this.activeRunAbort = planAbort;
|
|
20143
|
+
const steering = this.interactive;
|
|
20144
|
+
steering.steers.length = 0;
|
|
20145
|
+
steering.queuedSteers.length = 0;
|
|
20146
|
+
steering.preSessionSteers.length = 0;
|
|
20147
|
+
steering.harnessAbort = false;
|
|
20148
|
+
steering.userAborted = false;
|
|
20149
|
+
steering.steerInterrupt = new AbortController;
|
|
20150
|
+
steering.acceptingSteers = true;
|
|
19146
20151
|
const publishPreliminary = (details) => {
|
|
19147
20152
|
if (planAbort.signal.aborted || published)
|
|
19148
20153
|
return;
|
|
@@ -19178,6 +20183,9 @@ ${context.text}
|
|
|
19178
20183
|
session = await this.createSearchSession(resolved, buildSystemPrompt(this.currentSystemPromptConfig({ modelId: resolved.model.id })));
|
|
19179
20184
|
this.activeSession = session;
|
|
19180
20185
|
unsubscribers = this.configureSearchSession(session);
|
|
20186
|
+
for (const message of steering.preSessionSteers)
|
|
20187
|
+
session.agent.steer(message);
|
|
20188
|
+
steering.preSessionSteers.length = 0;
|
|
19181
20189
|
let timeout;
|
|
19182
20190
|
let timedOutAfterFastAnswer = false;
|
|
19183
20191
|
try {
|
|
@@ -19238,25 +20246,42 @@ Warning: the edited config did not validate, so the previous config was restored
|
|
|
19238
20246
|
}
|
|
19239
20247
|
return;
|
|
19240
20248
|
}
|
|
19241
|
-
|
|
20249
|
+
const steered = () => this.interactive.steers.length > 0;
|
|
20250
|
+
const stopped = () => this.interactive.userAborted;
|
|
20251
|
+
if (stopped())
|
|
20252
|
+
return;
|
|
20253
|
+
let directText;
|
|
20254
|
+
if (plan.route === "direct" && !steered()) {
|
|
19242
20255
|
setThinking(this.fastThinkingLevel);
|
|
19243
20256
|
await activeSession.prompt(this.buildDirectAnswerPrompt(trimmedQuery));
|
|
19244
|
-
|
|
19245
|
-
|
|
19246
|
-
|
|
20257
|
+
if (stopped())
|
|
20258
|
+
return;
|
|
20259
|
+
directText = finalAnswerText(transcript());
|
|
20260
|
+
if (!steered()) {
|
|
20261
|
+
if (directText !== undefined)
|
|
20262
|
+
captured = this.deriveAnswer(directText);
|
|
20263
|
+
return;
|
|
20264
|
+
}
|
|
20265
|
+
}
|
|
20266
|
+
const baseline = plan.route === "direct" ? "" : await this.prefetchInitialRetrievalContext(trimmedQuery, plan.queries, options, plan.datasources);
|
|
20267
|
+
if (stopped())
|
|
19247
20268
|
return;
|
|
20269
|
+
let fastText = directText;
|
|
20270
|
+
let preliminary;
|
|
20271
|
+
if (plan.route !== "direct") {
|
|
20272
|
+
setThinking(this.fastThinkingLevel);
|
|
20273
|
+
await activeSession.prompt(this.buildFastAnswerPrompt(trimmedQuery, options, baseline));
|
|
20274
|
+
if (stopped())
|
|
20275
|
+
return;
|
|
20276
|
+
fastText = finalAnswerText(transcript());
|
|
20277
|
+
preliminary = fastText === undefined ? undefined : this.deriveAnswer(fastText);
|
|
19248
20278
|
}
|
|
19249
|
-
const baseline = await this.prefetchInitialRetrievalContext(trimmedQuery, plan.queries, options, plan.datasources);
|
|
19250
|
-
setThinking(this.fastThinkingLevel);
|
|
19251
|
-
await activeSession.prompt(this.buildFastAnswerPrompt(trimmedQuery, options, baseline));
|
|
19252
|
-
const fastText = finalAnswerText(transcript());
|
|
19253
|
-
const preliminary = fastText === undefined ? undefined : this.deriveAnswer(fastText);
|
|
19254
20279
|
if (preliminary !== undefined) {
|
|
19255
20280
|
emitPreliminary(preliminary);
|
|
19256
|
-
const followUp = await this.shouldFollowUp(trimmedQuery, preliminary);
|
|
19257
|
-
if (planAbort.signal.aborted)
|
|
20281
|
+
const followUp = steered() || await this.shouldFollowUp(trimmedQuery, preliminary);
|
|
20282
|
+
if (planAbort.signal.aborted || stopped())
|
|
19258
20283
|
return;
|
|
19259
|
-
if (!followUp) {
|
|
20284
|
+
if (!followUp && !steered()) {
|
|
19260
20285
|
captured = preliminary;
|
|
19261
20286
|
return;
|
|
19262
20287
|
}
|
|
@@ -19264,14 +20289,23 @@ Warning: the edited config did not validate, so the previous config was restored
|
|
|
19264
20289
|
}
|
|
19265
20290
|
setThinking(this.finalThinkingLevel);
|
|
19266
20291
|
const fastAnswerDelivered = preliminary !== undefined && this.preliminaryCallback !== undefined;
|
|
19267
|
-
await activeSession.prompt(this.buildRefinementPrompt(trimmedQuery, options, fastText, fastAnswerDelivered));
|
|
20292
|
+
await activeSession.prompt(this.buildRefinementPrompt(trimmedQuery, options, fastText, fastAnswerDelivered, this.interactive.steers));
|
|
20293
|
+
while (!stopped() && !planAbort.signal.aborted && this.interactive.queuedSteers.length > 0) {
|
|
20294
|
+
await activeSession.prompt(buildSteerContinuationPrompt(this.interactive.steers));
|
|
20295
|
+
}
|
|
20296
|
+
steering.acceptingSteers = false;
|
|
20297
|
+
if (stopped())
|
|
20298
|
+
return;
|
|
19268
20299
|
const finalText = finalAnswerText(transcript());
|
|
19269
20300
|
if (finalText !== undefined)
|
|
19270
20301
|
captured = this.deriveAnswer(finalText);
|
|
19271
|
-
})()
|
|
20302
|
+
})().finally(() => {
|
|
20303
|
+
steering.acceptingSteers = false;
|
|
20304
|
+
}),
|
|
19272
20305
|
new Promise((resolve, reject) => {
|
|
19273
20306
|
timeout = setTimeout(() => {
|
|
19274
20307
|
planAbort.abort();
|
|
20308
|
+
this.interactive.harnessAbort = true;
|
|
19275
20309
|
Promise.resolve(session?.abort());
|
|
19276
20310
|
if (fastCaptured !== undefined && !this.remoteSession) {
|
|
19277
20311
|
timedOutAfterFastAnswer = true;
|
|
@@ -19718,12 +20752,13 @@ Warning: the edited config did not validate, so the previous config was restored
|
|
|
19718
20752
|
return selection.selected;
|
|
19719
20753
|
}
|
|
19720
20754
|
async prefetchInitialRetrievalContext(query, searchQueries, options, datasources = []) {
|
|
19721
|
-
const
|
|
20755
|
+
const signal = this.interactive.steerInterrupt.signal;
|
|
20756
|
+
const retrieveOptions = { topK: this.limits.prefetch.minSyncTopK, scope: options.scope, signal };
|
|
19722
20757
|
const formatSearchQueries = (list) => `Search queries (decomposed from the original question):
|
|
19723
20758
|
${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
|
|
19724
20759
|
`)}`;
|
|
19725
20760
|
const perQuery = await Promise.all(searchQueries.map((searchQuery) => Promise.all([
|
|
19726
|
-
this.jikjiClient === undefined ? Promise.resolve(undefined) : this.findJikji(searchQuery, { topK: this.limits.prefetch.jikjiTopK }).catch(() => {
|
|
20761
|
+
this.jikjiClient === undefined ? Promise.resolve(undefined) : this.findJikji(searchQuery, { topK: this.limits.prefetch.jikjiTopK, signal }).catch(() => {
|
|
19727
20762
|
return;
|
|
19728
20763
|
}),
|
|
19729
20764
|
this.minSyncMethod.retrieve(searchQuery, retrieveOptions).catch((error) => {
|
|
@@ -19733,7 +20768,8 @@ ${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
|
|
|
19733
20768
|
}),
|
|
19734
20769
|
Promise.all(datasources.map((datasourceId) => this.searchSingleDatasourceDocuments(datasourceId, searchQuery, {
|
|
19735
20770
|
topK: this.limits.singleDatasourceTopK,
|
|
19736
|
-
scope: options.scope
|
|
20771
|
+
scope: options.scope,
|
|
20772
|
+
signal
|
|
19737
20773
|
}).then((outcome) => outcome.results).catch(() => [])))
|
|
19738
20774
|
])));
|
|
19739
20775
|
for (const [index, [jikji, vector, perDatasource]] of perQuery.entries()) {
|
|
@@ -19783,7 +20819,7 @@ ${list.map((entry, index) => `[${index + 1}] ${entry}`).join(`
|
|
|
19783
20819
|
options.observedSources?.add(result.source);
|
|
19784
20820
|
for (const result of datasourceResults)
|
|
19785
20821
|
options.observedSources?.add(result.source);
|
|
19786
|
-
const reranked = await this.rerankPrefetchPool(query, jikjiPaths, minSyncResults, datasourceResults);
|
|
20822
|
+
const reranked = await this.rerankPrefetchPool(query, jikjiPaths, minSyncResults, datasourceResults, signal);
|
|
19787
20823
|
if (reranked !== undefined) {
|
|
19788
20824
|
const baseline = formatRerankedBaseline(reranked, reranked.map((result) => this.evidenceLedger.registerResult("baseline", result)));
|
|
19789
20825
|
return searchQueries.length > 1 ? `${formatSearchQueries(searchQueries)}
|
|
@@ -19815,7 +20851,7 @@ ${result.content.replace(/\s+/gu, " ")}`).join(`
|
|
|
19815
20851
|
|
|
19816
20852
|
`);
|
|
19817
20853
|
}
|
|
19818
|
-
async rerankPrefetchPool(query, answerPaths, minSyncResults, datasourceResults) {
|
|
20854
|
+
async rerankPrefetchPool(query, answerPaths, minSyncResults, datasourceResults, signal) {
|
|
19819
20855
|
const reranker = this.reranker;
|
|
19820
20856
|
if (reranker === undefined)
|
|
19821
20857
|
return;
|
|
@@ -19833,7 +20869,7 @@ ${result.content.replace(/\s+/gu, " ")}`).join(`
|
|
|
19833
20869
|
if (!reranker.describe().available)
|
|
19834
20870
|
return;
|
|
19835
20871
|
try {
|
|
19836
|
-
const reranked = await reranker.rerank(query, candidates, { topN: this.rerankTopN });
|
|
20872
|
+
const reranked = await reranker.rerank(query, candidates, { topN: this.rerankTopN, signal });
|
|
19837
20873
|
return reranked.length > 0 ? reranked : undefined;
|
|
19838
20874
|
} catch {
|
|
19839
20875
|
return;
|
|
@@ -19876,7 +20912,7 @@ ${baseline}
|
|
|
19876
20912
|
const fetch2 = names.has(WEB_FETCH_TOOL_NAME) ? ` (and ${WEB_FETCH_TOOL_NAME} to read a promising page)` : "";
|
|
19877
20913
|
return `When local evidence is missing, thin, or possibly outdated — or the question concerns current events, recent releases, prices, or other public facts — use ${WEB_SEARCH_TOOL_NAME}${fetch2} to confirm or correct claims and fill gaps; use URLs as result sources for web-derived evidence. `;
|
|
19878
20914
|
}
|
|
19879
|
-
buildRefinementPrompt(query, options, fastAnswer, fastAnswerDelivered) {
|
|
20915
|
+
buildRefinementPrompt(query, options, fastAnswer, fastAnswerDelivered, steers = []) {
|
|
19880
20916
|
const limit = typeof options.topK === "number" ? ` Cite at most ${options.topK} sources.` : "";
|
|
19881
20917
|
const scope = options.scope ? ` Restrict search to virtual path scope ${options.scope}.` : "";
|
|
19882
20918
|
const firstAnswer = formatFirstAnswerContext(fastAnswer, fastAnswerDelivered);
|
|
@@ -19899,7 +20935,7 @@ ${baseline}
|
|
|
19899
20935
|
|
|
19900
20936
|
` + `${firstAnswer}
|
|
19901
20937
|
|
|
19902
|
-
` + `Now verify it rigorously. ${this.discoveryHint((tools) => `Actively use ${tools} when discovering or exploring local files and folders. `)}Check important claims against source files with bash when needed, correct anything wrong or unsupported, fill gaps with retrieval tools, and resolve conflicts and freshness. ${this.webFallbackHint()}
|
|
20938
|
+
` + formatSteerContext(steers) + `Now verify it rigorously. ${this.discoveryHint((tools) => `Actively use ${tools} when discovering or exploring local files and folders. `)}Check important claims against source files with bash when needed, correct anything wrong or unsupported, fill gaps with retrieval tools, and resolve conflicts and freshness. ${this.webFallbackHint()}
|
|
19903
20939
|
|
|
19904
20940
|
` + `${answerRules}
|
|
19905
20941
|
|
|
@@ -20302,7 +21338,8 @@ ${initialRetrievalContext ?? "Pending; continue only with a brief progress state
|
|
|
20302
21338
|
const sourceRoots = planJikjiSourceRoots(this.searchPaths);
|
|
20303
21339
|
const findOpts = {
|
|
20304
21340
|
topK: opts?.topK,
|
|
20305
|
-
first: opts?.first
|
|
21341
|
+
first: opts?.first,
|
|
21342
|
+
...opts?.signal === undefined ? {} : { signal: opts.signal }
|
|
20306
21343
|
};
|
|
20307
21344
|
const diagnostics = [];
|
|
20308
21345
|
const okPacks = [];
|
|
@@ -20453,13 +21490,18 @@ ${initialRetrievalContext ?? "Pending; continue only with a brief progress state
|
|
|
20453
21490
|
return { results, diagnostics };
|
|
20454
21491
|
}
|
|
20455
21492
|
async searchAllDocuments(query, options = {}) {
|
|
20456
|
-
return this.retrieveWithDiagnostics(query, {
|
|
21493
|
+
return this.retrieveWithDiagnostics(query, {
|
|
21494
|
+
topK: options.topK,
|
|
21495
|
+
scope: options.scope,
|
|
21496
|
+
...options.signal === undefined ? {} : { signal: options.signal }
|
|
21497
|
+
});
|
|
20457
21498
|
}
|
|
20458
21499
|
async searchSingleDatasourceDocuments(datasourceId, query, options = {}) {
|
|
20459
21500
|
const retrievalOptions = {
|
|
20460
21501
|
...this.activeRetrievalOptions,
|
|
20461
21502
|
topK: options.topK,
|
|
20462
|
-
scope: options.scope
|
|
21503
|
+
scope: options.scope,
|
|
21504
|
+
...options.signal === undefined ? {} : { signal: options.signal }
|
|
20463
21505
|
};
|
|
20464
21506
|
const methods = this.methodRegistry.list().filter((method) => {
|
|
20465
21507
|
const descriptor = method.describe();
|
|
@@ -20562,6 +21604,49 @@ ${result.content.replace(/\s+/gu, " ")}`);
|
|
|
20562
21604
|
${lines.join(`
|
|
20563
21605
|
`)}`;
|
|
20564
21606
|
}
|
|
21607
|
+
function skippedBySteer(toolName) {
|
|
21608
|
+
return {
|
|
21609
|
+
content: [
|
|
21610
|
+
{
|
|
21611
|
+
type: "text",
|
|
21612
|
+
text: `Skipped due to the user's steering message. ${toolName} was stopped before it returned; do not count it as completed work. Apply the steering message, then retry the search if it is still needed.`
|
|
21613
|
+
}
|
|
21614
|
+
],
|
|
21615
|
+
details: { method: toolName, skipped: "steer" }
|
|
21616
|
+
};
|
|
21617
|
+
}
|
|
21618
|
+
function buildSteerMessage(text) {
|
|
21619
|
+
return `The user interrupted the running search with this correction:
|
|
21620
|
+
|
|
21621
|
+
${text}
|
|
21622
|
+
|
|
21623
|
+
` + `Treat it as direction for the search in progress, not as a new question: adjust which sources, queries, or claims you pursue, then continue. ` + `When finished, reply with the final answer to the original question, updated for this correction, and no tool call.`;
|
|
21624
|
+
}
|
|
21625
|
+
function frameSteers(messages, steerTexts) {
|
|
21626
|
+
if (steerTexts.size === 0)
|
|
21627
|
+
return messages;
|
|
21628
|
+
return messages.map((message) => {
|
|
21629
|
+
if (message.role !== "user" || typeof message.content === "string")
|
|
21630
|
+
return message;
|
|
21631
|
+
const [first, ...rest] = message.content;
|
|
21632
|
+
if (first?.type !== "text" || !steerTexts.has(first.text))
|
|
21633
|
+
return message;
|
|
21634
|
+
return { ...message, content: [{ ...first, text: buildSteerMessage(first.text) }, ...rest] };
|
|
21635
|
+
});
|
|
21636
|
+
}
|
|
21637
|
+
function formatSteerContext(steers) {
|
|
21638
|
+
if (steers.length === 0)
|
|
21639
|
+
return "";
|
|
21640
|
+
return `While the search ran, the user corrected it:
|
|
21641
|
+
${steers.map((steer) => `- ${steer}`).join(`
|
|
21642
|
+
`)}
|
|
21643
|
+
` + `The first answer predates these corrections; follow them over it.
|
|
21644
|
+
|
|
21645
|
+
`;
|
|
21646
|
+
}
|
|
21647
|
+
function buildSteerContinuationPrompt(steers) {
|
|
21648
|
+
return `${formatSteerContext(steers)}Apply the latest correction: search or verify further if it needs that, ` + `then reply with the complete final answer to the original question and no tool call.`;
|
|
21649
|
+
}
|
|
20565
21650
|
function formatFirstAnswerContext(fastAnswer, delivered) {
|
|
20566
21651
|
if (fastAnswer === undefined) {
|
|
20567
21652
|
return "(the fast phase produced no answer — write the complete answer)";
|
|
@@ -20970,23 +22055,175 @@ ${this.description}`
|
|
|
20970
22055
|
}
|
|
20971
22056
|
}
|
|
20972
22057
|
|
|
20973
|
-
// src/datasource/
|
|
20974
|
-
import {
|
|
20975
|
-
|
|
20976
|
-
|
|
20977
|
-
|
|
20978
|
-
|
|
20979
|
-
|
|
20980
|
-
|
|
20981
|
-
|
|
20982
|
-
|
|
20983
|
-
|
|
20984
|
-
|
|
20985
|
-
|
|
20986
|
-
|
|
20987
|
-
|
|
20988
|
-
|
|
20989
|
-
|
|
22058
|
+
// src/datasource/connector.ts
|
|
22059
|
+
import { createHash as createHash12 } from "node:crypto";
|
|
22060
|
+
function connectorFailureToDiagnosticCode(reason) {
|
|
22061
|
+
switch (reason) {
|
|
22062
|
+
case "auth":
|
|
22063
|
+
return "datasource-auth-error";
|
|
22064
|
+
case "permission":
|
|
22065
|
+
return "datasource-permission-denied";
|
|
22066
|
+
case "rate-limited":
|
|
22067
|
+
return "datasource-rate-limited";
|
|
22068
|
+
case "unavailable":
|
|
22069
|
+
case "not-configured":
|
|
22070
|
+
return "datasource-unavailable";
|
|
22071
|
+
case "empty":
|
|
22072
|
+
return "datasource-empty";
|
|
22073
|
+
case "api-error":
|
|
22074
|
+
case "invalid-data":
|
|
22075
|
+
return "datasource-index-failed";
|
|
22076
|
+
}
|
|
22077
|
+
}
|
|
22078
|
+
function boundDiagnosticText(value) {
|
|
22079
|
+
const trimmed = value.trim();
|
|
22080
|
+
return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
|
|
22081
|
+
}
|
|
22082
|
+
function sanitizeIdSegment(value) {
|
|
22083
|
+
const normalized = value.replace(/[^\p{L}\p{N}._-]+/gu, "-");
|
|
22084
|
+
let start = 0;
|
|
22085
|
+
while (start < normalized.length && (normalized[start] === "-" || normalized[start] === "."))
|
|
22086
|
+
start += 1;
|
|
22087
|
+
let end = normalized.length;
|
|
22088
|
+
while (end > start && (normalized[end - 1] === "-" || normalized[end - 1] === "."))
|
|
22089
|
+
end -= 1;
|
|
22090
|
+
const cleaned = normalized.slice(start, end);
|
|
22091
|
+
if (cleaned === value && cleaned.length > 0 && cleaned.length <= 80)
|
|
22092
|
+
return cleaned;
|
|
22093
|
+
const hash = createHash12("sha256").update(value).digest("hex").slice(0, 8);
|
|
22094
|
+
const stem = cleaned.slice(0, 60);
|
|
22095
|
+
return stem.length > 0 ? `${stem}-${hash}` : hash;
|
|
22096
|
+
}
|
|
22097
|
+
var init_connector = () => {};
|
|
22098
|
+
|
|
22099
|
+
// src/datasource/sidecar/config.ts
|
|
22100
|
+
import { readFileSync as readFileSync19 } from "node:fs";
|
|
22101
|
+
import { homedir as homedir3 } from "node:os";
|
|
22102
|
+
import { dirname as dirname17, isAbsolute as isAbsolute8, join as join30, resolve as resolve14, sep as sep7 } from "node:path";
|
|
22103
|
+
import { parse as parseToml2 } from "smol-toml";
|
|
22104
|
+
function resolveSidecarOptions(input) {
|
|
22105
|
+
const { connector, workspaceRoot } = input;
|
|
22106
|
+
const setting = connector?.sidecar;
|
|
22107
|
+
if (setting === false || workspaceRoot === undefined)
|
|
22108
|
+
return;
|
|
22109
|
+
const object = typeof setting === "object" && setting !== null && !Array.isArray(setting) ? setting : {};
|
|
22110
|
+
if (object.enabled === false)
|
|
22111
|
+
return;
|
|
22112
|
+
const configured = [object.crawlerDbPath, connector?.databasePath].find((value) => typeof value === "string" && value.length > 0);
|
|
22113
|
+
const crawlerDbPath = expandHome(configured ?? archivePathFromCrawlerConfig(input) ?? input.defaultCrawlerDb);
|
|
22114
|
+
const workspaceId = typeof connector?.workspace === "string" && connector.workspace.length > 0 ? connector.workspace : undefined;
|
|
22115
|
+
return {
|
|
22116
|
+
descriptor: input.descriptor,
|
|
22117
|
+
crawlerDbPath,
|
|
22118
|
+
sidecarDbPath: join30(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(input.registrationName), sanitizeIdSegment(input.instanceId), "sidecar.sqlite"),
|
|
22119
|
+
...workspaceId === undefined ? {} : { workspaceId },
|
|
22120
|
+
...object.semantic === false ? {} : { embedder: createGatewayEmbedder() }
|
|
22121
|
+
};
|
|
22122
|
+
}
|
|
22123
|
+
function archivePathFromCrawlerConfig(input) {
|
|
22124
|
+
const configPath = input.connector?.configPath;
|
|
22125
|
+
if (input.descriptor.archivePathInConfig !== true || typeof configPath !== "string" || configPath.length === 0) {
|
|
22126
|
+
return;
|
|
22127
|
+
}
|
|
22128
|
+
const file = expandHome(configPath);
|
|
22129
|
+
try {
|
|
22130
|
+
const dbPath = parseToml2(readFileSync19(file, "utf8")).db_path;
|
|
22131
|
+
if (typeof dbPath !== "string" || dbPath.length === 0)
|
|
22132
|
+
return;
|
|
22133
|
+
const expanded = expandHome(dbPath);
|
|
22134
|
+
return isAbsolute8(expanded) ? expanded : resolve14(dirname17(file), expanded);
|
|
22135
|
+
} catch {
|
|
22136
|
+
return;
|
|
22137
|
+
}
|
|
22138
|
+
}
|
|
22139
|
+
function expandHome(path) {
|
|
22140
|
+
if (path === "~")
|
|
22141
|
+
return homedir3();
|
|
22142
|
+
const prefixed = path.startsWith("~/") || sep7 === "\\" && path.startsWith("~\\");
|
|
22143
|
+
return prefixed ? join30(homedir3(), path.slice(2)) : path;
|
|
22144
|
+
}
|
|
22145
|
+
var init_config2 = __esm(() => {
|
|
22146
|
+
init_connector();
|
|
22147
|
+
});
|
|
22148
|
+
|
|
22149
|
+
// src/datasource/sidecar/descriptors.ts
|
|
22150
|
+
function chatCrawler(id) {
|
|
22151
|
+
return {
|
|
22152
|
+
id,
|
|
22153
|
+
keyExpression: "m.event_id",
|
|
22154
|
+
chunkId: (key) => key,
|
|
22155
|
+
fromClause: "messages m",
|
|
22156
|
+
liveWhere: "(m.deleted_at is null or m.deleted_at = 0)",
|
|
22157
|
+
textExpression: "coalesce(m.text, '')",
|
|
22158
|
+
titleExpression: "m.chat_name",
|
|
22159
|
+
epochSecondsExpression: "m.ts",
|
|
22160
|
+
fts: { table: "messages_fts", joinOn: "messages_fts.rowid = m.rowid" },
|
|
22161
|
+
metadata: [
|
|
22162
|
+
{ sql: "m.chat_jid", key: "chatId" },
|
|
22163
|
+
{ sql: "m.chat_name", key: "chatName" },
|
|
22164
|
+
{ sql: "m.sender_name", key: "senderName" },
|
|
22165
|
+
{ sql: "m.ts", key: "timestamp" }
|
|
22166
|
+
],
|
|
22167
|
+
channelIdExpression: "m.chat_jid",
|
|
22168
|
+
channelNameExpression: "m.chat_name",
|
|
22169
|
+
hierarchy: (metadata) => ["chats", str(metadata.chatName) ?? str(metadata.chatId) ?? "unknown"]
|
|
22170
|
+
};
|
|
22171
|
+
}
|
|
22172
|
+
var str = (value) => typeof value === "string" && value.length > 0 ? value : undefined, slack, CRAWLER_DESCRIPTORS;
|
|
22173
|
+
var init_descriptors = __esm(() => {
|
|
22174
|
+
slack = {
|
|
22175
|
+
id: "slack",
|
|
22176
|
+
archivePathInConfig: true,
|
|
22177
|
+
keyExpression: "m.channel_id || '|' || m.ts",
|
|
22178
|
+
chunkId: (_key, metadata) => `${metadata.channelId ?? ""}-${(metadata.timestamp ?? "").replaceAll(".", "-")}`,
|
|
22179
|
+
fromClause: "messages m left join channels c on c.id = m.channel_id and c.workspace_id = m.workspace_id " + "left join users u on u.id = m.user_id",
|
|
22180
|
+
liveWhere: "(m.deleted_ts is null or m.deleted_ts = '')",
|
|
22181
|
+
scopeExpression: "m.workspace_id",
|
|
22182
|
+
textExpression: "coalesce(nullif(m.normalized_text, ''), m.text)",
|
|
22183
|
+
titleExpression: "c.name",
|
|
22184
|
+
epochSecondsExpression: "cast(m.ts as real)",
|
|
22185
|
+
fts: { table: "message_fts", joinOn: "message_fts.message_key = m.channel_id || '|' || m.ts" },
|
|
22186
|
+
metadata: [
|
|
22187
|
+
{ sql: "m.channel_id", key: "channelId" },
|
|
22188
|
+
{ sql: "c.name", key: "channelName" },
|
|
22189
|
+
{ sql: "m.workspace_id", key: "workspaceId" },
|
|
22190
|
+
{ sql: "m.user_id", key: "userId" },
|
|
22191
|
+
{ sql: "coalesce(nullif(u.real_name, ''), u.name)", key: "senderName" },
|
|
22192
|
+
{ sql: "m.ts", key: "timestamp" }
|
|
22193
|
+
],
|
|
22194
|
+
channelIdExpression: "m.channel_id",
|
|
22195
|
+
channelNameExpression: "c.name",
|
|
22196
|
+
hierarchy: (metadata) => [
|
|
22197
|
+
"workspaces",
|
|
22198
|
+
str(metadata.workspaceId) ?? "unknown",
|
|
22199
|
+
"channels",
|
|
22200
|
+
str(metadata.channelName) ?? str(metadata.channelId) ?? "unknown"
|
|
22201
|
+
]
|
|
22202
|
+
};
|
|
22203
|
+
CRAWLER_DESCRIPTORS = {
|
|
22204
|
+
slack,
|
|
22205
|
+
telegram: chatCrawler("telegram"),
|
|
22206
|
+
whatsapp: chatCrawler("whatsapp")
|
|
22207
|
+
};
|
|
22208
|
+
});
|
|
22209
|
+
|
|
22210
|
+
// src/datasource/skills/clawgallery/client.ts
|
|
22211
|
+
import { spawn as spawn9 } from "node:child_process";
|
|
22212
|
+
import { existsSync as existsSync23 } from "node:fs";
|
|
22213
|
+
|
|
22214
|
+
class ClawGalleryClient {
|
|
22215
|
+
options;
|
|
22216
|
+
constructor(options = {}) {
|
|
22217
|
+
this.options = options;
|
|
22218
|
+
}
|
|
22219
|
+
async bootstrap(signal) {
|
|
22220
|
+
if (this.options.path !== undefined && !existsSync23(this.options.path)) {
|
|
22221
|
+
return {
|
|
22222
|
+
ok: false,
|
|
22223
|
+
reason: "nonzero-exit",
|
|
22224
|
+
stdout: "",
|
|
22225
|
+
stderr: "configured ClawGallery folder does not exist",
|
|
22226
|
+
code: null
|
|
20990
22227
|
};
|
|
20991
22228
|
}
|
|
20992
22229
|
const args = ["bootstrap"];
|
|
@@ -21085,7 +22322,7 @@ function spawnCli(binary, args, env, options, signal) {
|
|
|
21085
22322
|
const timeout = setTimeout(() => {
|
|
21086
22323
|
reason = "timeout";
|
|
21087
22324
|
terminate2(child);
|
|
21088
|
-
}, options.timeoutMs ??
|
|
22325
|
+
}, options.timeoutMs ?? DEFAULT_TIMEOUT_MS7);
|
|
21089
22326
|
const abort = () => {
|
|
21090
22327
|
reason = "aborted";
|
|
21091
22328
|
terminate2(child);
|
|
@@ -21191,16 +22428,16 @@ function failure2(result, reason) {
|
|
|
21191
22428
|
ok: false,
|
|
21192
22429
|
reason: reason ?? result.reason ?? "nonzero-exit",
|
|
21193
22430
|
stdout: result.stdout,
|
|
21194
|
-
stderr:
|
|
22431
|
+
stderr: boundDiagnosticText2(result.stderr),
|
|
21195
22432
|
code: result.code
|
|
21196
22433
|
};
|
|
21197
22434
|
}
|
|
21198
|
-
function
|
|
22435
|
+
function boundDiagnosticText2(value) {
|
|
21199
22436
|
if (value.length === 0)
|
|
21200
22437
|
return "";
|
|
21201
22438
|
return value.trim().slice(0, 4000);
|
|
21202
22439
|
}
|
|
21203
|
-
var DEFAULT_BINARY2 = "clawgallery",
|
|
22440
|
+
var DEFAULT_BINARY2 = "clawgallery", DEFAULT_TIMEOUT_MS7 = 60000, DEFAULT_MAX_BUFFER_BYTES2, SAFE_ENV;
|
|
21204
22441
|
var init_client5 = __esm(() => {
|
|
21205
22442
|
init_portable_spawn();
|
|
21206
22443
|
DEFAULT_MAX_BUFFER_BYTES2 = 4 * 1024 * 1024;
|
|
@@ -21494,7 +22731,7 @@ function statusToReason(status) {
|
|
|
21494
22731
|
}
|
|
21495
22732
|
async function requestText(url, options) {
|
|
21496
22733
|
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
21497
|
-
const timeoutMs = options.timeoutMs ??
|
|
22734
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
21498
22735
|
let response;
|
|
21499
22736
|
try {
|
|
21500
22737
|
response = await fetchImpl(url, {
|
|
@@ -21558,13 +22795,13 @@ function resolveToken(token, tokenEnv) {
|
|
|
21558
22795
|
}
|
|
21559
22796
|
return;
|
|
21560
22797
|
}
|
|
21561
|
-
var
|
|
22798
|
+
var DEFAULT_TIMEOUT_MS8 = 15000;
|
|
21562
22799
|
|
|
21563
22800
|
// src/datasource/skills/cloud-drive/rclone-connector.ts
|
|
21564
22801
|
import { spawn as spawn10 } from "node:child_process";
|
|
21565
|
-
import { mkdirSync as mkdirSync18, readFileSync as
|
|
22802
|
+
import { mkdirSync as mkdirSync18, readFileSync as readFileSync20, renameSync as renameSync10, rmSync as rmSync9, writeFileSync as writeFileSync17 } from "node:fs";
|
|
21566
22803
|
import { readFile as readFile4 } from "node:fs/promises";
|
|
21567
|
-
import { dirname as
|
|
22804
|
+
import { dirname as dirname18, join as join31, normalize as normalize4 } from "node:path";
|
|
21568
22805
|
|
|
21569
22806
|
class RcloneConnector {
|
|
21570
22807
|
options;
|
|
@@ -21580,7 +22817,7 @@ class RcloneConnector {
|
|
|
21580
22817
|
if (remote === undefined || remote.length === 0) {
|
|
21581
22818
|
return { ok: false, reason: "not-configured", message: "rclone remote not configured" };
|
|
21582
22819
|
}
|
|
21583
|
-
const timeoutMs = this.options.timeoutMs ??
|
|
22820
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS9;
|
|
21584
22821
|
const exportFormats = this.options.exportFormats ?? DEFAULT_EXPORT_FORMATS;
|
|
21585
22822
|
let listResult;
|
|
21586
22823
|
try {
|
|
@@ -21651,7 +22888,7 @@ class RcloneConnector {
|
|
|
21651
22888
|
return;
|
|
21652
22889
|
const destination = mirrorFilePath(paths.mirrorRoot, entry.path);
|
|
21653
22890
|
const temp = `${destination}.autorag-tmp`;
|
|
21654
|
-
mkdirSync18(
|
|
22891
|
+
mkdirSync18(dirname18(temp), { recursive: true });
|
|
21655
22892
|
let copied;
|
|
21656
22893
|
try {
|
|
21657
22894
|
copied = await this.runner(copyArgs(remote, entry.path, temp, exportFormats, this.options.concurrency, this.options.bandwidthLimit), timeoutMs);
|
|
@@ -21756,12 +22993,12 @@ function sameEntry(previous, current) {
|
|
|
21756
22993
|
return previous?.path === current.path && previous.size === current.size && previous.modTime === current.modTime && previous.remoteId === current.remoteId && JSON.stringify(previous.hashes) === JSON.stringify(current.hashes);
|
|
21757
22994
|
}
|
|
21758
22995
|
function mirrorPaths(workspaceRoot, skillName, instanceId) {
|
|
21759
|
-
const base =
|
|
21760
|
-
return { mirrorRoot:
|
|
22996
|
+
const base = join31(workspaceRoot, ".autorag", "datasources", skillName, instanceId);
|
|
22997
|
+
return { mirrorRoot: join31(base, "mirror"), manifestPath: join31(base, "manifest.json") };
|
|
21761
22998
|
}
|
|
21762
22999
|
function mirrorFilePath(root, path) {
|
|
21763
23000
|
const safe = normalize4(path).replace(/^(\.\.(?:[/\\]|$))+|^[/\\]+/gu, "");
|
|
21764
|
-
return
|
|
23001
|
+
return join31(root, safe);
|
|
21765
23002
|
}
|
|
21766
23003
|
function remoteTarget(remote, path) {
|
|
21767
23004
|
return remote.endsWith(":") || remote.endsWith("/") ? `${remote}${path}` : `${remote}/${path}`;
|
|
@@ -21780,14 +23017,14 @@ function copyArgs(remote, path, destination, exportFormats, concurrency, bandwid
|
|
|
21780
23017
|
}
|
|
21781
23018
|
function loadManifest2(path) {
|
|
21782
23019
|
try {
|
|
21783
|
-
const parsed = JSON.parse(
|
|
23020
|
+
const parsed = JSON.parse(readFileSync20(path, "utf8"));
|
|
21784
23021
|
return parsed.version === 1 && Array.isArray(parsed.entries) ? parsed : undefined;
|
|
21785
23022
|
} catch {
|
|
21786
23023
|
return;
|
|
21787
23024
|
}
|
|
21788
23025
|
}
|
|
21789
23026
|
function saveManifest(path, manifest) {
|
|
21790
|
-
mkdirSync18(
|
|
23027
|
+
mkdirSync18(dirname18(path), { recursive: true });
|
|
21791
23028
|
const temp = `${path}.autorag-tmp`;
|
|
21792
23029
|
writeFileSync17(temp, `${JSON.stringify(manifest)}
|
|
21793
23030
|
`, "utf8");
|
|
@@ -21889,7 +23126,7 @@ function runBinary(binary, args, timeoutMs, env, cwd) {
|
|
|
21889
23126
|
});
|
|
21890
23127
|
});
|
|
21891
23128
|
}
|
|
21892
|
-
var DEFAULT_BINARY3 = "rclone", DEFAULT_EXTENSIONS, DEFAULT_EXPORT_FORMATS = "txt,csv", DEFAULT_MAX_DOCUMENTS = 200, DEFAULT_MAX_BYTES_PER_FILE,
|
|
23129
|
+
var DEFAULT_BINARY3 = "rclone", DEFAULT_EXTENSIONS, DEFAULT_EXPORT_FORMATS = "txt,csv", DEFAULT_MAX_DOCUMENTS = 200, DEFAULT_MAX_BYTES_PER_FILE, DEFAULT_TIMEOUT_MS9 = 60000, MAX_CONTENT_CHARS2 = 1e5;
|
|
21893
23130
|
var init_rclone_connector = __esm(() => {
|
|
21894
23131
|
init_defaults();
|
|
21895
23132
|
init_portable_spawn();
|
|
@@ -21897,52 +23134,11 @@ var init_rclone_connector = __esm(() => {
|
|
|
21897
23134
|
DEFAULT_MAX_BYTES_PER_FILE = 2 * 1024 * 1024;
|
|
21898
23135
|
});
|
|
21899
23136
|
|
|
21900
|
-
// src/datasource/connector.ts
|
|
21901
|
-
import { createHash as createHash12 } from "node:crypto";
|
|
21902
|
-
function connectorFailureToDiagnosticCode(reason) {
|
|
21903
|
-
switch (reason) {
|
|
21904
|
-
case "auth":
|
|
21905
|
-
return "datasource-auth-error";
|
|
21906
|
-
case "permission":
|
|
21907
|
-
return "datasource-permission-denied";
|
|
21908
|
-
case "rate-limited":
|
|
21909
|
-
return "datasource-rate-limited";
|
|
21910
|
-
case "unavailable":
|
|
21911
|
-
case "not-configured":
|
|
21912
|
-
return "datasource-unavailable";
|
|
21913
|
-
case "empty":
|
|
21914
|
-
return "datasource-empty";
|
|
21915
|
-
case "api-error":
|
|
21916
|
-
case "invalid-data":
|
|
21917
|
-
return "datasource-index-failed";
|
|
21918
|
-
}
|
|
21919
|
-
}
|
|
21920
|
-
function boundDiagnosticText2(value) {
|
|
21921
|
-
const trimmed = value.trim();
|
|
21922
|
-
return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
|
|
21923
|
-
}
|
|
21924
|
-
function sanitizeIdSegment(value) {
|
|
21925
|
-
const normalized = value.replace(/[^\p{L}\p{N}._-]+/gu, "-");
|
|
21926
|
-
let start = 0;
|
|
21927
|
-
while (start < normalized.length && (normalized[start] === "-" || normalized[start] === "."))
|
|
21928
|
-
start += 1;
|
|
21929
|
-
let end = normalized.length;
|
|
21930
|
-
while (end > start && (normalized[end - 1] === "-" || normalized[end - 1] === "."))
|
|
21931
|
-
end -= 1;
|
|
21932
|
-
const cleaned = normalized.slice(start, end);
|
|
21933
|
-
if (cleaned === value && cleaned.length > 0 && cleaned.length <= 80)
|
|
21934
|
-
return cleaned;
|
|
21935
|
-
const hash = createHash12("sha256").update(value).digest("hex").slice(0, 8);
|
|
21936
|
-
const stem = cleaned.slice(0, 60);
|
|
21937
|
-
return stem.length > 0 ? `${stem}-${hash}` : hash;
|
|
21938
|
-
}
|
|
21939
|
-
var init_connector = () => {};
|
|
21940
|
-
|
|
21941
23137
|
// src/datasource/chunk-store.ts
|
|
21942
|
-
import { existsSync as existsSync24, mkdirSync as mkdirSync19, readFileSync as
|
|
21943
|
-
import { dirname as
|
|
23138
|
+
import { existsSync as existsSync24, mkdirSync as mkdirSync19, readFileSync as readFileSync21, writeFileSync as writeFileSync18 } from "node:fs";
|
|
23139
|
+
import { dirname as dirname19, join as join32 } from "node:path";
|
|
21944
23140
|
function datasourceChunkStorePath(workspaceRoot, skillName, instanceId) {
|
|
21945
|
-
return
|
|
23141
|
+
return join32(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId), "chunks.json");
|
|
21946
23142
|
}
|
|
21947
23143
|
|
|
21948
23144
|
class DatasourceChunkStore {
|
|
@@ -21981,7 +23177,7 @@ class DatasourceChunkStore {
|
|
|
21981
23177
|
try {
|
|
21982
23178
|
if (!existsSync24(this.storePath))
|
|
21983
23179
|
return false;
|
|
21984
|
-
const parsed = JSON.parse(
|
|
23180
|
+
const parsed = JSON.parse(readFileSync21(this.storePath, "utf8"));
|
|
21985
23181
|
if (parsed.version !== STORE_VERSION || !Array.isArray(parsed.chunks))
|
|
21986
23182
|
return false;
|
|
21987
23183
|
this.chunkList = parsed.chunks.filter((chunk) => typeof chunk?.chunkId === "string" && typeof chunk?.content === "string");
|
|
@@ -22027,7 +23223,7 @@ class DatasourceChunkStore {
|
|
|
22027
23223
|
if (this.storePath === undefined)
|
|
22028
23224
|
return;
|
|
22029
23225
|
try {
|
|
22030
|
-
mkdirSync19(
|
|
23226
|
+
mkdirSync19(dirname19(this.storePath), { recursive: true });
|
|
22031
23227
|
const payload = {
|
|
22032
23228
|
version: STORE_VERSION,
|
|
22033
23229
|
skill: this.skillName,
|
|
@@ -22169,7 +23365,7 @@ class ConnectorDatasourceSkill {
|
|
|
22169
23365
|
}
|
|
22170
23366
|
if (!fetched.ok) {
|
|
22171
23367
|
const code = connectorFailureToDiagnosticCode(fetched.reason);
|
|
22172
|
-
const message =
|
|
23368
|
+
const message = boundDiagnosticText(fetched.message ?? fetched.reason);
|
|
22173
23369
|
return this.fail(code, `${fetched.reason}: ${message}`);
|
|
22174
23370
|
}
|
|
22175
23371
|
const documents = this.applyDedupeWindow(fetched.documents);
|
|
@@ -22181,7 +23377,7 @@ class ConnectorDatasourceSkill {
|
|
|
22181
23377
|
const diagnostics = (fetched.warnings ?? []).map((warning) => ({
|
|
22182
23378
|
code: "datasource-index-failed",
|
|
22183
23379
|
severity: "warning",
|
|
22184
|
-
message:
|
|
23380
|
+
message: boundDiagnosticText(warning),
|
|
22185
23381
|
instanceId: this.instanceId,
|
|
22186
23382
|
source: this.definition.skillName
|
|
22187
23383
|
}));
|
|
@@ -22298,7 +23494,7 @@ class ConnectorDatasourceSkill {
|
|
|
22298
23494
|
return out;
|
|
22299
23495
|
}
|
|
22300
23496
|
fail(code, message) {
|
|
22301
|
-
const sanitized =
|
|
23497
|
+
const sanitized = boundDiagnosticText(message);
|
|
22302
23498
|
this.lastError = sanitized;
|
|
22303
23499
|
const diagnostic = {
|
|
22304
23500
|
code,
|
|
@@ -22490,8 +23686,8 @@ var init_types4 = __esm(() => {
|
|
|
22490
23686
|
|
|
22491
23687
|
// src/datasource/skills/discrawl/client.ts
|
|
22492
23688
|
import { spawn as spawn12 } from "node:child_process";
|
|
22493
|
-
import { mkdirSync as mkdirSync20, readFileSync as
|
|
22494
|
-
import { dirname as
|
|
23689
|
+
import { mkdirSync as mkdirSync20, readFileSync as readFileSync22, writeFileSync as writeFileSync19 } from "node:fs";
|
|
23690
|
+
import { dirname as dirname20 } from "node:path";
|
|
22495
23691
|
|
|
22496
23692
|
class DiscrawlClient {
|
|
22497
23693
|
options;
|
|
@@ -22507,7 +23703,7 @@ class DiscrawlClient {
|
|
|
22507
23703
|
const path = `${workspace}/.autorag/datasources/discrawl/config.toml`;
|
|
22508
23704
|
let existing = "";
|
|
22509
23705
|
try {
|
|
22510
|
-
existing =
|
|
23706
|
+
existing = readFileSync22(path, "utf8");
|
|
22511
23707
|
} catch {}
|
|
22512
23708
|
const managed = existing.startsWith(`# AutoRAG managed discrawl embeddings v1
|
|
22513
23709
|
`);
|
|
@@ -22515,7 +23711,7 @@ class DiscrawlClient {
|
|
|
22515
23711
|
return { configured: false, rebuildRequired: false };
|
|
22516
23712
|
const previous = managed ? parseManagedEmbedding(existing) : undefined;
|
|
22517
23713
|
const config = managedDiscrawlConfig({ ...runtime, baseUrl: nativeGatewayBaseUrl(runtime.baseUrl) });
|
|
22518
|
-
mkdirSync20(
|
|
23714
|
+
mkdirSync20(dirname20(path), { recursive: true });
|
|
22519
23715
|
if (existing !== config)
|
|
22520
23716
|
writeFileSync19(path, config, "utf8");
|
|
22521
23717
|
return {
|
|
@@ -23540,8 +24736,8 @@ var init_github = __esm(() => {
|
|
|
23540
24736
|
|
|
23541
24737
|
// src/datasource/skills/github-gist/connector.ts
|
|
23542
24738
|
import { execFile as execFile2 } from "node:child_process";
|
|
23543
|
-
import { existsSync as existsSync25, mkdirSync as mkdirSync21, readFileSync as
|
|
23544
|
-
import { dirname as
|
|
24739
|
+
import { existsSync as existsSync25, mkdirSync as mkdirSync21, readFileSync as readFileSync23, writeFileSync as writeFileSync20 } from "node:fs";
|
|
24740
|
+
import { dirname as dirname21 } from "node:path";
|
|
23545
24741
|
async function defaultGhTokenRunner() {
|
|
23546
24742
|
return new Promise((resolve) => {
|
|
23547
24743
|
execFile2("gh", ["auth", "token"], { timeout: GH_TOKEN_TIMEOUT_MS }, (error, stdout) => {
|
|
@@ -23663,7 +24859,7 @@ class GitHubGistConnector {
|
|
|
23663
24859
|
if (statePath === undefined || !existsSync25(statePath))
|
|
23664
24860
|
return { version: STATE_VERSION, gists: {} };
|
|
23665
24861
|
try {
|
|
23666
|
-
const parsed = JSON.parse(
|
|
24862
|
+
const parsed = JSON.parse(readFileSync23(statePath, "utf8"));
|
|
23667
24863
|
if (parsed.version !== STATE_VERSION || typeof parsed.gists !== "object" || parsed.gists === null) {
|
|
23668
24864
|
return { version: STATE_VERSION, gists: {} };
|
|
23669
24865
|
}
|
|
@@ -23677,7 +24873,7 @@ class GitHubGistConnector {
|
|
|
23677
24873
|
if (statePath === undefined)
|
|
23678
24874
|
return;
|
|
23679
24875
|
try {
|
|
23680
|
-
mkdirSync21(
|
|
24876
|
+
mkdirSync21(dirname21(statePath), { recursive: true });
|
|
23681
24877
|
writeFileSync20(statePath, `${JSON.stringify(state)}
|
|
23682
24878
|
`, "utf8");
|
|
23683
24879
|
} catch {}
|
|
@@ -23723,8 +24919,8 @@ var init_connector3 = () => {};
|
|
|
23723
24919
|
|
|
23724
24920
|
// src/datasource/skills/github-gist/semantic.ts
|
|
23725
24921
|
import { createHash as createHash13 } from "node:crypto";
|
|
23726
|
-
import { existsSync as existsSync26, mkdirSync as mkdirSync22, readFileSync as
|
|
23727
|
-
import { dirname as
|
|
24922
|
+
import { existsSync as existsSync26, mkdirSync as mkdirSync22, readFileSync as readFileSync24, writeFileSync as writeFileSync21 } from "node:fs";
|
|
24923
|
+
import { dirname as dirname22 } from "node:path";
|
|
23728
24924
|
function contentHash(chunk) {
|
|
23729
24925
|
return createHash13("sha256").update(`${chunk.title ?? ""}
|
|
23730
24926
|
${chunk.content}`).digest("hex");
|
|
@@ -23817,7 +25013,7 @@ ${chunk.content}`));
|
|
|
23817
25013
|
if (this.statePath === undefined || !existsSync26(this.statePath))
|
|
23818
25014
|
return;
|
|
23819
25015
|
try {
|
|
23820
|
-
const parsed = JSON.parse(
|
|
25016
|
+
const parsed = JSON.parse(readFileSync24(this.statePath, "utf8"));
|
|
23821
25017
|
if (parsed.version !== VECTORS_VERSION || typeof parsed.entries !== "object" || parsed.entries === null)
|
|
23822
25018
|
return;
|
|
23823
25019
|
if (typeof parsed.identity?.dimension !== "number")
|
|
@@ -23832,7 +25028,7 @@ ${chunk.content}`));
|
|
|
23832
25028
|
if (this.identity === undefined)
|
|
23833
25029
|
return;
|
|
23834
25030
|
try {
|
|
23835
|
-
mkdirSync22(
|
|
25031
|
+
mkdirSync22(dirname22(this.statePath), { recursive: true });
|
|
23836
25032
|
const payload = {
|
|
23837
25033
|
version: VECTORS_VERSION,
|
|
23838
25034
|
identity: this.identity,
|
|
@@ -23915,9 +25111,9 @@ var init_semantic = __esm(() => {
|
|
|
23915
25111
|
});
|
|
23916
25112
|
|
|
23917
25113
|
// src/datasource/skills/github-gist/skill.ts
|
|
23918
|
-
import { join as
|
|
25114
|
+
import { join as join33 } from "node:path";
|
|
23919
25115
|
function gistDatasourceDir(workspaceRoot, skillName, instanceId) {
|
|
23920
|
-
return
|
|
25116
|
+
return join33(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId));
|
|
23921
25117
|
}
|
|
23922
25118
|
var GITHUB_GIST_SKILL_DEFINITION, DEFAULT_INSTANCE_ID3 = "default", GitHubGistSkill;
|
|
23923
25119
|
var init_skill5 = __esm(() => {
|
|
@@ -23945,7 +25141,7 @@ var init_skill5 = __esm(() => {
|
|
|
23945
25141
|
constructor(options = {}) {
|
|
23946
25142
|
const { connector, connectorOptions, semantic, ...rest } = options;
|
|
23947
25143
|
const instanceId = options.instanceId ?? DEFAULT_INSTANCE_ID3;
|
|
23948
|
-
const statePath = connectorOptions?.statePath ?? (options.workspaceRoot === undefined ? undefined :
|
|
25144
|
+
const statePath = connectorOptions?.statePath ?? (options.workspaceRoot === undefined ? undefined : join33(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "state.json"));
|
|
23949
25145
|
super(GITHUB_GIST_SKILL_DEFINITION, {
|
|
23950
25146
|
...rest,
|
|
23951
25147
|
connector: connector ?? new GitHubGistConnector({
|
|
@@ -23955,7 +25151,7 @@ var init_skill5 = __esm(() => {
|
|
|
23955
25151
|
});
|
|
23956
25152
|
if (semantic?.enabled !== false) {
|
|
23957
25153
|
this.semanticIndex = new GistSemanticIndex({
|
|
23958
|
-
statePath: semantic?.statePath ?? (options.workspaceRoot === undefined ? undefined :
|
|
25154
|
+
statePath: semantic?.statePath ?? (options.workspaceRoot === undefined ? undefined : join33(gistDatasourceDir(options.workspaceRoot, GITHUB_GIST_SKILL_DEFINITION.skillName, instanceId), "vectors.json"))
|
|
23959
25155
|
});
|
|
23960
25156
|
this.semanticEmbedder = semantic?.embedder ?? createGatewayEmbedder(semantic?.embedderOptions ?? {});
|
|
23961
25157
|
this.semanticMethod = new GitHubGistSemanticMethod({
|
|
@@ -23983,7 +25179,7 @@ var init_skill5 = __esm(() => {
|
|
|
23983
25179
|
{
|
|
23984
25180
|
code: "datasource-index-failed",
|
|
23985
25181
|
severity: "warning",
|
|
23986
|
-
message:
|
|
25182
|
+
message: boundDiagnosticText(`semantic-unavailable: ${sync.message}`),
|
|
23987
25183
|
instanceId: this.instanceId,
|
|
23988
25184
|
source: this.describe().name
|
|
23989
25185
|
}
|
|
@@ -24678,7 +25874,7 @@ var init_types5 = __esm(() => {
|
|
|
24678
25874
|
// src/datasource/skills/lazykatok/client.ts
|
|
24679
25875
|
import { spawn as spawn14 } from "node:child_process";
|
|
24680
25876
|
import { existsSync as existsSync27 } from "node:fs";
|
|
24681
|
-
import { delimiter as delimiter4, join as
|
|
25877
|
+
import { delimiter as delimiter4, join as join34 } from "node:path";
|
|
24682
25878
|
|
|
24683
25879
|
class LazykatokClient {
|
|
24684
25880
|
options;
|
|
@@ -24866,7 +26062,7 @@ function lookupInPath2(executable, env) {
|
|
|
24866
26062
|
for (const dir of pathEnv.split(delimiter4)) {
|
|
24867
26063
|
if (dir.length === 0)
|
|
24868
26064
|
continue;
|
|
24869
|
-
const candidate =
|
|
26065
|
+
const candidate = join34(dir, executable);
|
|
24870
26066
|
if (existsSync27(candidate))
|
|
24871
26067
|
return candidate;
|
|
24872
26068
|
}
|
|
@@ -25356,7 +26552,7 @@ var init_lazykatok = __esm(() => {
|
|
|
25356
26552
|
|
|
25357
26553
|
// src/datasource/skills/mail-export/connector.ts
|
|
25358
26554
|
import { readdir as readdir3, readFile as readFile5, stat as stat6 } from "node:fs/promises";
|
|
25359
|
-
import { basename as basename12, dirname as
|
|
26555
|
+
import { basename as basename12, dirname as dirname23, extname as extname7, join as join35 } from "node:path";
|
|
25360
26556
|
import { simpleParser as simpleParser2 } from "mailparser";
|
|
25361
26557
|
|
|
25362
26558
|
class MailExportConnector {
|
|
@@ -25382,7 +26578,7 @@ class MailExportConnector {
|
|
|
25382
26578
|
continue;
|
|
25383
26579
|
const extension = extname7(entry.name).toLowerCase();
|
|
25384
26580
|
if (extension === ".eml" || extension === ".mbox") {
|
|
25385
|
-
files.push(
|
|
26581
|
+
files.push(join35(entry.parentPath, entry.name));
|
|
25386
26582
|
}
|
|
25387
26583
|
}
|
|
25388
26584
|
} else if (info.isFile()) {
|
|
@@ -25417,7 +26613,7 @@ class MailExportConnector {
|
|
|
25417
26613
|
continue;
|
|
25418
26614
|
}
|
|
25419
26615
|
const stem = basename12(file, extname7(file));
|
|
25420
|
-
const mailboxDir = basename12(
|
|
26616
|
+
const mailboxDir = basename12(dirname23(file));
|
|
25421
26617
|
const hierarchy = ["mailboxes", mailboxDir, stem];
|
|
25422
26618
|
const extension = extname7(file).toLowerCase();
|
|
25423
26619
|
const rawMessages = extension === ".mbox" ? splitMbox(bytes.toString("utf8")) : [bytes.toString("utf8")];
|
|
@@ -25538,7 +26734,7 @@ function ensurePrivateMailcrawlDataDir(path) {
|
|
|
25538
26734
|
if (process.platform !== "win32")
|
|
25539
26735
|
chmodSync(path, 448);
|
|
25540
26736
|
}
|
|
25541
|
-
var
|
|
26737
|
+
var init_config3 = () => {};
|
|
25542
26738
|
|
|
25543
26739
|
// src/datasource/skills/mailcrawl/client.ts
|
|
25544
26740
|
import { spawn as spawn15 } from "node:child_process";
|
|
@@ -25864,7 +27060,7 @@ function truncateUtf8(value, maxBytes) {
|
|
|
25864
27060
|
var DEFAULT_BINARY4 = "mailcrawl", DEFAULT_SEARCH_TIMEOUT_MS3 = 60000, DEFAULT_INDEX_TIMEOUT_MS3 = 1800000, DEFAULT_MAX_BUFFER_BYTES4 = 1048576, LOOPBACK_HOSTS2, SAFE_ENV_KEYS2, SAFE_MAILCRAWL_ENV_KEYS, REMOTE_EMBEDDING_ENV_KEYS;
|
|
25865
27061
|
var init_client9 = __esm(() => {
|
|
25866
27062
|
init_portable_spawn();
|
|
25867
|
-
|
|
27063
|
+
init_config3();
|
|
25868
27064
|
LOOPBACK_HOSTS2 = new Set(["127.0.0.1", "localhost", "[::1]", "::1"]);
|
|
25869
27065
|
SAFE_ENV_KEYS2 = new Set(["HOME", "LANG", "LC_ALL", "PATH", "TMPDIR", "TMP", "TEMP", "NO_COLOR"]);
|
|
25870
27066
|
SAFE_MAILCRAWL_ENV_KEYS = new Set([
|
|
@@ -26111,7 +27307,7 @@ var TAGS;
|
|
|
26111
27307
|
var init_skill9 = __esm(() => {
|
|
26112
27308
|
init_scope2();
|
|
26113
27309
|
init_client9();
|
|
26114
|
-
|
|
27310
|
+
init_config3();
|
|
26115
27311
|
init_methods4();
|
|
26116
27312
|
TAGS = ["mailcrawl", "email", "pii"];
|
|
26117
27313
|
});
|
|
@@ -26123,6 +27319,504 @@ var init_mailcrawl = __esm(() => {
|
|
|
26123
27319
|
init_skill9();
|
|
26124
27320
|
});
|
|
26125
27321
|
|
|
27322
|
+
// src/datasource/sidecar/fusion.ts
|
|
27323
|
+
function encodeVector(vector) {
|
|
27324
|
+
const floats = Float32Array.from(vector);
|
|
27325
|
+
return new Uint8Array(floats.buffer, floats.byteOffset, floats.byteLength);
|
|
27326
|
+
}
|
|
27327
|
+
function decodeVector(blob) {
|
|
27328
|
+
if (blob.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0)
|
|
27329
|
+
return;
|
|
27330
|
+
return new Float32Array(new Uint8Array(blob).buffer);
|
|
27331
|
+
}
|
|
27332
|
+
function cosine3(a, b) {
|
|
27333
|
+
if (a.length === 0 || a.length !== b.length)
|
|
27334
|
+
return 0;
|
|
27335
|
+
let dot = 0;
|
|
27336
|
+
let normA = 0;
|
|
27337
|
+
let normB = 0;
|
|
27338
|
+
for (let index = 0;index < a.length; index += 1) {
|
|
27339
|
+
const x = a[index] ?? 0;
|
|
27340
|
+
const y = b[index] ?? 0;
|
|
27341
|
+
dot += x * y;
|
|
27342
|
+
normA += x * x;
|
|
27343
|
+
normB += y * y;
|
|
27344
|
+
}
|
|
27345
|
+
if (normA === 0 || normB === 0)
|
|
27346
|
+
return 0;
|
|
27347
|
+
return dot / Math.sqrt(normA * normB);
|
|
27348
|
+
}
|
|
27349
|
+
function topKByCosine(query, rows, k) {
|
|
27350
|
+
if (k <= 0)
|
|
27351
|
+
return [];
|
|
27352
|
+
const scored = [];
|
|
27353
|
+
for (const row of rows) {
|
|
27354
|
+
const score = cosine3(query, row.vector);
|
|
27355
|
+
if (score > 0)
|
|
27356
|
+
scored.push({ key: row.key, score });
|
|
27357
|
+
}
|
|
27358
|
+
scored.sort((a, b) => b.score - a.score || a.key.localeCompare(b.key));
|
|
27359
|
+
return scored.slice(0, k);
|
|
27360
|
+
}
|
|
27361
|
+
function reciprocalRankFusion(lists) {
|
|
27362
|
+
const scores = new Map;
|
|
27363
|
+
for (const list of lists) {
|
|
27364
|
+
const seen = new Set;
|
|
27365
|
+
for (const [index, key] of list.entries()) {
|
|
27366
|
+
if (seen.has(key))
|
|
27367
|
+
continue;
|
|
27368
|
+
seen.add(key);
|
|
27369
|
+
scores.set(key, (scores.get(key) ?? 0) + 1 / (RRF_K2 + index + 1));
|
|
27370
|
+
}
|
|
27371
|
+
}
|
|
27372
|
+
return [...scores.entries()].map(([key, score]) => ({ key, score })).sort((a, b) => b.score - a.score);
|
|
27373
|
+
}
|
|
27374
|
+
var RRF_K2 = 60;
|
|
27375
|
+
|
|
27376
|
+
// src/datasource/sidecar/index.ts
|
|
27377
|
+
import { createHash as createHash14 } from "node:crypto";
|
|
27378
|
+
import { mkdirSync as mkdirSync24, statSync as statSync3 } from "node:fs";
|
|
27379
|
+
import { dirname as dirname24 } from "node:path";
|
|
27380
|
+
import { DatabaseSync as DatabaseSync2 } from "node:sqlite";
|
|
27381
|
+
import { pathToFileURL } from "node:url";
|
|
27382
|
+
|
|
27383
|
+
class CrawlerSidecar {
|
|
27384
|
+
descriptor;
|
|
27385
|
+
options;
|
|
27386
|
+
select;
|
|
27387
|
+
where;
|
|
27388
|
+
opened;
|
|
27389
|
+
attached;
|
|
27390
|
+
lastSemantic;
|
|
27391
|
+
constructor(options) {
|
|
27392
|
+
this.options = options;
|
|
27393
|
+
const { descriptor } = options;
|
|
27394
|
+
this.descriptor = descriptor;
|
|
27395
|
+
const metadataColumns = descriptor.metadata.map((column, index) => `${column.sql} as m${index}`);
|
|
27396
|
+
this.select = [
|
|
27397
|
+
`${descriptor.keyExpression} as k`,
|
|
27398
|
+
`${descriptor.textExpression} as content`,
|
|
27399
|
+
`${descriptor.titleExpression ?? "null"} as title`,
|
|
27400
|
+
`${descriptor.epochSecondsExpression ?? "null"} as epoch`,
|
|
27401
|
+
...metadataColumns
|
|
27402
|
+
].join(", ");
|
|
27403
|
+
const scope = descriptor.scopeExpression !== undefined && options.workspaceId !== undefined;
|
|
27404
|
+
const allowlist = [
|
|
27405
|
+
...(options.channelIds?.length ?? 0) > 0 ? [`${descriptor.channelIdExpression} in (${options.channelIds?.map(() => "?").join(", ")})`] : [],
|
|
27406
|
+
...(options.channelNames?.length ?? 0) > 0 ? [`${descriptor.channelNameExpression} in (${options.channelNames?.map(() => "?").join(", ")})`] : []
|
|
27407
|
+
];
|
|
27408
|
+
this.where = [
|
|
27409
|
+
descriptor.liveWhere,
|
|
27410
|
+
...scope ? [`${descriptor.scopeExpression} = ?`] : [],
|
|
27411
|
+
...allowlist.length > 0 ? [`(${allowlist.join(" or ")})`] : []
|
|
27412
|
+
].join(" and ");
|
|
27413
|
+
}
|
|
27414
|
+
get db() {
|
|
27415
|
+
if (this.opened !== undefined)
|
|
27416
|
+
return this.opened;
|
|
27417
|
+
mkdirSync24(dirname24(this.options.sidecarDbPath), { recursive: true });
|
|
27418
|
+
const db = new DatabaseSync2(this.options.sidecarDbPath);
|
|
27419
|
+
db.exec(`pragma busy_timeout = ${BUSY_TIMEOUT_MS}`);
|
|
27420
|
+
db.exec("pragma journal_mode = wal");
|
|
27421
|
+
db.exec(`
|
|
27422
|
+
create table if not exists sidecar_meta (k text primary key, v text not null);
|
|
27423
|
+
create table if not exists sidecar_vec (message_key text primary key, hash text not null, vec blob not null);
|
|
27424
|
+
`);
|
|
27425
|
+
this.opened = db;
|
|
27426
|
+
return db;
|
|
27427
|
+
}
|
|
27428
|
+
attachCrawler() {
|
|
27429
|
+
let file;
|
|
27430
|
+
try {
|
|
27431
|
+
const { dev, ino } = statSync3(this.options.crawlerDbPath);
|
|
27432
|
+
file = { dev, ino };
|
|
27433
|
+
} catch {
|
|
27434
|
+
this.detachCrawler();
|
|
27435
|
+
return false;
|
|
27436
|
+
}
|
|
27437
|
+
if (this.attached !== undefined) {
|
|
27438
|
+
if (this.attached.dev === file.dev && this.attached.ino === file.ino)
|
|
27439
|
+
return true;
|
|
27440
|
+
this.detachCrawler();
|
|
27441
|
+
}
|
|
27442
|
+
const crawlerUri = `${pathToFileURL(this.options.crawlerDbPath).href}?mode=ro`.replaceAll("'", "''");
|
|
27443
|
+
try {
|
|
27444
|
+
this.db.exec(`attach database '${crawlerUri}' as crawler`);
|
|
27445
|
+
} catch {
|
|
27446
|
+
return false;
|
|
27447
|
+
}
|
|
27448
|
+
const { descriptor } = this.options;
|
|
27449
|
+
try {
|
|
27450
|
+
this.db.prepare(`select ${this.select} from ${descriptor.fromClause} where ${this.where} limit 0`).all(...this.whereParams());
|
|
27451
|
+
this.db.prepare(`select 1 from ${descriptor.fts.table} limit 0`).all();
|
|
27452
|
+
} catch {
|
|
27453
|
+
this.db.exec("detach database crawler");
|
|
27454
|
+
return false;
|
|
27455
|
+
}
|
|
27456
|
+
this.attached = file;
|
|
27457
|
+
this.lastSemantic = undefined;
|
|
27458
|
+
return true;
|
|
27459
|
+
}
|
|
27460
|
+
detachCrawler() {
|
|
27461
|
+
if (this.attached === undefined)
|
|
27462
|
+
return;
|
|
27463
|
+
this.attached = undefined;
|
|
27464
|
+
this.lastSemantic = undefined;
|
|
27465
|
+
try {
|
|
27466
|
+
this.opened?.exec("detach database crawler");
|
|
27467
|
+
} catch {}
|
|
27468
|
+
}
|
|
27469
|
+
get semanticEnabled() {
|
|
27470
|
+
return this.options.embedder !== undefined;
|
|
27471
|
+
}
|
|
27472
|
+
close() {
|
|
27473
|
+
this.opened?.close();
|
|
27474
|
+
this.opened = undefined;
|
|
27475
|
+
this.attached = undefined;
|
|
27476
|
+
this.lastSemantic = undefined;
|
|
27477
|
+
}
|
|
27478
|
+
whereParams() {
|
|
27479
|
+
const { descriptor, workspaceId, channelIds = [], channelNames = [] } = this.options;
|
|
27480
|
+
return [
|
|
27481
|
+
...descriptor.scopeExpression !== undefined && workspaceId !== undefined ? [workspaceId] : [],
|
|
27482
|
+
...channelIds,
|
|
27483
|
+
...channelNames
|
|
27484
|
+
];
|
|
27485
|
+
}
|
|
27486
|
+
searchLexical(query, topK) {
|
|
27487
|
+
const match = toMatchQuery(query);
|
|
27488
|
+
if (match === undefined || topK <= 0 || !this.attachCrawler())
|
|
27489
|
+
return [];
|
|
27490
|
+
const { descriptor } = this.options;
|
|
27491
|
+
const rows = this.db.prepare(`select ${this.select}, bm25(${descriptor.fts.table}) as rank
|
|
27492
|
+
from ${descriptor.fromClause} join ${descriptor.fts.table} on ${descriptor.fts.joinOn}
|
|
27493
|
+
where ${descriptor.fts.table} match ? and ${this.where}
|
|
27494
|
+
order by rank limit ?`).all(match, ...this.whereParams(), topK);
|
|
27495
|
+
return rows.map((row) => this.toHit(this.toRow(row), -Number(row.rank)));
|
|
27496
|
+
}
|
|
27497
|
+
async searchSemantic(query, topK) {
|
|
27498
|
+
const trimmed = query.trim();
|
|
27499
|
+
if (this.options.embedder === undefined || trimmed.length === 0 || topK <= 0 || !this.attachCrawler())
|
|
27500
|
+
return [];
|
|
27501
|
+
const ranked = await this.rankSemantic(trimmed, topK * POOL_FACTOR);
|
|
27502
|
+
if (ranked.length === 0)
|
|
27503
|
+
return [];
|
|
27504
|
+
const live = this.hydrate(ranked.map((entry) => entry.key));
|
|
27505
|
+
const hits = [];
|
|
27506
|
+
for (const entry of ranked) {
|
|
27507
|
+
const row = live.get(entry.key);
|
|
27508
|
+
if (row !== undefined)
|
|
27509
|
+
hits.push(this.toHit(row, entry.score));
|
|
27510
|
+
if (hits.length >= topK)
|
|
27511
|
+
break;
|
|
27512
|
+
}
|
|
27513
|
+
return hits;
|
|
27514
|
+
}
|
|
27515
|
+
async rankSemantic(query, pool) {
|
|
27516
|
+
const now = Date.now();
|
|
27517
|
+
let cached = this.lastSemantic;
|
|
27518
|
+
if (cached === undefined || cached.query !== query || now - cached.at > SEMANTIC_REUSE_MS) {
|
|
27519
|
+
const embedder = this.options.embedder;
|
|
27520
|
+
if (embedder === undefined)
|
|
27521
|
+
return [];
|
|
27522
|
+
cached = {
|
|
27523
|
+
query,
|
|
27524
|
+
at: now,
|
|
27525
|
+
vector: embedder.embed([query]).then((vectors) => vectors[0], () => {
|
|
27526
|
+
return;
|
|
27527
|
+
}),
|
|
27528
|
+
ranked: undefined
|
|
27529
|
+
};
|
|
27530
|
+
this.lastSemantic = cached;
|
|
27531
|
+
}
|
|
27532
|
+
const vector = await cached.vector;
|
|
27533
|
+
if (vector === undefined) {
|
|
27534
|
+
if (this.lastSemantic === cached)
|
|
27535
|
+
this.lastSemantic = undefined;
|
|
27536
|
+
return [];
|
|
27537
|
+
}
|
|
27538
|
+
if (cached.ranked === undefined || cached.ranked.pool < pool) {
|
|
27539
|
+
const wide = pool * POOL_FACTOR;
|
|
27540
|
+
cached.ranked = { pool: wide, keys: topKByCosine(vector, this.storedVectors(), wide) };
|
|
27541
|
+
}
|
|
27542
|
+
return cached.ranked.keys.slice(0, pool);
|
|
27543
|
+
}
|
|
27544
|
+
async searchHybrid(query, topK) {
|
|
27545
|
+
if (topK <= 0)
|
|
27546
|
+
return [];
|
|
27547
|
+
const pool = topK * POOL_FACTOR;
|
|
27548
|
+
const lexical = this.searchLexical(query, pool);
|
|
27549
|
+
const semantic = await this.searchSemantic(query, pool);
|
|
27550
|
+
if (semantic.length === 0)
|
|
27551
|
+
return lexical.slice(0, topK);
|
|
27552
|
+
const byKey = new Map;
|
|
27553
|
+
for (const hit of [...semantic, ...lexical])
|
|
27554
|
+
byKey.set(hit.key, hit);
|
|
27555
|
+
const fused = [];
|
|
27556
|
+
for (const entry of reciprocalRankFusion([lexical.map((hit) => hit.key), semantic.map((hit) => hit.key)])) {
|
|
27557
|
+
const hit = byKey.get(entry.key);
|
|
27558
|
+
if (hit !== undefined)
|
|
27559
|
+
fused.push({ ...hit, score: entry.score });
|
|
27560
|
+
if (fused.length >= topK)
|
|
27561
|
+
break;
|
|
27562
|
+
}
|
|
27563
|
+
return fused;
|
|
27564
|
+
}
|
|
27565
|
+
async sync() {
|
|
27566
|
+
const { embedder } = this.options;
|
|
27567
|
+
if (embedder === undefined)
|
|
27568
|
+
return { ok: false, message: "no embedder configured" };
|
|
27569
|
+
if (!this.attachCrawler())
|
|
27570
|
+
return { ok: false, message: "crawler archive not available yet" };
|
|
27571
|
+
let identity;
|
|
27572
|
+
try {
|
|
27573
|
+
identity = await embedder.identity();
|
|
27574
|
+
} catch (error) {
|
|
27575
|
+
return { ok: false, message: errorMessage2(error) };
|
|
27576
|
+
}
|
|
27577
|
+
this.adoptIdentity(identity);
|
|
27578
|
+
const pruned = this.prune();
|
|
27579
|
+
const pending = this.pendingKeys();
|
|
27580
|
+
const batchSize = Math.max(1, this.options.batchSize ?? DEFAULT_BATCH_SIZE);
|
|
27581
|
+
const budget = Math.max(1, this.options.maxEmbedPerSync ?? DEFAULT_MAX_EMBED_PER_SYNC);
|
|
27582
|
+
const target = Math.min(pending.length, budget);
|
|
27583
|
+
const embed = async (rows) => {
|
|
27584
|
+
const vectors = await embedder.embed(rows.map((row) => row.input));
|
|
27585
|
+
if (vectors.length !== rows.length) {
|
|
27586
|
+
throw new Error(`embedder returned ${vectors.length} vectors for ${rows.length} messages`);
|
|
27587
|
+
}
|
|
27588
|
+
const wrong = vectors.find((vector) => vector.length !== identity.dimension);
|
|
27589
|
+
if (wrong !== undefined) {
|
|
27590
|
+
throw new Error(`embedder returned a ${wrong.length}-dimension vector, expected ${identity.dimension}`);
|
|
27591
|
+
}
|
|
27592
|
+
return vectors;
|
|
27593
|
+
};
|
|
27594
|
+
let embedded = 0;
|
|
27595
|
+
let skipped = 0;
|
|
27596
|
+
for (let offset = 0;offset < target; offset += batchSize) {
|
|
27597
|
+
const rows = [...this.hydrate(pending.slice(offset, Math.min(offset + batchSize, target))).values()];
|
|
27598
|
+
if (rows.length === 0)
|
|
27599
|
+
continue;
|
|
27600
|
+
try {
|
|
27601
|
+
this.store(rows, await embed(rows));
|
|
27602
|
+
embedded += rows.length;
|
|
27603
|
+
} catch (batchError) {
|
|
27604
|
+
const rejected = [];
|
|
27605
|
+
let lastError = batchError;
|
|
27606
|
+
for (const row of rows) {
|
|
27607
|
+
try {
|
|
27608
|
+
this.store([row], await embed([row]));
|
|
27609
|
+
embedded += 1;
|
|
27610
|
+
} catch (error) {
|
|
27611
|
+
rejected.push(row);
|
|
27612
|
+
lastError = error;
|
|
27613
|
+
}
|
|
27614
|
+
}
|
|
27615
|
+
if (rejected.length === 0)
|
|
27616
|
+
continue;
|
|
27617
|
+
if (embedded === 0)
|
|
27618
|
+
return { ok: false, message: errorMessage2(lastError) };
|
|
27619
|
+
this.store(rejected, undefined);
|
|
27620
|
+
skipped += rejected.length;
|
|
27621
|
+
}
|
|
27622
|
+
}
|
|
27623
|
+
return { ok: true, embedded, skipped, pruned, remaining: pending.length - embedded - skipped };
|
|
27624
|
+
}
|
|
27625
|
+
adoptIdentity(identity) {
|
|
27626
|
+
const serialized = JSON.stringify(identity);
|
|
27627
|
+
const current = this.db.prepare("select v from sidecar_meta where k = ?").get(IDENTITY_KEY);
|
|
27628
|
+
if (current?.v === serialized)
|
|
27629
|
+
return;
|
|
27630
|
+
this.transaction(() => {
|
|
27631
|
+
this.db.exec("delete from sidecar_vec");
|
|
27632
|
+
this.db.prepare("insert or replace into sidecar_meta (k, v) values (?, ?)").run(IDENTITY_KEY, serialized);
|
|
27633
|
+
});
|
|
27634
|
+
this.lastSemantic = undefined;
|
|
27635
|
+
}
|
|
27636
|
+
prune() {
|
|
27637
|
+
const { descriptor } = this.options;
|
|
27638
|
+
const result = this.db.prepare(`delete from sidecar_vec where message_key not in (
|
|
27639
|
+
select ${descriptor.keyExpression} from ${descriptor.fromClause} where ${this.where}
|
|
27640
|
+
)`).run(...this.whereParams());
|
|
27641
|
+
if (Number(result.changes) > 0)
|
|
27642
|
+
this.lastSemantic = undefined;
|
|
27643
|
+
return Number(result.changes);
|
|
27644
|
+
}
|
|
27645
|
+
pendingKeys() {
|
|
27646
|
+
const { descriptor } = this.options;
|
|
27647
|
+
const rows = this.db.prepare(`select ${this.select}, v.hash as stored_hash
|
|
27648
|
+
from ${descriptor.fromClause}
|
|
27649
|
+
left join sidecar_vec v on v.message_key = ${descriptor.keyExpression}
|
|
27650
|
+
where ${this.where}`).iterate(...this.whereParams());
|
|
27651
|
+
const pending = [];
|
|
27652
|
+
for (const row of rows) {
|
|
27653
|
+
if (row.stored_hash !== this.toRow(row).hash)
|
|
27654
|
+
pending.push(String(row.k));
|
|
27655
|
+
}
|
|
27656
|
+
return pending;
|
|
27657
|
+
}
|
|
27658
|
+
hydrate(keys) {
|
|
27659
|
+
const { descriptor } = this.options;
|
|
27660
|
+
const out = new Map;
|
|
27661
|
+
if (keys.length === 0)
|
|
27662
|
+
return out;
|
|
27663
|
+
const rows = this.db.prepare(`select ${this.select} from ${descriptor.fromClause}
|
|
27664
|
+
where ${descriptor.keyExpression} in (${keys.map(() => "?").join(", ")}) and ${this.where}`).all(...keys, ...this.whereParams());
|
|
27665
|
+
const byKey = new Map(rows.map((row) => [String(row.k), this.toRow(row)]));
|
|
27666
|
+
for (const key of keys) {
|
|
27667
|
+
const row = byKey.get(key);
|
|
27668
|
+
if (row !== undefined)
|
|
27669
|
+
out.set(key, row);
|
|
27670
|
+
}
|
|
27671
|
+
return out;
|
|
27672
|
+
}
|
|
27673
|
+
store(rows, vectors) {
|
|
27674
|
+
const upsert = this.db.prepare("insert or replace into sidecar_vec (message_key, hash, vec) values (?, ?, ?)");
|
|
27675
|
+
this.transaction(() => {
|
|
27676
|
+
for (const [index, row] of rows.entries()) {
|
|
27677
|
+
upsert.run(row.key, row.hash, encodeVector(vectors?.[index] ?? []));
|
|
27678
|
+
}
|
|
27679
|
+
});
|
|
27680
|
+
this.lastSemantic = undefined;
|
|
27681
|
+
}
|
|
27682
|
+
*storedVectors() {
|
|
27683
|
+
const rows = this.db.prepare("select message_key, vec from sidecar_vec where length(vec) > 0").iterate();
|
|
27684
|
+
for (const row of rows) {
|
|
27685
|
+
const vector = row.vec instanceof Uint8Array ? decodeVector(row.vec) : undefined;
|
|
27686
|
+
if (vector !== undefined)
|
|
27687
|
+
yield { key: String(row.message_key), vector };
|
|
27688
|
+
}
|
|
27689
|
+
}
|
|
27690
|
+
transaction(work) {
|
|
27691
|
+
this.db.exec("begin immediate");
|
|
27692
|
+
try {
|
|
27693
|
+
work();
|
|
27694
|
+
this.db.exec("commit");
|
|
27695
|
+
} catch (error) {
|
|
27696
|
+
this.db.exec("rollback");
|
|
27697
|
+
throw error;
|
|
27698
|
+
}
|
|
27699
|
+
}
|
|
27700
|
+
toRow(row) {
|
|
27701
|
+
const metadata = {};
|
|
27702
|
+
for (const [index, column] of this.options.descriptor.metadata.entries()) {
|
|
27703
|
+
const value = row[`m${index}`];
|
|
27704
|
+
if (typeof value === "string" && value.length > 0 || typeof value === "number")
|
|
27705
|
+
metadata[column.key] = String(value);
|
|
27706
|
+
}
|
|
27707
|
+
const title = typeof row.title === "string" && row.title.length > 0 ? row.title : undefined;
|
|
27708
|
+
const epoch = typeof row.epoch === "number" && Number.isFinite(row.epoch) ? row.epoch : undefined;
|
|
27709
|
+
const content = String(row.content ?? "");
|
|
27710
|
+
const input = `${title ?? ""}
|
|
27711
|
+
${content}`.slice(0, MAX_EMBED_INPUT_CHARS);
|
|
27712
|
+
return {
|
|
27713
|
+
key: String(row.k),
|
|
27714
|
+
content,
|
|
27715
|
+
title,
|
|
27716
|
+
epoch,
|
|
27717
|
+
metadata,
|
|
27718
|
+
input,
|
|
27719
|
+
hash: createHash14("sha1").update(input).digest("hex")
|
|
27720
|
+
};
|
|
27721
|
+
}
|
|
27722
|
+
toHit(row, score) {
|
|
27723
|
+
return {
|
|
27724
|
+
key: row.key,
|
|
27725
|
+
content: row.content,
|
|
27726
|
+
score,
|
|
27727
|
+
...row.title !== undefined ? { title: row.title } : {},
|
|
27728
|
+
...row.epoch !== undefined ? { publishedAt: Math.round(row.epoch * 1000) } : {},
|
|
27729
|
+
metadata: row.metadata
|
|
27730
|
+
};
|
|
27731
|
+
}
|
|
27732
|
+
}
|
|
27733
|
+
function toMatchQuery(text) {
|
|
27734
|
+
const terms = [...new Set(text.match(/[\p{L}\p{N}_]+/gu) ?? [])];
|
|
27735
|
+
return terms.length === 0 ? undefined : terms.map((term) => `"${term}"`).join(" OR ");
|
|
27736
|
+
}
|
|
27737
|
+
function errorMessage2(error) {
|
|
27738
|
+
return error instanceof Error ? error.message : String(error);
|
|
27739
|
+
}
|
|
27740
|
+
var DEFAULT_BATCH_SIZE = 256, DEFAULT_MAX_EMBED_PER_SYNC = 2048, MAX_EMBED_INPUT_CHARS = 8192, BUSY_TIMEOUT_MS = 5000, POOL_FACTOR = 3, SEMANTIC_REUSE_MS = 1e4, IDENTITY_KEY = "identity";
|
|
27741
|
+
var init_sidecar = () => {};
|
|
27742
|
+
|
|
27743
|
+
// src/datasource/sidecar/methods.ts
|
|
27744
|
+
function sidecarMethodName(datasourceId, mode) {
|
|
27745
|
+
return mode === "bm25" ? `${datasourceId}-sidecar-bm25` : `${datasourceId}-${mode}`;
|
|
27746
|
+
}
|
|
27747
|
+
|
|
27748
|
+
class SidecarRetrievalMethod {
|
|
27749
|
+
options;
|
|
27750
|
+
constructor(options) {
|
|
27751
|
+
this.options = options;
|
|
27752
|
+
}
|
|
27753
|
+
describe() {
|
|
27754
|
+
const { datasourceId, skillType, mode, tags } = this.options;
|
|
27755
|
+
const { type, text, capabilities } = DESCRIPTIONS[mode];
|
|
27756
|
+
return {
|
|
27757
|
+
name: sidecarMethodName(datasourceId, mode),
|
|
27758
|
+
type,
|
|
27759
|
+
description: `${text} over ${skillType} via a read-only sidecar index`,
|
|
27760
|
+
status: "active",
|
|
27761
|
+
capabilities: [...capabilities, "scoped", "path-opaque-sources"],
|
|
27762
|
+
datasourceId,
|
|
27763
|
+
tags
|
|
27764
|
+
};
|
|
27765
|
+
}
|
|
27766
|
+
async retrieve(query, options) {
|
|
27767
|
+
const trimmed = query.trim();
|
|
27768
|
+
if (trimmed.length === 0)
|
|
27769
|
+
return [];
|
|
27770
|
+
const topK = options.topK ?? DEFAULT_TOP_K5;
|
|
27771
|
+
const { sidecar, mode } = this.options;
|
|
27772
|
+
const hits = mode === "bm25" ? sidecar.searchLexical(trimmed, topK) : mode === "semantic" ? await sidecar.searchSemantic(trimmed, topK) : await sidecar.searchHybrid(trimmed, topK);
|
|
27773
|
+
const results = [];
|
|
27774
|
+
for (const hit of hits) {
|
|
27775
|
+
const result = this.toResult(hit, options);
|
|
27776
|
+
if (result !== undefined)
|
|
27777
|
+
results.push(result);
|
|
27778
|
+
}
|
|
27779
|
+
return results;
|
|
27780
|
+
}
|
|
27781
|
+
toResult(hit, options) {
|
|
27782
|
+
const { descriptor, datasourceId, instanceId, mode } = this.options;
|
|
27783
|
+
const chunkId = descriptor.chunkId(hit.key, hit.metadata);
|
|
27784
|
+
const source = datasourceSourcePath(datasourceId, instanceId, chunkId);
|
|
27785
|
+
if (!matchesDatasourceScope(source, options.scope))
|
|
27786
|
+
return;
|
|
27787
|
+
const method = sidecarMethodName(datasourceId, mode);
|
|
27788
|
+
return {
|
|
27789
|
+
id: `${datasourceId}:${instanceId}:${chunkId}`,
|
|
27790
|
+
content: hit.content,
|
|
27791
|
+
source,
|
|
27792
|
+
score: hit.score,
|
|
27793
|
+
metadata: {
|
|
27794
|
+
...hit.metadata,
|
|
27795
|
+
method,
|
|
27796
|
+
datasourceId,
|
|
27797
|
+
instanceId,
|
|
27798
|
+
backend: "sidecar",
|
|
27799
|
+
...hit.title !== undefined ? { title: hit.title } : {},
|
|
27800
|
+
hierarchy: descriptor.hierarchy(hit.metadata).join("/"),
|
|
27801
|
+
...hit.publishedAt !== undefined ? { publishedAt: hit.publishedAt } : {}
|
|
27802
|
+
}
|
|
27803
|
+
};
|
|
27804
|
+
}
|
|
27805
|
+
}
|
|
27806
|
+
var DEFAULT_TOP_K5 = 20, DESCRIPTIONS;
|
|
27807
|
+
var init_methods5 = __esm(() => {
|
|
27808
|
+
init_scope2();
|
|
27809
|
+
DESCRIPTIONS = {
|
|
27810
|
+
bm25: { type: "bm25", text: "BM25-ranked FTS5 retrieval", capabilities: ["lexical", "fts5", "bm25"] },
|
|
27811
|
+
semantic: { type: "vector", text: "Semantic (embedding) retrieval", capabilities: ["semantic"] },
|
|
27812
|
+
hybrid: {
|
|
27813
|
+
type: "hybrid",
|
|
27814
|
+
text: "Hybrid BM25 + semantic retrieval (reciprocal rank fusion)",
|
|
27815
|
+
capabilities: ["lexical", "semantic", "hybrid"]
|
|
27816
|
+
}
|
|
27817
|
+
};
|
|
27818
|
+
});
|
|
27819
|
+
|
|
26126
27820
|
// src/datasource/crawler-skill.ts
|
|
26127
27821
|
class CrawlerDatasourceSkill {
|
|
26128
27822
|
definition;
|
|
@@ -26133,6 +27827,7 @@ class CrawlerDatasourceSkill {
|
|
|
26133
27827
|
tags;
|
|
26134
27828
|
channelIds;
|
|
26135
27829
|
channelNames;
|
|
27830
|
+
sidecar;
|
|
26136
27831
|
lastIndexedAt;
|
|
26137
27832
|
constructor(definition, options) {
|
|
26138
27833
|
this.definition = options.datasourceId === undefined ? definition : {
|
|
@@ -26148,6 +27843,11 @@ class CrawlerDatasourceSkill {
|
|
|
26148
27843
|
this.lastIndexedAt = options.lastIndexedAt;
|
|
26149
27844
|
this.channelIds = new Set(options.channelIds ?? []);
|
|
26150
27845
|
this.channelNames = new Set(options.channelNames ?? []);
|
|
27846
|
+
this.sidecar = options.sidecar === undefined ? undefined : new CrawlerSidecar({
|
|
27847
|
+
...options.sidecar,
|
|
27848
|
+
channelIds: options.channelIds ?? [],
|
|
27849
|
+
channelNames: options.channelNames ?? []
|
|
27850
|
+
});
|
|
26151
27851
|
}
|
|
26152
27852
|
describe() {
|
|
26153
27853
|
return {
|
|
@@ -26183,26 +27883,89 @@ class CrawlerDatasourceSkill {
|
|
|
26183
27883
|
skill: this.definition.datasourceId,
|
|
26184
27884
|
chunkCount: result.count,
|
|
26185
27885
|
indexedAt: this.lastIndexedAt,
|
|
26186
|
-
diagnostics:
|
|
27886
|
+
diagnostics: await this.syncSidecar()
|
|
26187
27887
|
};
|
|
26188
27888
|
} catch {
|
|
26189
27889
|
return this.fail("datasource-unavailable", "spawn-error");
|
|
26190
27890
|
}
|
|
26191
27891
|
}
|
|
27892
|
+
async syncSidecar() {
|
|
27893
|
+
if (this.sidecar === undefined || !this.sidecar.semanticEnabled)
|
|
27894
|
+
return [];
|
|
27895
|
+
const base = { instanceId: this.instanceId, source: this.definition.datasourceId };
|
|
27896
|
+
let result;
|
|
27897
|
+
try {
|
|
27898
|
+
result = await this.sidecar.sync();
|
|
27899
|
+
} catch (error) {
|
|
27900
|
+
result = { ok: false, message: error instanceof Error ? error.message : String(error) };
|
|
27901
|
+
}
|
|
27902
|
+
if (!result.ok) {
|
|
27903
|
+
return [
|
|
27904
|
+
{
|
|
27905
|
+
code: "datasource-index-failed",
|
|
27906
|
+
severity: "warning",
|
|
27907
|
+
message: `semantic-unavailable: ${result.message}`.slice(0, 300),
|
|
27908
|
+
...base
|
|
27909
|
+
}
|
|
27910
|
+
];
|
|
27911
|
+
}
|
|
27912
|
+
const diagnostics = [];
|
|
27913
|
+
if (result.skipped > 0) {
|
|
27914
|
+
diagnostics.push({
|
|
27915
|
+
code: "datasource-index-partial",
|
|
27916
|
+
severity: "warning",
|
|
27917
|
+
message: `semantic index skipped ${result.skipped} message(s) the embedder rejected; they stay searchable by BM25 and are retried when edited`,
|
|
27918
|
+
...base
|
|
27919
|
+
});
|
|
27920
|
+
}
|
|
27921
|
+
if (result.remaining > 0) {
|
|
27922
|
+
diagnostics.push({
|
|
27923
|
+
code: "datasource-index-partial",
|
|
27924
|
+
severity: "info",
|
|
27925
|
+
message: `semantic index catching up: ${result.embedded} embedded this refresh, ${result.remaining} remaining; hybrid search uses BM25 for the rest until a later refresh`,
|
|
27926
|
+
...base
|
|
27927
|
+
});
|
|
27928
|
+
}
|
|
27929
|
+
return diagnostics;
|
|
27930
|
+
}
|
|
27931
|
+
close() {
|
|
27932
|
+
this.sidecar?.close();
|
|
27933
|
+
}
|
|
26192
27934
|
retrievalMethods() {
|
|
27935
|
+
const cli = this.cliMethod();
|
|
27936
|
+
const { sidecar } = this;
|
|
27937
|
+
if (sidecar === undefined)
|
|
27938
|
+
return [cli];
|
|
27939
|
+
const shared = {
|
|
27940
|
+
sidecar,
|
|
27941
|
+
descriptor: sidecar.descriptor,
|
|
27942
|
+
datasourceId: this.definition.datasourceId,
|
|
27943
|
+
skillType: this.definition.skillType,
|
|
27944
|
+
instanceId: this.instanceId,
|
|
27945
|
+
tags: this.tags
|
|
27946
|
+
};
|
|
27947
|
+
const bm25 = new SidecarRetrievalMethod({ ...shared, mode: "bm25" });
|
|
27948
|
+
if (!sidecar.semanticEnabled)
|
|
27949
|
+
return [cli, bm25];
|
|
26193
27950
|
return [
|
|
26194
|
-
|
|
26195
|
-
|
|
26196
|
-
|
|
26197
|
-
|
|
26198
|
-
instanceId: this.instanceId,
|
|
26199
|
-
tags: this.tags,
|
|
26200
|
-
backendName: this.definition.backendName,
|
|
26201
|
-
channelIds: this.channelIds,
|
|
26202
|
-
channelNames: this.channelNames
|
|
26203
|
-
})
|
|
27951
|
+
cli,
|
|
27952
|
+
bm25,
|
|
27953
|
+
new SidecarRetrievalMethod({ ...shared, mode: "semantic" }),
|
|
27954
|
+
new SidecarRetrievalMethod({ ...shared, mode: "hybrid" })
|
|
26204
27955
|
];
|
|
26205
27956
|
}
|
|
27957
|
+
cliMethod() {
|
|
27958
|
+
return new CrawlerLexicalMethod({
|
|
27959
|
+
client: this.client,
|
|
27960
|
+
datasourceId: this.definition.datasourceId,
|
|
27961
|
+
skillType: this.definition.skillType,
|
|
27962
|
+
instanceId: this.instanceId,
|
|
27963
|
+
tags: this.tags,
|
|
27964
|
+
backendName: this.definition.backendName,
|
|
27965
|
+
channelIds: this.channelIds,
|
|
27966
|
+
channelNames: this.channelNames
|
|
27967
|
+
});
|
|
27968
|
+
}
|
|
26206
27969
|
describeSources() {
|
|
26207
27970
|
return this.instances.map((instanceId) => ({
|
|
26208
27971
|
source: datasourceSourcePath(this.definition.datasourceId, instanceId),
|
|
@@ -26333,6 +28096,8 @@ var DEFAULT_INSTANCE_ID5 = "default", DEFAULT_POLLING_INTERVAL_MS4;
|
|
|
26333
28096
|
var init_crawler_skill = __esm(() => {
|
|
26334
28097
|
init_errors4();
|
|
26335
28098
|
init_scope2();
|
|
28099
|
+
init_sidecar();
|
|
28100
|
+
init_methods5();
|
|
26336
28101
|
DEFAULT_POLLING_INTERVAL_MS4 = 15 * 60 * 1000;
|
|
26337
28102
|
});
|
|
26338
28103
|
|
|
@@ -26485,15 +28250,15 @@ var init_notion = __esm(() => {
|
|
|
26485
28250
|
});
|
|
26486
28251
|
|
|
26487
28252
|
// src/datasource/skills/obsidian/paths.ts
|
|
26488
|
-
import { join as
|
|
28253
|
+
import { join as join36 } from "node:path";
|
|
26489
28254
|
function obsidianDatasourceRoot(workspaceRoot, instanceId) {
|
|
26490
|
-
return
|
|
28255
|
+
return join36(workspaceRoot, AUTORAG_DIRNAME, DATASOURCES_DIRNAME, OBSIDIAN_DATASOURCE_DIRNAME, instanceId);
|
|
26491
28256
|
}
|
|
26492
28257
|
function obsidianQmdConfigDir(workspaceRoot, instanceId) {
|
|
26493
|
-
return
|
|
28258
|
+
return join36(obsidianDatasourceRoot(workspaceRoot, instanceId), "config");
|
|
26494
28259
|
}
|
|
26495
28260
|
function obsidianQmdCacheDir(workspaceRoot, instanceId) {
|
|
26496
|
-
return
|
|
28261
|
+
return join36(obsidianDatasourceRoot(workspaceRoot, instanceId), "cache");
|
|
26497
28262
|
}
|
|
26498
28263
|
function stripEdgeDashes(value) {
|
|
26499
28264
|
let start = 0;
|
|
@@ -26516,8 +28281,8 @@ var DEFAULT_QMD_BINARY = "qmd", DEFAULT_QMD_TIMEOUT_MS = 60000, DEFAULT_QMD_MAX_
|
|
|
26516
28281
|
|
|
26517
28282
|
// src/datasource/skills/obsidian/client.ts
|
|
26518
28283
|
import { spawn as spawn16 } from "node:child_process";
|
|
26519
|
-
import { mkdirSync as
|
|
26520
|
-
import { isAbsolute as
|
|
28284
|
+
import { mkdirSync as mkdirSync25, writeFileSync as writeFileSync22 } from "node:fs";
|
|
28285
|
+
import { isAbsolute as isAbsolute9, resolve as resolve15 } from "node:path";
|
|
26521
28286
|
|
|
26522
28287
|
class QmdClient {
|
|
26523
28288
|
options;
|
|
@@ -26536,10 +28301,10 @@ class QmdClient {
|
|
|
26536
28301
|
const cacheDir = obsidianQmdCacheDir(workspaceRoot, instanceId);
|
|
26537
28302
|
try {
|
|
26538
28303
|
if (this.options.configPath === undefined) {
|
|
26539
|
-
|
|
26540
|
-
writeFileSync22(
|
|
28304
|
+
mkdirSync25(configDir, { recursive: true });
|
|
28305
|
+
writeFileSync22(resolve15(configDir, "index.yml"), renderIndexYaml(collectionName, vaultPath), "utf8");
|
|
26541
28306
|
}
|
|
26542
|
-
|
|
28307
|
+
mkdirSync25(cacheDir, { recursive: true });
|
|
26543
28308
|
} catch (error) {
|
|
26544
28309
|
return failure4("spawn-error", error instanceof Error ? error.message : "failed to write qmd config");
|
|
26545
28310
|
}
|
|
@@ -26613,7 +28378,7 @@ function resolveVaultPath(vaultPath) {
|
|
|
26613
28378
|
if (vaultPath === undefined || vaultPath.trim().length === 0)
|
|
26614
28379
|
return;
|
|
26615
28380
|
const trimmed = vaultPath.trim();
|
|
26616
|
-
return
|
|
28381
|
+
return isAbsolute9(trimmed) ? trimmed : resolve15(trimmed);
|
|
26617
28382
|
}
|
|
26618
28383
|
function controlledEnv5(options) {
|
|
26619
28384
|
const instanceId = options.instanceId ?? "default";
|
|
@@ -26932,7 +28697,7 @@ async function retrieveObsidian(client, mode, methodName, instanceId, query, opt
|
|
|
26932
28697
|
const trimmed = query.trim();
|
|
26933
28698
|
if (trimmed.length === 0)
|
|
26934
28699
|
return [];
|
|
26935
|
-
const topK = options.topK ??
|
|
28700
|
+
const topK = options.topK ?? DEFAULT_TOP_K6;
|
|
26936
28701
|
const searchOptions = { topK, signal: options.signal };
|
|
26937
28702
|
const result = await client.search(mode, trimmed, searchOptions);
|
|
26938
28703
|
if (!result.ok)
|
|
@@ -26967,8 +28732,8 @@ function toRetrievalResult3(hit, source, methodName, instanceId, mode) {
|
|
|
26967
28732
|
}
|
|
26968
28733
|
};
|
|
26969
28734
|
}
|
|
26970
|
-
var OBSIDIAN_DATASOURCE_ID = "obsidian", DEFAULT_OBSIDIAN_TAGS,
|
|
26971
|
-
var
|
|
28735
|
+
var OBSIDIAN_DATASOURCE_ID = "obsidian", DEFAULT_OBSIDIAN_TAGS, DEFAULT_TOP_K6 = 20;
|
|
28736
|
+
var init_methods6 = __esm(() => {
|
|
26972
28737
|
init_errors4();
|
|
26973
28738
|
init_scope2();
|
|
26974
28739
|
DEFAULT_OBSIDIAN_TAGS = ["obsidian", "notes"];
|
|
@@ -27158,7 +28923,7 @@ var OBSIDIAN_DATASOURCE_ID2 = "obsidian", OBSIDIAN_SKILL_TYPE = "obsidian-vault"
|
|
|
27158
28923
|
var init_skill10 = __esm(() => {
|
|
27159
28924
|
init_scope2();
|
|
27160
28925
|
init_client10();
|
|
27161
|
-
|
|
28926
|
+
init_methods6();
|
|
27162
28927
|
init_paths5();
|
|
27163
28928
|
DEFAULT_POLLING_INTERVAL_MS5 = 15 * 60 * 1000;
|
|
27164
28929
|
DEFAULT_OBSIDIAN_TAGS2 = ["obsidian", "notes"];
|
|
@@ -27167,7 +28932,7 @@ var init_skill10 = __esm(() => {
|
|
|
27167
28932
|
// src/datasource/skills/obsidian/index.ts
|
|
27168
28933
|
var init_obsidian = __esm(() => {
|
|
27169
28934
|
init_client10();
|
|
27170
|
-
|
|
28935
|
+
init_methods6();
|
|
27171
28936
|
init_paths5();
|
|
27172
28937
|
init_skill10();
|
|
27173
28938
|
});
|
|
@@ -27507,7 +29272,7 @@ var init_slack = __esm(() => {
|
|
|
27507
29272
|
|
|
27508
29273
|
// src/datasource/skills/spotlight/connector.ts
|
|
27509
29274
|
import { spawn as spawn17 } from "node:child_process";
|
|
27510
|
-
import { createHash as
|
|
29275
|
+
import { createHash as createHash15 } from "node:crypto";
|
|
27511
29276
|
import { readFile as fsReadFile, stat as stat7 } from "node:fs/promises";
|
|
27512
29277
|
import { basename as basename13, extname as extname8 } from "node:path";
|
|
27513
29278
|
|
|
@@ -27530,7 +29295,7 @@ class SpotlightConnector {
|
|
|
27530
29295
|
if (queries.length === 0) {
|
|
27531
29296
|
return { ok: false, reason: "not-configured", message: "no spotlight queries configured" };
|
|
27532
29297
|
}
|
|
27533
|
-
const timeoutMs = this.options.timeoutMs ??
|
|
29298
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS10;
|
|
27534
29299
|
const status = await this.run("mdutil", ["-s", "/"], timeoutMs);
|
|
27535
29300
|
if (status.ok && /indexing disabled/iu.test(status.stdout)) {
|
|
27536
29301
|
return {
|
|
@@ -27592,7 +29357,7 @@ class SpotlightConnector {
|
|
|
27592
29357
|
if (content.length === 0)
|
|
27593
29358
|
continue;
|
|
27594
29359
|
documents.push({
|
|
27595
|
-
docId:
|
|
29360
|
+
docId: createHash15("sha256").update(path).digest("hex").slice(0, 16),
|
|
27596
29361
|
hierarchy: ["files"],
|
|
27597
29362
|
title: basename13(path),
|
|
27598
29363
|
content,
|
|
@@ -27625,7 +29390,7 @@ class SpotlightConnector {
|
|
|
27625
29390
|
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
27626
29391
|
}
|
|
27627
29392
|
}
|
|
27628
|
-
var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS5 = 300, DEFAULT_MAX_BYTES_PER_FILE3,
|
|
29393
|
+
var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS5 = 300, DEFAULT_MAX_BYTES_PER_FILE3, DEFAULT_TIMEOUT_MS10 = 15000, MAX_CONTENT_CHARS4 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve) => {
|
|
27629
29394
|
const child = spawn17(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
|
|
27630
29395
|
let stdout = "";
|
|
27631
29396
|
let stderr = "";
|
|
@@ -27742,10 +29507,12 @@ function parseHits3(stdout) {
|
|
|
27742
29507
|
for (const [index, row] of rows.entries()) {
|
|
27743
29508
|
if (!isRecord18(row))
|
|
27744
29509
|
return;
|
|
27745
|
-
const id = stringValue4(row, ["
|
|
27746
|
-
|
|
27747
|
-
if (id === undefined || content === undefined)
|
|
29510
|
+
const id = stringValue4(row, ["event_id", "message_id", "id"]);
|
|
29511
|
+
if (id === undefined)
|
|
27748
29512
|
return;
|
|
29513
|
+
const content = stringValue4(row, ["snippet", "text", "content"]);
|
|
29514
|
+
if (content === undefined)
|
|
29515
|
+
continue;
|
|
27749
29516
|
const chatId = stringValue4(row, ["chat_jid", "chat_id"]);
|
|
27750
29517
|
const chatName = stringValue4(row, ["chat_name"]);
|
|
27751
29518
|
const senderName = stringValue4(row, ["sender_name"]);
|
|
@@ -27862,10 +29629,12 @@ function parseHits4(stdout) {
|
|
|
27862
29629
|
for (const [index, row] of rows.entries()) {
|
|
27863
29630
|
if (!isRecord19(row))
|
|
27864
29631
|
return;
|
|
27865
|
-
const id = stringValue5(row, ["
|
|
27866
|
-
|
|
27867
|
-
if (id === undefined || content === undefined)
|
|
29632
|
+
const id = stringValue5(row, ["event_id", "message_id", "id"]);
|
|
29633
|
+
if (id === undefined)
|
|
27868
29634
|
return;
|
|
29635
|
+
const content = stringValue5(row, ["snippet", "text", "content"]);
|
|
29636
|
+
if (content === undefined)
|
|
29637
|
+
continue;
|
|
27869
29638
|
const chatId = stringValue5(row, ["chat_jid", "chat_id"]);
|
|
27870
29639
|
const chatName = stringValue5(row, ["chat_name"]);
|
|
27871
29640
|
const senderName = stringValue5(row, ["sender_name"]);
|
|
@@ -27955,6 +29724,31 @@ var init_wacrawl = __esm(() => {
|
|
|
27955
29724
|
});
|
|
27956
29725
|
|
|
27957
29726
|
// src/datasource/skills/factory.ts
|
|
29727
|
+
import { homedir as homedir4 } from "node:os";
|
|
29728
|
+
import { join as join37 } from "node:path";
|
|
29729
|
+
function chatCrawlerArgs(descriptor, defaultCrawlerDb, config, workspaceRoot, registrationName) {
|
|
29730
|
+
const sidecar = resolveSidecarOptions({
|
|
29731
|
+
descriptor,
|
|
29732
|
+
connector: config.connector,
|
|
29733
|
+
workspaceRoot,
|
|
29734
|
+
registrationName,
|
|
29735
|
+
instanceId: config.instanceId ?? "default",
|
|
29736
|
+
defaultCrawlerDb
|
|
29737
|
+
});
|
|
29738
|
+
return {
|
|
29739
|
+
datasourceId: registrationName,
|
|
29740
|
+
...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
|
|
29741
|
+
...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
|
|
29742
|
+
...config.tags !== undefined ? { tags: config.tags } : {},
|
|
29743
|
+
channelIds: config.channels?.ids,
|
|
29744
|
+
channelNames: config.channels?.names,
|
|
29745
|
+
...sidecar === undefined ? {} : { sidecar },
|
|
29746
|
+
connectorOptions: {
|
|
29747
|
+
...config.connector,
|
|
29748
|
+
...workspaceRoot === undefined ? {} : { workspacePath: workspaceRoot }
|
|
29749
|
+
}
|
|
29750
|
+
};
|
|
29751
|
+
}
|
|
27958
29752
|
function larkConnectorOptions(connector) {
|
|
27959
29753
|
if (connector === undefined)
|
|
27960
29754
|
return {};
|
|
@@ -28015,6 +29809,8 @@ function buildDatasourceSkills(config, workspaceRoot, embeddingRuntime) {
|
|
|
28015
29809
|
var BUILDERS, BUILTIN_DATASOURCE_SKILL_NAMES;
|
|
28016
29810
|
var init_factory = __esm(() => {
|
|
28017
29811
|
init_aliased_skill();
|
|
29812
|
+
init_config2();
|
|
29813
|
+
init_descriptors();
|
|
28018
29814
|
init_clawgallery();
|
|
28019
29815
|
init_cloud_drive();
|
|
28020
29816
|
init_discrawl();
|
|
@@ -28032,42 +29828,9 @@ var init_factory = __esm(() => {
|
|
|
28032
29828
|
init_telecrawl();
|
|
28033
29829
|
init_wacrawl();
|
|
28034
29830
|
BUILDERS = {
|
|
28035
|
-
telegram: (config,
|
|
28036
|
-
|
|
28037
|
-
|
|
28038
|
-
...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
|
|
28039
|
-
...config.tags !== undefined ? { tags: config.tags } : {},
|
|
28040
|
-
channelIds: config.channels?.ids,
|
|
28041
|
-
channelNames: config.channels?.names,
|
|
28042
|
-
connectorOptions: {
|
|
28043
|
-
...config.connector,
|
|
28044
|
-
..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
|
|
28045
|
-
}
|
|
28046
|
-
}),
|
|
28047
|
-
whatsapp: (config, _workspaceRoot, registrationName) => new WacrawlSkill({
|
|
28048
|
-
datasourceId: registrationName,
|
|
28049
|
-
...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
|
|
28050
|
-
...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
|
|
28051
|
-
...config.tags !== undefined ? { tags: config.tags } : {},
|
|
28052
|
-
channelIds: config.channels?.ids,
|
|
28053
|
-
channelNames: config.channels?.names,
|
|
28054
|
-
connectorOptions: {
|
|
28055
|
-
...config.connector,
|
|
28056
|
-
..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
|
|
28057
|
-
}
|
|
28058
|
-
}),
|
|
28059
|
-
slack: (config, _workspaceRoot, registrationName) => new SlackSkill({
|
|
28060
|
-
datasourceId: registrationName,
|
|
28061
|
-
...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
|
|
28062
|
-
...config.pollingIntervalMs !== undefined ? { pollingIntervalMs: config.pollingIntervalMs } : {},
|
|
28063
|
-
...config.tags !== undefined ? { tags: config.tags } : {},
|
|
28064
|
-
channelIds: config.channels?.ids,
|
|
28065
|
-
channelNames: config.channels?.names,
|
|
28066
|
-
connectorOptions: {
|
|
28067
|
-
...config.connector,
|
|
28068
|
-
..._workspaceRoot === undefined ? {} : { workspacePath: _workspaceRoot }
|
|
28069
|
-
}
|
|
28070
|
-
}),
|
|
29831
|
+
telegram: (config, workspaceRoot, registrationName) => new TelecrawlSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.telegram, join37(homedir4(), ".telecrawl", "telecrawl.db"), config, workspaceRoot, registrationName)),
|
|
29832
|
+
whatsapp: (config, workspaceRoot, registrationName) => new WacrawlSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.whatsapp, join37(homedir4(), ".wacrawl", "wacrawl.db"), config, workspaceRoot, registrationName)),
|
|
29833
|
+
slack: (config, workspaceRoot, registrationName) => new SlackSkill(chatCrawlerArgs(CRAWLER_DESCRIPTORS.slack, join37(homedir4(), ".slacrawl", "slacrawl.db"), config, workspaceRoot, registrationName)),
|
|
28071
29834
|
discord: (config, workspaceRoot, registrationName, embeddingRuntime) => {
|
|
28072
29835
|
const connector = config.connector ?? {};
|
|
28073
29836
|
const clientOptions = {
|
|
@@ -28186,8 +29949,8 @@ var init_factory = __esm(() => {
|
|
|
28186
29949
|
|
|
28187
29950
|
// src/cli/config.ts
|
|
28188
29951
|
import { randomUUID as randomUUID9 } from "node:crypto";
|
|
28189
|
-
import { existsSync as existsSync28, mkdirSync as
|
|
28190
|
-
import { basename as basename14, dirname as
|
|
29952
|
+
import { existsSync as existsSync28, mkdirSync as mkdirSync26, readFileSync as readFileSync25, renameSync as renameSync11, unlinkSync as unlinkSync5, writeFileSync as writeFileSync23 } from "node:fs";
|
|
29953
|
+
import { basename as basename14, dirname as dirname25, isAbsolute as isAbsolute10, join as join38, resolve as resolve16 } from "node:path";
|
|
28191
29954
|
import { findEnvKeys, getEnvApiKey } from "@earendil-works/pi-ai/compat";
|
|
28192
29955
|
import { getAgentDir as getAgentDir3, ModelRuntime as ModelRuntime2, SettingsManager as SettingsManager2 } from "@earendil-works/pi-coding-agent";
|
|
28193
29956
|
function resolveConfigPath(input) {
|
|
@@ -28203,9 +29966,9 @@ function resolveConfigPath(input) {
|
|
|
28203
29966
|
return { configPath: envConfig, explicit: true };
|
|
28204
29967
|
}
|
|
28205
29968
|
return {
|
|
28206
|
-
configPath:
|
|
29969
|
+
configPath: join38(resolveAutoRAGHome(env), DEFAULT_CONFIG_FILENAME),
|
|
28207
29970
|
explicit: false,
|
|
28208
|
-
legacyPath:
|
|
29971
|
+
legacyPath: join38(cwd, LEGACY_CONFIG_FILENAME)
|
|
28209
29972
|
};
|
|
28210
29973
|
}
|
|
28211
29974
|
function readConfigFile(configPath, explicit) {
|
|
@@ -28223,7 +29986,7 @@ function readConfigFile(configPath, explicit) {
|
|
|
28223
29986
|
}
|
|
28224
29987
|
let text;
|
|
28225
29988
|
try {
|
|
28226
|
-
text =
|
|
29989
|
+
text = readFileSync25(configPath, "utf8");
|
|
28227
29990
|
} catch (err) {
|
|
28228
29991
|
throw new ConfigError(`Failed to read config file: ${err.message}`);
|
|
28229
29992
|
}
|
|
@@ -28261,7 +30024,7 @@ function acquireConfigWriteLock(configPath) {
|
|
|
28261
30024
|
});
|
|
28262
30025
|
}
|
|
28263
30026
|
function replaceFileAtomically(path, contents, assertCommitAllowed) {
|
|
28264
|
-
const temporaryPath =
|
|
30027
|
+
const temporaryPath = join38(dirname25(path), `.${basename14(path)}.${process.pid}.${randomUUID9()}.tmp`);
|
|
28265
30028
|
try {
|
|
28266
30029
|
writeFileSync23(temporaryPath, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
|
|
28267
30030
|
assertCommitAllowed?.();
|
|
@@ -28274,12 +30037,12 @@ function migrateLegacyConfig(configPath, legacyPath) {
|
|
|
28274
30037
|
if (existsSync28(configPath) || !existsSync28(legacyPath))
|
|
28275
30038
|
return;
|
|
28276
30039
|
const legacy = readConfigFile(legacyPath, true);
|
|
28277
|
-
const legacyBytes =
|
|
28278
|
-
const migrated = normalizeLegacyConfigPaths(legacy ?? {},
|
|
30040
|
+
const legacyBytes = readFileSync25(legacyPath);
|
|
30041
|
+
const migrated = normalizeLegacyConfigPaths(legacy ?? {}, dirname25(legacyPath));
|
|
28279
30042
|
const migratedPathsUnchanged = legacy?.workspacePath === migrated.workspacePath && legacy?.memoryPath === migrated.memoryPath && JSON.stringify(legacy?.searchPaths) === JSON.stringify(migrated.searchPaths);
|
|
28280
30043
|
const migratedBytes = migratedPathsUnchanged ? legacyBytes : `${JSON.stringify(migrated, null, 2)}
|
|
28281
30044
|
`;
|
|
28282
|
-
|
|
30045
|
+
mkdirSync26(dirname25(configPath), { recursive: true });
|
|
28283
30046
|
const lock = acquireConfigWriteLock(configPath);
|
|
28284
30047
|
try {
|
|
28285
30048
|
const winner = readConfigFile(configPath, false);
|
|
@@ -28309,13 +30072,13 @@ function resolveConfigFileReadOnly(configPath, explicit, legacyPath) {
|
|
|
28309
30072
|
const legacy = readConfigFile(legacyPath, false);
|
|
28310
30073
|
if (legacy === undefined)
|
|
28311
30074
|
return {};
|
|
28312
|
-
return normalizeLegacyConfigPaths(legacy,
|
|
30075
|
+
return normalizeLegacyConfigPaths(legacy, dirname25(legacyPath));
|
|
28313
30076
|
}
|
|
28314
30077
|
function resolveSearchPaths(searchPaths, origin) {
|
|
28315
30078
|
return searchPaths.map((searchPath) => resolvePersistedPath(searchPath, origin));
|
|
28316
30079
|
}
|
|
28317
30080
|
function resolvePersistedPath(path, origin) {
|
|
28318
|
-
return
|
|
30081
|
+
return isAbsolute10(path) ? path : resolve16(origin, path);
|
|
28319
30082
|
}
|
|
28320
30083
|
function normalizeLegacyConfigPaths(partial, origin) {
|
|
28321
30084
|
const workspacePath = resolvePersistedPath(partial.workspacePath ?? ".", origin);
|
|
@@ -28502,7 +30265,7 @@ function normalizeEmbedder(raw, path) {
|
|
|
28502
30265
|
out.passagePrefix = record.passagePrefix;
|
|
28503
30266
|
}
|
|
28504
30267
|
if (record.profile !== undefined) {
|
|
28505
|
-
if (record.profile !== "qwen3-embedding-0.6b"
|
|
30268
|
+
if (record.profile !== "qwen3-embedding-0.6b") {
|
|
28506
30269
|
throw new ConfigError(`${path}.profile must be a supported runtime profile`);
|
|
28507
30270
|
}
|
|
28508
30271
|
out.profile = record.profile;
|
|
@@ -28745,10 +30508,10 @@ function resolveConfig(input) {
|
|
|
28745
30508
|
const file = readOnly ? resolveConfigFileReadOnly(configPath, explicit, legacyPath) : !explicit && legacyPath ? migrateLegacyConfig(configPath, legacyPath) ?? readConfigFile(configPath, explicit) ?? {} : readConfigFile(configPath, explicit) ?? {};
|
|
28746
30509
|
const defaultSearchPaths = ["."];
|
|
28747
30510
|
const defaultWorkspacePath = cwd;
|
|
28748
|
-
const defaultMemoryPath =
|
|
30511
|
+
const defaultMemoryPath = join38(resolveAutoRAGHome(env), "memory.json");
|
|
28749
30512
|
const flagSearchPaths = parseCsv(flagString(flags, "search-paths"));
|
|
28750
30513
|
const envSearchPaths = parseCsv(envString(env, "AUTORAG_SEARCH_PATHS"));
|
|
28751
|
-
const configOrigin =
|
|
30514
|
+
const configOrigin = dirname25(resolve16(configPath));
|
|
28752
30515
|
const fileWorkspacePath = typeof file.workspacePath === "string" ? resolvePersistedPath(file.workspacePath, configOrigin) : undefined;
|
|
28753
30516
|
const fileSearchPaths = file.searchPaths ? resolveSearchPaths(file.searchPaths, fileWorkspacePath ?? configOrigin) : undefined;
|
|
28754
30517
|
if (file.excludePaths !== undefined && (!Array.isArray(file.excludePaths) || file.excludePaths.some((path) => typeof path !== "string" || path.length === 0))) {
|
|
@@ -28771,7 +30534,7 @@ function resolveConfig(input) {
|
|
|
28771
30534
|
const flagModelId = pickString(flags, env, "model-id", "AUTORAG_MODEL_ID", undefined);
|
|
28772
30535
|
const model = applyRoleFlagOverrides(fileModel, flagModelProvider, flagModelId, "model");
|
|
28773
30536
|
const config = {
|
|
28774
|
-
configPath:
|
|
30537
|
+
configPath: resolve16(configPath),
|
|
28775
30538
|
searchPaths,
|
|
28776
30539
|
workspacePath,
|
|
28777
30540
|
memoryPath,
|
|
@@ -28947,36 +30710,38 @@ function buildWebSearchAgentOption(raw) {
|
|
|
28947
30710
|
function normalizeRerankConfig(raw, path) {
|
|
28948
30711
|
if (raw === false)
|
|
28949
30712
|
return false;
|
|
28950
|
-
|
|
28951
|
-
|
|
28952
|
-
|
|
28953
|
-
|
|
28954
|
-
|
|
28955
|
-
|
|
28956
|
-
|
|
28957
|
-
|
|
28958
|
-
|
|
28959
|
-
|
|
28960
|
-
}
|
|
28961
|
-
const record = raw;
|
|
28962
|
-
for (const key of Object.keys(record)) {
|
|
28963
|
-
if (!RERANK_ALLOWLIST.has(key))
|
|
28964
|
-
throw new ConfigError(`${path}.${key} is not a recognized field`);
|
|
28965
|
-
}
|
|
28966
|
-
if (record.enabled !== undefined) {
|
|
28967
|
-
if (typeof record.enabled !== "boolean")
|
|
28968
|
-
throw new ConfigError(`${path}.enabled must be a boolean`);
|
|
28969
|
-
out.enabled = record.enabled;
|
|
30713
|
+
let record = {};
|
|
30714
|
+
if (raw !== undefined && raw !== null) {
|
|
30715
|
+
if (typeof raw !== "object" || Array.isArray(raw)) {
|
|
30716
|
+
throw new ConfigError(`${path} must be an object or false`);
|
|
30717
|
+
}
|
|
30718
|
+
record = raw;
|
|
30719
|
+
for (const key of Object.keys(record)) {
|
|
30720
|
+
if (!RERANK_ALLOWLIST.has(key))
|
|
30721
|
+
throw new ConfigError(`${path}.${key} is not a recognized field`);
|
|
30722
|
+
}
|
|
28970
30723
|
}
|
|
30724
|
+
let provider = DEFAULT_RERANK_PROVIDER;
|
|
28971
30725
|
if (record.provider !== undefined) {
|
|
28972
30726
|
if (typeof record.provider !== "string" || record.provider.trim() === "") {
|
|
28973
30727
|
throw new ConfigError(`${path}.provider must be a non-empty string`);
|
|
28974
30728
|
}
|
|
28975
|
-
|
|
30729
|
+
provider = record.provider.trim();
|
|
28976
30730
|
}
|
|
28977
|
-
if (!SUPPORTED_RERANK_PROVIDERS.includes(
|
|
30731
|
+
if (!SUPPORTED_RERANK_PROVIDERS.includes(provider)) {
|
|
28978
30732
|
throw new ConfigError(`${path}.provider must be one of: ${SUPPORTED_RERANK_PROVIDERS.join(", ")}`);
|
|
28979
30733
|
}
|
|
30734
|
+
const out = {
|
|
30735
|
+
provider,
|
|
30736
|
+
model: defaultRerankModel(provider),
|
|
30737
|
+
apiKeyEnv: defaultRerankApiKeyEnv(provider),
|
|
30738
|
+
topN: DEFAULT_RERANK_TOP_N
|
|
30739
|
+
};
|
|
30740
|
+
if (record.enabled !== undefined) {
|
|
30741
|
+
if (typeof record.enabled !== "boolean")
|
|
30742
|
+
throw new ConfigError(`${path}.enabled must be a boolean`);
|
|
30743
|
+
out.enabled = record.enabled;
|
|
30744
|
+
}
|
|
28980
30745
|
if (record.model !== undefined) {
|
|
28981
30746
|
if (typeof record.model !== "string" || record.model.trim() === "") {
|
|
28982
30747
|
throw new ConfigError(`${path}.model must be a non-empty string`);
|
|
@@ -29106,7 +30871,8 @@ function buildModelFromConfiguredEndpoint(reference) {
|
|
|
29106
30871
|
input: reference.input ?? ["text"],
|
|
29107
30872
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
29108
30873
|
contextWindow: reference.contextWindow ?? 128000,
|
|
29109
|
-
maxTokens: reference.maxTokens ?? 16384
|
|
30874
|
+
maxTokens: reference.maxTokens ?? 16384,
|
|
30875
|
+
...reference.provider === PROVIDER_ID ? { inputLimits: AUTORAG_INPUT_LIMITS } : {}
|
|
29110
30876
|
};
|
|
29111
30877
|
}
|
|
29112
30878
|
function mergeCatalogModel(catalog, reference) {
|
|
@@ -29157,8 +30923,8 @@ function getModelRuntime(agentDir) {
|
|
|
29157
30923
|
return cached;
|
|
29158
30924
|
const created = (async () => {
|
|
29159
30925
|
const runtime = await ModelRuntime2.create({
|
|
29160
|
-
authPath:
|
|
29161
|
-
modelsPath:
|
|
30926
|
+
authPath: join38(agentDir, "auth.json"),
|
|
30927
|
+
modelsPath: join38(agentDir, "models.json"),
|
|
29162
30928
|
allowModelNetwork: false
|
|
29163
30929
|
});
|
|
29164
30930
|
await registerAutoRAGProvider(runtime);
|
|
@@ -29417,7 +31183,7 @@ function writeConfigObject(path, config) {
|
|
|
29417
31183
|
if (config === null || typeof config !== "object" || Array.isArray(config)) {
|
|
29418
31184
|
throw new ConfigError("Config file must be a JSON object");
|
|
29419
31185
|
}
|
|
29420
|
-
|
|
31186
|
+
mkdirSync26(dirname25(path), { recursive: true });
|
|
29421
31187
|
const contents = `${JSON.stringify(config, null, 2)}
|
|
29422
31188
|
`;
|
|
29423
31189
|
const lock = acquireConfigWriteLock(path);
|
|
@@ -29428,9 +31194,9 @@ function writeConfigObject(path, config) {
|
|
|
29428
31194
|
}
|
|
29429
31195
|
}
|
|
29430
31196
|
function writeDefaultConfig(path, partial, opts = {}) {
|
|
29431
|
-
const cwd =
|
|
31197
|
+
const cwd = resolve16(opts.cwd ?? process.cwd());
|
|
29432
31198
|
const workspacePath = resolvePersistedPath(partial.workspacePath ?? ".", cwd);
|
|
29433
|
-
const memoryPath = partial.memoryPath === undefined ?
|
|
31199
|
+
const memoryPath = partial.memoryPath === undefined ? join38(resolveAutoRAGHome(opts.env), "memory.json") : resolvePersistedPath(partial.memoryPath, workspacePath);
|
|
29434
31200
|
const full = {
|
|
29435
31201
|
searchPaths: resolveSearchPaths(partial.searchPaths ?? ["."], cwd),
|
|
29436
31202
|
workspacePath,
|
|
@@ -29464,7 +31230,7 @@ function writeDefaultConfig(path, partial, opts = {}) {
|
|
|
29464
31230
|
full.p2p = { enabled: false };
|
|
29465
31231
|
full.jev = partial.jev ?? { backend: DEFAULT_JEV_BACKEND };
|
|
29466
31232
|
full.queryDecomposition = partial.queryDecomposition ?? { model: { ...DEFAULT_QUERY_DECOMPOSITION_MODEL } };
|
|
29467
|
-
|
|
31233
|
+
mkdirSync26(dirname25(path), { recursive: true });
|
|
29468
31234
|
const contents = `${JSON.stringify(full, null, 2)}
|
|
29469
31235
|
`;
|
|
29470
31236
|
const lock = acquireConfigWriteLock(path);
|
|
@@ -29492,8 +31258,9 @@ function writeDefaultConfig(path, partial, opts = {}) {
|
|
|
29492
31258
|
}
|
|
29493
31259
|
}
|
|
29494
31260
|
var DEFAULT_CONFIG_FILENAME = "config.json", LEGACY_CONFIG_FILENAME = "autorag.config.json", CONFIG_LOCK_RETRY_MS = 10, CONFIG_LOCK_TIMEOUT_MS = 1e4, CONFIG_LOCK_STALE_MS = 30000, ConfigError, MINSYNC_REQUIRED_MESSAGE = "MinSync is required and cannot be disabled; remove `minSync: false` / `minSync.enabled: false` from the config.", AGENT_MODEL_APIS, AGENT_MODEL_INPUTS, API_KEY_ENV_PATTERN2, POSITIVE_INT_FIELDS, EMBEDDER_ALLOWLIST, MINSYNC_ALLOWLIST, MINSYNC_LEGACY_IGNORED, P2P_ALLOWLIST, P2P_QUOTA_ALLOWLIST, LIMITS_ALLOWLIST, LIMITS_PREFETCH_ALLOWLIST, JEV_CONFIG_FIELDS, JEV_BACKEND_NAMES, DEFAULT_JEV_BACKEND = "openrouter", DEFAULT_QUERY_DECOMPOSITION_MODEL, RERANK_ALLOWLIST, UNKNOWN_MODEL_HINT = "Add baseUrl (and optional api/apiKeyEnv) for an OpenAI-compatible endpoint outside the pi-ai catalog, or use a pi-ai catalog model id.", modelRuntimeCache;
|
|
29495
|
-
var
|
|
31261
|
+
var init_config4 = __esm(() => {
|
|
29496
31262
|
init_local_model();
|
|
31263
|
+
init_config();
|
|
29497
31264
|
init_provider();
|
|
29498
31265
|
init_home();
|
|
29499
31266
|
init_factory();
|
|
@@ -29575,7 +31342,7 @@ var init_config3 = __esm(() => {
|
|
|
29575
31342
|
model: true,
|
|
29576
31343
|
confidenceThreshold: true
|
|
29577
31344
|
};
|
|
29578
|
-
JEV_BACKEND_NAMES = ["typesafe", "openrouter", "vercel"];
|
|
31345
|
+
JEV_BACKEND_NAMES = ["typesafe", "openrouter", "vercel", "autorag"];
|
|
29579
31346
|
DEFAULT_QUERY_DECOMPOSITION_MODEL = { provider: "openrouter", id: "qwen/qwen3.7-flash" };
|
|
29580
31347
|
RERANK_ALLOWLIST = new Set(["enabled", "provider", "model", "apiKeyEnv", "baseUrl", "topN", "timeoutMs"]);
|
|
29581
31348
|
modelRuntimeCache = new Map;
|
|
@@ -29665,9 +31432,9 @@ function createAutoRAGLite(options = {}) {
|
|
|
29665
31432
|
var init_core = __esm(() => {
|
|
29666
31433
|
init_agent();
|
|
29667
31434
|
init_search_documents();
|
|
29668
|
-
|
|
31435
|
+
init_config4();
|
|
29669
31436
|
init_memory();
|
|
29670
|
-
|
|
31437
|
+
init_config4();
|
|
29671
31438
|
});
|
|
29672
31439
|
|
|
29673
31440
|
// src/cli/output.ts
|
|
@@ -30112,7 +31879,7 @@ async function runEvidence(ctx) {
|
|
|
30112
31879
|
}
|
|
30113
31880
|
var init_evidence = __esm(() => {
|
|
30114
31881
|
init_memory();
|
|
30115
|
-
|
|
31882
|
+
init_config4();
|
|
30116
31883
|
});
|
|
30117
31884
|
|
|
30118
31885
|
// src/agent/results.ts
|
|
@@ -30170,12 +31937,12 @@ var init_results = __esm(() => {
|
|
|
30170
31937
|
});
|
|
30171
31938
|
|
|
30172
31939
|
// src/cli/commands/report.ts
|
|
30173
|
-
import { readFileSync as
|
|
31940
|
+
import { readFileSync as readFileSync26 } from "node:fs";
|
|
30174
31941
|
import { Value as Value3 } from "typebox/value";
|
|
30175
31942
|
function readReportInput(input) {
|
|
30176
31943
|
if (typeof input === "string" && input.length > 0)
|
|
30177
|
-
return
|
|
30178
|
-
return
|
|
31944
|
+
return readFileSync26(input, "utf8");
|
|
31945
|
+
return readFileSync26(0, "utf8");
|
|
30179
31946
|
}
|
|
30180
31947
|
function describeSchemaErrors(value) {
|
|
30181
31948
|
const errors = Value3.Errors(reportSchema, value);
|
|
@@ -30266,7 +32033,7 @@ var MAX_REPORTED_SCHEMA_ERRORS = 8;
|
|
|
30266
32033
|
var init_report = __esm(() => {
|
|
30267
32034
|
init_results();
|
|
30268
32035
|
init_core();
|
|
30269
|
-
|
|
32036
|
+
init_config4();
|
|
30270
32037
|
});
|
|
30271
32038
|
|
|
30272
32039
|
// src/mcp/index.ts
|
|
@@ -30279,8 +32046,8 @@ import { StdioServerTransport } from "@modelcontextprotocol/server/stdio";
|
|
|
30279
32046
|
init_evidence();
|
|
30280
32047
|
init_report();
|
|
30281
32048
|
init_dupey();
|
|
30282
|
-
import { readFileSync as
|
|
30283
|
-
import { resolve as
|
|
32049
|
+
import { readFileSync as readFileSync27 } from "node:fs";
|
|
32050
|
+
import { resolve as resolve18 } from "node:path";
|
|
30284
32051
|
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
30285
32052
|
import { McpServer } from "@modelcontextprotocol/server";
|
|
30286
32053
|
import { z as z2 } from "zod";
|
|
@@ -30288,7 +32055,7 @@ import { z as z2 } from "zod";
|
|
|
30288
32055
|
// src/filesystem/name-search.ts
|
|
30289
32056
|
import { realpathSync as realpathSync6 } from "node:fs";
|
|
30290
32057
|
import { lstat, readdir as readdir4 } from "node:fs/promises";
|
|
30291
|
-
import { basename as basename15, dirname as
|
|
32058
|
+
import { basename as basename15, dirname as dirname26, isAbsolute as isAbsolute11, join as join39, relative as relative6, resolve as resolve17, sep as sep8 } from "node:path";
|
|
30292
32059
|
var SKIP_DIR_NAMES2 = {
|
|
30293
32060
|
".git": true,
|
|
30294
32061
|
".autorag": true,
|
|
@@ -30316,27 +32083,27 @@ function fsErrorCode(error) {
|
|
|
30316
32083
|
return typeof code === "string" ? code : undefined;
|
|
30317
32084
|
}
|
|
30318
32085
|
function isContained(childReal, parentReal) {
|
|
30319
|
-
const prefix = parentReal.endsWith(
|
|
32086
|
+
const prefix = parentReal.endsWith(sep8) ? parentReal : `${parentReal}${sep8}`;
|
|
30320
32087
|
return childReal === parentReal || childReal.startsWith(prefix);
|
|
30321
32088
|
}
|
|
30322
32089
|
function pinExcludedPath2(path) {
|
|
30323
|
-
const resolvedPath =
|
|
32090
|
+
const resolvedPath = resolve17(path);
|
|
30324
32091
|
try {
|
|
30325
|
-
return
|
|
32092
|
+
return join39(realpathSync6(dirname26(resolvedPath)), basename15(resolvedPath));
|
|
30326
32093
|
} catch {
|
|
30327
32094
|
return resolvedPath;
|
|
30328
32095
|
}
|
|
30329
32096
|
}
|
|
30330
32097
|
function isExcluded2(absPath, excluded) {
|
|
30331
32098
|
for (const excludedPath of excluded) {
|
|
30332
|
-
const prefix = excludedPath.endsWith(
|
|
32099
|
+
const prefix = excludedPath.endsWith(sep8) ? excludedPath : `${excludedPath}${sep8}`;
|
|
30333
32100
|
if (absPath === excludedPath || absPath.startsWith(prefix))
|
|
30334
32101
|
return true;
|
|
30335
32102
|
}
|
|
30336
32103
|
return false;
|
|
30337
32104
|
}
|
|
30338
32105
|
function matchesEntry(context, absPath, name) {
|
|
30339
|
-
const haystackRaw = context.matchPath ? relative6(context.rootReal, absPath).split(
|
|
32106
|
+
const haystackRaw = context.matchPath ? relative6(context.rootReal, absPath).split(sep8).join("/") : name;
|
|
30340
32107
|
const haystack = context.matchCase ? haystackRaw : haystackRaw.toLowerCase();
|
|
30341
32108
|
const needle = context.matchCase ? context.query : context.query.toLowerCase();
|
|
30342
32109
|
return haystack.includes(needle);
|
|
@@ -30376,7 +32143,7 @@ async function walkDirectory(context, directory) {
|
|
|
30376
32143
|
}
|
|
30377
32144
|
if (entry.isSymbolicLink())
|
|
30378
32145
|
continue;
|
|
30379
|
-
const absPath =
|
|
32146
|
+
const absPath = join39(directory, entry.name);
|
|
30380
32147
|
if (entry.isDirectory()) {
|
|
30381
32148
|
if (SKIP_DIR_NAMES2[entry.name] === true)
|
|
30382
32149
|
continue;
|
|
@@ -30413,7 +32180,7 @@ async function pinConfiguredRoots(roots, diagnostics) {
|
|
|
30413
32180
|
for (const root of roots) {
|
|
30414
32181
|
if (typeof root !== "string" || root.length === 0)
|
|
30415
32182
|
continue;
|
|
30416
|
-
const resolvedRoot =
|
|
32183
|
+
const resolvedRoot = resolve17(root);
|
|
30417
32184
|
let real;
|
|
30418
32185
|
try {
|
|
30419
32186
|
real = realpathSync6(resolvedRoot);
|
|
@@ -30435,7 +32202,7 @@ async function pinConfiguredRoots(roots, diagnostics) {
|
|
|
30435
32202
|
return outermost;
|
|
30436
32203
|
}
|
|
30437
32204
|
async function resolveRequestRoots(requestRoot, pinnedRoots) {
|
|
30438
|
-
const candidates =
|
|
32205
|
+
const candidates = isAbsolute11(requestRoot) ? [requestRoot] : pinnedRoots.map((root) => resolve17(root, requestRoot));
|
|
30439
32206
|
const allowed = [];
|
|
30440
32207
|
for (const candidate of candidates) {
|
|
30441
32208
|
let real;
|
|
@@ -30477,7 +32244,7 @@ async function filterFileNameSearchMatches(roots, candidates, excludePaths = [])
|
|
|
30477
32244
|
continue;
|
|
30478
32245
|
if (isExcluded2(real, excluded))
|
|
30479
32246
|
continue;
|
|
30480
|
-
const segments = real.split(
|
|
32247
|
+
const segments = real.split(sep8);
|
|
30481
32248
|
if (segments.some((segment) => SKIP_DIR_NAMES2[segment] === true))
|
|
30482
32249
|
continue;
|
|
30483
32250
|
matches.push({ path: real, type: stat.isDirectory() ? "folder" : "file" });
|
|
@@ -30626,7 +32393,7 @@ function isToolEnabled(name, options) {
|
|
|
30626
32393
|
}
|
|
30627
32394
|
function readPackageVersion() {
|
|
30628
32395
|
try {
|
|
30629
|
-
const manifest = JSON.parse(
|
|
32396
|
+
const manifest = JSON.parse(readFileSync27(fileURLToPath3(new URL("../../package.json", import.meta.url)), "utf8"));
|
|
30630
32397
|
if (typeof manifest === "object" && manifest !== null && "version" in manifest && typeof manifest.version === "string") {
|
|
30631
32398
|
return manifest.version;
|
|
30632
32399
|
}
|
|
@@ -30865,7 +32632,7 @@ function createAutoRAGMcpServer(lite, options = {}) {
|
|
|
30865
32632
|
...configuredDupey.timeoutMs !== undefined ? { timeoutMs: configuredDupey.timeoutMs } : {},
|
|
30866
32633
|
...configuredDupey.run !== undefined ? { run: configuredDupey.run } : {}
|
|
30867
32634
|
};
|
|
30868
|
-
const roots = lite.config.searchPaths.map((path) =>
|
|
32635
|
+
const roots = lite.config.searchPaths.map((path) => resolve18(path));
|
|
30869
32636
|
const scans = await Promise.all(roots.map((root) => scanWithDupey(root, dupeyOptions)));
|
|
30870
32637
|
return jsonResult({
|
|
30871
32638
|
ok: true,
|