@autorag/librarian 2.4.0 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -2
- package/dist/agent/agent.d.ts +31 -11
- package/dist/agent/index.d.ts +1 -1
- package/dist/agent/search-documents.d.ts +11 -2
- package/dist/cli/commands/evidence.d.ts +3 -0
- package/dist/cli/commands/index.d.ts +1 -1
- package/dist/cli/commands/refresh.d.ts +3 -4
- package/dist/cli/commands/search.d.ts +2 -2
- package/dist/cli/commands/tui.d.ts +1 -1
- package/dist/cli/commands/watch.d.ts +1 -1
- package/dist/cli/config.d.ts +0 -11
- package/dist/cli/index.js +771 -1044
- package/dist/datasource/skills/gmail/index.d.ts +0 -1
- package/dist/datasource/skills/gmail/skill.d.ts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +402 -664
- package/dist/jikji/client.d.ts +2 -0
- package/dist/minsync/index.d.ts +1 -1
- package/dist/minsync/method.d.ts +6 -18
- package/dist/retrieval/index.d.ts +0 -2
- package/dist/retrieval/registry.d.ts +2 -2
- package/dist/retrieval/types.d.ts +1 -1
- package/dist/ui/discover.d.ts +0 -2
- package/package.json +1 -1
- package/skills/autorag/SKILL.md +19 -6
- package/skills/autorag-setup/SKILL.md +10 -4
- package/dist/agent/search-bm25-tool.d.ts +0 -26
- package/dist/datasource/skills/gmail/himalaya-connector.d.ts +0 -51
- package/dist/retrieval/methods/bm25.d.ts +0 -23
package/dist/index.js
CHANGED
|
@@ -1,20 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
enumerable: true,
|
|
12
|
-
configurable: true,
|
|
13
|
-
set: __exportSetter.bind(all, name)
|
|
14
|
-
});
|
|
1
|
+
var __esm = (fn, res, err) => () => {
|
|
2
|
+
if (fn)
|
|
3
|
+
try {
|
|
4
|
+
res = fn(fn = 0);
|
|
5
|
+
} catch (e) {
|
|
6
|
+
err = [e];
|
|
7
|
+
}
|
|
8
|
+
if (err)
|
|
9
|
+
throw err[0];
|
|
10
|
+
return res;
|
|
15
11
|
};
|
|
16
|
-
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
17
|
-
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
18
12
|
|
|
19
13
|
// src/config/home.ts
|
|
20
14
|
import { homedir } from "node:os";
|
|
@@ -844,6 +838,7 @@ var init_installer = __esm(() => {
|
|
|
844
838
|
// src/jikji/client.ts
|
|
845
839
|
import { spawn as spawn3 } from "node:child_process";
|
|
846
840
|
import { existsSync as existsSync4 } from "node:fs";
|
|
841
|
+
import { join as join5 } from "node:path";
|
|
847
842
|
|
|
848
843
|
class JikjiClient {
|
|
849
844
|
options;
|
|
@@ -851,6 +846,9 @@ class JikjiClient {
|
|
|
851
846
|
constructor(options = {}) {
|
|
852
847
|
this.options = options;
|
|
853
848
|
}
|
|
849
|
+
isPrepared(root) {
|
|
850
|
+
return existsSync4(join5(root, ".jikji", "doc_meta")) || existsSync4(join5(root, ".jikji", "doc_text"));
|
|
851
|
+
}
|
|
854
852
|
async resolveCommand() {
|
|
855
853
|
if (this.options.binaryPath !== undefined && this.options.binaryPath !== DEFAULT_BINARY) {
|
|
856
854
|
return commandFor(this.options.binaryPath);
|
|
@@ -942,7 +940,7 @@ class JikjiClient {
|
|
|
942
940
|
}
|
|
943
941
|
}
|
|
944
942
|
function spawnJikji(request) {
|
|
945
|
-
return new Promise((
|
|
943
|
+
return new Promise((resolve) => {
|
|
946
944
|
const options = request.options;
|
|
947
945
|
const portable = portableSpawnCommand(request.command, request.args);
|
|
948
946
|
const child = spawn3(portable.command, [...portable.args], {
|
|
@@ -987,7 +985,7 @@ function spawnJikji(request) {
|
|
|
987
985
|
settled = true;
|
|
988
986
|
clearTimeout(timeout);
|
|
989
987
|
request.signal?.removeEventListener("abort", abortHandler);
|
|
990
|
-
|
|
988
|
+
resolve({ ok: false, reason: "spawn-error", stdout: stdout.text, stderr: error.message, code: null });
|
|
991
989
|
});
|
|
992
990
|
child.on("close", (code) => {
|
|
993
991
|
if (settled)
|
|
@@ -996,10 +994,10 @@ function spawnJikji(request) {
|
|
|
996
994
|
clearTimeout(timeout);
|
|
997
995
|
request.signal?.removeEventListener("abort", abortHandler);
|
|
998
996
|
if (finalReason !== undefined) {
|
|
999
|
-
|
|
997
|
+
resolve({ ok: false, reason: finalReason, stdout: stdout.text, stderr: stderr.text, code });
|
|
1000
998
|
return;
|
|
1001
999
|
}
|
|
1002
|
-
|
|
1000
|
+
resolve({ ok: code === 0, stdout: stdout.text, stderr: stderr.text, code });
|
|
1003
1001
|
});
|
|
1004
1002
|
});
|
|
1005
1003
|
}
|
|
@@ -1141,7 +1139,7 @@ var init_jikji = __esm(() => {
|
|
|
1141
1139
|
|
|
1142
1140
|
// src/manifest/loader.ts
|
|
1143
1141
|
import { existsSync as existsSync5, readdirSync, readFileSync as readFileSync3 } from "node:fs";
|
|
1144
|
-
import { extname as extname2, join as
|
|
1142
|
+
import { extname as extname2, join as join6 } from "node:path";
|
|
1145
1143
|
import { parse as parseYaml } from "yaml";
|
|
1146
1144
|
function isValidManifest(data) {
|
|
1147
1145
|
if (typeof data !== "object" || data === null)
|
|
@@ -1187,7 +1185,7 @@ function loadManifests(dirPath) {
|
|
|
1187
1185
|
const ext = extname2(entry).toLowerCase();
|
|
1188
1186
|
if (ext !== ".yaml" && ext !== ".yml" && ext !== ".json")
|
|
1189
1187
|
continue;
|
|
1190
|
-
const filePath =
|
|
1188
|
+
const filePath = join6(dirPath, entry);
|
|
1191
1189
|
try {
|
|
1192
1190
|
const manifest = loadManifest(filePath);
|
|
1193
1191
|
manifests.push(manifest);
|
|
@@ -1316,7 +1314,7 @@ import {
|
|
|
1316
1314
|
unlinkSync,
|
|
1317
1315
|
writeSync
|
|
1318
1316
|
} from "node:fs";
|
|
1319
|
-
import { join as
|
|
1317
|
+
import { join as join7 } from "node:path";
|
|
1320
1318
|
function hasErrorCode(error, code) {
|
|
1321
1319
|
return error instanceof Error && "code" in error && error.code === code;
|
|
1322
1320
|
}
|
|
@@ -1465,7 +1463,7 @@ function reapStaleLockDirectory(lockPath, staleMs) {
|
|
|
1465
1463
|
return removeEmptyLockDirectory(lockPath);
|
|
1466
1464
|
const staleSnapshots = [];
|
|
1467
1465
|
for (const name of names) {
|
|
1468
|
-
const inspection = inspectRegularFile(
|
|
1466
|
+
const inspection = inspectRegularFile(join7(lockPath, name));
|
|
1469
1467
|
if (inspection.kind === "missing")
|
|
1470
1468
|
continue;
|
|
1471
1469
|
if (inspection.kind !== "file" || !isStaleSnapshot(inspection, staleMs))
|
|
@@ -1527,7 +1525,7 @@ function tryCreateOwnerMarker(lockPath, owner, contents) {
|
|
|
1527
1525
|
throw error;
|
|
1528
1526
|
}
|
|
1529
1527
|
const markerName = `owner-${owner.token}.json`;
|
|
1530
|
-
const markerPath =
|
|
1528
|
+
const markerPath = join7(lockPath, markerName);
|
|
1531
1529
|
let acquired = false;
|
|
1532
1530
|
try {
|
|
1533
1531
|
try {
|
|
@@ -1981,11 +1979,11 @@ class RetrievalMemory {
|
|
|
1981
1979
|
timestamp: Date.now()
|
|
1982
1980
|
});
|
|
1983
1981
|
const evidenceWeight = curated.evidenceIds.length > 0 ? explicitWeight / curated.evidenceIds.length : 0;
|
|
1984
|
-
for (const
|
|
1985
|
-
const evidence = this.data.evidenceChunks.find((chunk) => chunk.stableEvidenceId ===
|
|
1982
|
+
for (const stableEvidenceId of curated.evidenceIds) {
|
|
1983
|
+
const evidence = this.data.evidenceChunks.find((chunk) => chunk.stableEvidenceId === stableEvidenceId);
|
|
1986
1984
|
this.data.feedbackSignals.push({
|
|
1987
1985
|
id: randomUUID3(),
|
|
1988
|
-
target: { type: "evidence_chunk", stableEvidenceId
|
|
1986
|
+
target: { type: "evidence_chunk", stableEvidenceId },
|
|
1989
1987
|
query: curated.query,
|
|
1990
1988
|
method: evidence?.method,
|
|
1991
1989
|
sentiment,
|
|
@@ -2071,8 +2069,8 @@ class RetrievalMemory {
|
|
|
2071
2069
|
for (const signal of this.data.feedbackSignals) {
|
|
2072
2070
|
if (signal.target.type !== "evidence_chunk")
|
|
2073
2071
|
continue;
|
|
2074
|
-
const
|
|
2075
|
-
const evidence = this.data.evidenceChunks.find((chunk) => chunk.stableEvidenceId ===
|
|
2072
|
+
const stableEvidenceId = signal.target.stableEvidenceId;
|
|
2073
|
+
const evidence = this.data.evidenceChunks.find((chunk) => chunk.stableEvidenceId === stableEvidenceId);
|
|
2076
2074
|
if (!evidence)
|
|
2077
2075
|
continue;
|
|
2078
2076
|
add(documentAreas, evidence.documentArea, signal);
|
|
@@ -2434,10 +2432,10 @@ var init_memory = __esm(() => {
|
|
|
2434
2432
|
|
|
2435
2433
|
// src/minsync/embedder-config.ts
|
|
2436
2434
|
import { readFileSync as readFileSync6, writeFileSync as writeFileSync3 } from "node:fs";
|
|
2437
|
-
import { join as
|
|
2435
|
+
import { join as join8 } from "node:path";
|
|
2438
2436
|
import { parse, stringify } from "smol-toml";
|
|
2439
2437
|
function minSyncConfigPath(workspacePath) {
|
|
2440
|
-
return
|
|
2438
|
+
return join8(workspacePath, MINSYNC_CONFIG_DIR, MINSYNC_CONFIG_FILE);
|
|
2441
2439
|
}
|
|
2442
2440
|
function configuredMaxChunkSize(workspacePath) {
|
|
2443
2441
|
let raw;
|
|
@@ -2505,7 +2503,7 @@ var init_embedder_config = () => {};
|
|
|
2505
2503
|
// src/minsync/process.ts
|
|
2506
2504
|
import { spawn as spawn4 } from "node:child_process";
|
|
2507
2505
|
function spawnProcess(command, args, cwd, options = {}) {
|
|
2508
|
-
return new Promise((
|
|
2506
|
+
return new Promise((resolve) => {
|
|
2509
2507
|
const portable = portableSpawnCommand(command, args);
|
|
2510
2508
|
const child = spawn4(portable.command, [...portable.args], { cwd, stdio: ["ignore", "pipe", "pipe"] });
|
|
2511
2509
|
let stdout = "";
|
|
@@ -2526,15 +2524,15 @@ function spawnProcess(command, args, cwd, options = {}) {
|
|
|
2526
2524
|
child.on("error", (error) => {
|
|
2527
2525
|
if (timer)
|
|
2528
2526
|
clearTimeout(timer);
|
|
2529
|
-
|
|
2527
|
+
resolve({ ok: false, stdout, stderr: error.message, code: null });
|
|
2530
2528
|
});
|
|
2531
2529
|
child.on("close", (code) => {
|
|
2532
2530
|
if (timer)
|
|
2533
2531
|
clearTimeout(timer);
|
|
2534
2532
|
if (timedOut) {
|
|
2535
|
-
|
|
2533
|
+
resolve({ ok: false, stdout, stderr: "process timed out", code: null, timedOut: true });
|
|
2536
2534
|
} else {
|
|
2537
|
-
|
|
2535
|
+
resolve({ ok: code === 0, stdout, stderr, code });
|
|
2538
2536
|
}
|
|
2539
2537
|
});
|
|
2540
2538
|
});
|
|
@@ -2545,7 +2543,7 @@ var init_process = __esm(() => {
|
|
|
2545
2543
|
|
|
2546
2544
|
// src/minsync/client.ts
|
|
2547
2545
|
import { existsSync as existsSync7, readFileSync as readFileSync7, rmSync, writeFileSync as writeFileSync4 } from "node:fs";
|
|
2548
|
-
import { join as
|
|
2546
|
+
import { join as join9 } from "node:path";
|
|
2549
2547
|
|
|
2550
2548
|
class MinSyncClient {
|
|
2551
2549
|
binaryPath;
|
|
@@ -2579,7 +2577,7 @@ class MinSyncClient {
|
|
|
2579
2577
|
}
|
|
2580
2578
|
const spawnOpts = this.embedder?.timeoutMs !== undefined ? { timeoutMs: this.embedder.timeoutMs } : {};
|
|
2581
2579
|
const initialized = existsSync7(minSyncConfigPath(this.workspacePath));
|
|
2582
|
-
const cursorPath =
|
|
2580
|
+
const cursorPath = join9(this.workspacePath, ".minsync", "cursor.json");
|
|
2583
2581
|
if (!initialized) {
|
|
2584
2582
|
const initArgs = ["init", "--format", "json"];
|
|
2585
2583
|
if (this.embedder?.id) {
|
|
@@ -2724,10 +2722,10 @@ import { copyFileSync, createWriteStream, existsSync as existsSync8, mkdirSync a
|
|
|
2724
2722
|
import { chmod, mkdtemp } from "node:fs/promises";
|
|
2725
2723
|
import { get } from "node:https";
|
|
2726
2724
|
import { tmpdir } from "node:os";
|
|
2727
|
-
import { basename as basename2, dirname as dirname4, join as
|
|
2725
|
+
import { basename as basename2, dirname as dirname4, join as join10 } from "node:path";
|
|
2728
2726
|
import { pipeline } from "node:stream/promises";
|
|
2729
2727
|
async function ensureMinSyncBinary(options) {
|
|
2730
|
-
const binaryPath =
|
|
2728
|
+
const binaryPath = join10(options.root, ".autorag", "bin", executableName(options.platform ?? process.platform));
|
|
2731
2729
|
if (existsSync8(binaryPath))
|
|
2732
2730
|
return { binaryPath, version: "cached" };
|
|
2733
2731
|
const releaseProvider = options.releaseProvider ?? fetchLatestMinSyncRelease;
|
|
@@ -2748,14 +2746,14 @@ async function ensureMinSyncBinary(options) {
|
|
|
2748
2746
|
return { binaryPath, version: release.tagName };
|
|
2749
2747
|
}
|
|
2750
2748
|
async function installMinSyncFromCargo(options, destination) {
|
|
2751
|
-
const cargoRoot =
|
|
2749
|
+
const cargoRoot = join10(options.root, ".autorag", "minsync-cargo");
|
|
2752
2750
|
mkdirSync4(dirname4(destination), { recursive: true });
|
|
2753
2751
|
mkdirSync4(cargoRoot, { recursive: true });
|
|
2754
2752
|
const result = await spawnProcess("cargo", ["install", "minsync", "--version", MINSYNC_VERSION, "--locked", "--root", cargoRoot], options.root);
|
|
2755
2753
|
if (!result.ok) {
|
|
2756
2754
|
throw new MinSyncReleaseError(result.stderr || `Could not install MinSync ${MINSYNC_VERSION} from crates.io`);
|
|
2757
2755
|
}
|
|
2758
|
-
const installedBinary =
|
|
2756
|
+
const installedBinary = join10(cargoRoot, "bin", executableName(options.platform ?? process.platform));
|
|
2759
2757
|
if (!existsSync8(installedBinary)) {
|
|
2760
2758
|
throw new MinSyncReleaseError(`Cargo did not produce the MinSync ${MINSYNC_VERSION} binary`);
|
|
2761
2759
|
}
|
|
@@ -2786,9 +2784,9 @@ function selectReleaseAsset(release, platform, arch) {
|
|
|
2786
2784
|
}
|
|
2787
2785
|
async function installReleaseAsset(asset, destination) {
|
|
2788
2786
|
const expectedSha256 = requireSha256(asset);
|
|
2789
|
-
const tempDir = await mkdtemp(
|
|
2787
|
+
const tempDir = await mkdtemp(join10(tmpdir(), "autorag-minsync-install-"));
|
|
2790
2788
|
try {
|
|
2791
|
-
const archive =
|
|
2789
|
+
const archive = join10(tempDir, asset.name);
|
|
2792
2790
|
await downloadFile(asset.downloadUrl, archive);
|
|
2793
2791
|
await verifySha256(archive, expectedSha256);
|
|
2794
2792
|
if (!asset.name.endsWith(".tar.gz") && !asset.name.endsWith(".zip")) {
|
|
@@ -2797,7 +2795,7 @@ async function installReleaseAsset(asset, destination) {
|
|
|
2797
2795
|
const extracted = await spawnProcess("tar", ["-xf", archive, "-C", tempDir], tempDir);
|
|
2798
2796
|
if (!extracted.ok)
|
|
2799
2797
|
throw new MinSyncReleaseError(extracted.stderr || `Could not extract ${asset.name}`);
|
|
2800
|
-
renameSync3(
|
|
2798
|
+
renameSync3(join10(tempDir, basename2(destination)), destination);
|
|
2801
2799
|
} finally {
|
|
2802
2800
|
rmSync2(tempDir, { recursive: true, force: true });
|
|
2803
2801
|
}
|
|
@@ -2814,11 +2812,11 @@ async function verifySha256(path, expected) {
|
|
|
2814
2812
|
throw new MinSyncReleaseError(`MinSync asset sha256 mismatch: expected ${expected}, got ${actual}`);
|
|
2815
2813
|
}
|
|
2816
2814
|
function readHttpsText(url, redirectCount = 0) {
|
|
2817
|
-
return new Promise((
|
|
2815
|
+
return new Promise((resolve, reject) => {
|
|
2818
2816
|
get(url, { headers: { "User-Agent": "AutoRAG-MinSync" } }, (response) => {
|
|
2819
2817
|
if (isRedirect(response.statusCode) && response.headers.location) {
|
|
2820
2818
|
response.resume();
|
|
2821
|
-
readHttpsText(new URL(response.headers.location, url).toString(), redirectCount + 1).then(
|
|
2819
|
+
readHttpsText(new URL(response.headers.location, url).toString(), redirectCount + 1).then(resolve, reject);
|
|
2822
2820
|
return;
|
|
2823
2821
|
}
|
|
2824
2822
|
if (redirectCount > 5) {
|
|
@@ -2837,17 +2835,17 @@ function readHttpsText(url, redirectCount = 0) {
|
|
|
2837
2835
|
body += chunk;
|
|
2838
2836
|
});
|
|
2839
2837
|
response.on("end", () => {
|
|
2840
|
-
|
|
2838
|
+
resolve(body);
|
|
2841
2839
|
});
|
|
2842
2840
|
}).on("error", reject);
|
|
2843
2841
|
});
|
|
2844
2842
|
}
|
|
2845
2843
|
function openHttps(url, redirectCount = 0) {
|
|
2846
|
-
return new Promise((
|
|
2844
|
+
return new Promise((resolve, reject) => {
|
|
2847
2845
|
get(url, { headers: { "User-Agent": "AutoRAG-MinSync" } }, (response) => {
|
|
2848
2846
|
if (isRedirect(response.statusCode) && response.headers.location) {
|
|
2849
2847
|
response.resume();
|
|
2850
|
-
openHttps(new URL(response.headers.location, url).toString(), redirectCount + 1).then(
|
|
2848
|
+
openHttps(new URL(response.headers.location, url).toString(), redirectCount + 1).then(resolve, reject);
|
|
2851
2849
|
return;
|
|
2852
2850
|
}
|
|
2853
2851
|
if (redirectCount > 5) {
|
|
@@ -2860,7 +2858,7 @@ function openHttps(url, redirectCount = 0) {
|
|
|
2860
2858
|
response.resume();
|
|
2861
2859
|
return;
|
|
2862
2860
|
}
|
|
2863
|
-
|
|
2861
|
+
resolve(response);
|
|
2864
2862
|
}).on("error", reject);
|
|
2865
2863
|
});
|
|
2866
2864
|
}
|
|
@@ -2907,28 +2905,6 @@ var init_installer2 = __esm(() => {
|
|
|
2907
2905
|
};
|
|
2908
2906
|
});
|
|
2909
2907
|
|
|
2910
|
-
// src/retrieval/methods/bm25.ts
|
|
2911
|
-
import { existsSync as existsSync9, rmSync as rmSync3 } from "node:fs";
|
|
2912
|
-
import { join as join10 } from "node:path";
|
|
2913
|
-
function removeLegacyBm25Artifacts(root) {
|
|
2914
|
-
rmSync3(join10(root, BM25_SUBDIR), { recursive: true, force: true });
|
|
2915
|
-
}
|
|
2916
|
-
function hasLegacyBm25Artifacts(root) {
|
|
2917
|
-
return existsSync9(join10(root, BM25_SUBDIR));
|
|
2918
|
-
}
|
|
2919
|
-
var BM25_SUBDIR, BM25UnavailableError;
|
|
2920
|
-
var init_bm25 = __esm(() => {
|
|
2921
|
-
BM25_SUBDIR = join10(".autorag", "bm25");
|
|
2922
|
-
BM25UnavailableError = class BM25UnavailableError extends Error {
|
|
2923
|
-
readiness;
|
|
2924
|
-
constructor(readiness, message) {
|
|
2925
|
-
super(message);
|
|
2926
|
-
this.name = "BM25UnavailableError";
|
|
2927
|
-
this.readiness = readiness;
|
|
2928
|
-
}
|
|
2929
|
-
};
|
|
2930
|
-
});
|
|
2931
|
-
|
|
2932
2908
|
// src/minsync/paths.ts
|
|
2933
2909
|
import { join as join11 } from "node:path";
|
|
2934
2910
|
function minSyncWorkspaceRoot(root) {
|
|
@@ -2947,13 +2923,13 @@ var init_paths2 = __esm(() => {
|
|
|
2947
2923
|
import { randomUUID as randomUUID4 } from "node:crypto";
|
|
2948
2924
|
import {
|
|
2949
2925
|
copyFileSync as copyFileSync2,
|
|
2950
|
-
existsSync as
|
|
2926
|
+
existsSync as existsSync9,
|
|
2951
2927
|
lstatSync as lstatSync2,
|
|
2952
2928
|
mkdirSync as mkdirSync5,
|
|
2953
2929
|
readdirSync as readdirSync3,
|
|
2954
2930
|
readFileSync as readFileSync8,
|
|
2955
2931
|
renameSync as renameSync4,
|
|
2956
|
-
rmSync as
|
|
2932
|
+
rmSync as rmSync3,
|
|
2957
2933
|
writeFileSync as writeFileSync5
|
|
2958
2934
|
} from "node:fs";
|
|
2959
2935
|
import { basename as basename3, dirname as dirname5, isAbsolute as isAbsolute5, join as join12, relative as relative4, sep as sep3 } from "node:path";
|
|
@@ -2961,7 +2937,7 @@ function syncMinSyncWorkspace(root, options = {}) {
|
|
|
2961
2937
|
const workspacePath = options.workspacePath ?? minSyncWorkspaceRoot(root);
|
|
2962
2938
|
const filesRoot = join12(workspacePath, "files");
|
|
2963
2939
|
const index = loadMirrorIndex(root);
|
|
2964
|
-
const entries = Object.values(index.entries).sort((a, b) => a.virtualPath.localeCompare(b.virtualPath)).filter((entry) => normalizeVirtualPath(entry.virtualPath) === entry.virtualPath &&
|
|
2940
|
+
const entries = Object.values(index.entries).sort((a, b) => a.virtualPath.localeCompare(b.virtualPath)).filter((entry) => normalizeVirtualPath(entry.virtualPath) === entry.virtualPath && existsSync9(entry.outputPath)).map((entry) => {
|
|
2965
2941
|
const minSyncPath = minSyncDocumentPath(workspacePath, entry.virtualPath);
|
|
2966
2942
|
return {
|
|
2967
2943
|
virtualPath: entry.virtualPath,
|
|
@@ -3023,7 +2999,7 @@ function isMinSyncStagingState(value) {
|
|
|
3023
2999
|
function ensureManagedFilesRoot(filesRoot, rebuild) {
|
|
3024
3000
|
const status = lstatSync2(filesRoot, { throwIfNoEntry: false });
|
|
3025
3001
|
if (status !== undefined && (rebuild || !status.isDirectory())) {
|
|
3026
|
-
|
|
3002
|
+
rmSync3(filesRoot, { recursive: status.isDirectory(), force: true });
|
|
3027
3003
|
}
|
|
3028
3004
|
mkdirSync5(filesRoot, { recursive: true });
|
|
3029
3005
|
}
|
|
@@ -3040,7 +3016,7 @@ function ensureManagedDirectory(filesRoot, directory) {
|
|
|
3040
3016
|
if (status?.isDirectory())
|
|
3041
3017
|
continue;
|
|
3042
3018
|
if (status !== undefined) {
|
|
3043
|
-
|
|
3019
|
+
rmSync3(current, { force: true });
|
|
3044
3020
|
}
|
|
3045
3021
|
mkdirSync5(current);
|
|
3046
3022
|
}
|
|
@@ -3051,7 +3027,7 @@ function copyFileAtomically(source, destination) {
|
|
|
3051
3027
|
copyFileSync2(source, temporaryPath);
|
|
3052
3028
|
renameSync4(temporaryPath, destination);
|
|
3053
3029
|
} finally {
|
|
3054
|
-
|
|
3030
|
+
rmSync3(temporaryPath, { force: true });
|
|
3055
3031
|
}
|
|
3056
3032
|
}
|
|
3057
3033
|
function isRegularFile(path) {
|
|
@@ -3063,7 +3039,7 @@ function removeUnexpectedStagedFiles(directory, desiredPaths) {
|
|
|
3063
3039
|
if (entry.isDirectory()) {
|
|
3064
3040
|
removeUnexpectedStagedFiles(path, desiredPaths);
|
|
3065
3041
|
} else if (!desiredPaths.has(path)) {
|
|
3066
|
-
|
|
3042
|
+
rmSync3(path, { force: true });
|
|
3067
3043
|
}
|
|
3068
3044
|
}
|
|
3069
3045
|
}
|
|
@@ -3098,7 +3074,7 @@ var init_workspace = __esm(() => {
|
|
|
3098
3074
|
});
|
|
3099
3075
|
|
|
3100
3076
|
// src/minsync/method.ts
|
|
3101
|
-
import { accessSync, constants, existsSync as
|
|
3077
|
+
import { accessSync, constants, existsSync as existsSync10, realpathSync as realpathSync3 } from "node:fs";
|
|
3102
3078
|
import { basename as basename4, delimiter as delimiter2, join as join13, normalize } from "node:path";
|
|
3103
3079
|
function degrade(workspacePath, reason) {
|
|
3104
3080
|
return { ok: false, synced: 0, workspacePath, reason };
|
|
@@ -3171,7 +3147,10 @@ class MinSyncVectorMethod {
|
|
|
3171
3147
|
return binaryResult;
|
|
3172
3148
|
}
|
|
3173
3149
|
isBinaryMissing() {
|
|
3174
|
-
return this.binaryPath !== undefined && !
|
|
3150
|
+
return this.binaryPath !== undefined && !existsSync10(this.binaryPath);
|
|
3151
|
+
}
|
|
3152
|
+
isReady() {
|
|
3153
|
+
return existsSync10(join13(this.workspacePath, ".minsync", "cursor.json"));
|
|
3175
3154
|
}
|
|
3176
3155
|
async retrieve(query, options) {
|
|
3177
3156
|
const topK = options.topK ?? 20;
|
|
@@ -3211,16 +3190,15 @@ class MinSyncVectorMethod {
|
|
|
3211
3190
|
return results;
|
|
3212
3191
|
}
|
|
3213
3192
|
async resolveBinary() {
|
|
3214
|
-
if (this.binaryPath &&
|
|
3193
|
+
if (this.binaryPath && existsSync10(this.binaryPath))
|
|
3215
3194
|
return this.binaryPath;
|
|
3216
|
-
}
|
|
3217
3195
|
if (this.binaryPath !== undefined)
|
|
3218
3196
|
return;
|
|
3219
3197
|
const pathBinary = lookupInPath(process.env);
|
|
3220
3198
|
if (pathBinary)
|
|
3221
3199
|
return pathBinary;
|
|
3222
3200
|
const cachedBinary = join13(this.root, ".autorag", "bin", executableName(process.platform));
|
|
3223
|
-
if (
|
|
3201
|
+
if (existsSync10(cachedBinary))
|
|
3224
3202
|
return cachedBinary;
|
|
3225
3203
|
if (this.autoInstall) {
|
|
3226
3204
|
try {
|
|
@@ -3233,63 +3211,13 @@ class MinSyncVectorMethod {
|
|
|
3233
3211
|
return;
|
|
3234
3212
|
}
|
|
3235
3213
|
}
|
|
3236
|
-
var
|
|
3214
|
+
var MinSyncHybridMethod;
|
|
3237
3215
|
var init_method = __esm(() => {
|
|
3238
|
-
init_bm25();
|
|
3239
3216
|
init_scope();
|
|
3240
3217
|
init_client2();
|
|
3241
3218
|
init_installer2();
|
|
3242
3219
|
init_paths2();
|
|
3243
3220
|
init_workspace();
|
|
3244
|
-
MinSyncBM25Method = class MinSyncBM25Method extends MinSyncVectorMethod {
|
|
3245
|
-
status = { readiness: "index_missing", engine: "minsync" };
|
|
3246
|
-
constructor(options) {
|
|
3247
|
-
super({ ...options, mode: "bm25" });
|
|
3248
|
-
}
|
|
3249
|
-
describe() {
|
|
3250
|
-
return {
|
|
3251
|
-
name: "bm25",
|
|
3252
|
-
type: "bm25",
|
|
3253
|
-
description: "MinSync-backed BM25 lexical retrieval over parsed markdown mirror chunks",
|
|
3254
|
-
status: this.status.readiness === "ready" ? "active" : "stub",
|
|
3255
|
-
capabilities: [
|
|
3256
|
-
"lexical",
|
|
3257
|
-
"incremental",
|
|
3258
|
-
"parsed-mirrors",
|
|
3259
|
-
"virtual-paths",
|
|
3260
|
-
`readiness:${this.status.readiness}`
|
|
3261
|
-
]
|
|
3262
|
-
};
|
|
3263
|
-
}
|
|
3264
|
-
async retrieve(query, options) {
|
|
3265
|
-
const binaryResult = await this.resolveBinaryForRetrieve();
|
|
3266
|
-
if (binaryResult === undefined) {
|
|
3267
|
-
const readiness = this.status.readiness === "ready" ? "dependency_unavailable" : this.status.readiness;
|
|
3268
|
-
throw new BM25UnavailableError(readiness === "index_missing" ? "dependency_unavailable" : readiness, this.status.message ?? "MinSync BM25 is unavailable");
|
|
3269
|
-
}
|
|
3270
|
-
return super.retrieve(query, options);
|
|
3271
|
-
}
|
|
3272
|
-
syncFromMinSync(result) {
|
|
3273
|
-
this.status = result.ok ? { readiness: "ready", engine: "minsync" } : {
|
|
3274
|
-
readiness: result.reason === "missing-binary" ? "dependency_unavailable" : "error",
|
|
3275
|
-
engine: "minsync",
|
|
3276
|
-
message: result.reason
|
|
3277
|
-
};
|
|
3278
|
-
return {
|
|
3279
|
-
indexPath: result.workspacePath,
|
|
3280
|
-
indexedChunks: result.synced,
|
|
3281
|
-
readiness: this.status.readiness,
|
|
3282
|
-
engine: this.status.engine
|
|
3283
|
-
};
|
|
3284
|
-
}
|
|
3285
|
-
getStatus() {
|
|
3286
|
-
return this.status;
|
|
3287
|
-
}
|
|
3288
|
-
async resolveBinaryForRetrieve() {
|
|
3289
|
-
const binaryResult = await this.resolveBinary();
|
|
3290
|
-
return typeof binaryResult === "string" ? binaryResult : undefined;
|
|
3291
|
-
}
|
|
3292
|
-
};
|
|
3293
3221
|
MinSyncHybridMethod = class MinSyncHybridMethod extends MinSyncVectorMethod {
|
|
3294
3222
|
constructor(options) {
|
|
3295
3223
|
super({ ...options, mode: "hybrid" });
|
|
@@ -3473,7 +3401,7 @@ function renderTable(table) {
|
|
|
3473
3401
|
return renderMergedTable(cells);
|
|
3474
3402
|
const cellsByPosition = new Map(cells.map((cell) => [`${cell.row}:${cell.column}`, cell.text]));
|
|
3475
3403
|
const header = Array.from({ length: table.columnCount }, (_, column) => `Column ${column + 1}`);
|
|
3476
|
-
const rows = Array.from({ length: table.rowCount }, (_, row) => Array.from({ length: table.columnCount }, (
|
|
3404
|
+
const rows = Array.from({ length: table.rowCount }, (_, row) => Array.from({ length: table.columnCount }, (_, column) => cellsByPosition.get(`${row}:${column}`) ?? ""));
|
|
3477
3405
|
return [
|
|
3478
3406
|
`| ${header.join(" | ")} |`,
|
|
3479
3407
|
`| ${header.map(() => "---").join(" | ")} |`,
|
|
@@ -3934,7 +3862,7 @@ async function readZipXmlText(bytes, pathPattern) {
|
|
|
3934
3862
|
}
|
|
3935
3863
|
async function readXmlFile(file, remainingBytes) {
|
|
3936
3864
|
const stream = file.nodeStream("nodebuffer");
|
|
3937
|
-
return new Promise((
|
|
3865
|
+
return new Promise((resolve, reject) => {
|
|
3938
3866
|
const chunks = [];
|
|
3939
3867
|
let totalBytes = 0;
|
|
3940
3868
|
stream.on("data", (chunk) => {
|
|
@@ -3948,7 +3876,7 @@ async function readXmlFile(file, remainingBytes) {
|
|
|
3948
3876
|
chunks.push(buffer);
|
|
3949
3877
|
});
|
|
3950
3878
|
stream.on("error", reject);
|
|
3951
|
-
stream.on("end", () =>
|
|
3879
|
+
stream.on("end", () => resolve(Buffer2.concat(chunks).toString("utf8")));
|
|
3952
3880
|
});
|
|
3953
3881
|
}
|
|
3954
3882
|
function uncompressedSize(file) {
|
|
@@ -4054,8 +3982,8 @@ function tesseractOcr(input) {
|
|
|
4054
3982
|
let cleanupReject = () => {
|
|
4055
3983
|
return;
|
|
4056
3984
|
};
|
|
4057
|
-
const cleanup = new Promise((
|
|
4058
|
-
cleanupResolve =
|
|
3985
|
+
const cleanup = new Promise((resolve, reject) => {
|
|
3986
|
+
cleanupResolve = resolve;
|
|
4059
3987
|
cleanupReject = reject;
|
|
4060
3988
|
});
|
|
4061
3989
|
const result = runTesseractOcr(input, cleanupResolve, cleanupReject);
|
|
@@ -4064,7 +3992,7 @@ function tesseractOcr(input) {
|
|
|
4064
3992
|
async function runTesseractOcr(input, cleanupResolve, cleanupReject) {
|
|
4065
3993
|
let worker;
|
|
4066
3994
|
let termination;
|
|
4067
|
-
const
|
|
3995
|
+
const terminate = async () => {
|
|
4068
3996
|
if (worker === undefined)
|
|
4069
3997
|
return;
|
|
4070
3998
|
termination ??= worker.terminate().then(() => {
|
|
@@ -4074,7 +4002,7 @@ async function runTesseractOcr(input, cleanupResolve, cleanupReject) {
|
|
|
4074
4002
|
};
|
|
4075
4003
|
const abort = () => {
|
|
4076
4004
|
if (worker !== undefined) {
|
|
4077
|
-
|
|
4005
|
+
terminate().then(cleanupResolve, cleanupReject);
|
|
4078
4006
|
}
|
|
4079
4007
|
};
|
|
4080
4008
|
input.signal.addEventListener("abort", abort, { once: true });
|
|
@@ -4086,18 +4014,18 @@ async function runTesseractOcr(input, cleanupResolve, cleanupReject) {
|
|
|
4086
4014
|
return result.data.text;
|
|
4087
4015
|
} finally {
|
|
4088
4016
|
input.signal.removeEventListener("abort", abort);
|
|
4089
|
-
await
|
|
4017
|
+
await terminate().then(cleanupResolve, cleanupReject);
|
|
4090
4018
|
}
|
|
4091
4019
|
}
|
|
4092
4020
|
function withTimeout(operation, timeoutMs, onTimeout) {
|
|
4093
|
-
return new Promise((
|
|
4021
|
+
return new Promise((resolve, reject) => {
|
|
4094
4022
|
const timeout = setTimeout(() => {
|
|
4095
4023
|
onTimeout();
|
|
4096
4024
|
operation.cleanup.finally(() => reject(new Error(`OCR timed out after ${timeoutMs}ms`)));
|
|
4097
4025
|
}, timeoutMs);
|
|
4098
4026
|
operation.result.then((value) => {
|
|
4099
4027
|
clearTimeout(timeout);
|
|
4100
|
-
|
|
4028
|
+
resolve(value);
|
|
4101
4029
|
}, (error) => {
|
|
4102
4030
|
clearTimeout(timeout);
|
|
4103
4031
|
operation.cleanup.then(() => reject(error), (cleanupError) => reject(cleanupError));
|
|
@@ -4380,17 +4308,17 @@ import { extname as extname5 } from "node:path";
|
|
|
4380
4308
|
class ParserRegistry {
|
|
4381
4309
|
parsersByExtension = new Map;
|
|
4382
4310
|
constructor(parsers = []) {
|
|
4383
|
-
for (const
|
|
4384
|
-
this.register(
|
|
4311
|
+
for (const parser of parsers) {
|
|
4312
|
+
this.register(parser);
|
|
4385
4313
|
}
|
|
4386
4314
|
}
|
|
4387
|
-
register(
|
|
4388
|
-
for (const extension of
|
|
4315
|
+
register(parser) {
|
|
4316
|
+
for (const extension of parser.extensions) {
|
|
4389
4317
|
const normalized = normalizeExtension(extension);
|
|
4390
4318
|
if (this.parsersByExtension.has(normalized)) {
|
|
4391
4319
|
throw new Error(`Parser extension "${normalized}" is already registered`);
|
|
4392
4320
|
}
|
|
4393
|
-
this.parsersByExtension.set(normalized,
|
|
4321
|
+
this.parsersByExtension.set(normalized, parser);
|
|
4394
4322
|
}
|
|
4395
4323
|
}
|
|
4396
4324
|
getForVirtualPath(virtualPath) {
|
|
@@ -4433,14 +4361,14 @@ var init_defaults = __esm(() => {
|
|
|
4433
4361
|
|
|
4434
4362
|
// src/mirror/sync.ts
|
|
4435
4363
|
import { randomUUID as randomUUID5 } from "node:crypto";
|
|
4436
|
-
import { existsSync as
|
|
4364
|
+
import { existsSync as existsSync11, mkdirSync as mkdirSync6, renameSync as renameSync5, rmSync as rmSync4, writeFileSync as writeFileSync6 } from "node:fs";
|
|
4437
4365
|
import { opendir, readFile as readFile3, stat as stat2 } from "node:fs/promises";
|
|
4438
4366
|
import { dirname as dirname6, extname as extname6, resolve as resolve8 } from "node:path";
|
|
4439
|
-
function parserFailureCode(
|
|
4440
|
-
return
|
|
4367
|
+
function parserFailureCode(parser, error) {
|
|
4368
|
+
return parser.name === "opendataloader-pdf" && /UnsupportedClassVersionError/.test(error.message) ? "pdf-java-version" : "parser-failed";
|
|
4441
4369
|
}
|
|
4442
|
-
function parserFailureMessage(
|
|
4443
|
-
if (parserFailureCode(
|
|
4370
|
+
function parserFailureMessage(parser, error) {
|
|
4371
|
+
if (parserFailureCode(parser, error) === "pdf-java-version") {
|
|
4444
4372
|
return "PDF parsing requires Java 11 or newer; the Java runtime selected from PATH is too old.";
|
|
4445
4373
|
}
|
|
4446
4374
|
return "The registered parser failed on this file; it was skipped during indexing.";
|
|
@@ -4480,8 +4408,8 @@ async function syncParsedMirrors(options) {
|
|
|
4480
4408
|
});
|
|
4481
4409
|
continue;
|
|
4482
4410
|
}
|
|
4483
|
-
const
|
|
4484
|
-
if (!
|
|
4411
|
+
const parser = registry.getForVirtualPath(entry.virtualPath);
|
|
4412
|
+
if (!parser) {
|
|
4485
4413
|
deleted += removePrevious(options.root, previous, entry.virtualPath);
|
|
4486
4414
|
handledPrevious.add(entry.virtualPath);
|
|
4487
4415
|
skipped += 1;
|
|
@@ -4504,12 +4432,12 @@ async function syncParsedMirrors(options) {
|
|
|
4504
4432
|
}
|
|
4505
4433
|
const previousEntry = previous.entries[entry.virtualPath];
|
|
4506
4434
|
const outputPath = parsedOutputPath(options.root, entry.virtualPath);
|
|
4507
|
-
const unchanged = !options.force && previousEntry?.sourceMtimeNs === entry.mtimeNs && previousEntry.sourceSizeBytes === entry.sizeBytes && previousEntry.parserName ===
|
|
4435
|
+
const unchanged = !options.force && previousEntry?.sourceMtimeNs === entry.mtimeNs && previousEntry.sourceSizeBytes === entry.sizeBytes && previousEntry.parserName === parser.name && existsSync11(outputPath);
|
|
4508
4436
|
if (!unchanged) {
|
|
4509
4437
|
let parsed;
|
|
4510
4438
|
try {
|
|
4511
4439
|
const bytes = await readFile3(entry.sourcePath);
|
|
4512
|
-
parsed = await
|
|
4440
|
+
parsed = await parser.parse({ virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes });
|
|
4513
4441
|
} catch (error) {
|
|
4514
4442
|
if (!(error instanceof ParseError))
|
|
4515
4443
|
throw error;
|
|
@@ -4517,9 +4445,9 @@ async function syncParsedMirrors(options) {
|
|
|
4517
4445
|
handledPrevious.add(entry.virtualPath);
|
|
4518
4446
|
skipped += 1;
|
|
4519
4447
|
diagnostics.push({
|
|
4520
|
-
code: parserFailureCode(
|
|
4448
|
+
code: parserFailureCode(parser, error),
|
|
4521
4449
|
severity: "warning",
|
|
4522
|
-
message: parserFailureMessage(
|
|
4450
|
+
message: parserFailureMessage(parser, error),
|
|
4523
4451
|
source: entry.virtualPath
|
|
4524
4452
|
});
|
|
4525
4453
|
sinceCheckpoint += 1;
|
|
@@ -4543,7 +4471,7 @@ async function syncParsedMirrors(options) {
|
|
|
4543
4471
|
virtualPath: entry.virtualPath,
|
|
4544
4472
|
sourcePath: entry.sourcePath,
|
|
4545
4473
|
outputPath,
|
|
4546
|
-
parserName:
|
|
4474
|
+
parserName: parser.name,
|
|
4547
4475
|
sourceMtimeNs: entry.mtimeNs,
|
|
4548
4476
|
sourceSizeBytes: entry.sizeBytes,
|
|
4549
4477
|
updatedAt: unchanged ? previousEntry?.updatedAt ?? new Date().toISOString() : new Date().toISOString()
|
|
@@ -4602,8 +4530,8 @@ async function detectMirrorStaleness(options) {
|
|
|
4602
4530
|
}
|
|
4603
4531
|
function supportedExtensionSet(registry) {
|
|
4604
4532
|
const extensions = new Set;
|
|
4605
|
-
for (const
|
|
4606
|
-
for (const extension of
|
|
4533
|
+
for (const parser of registry.list()) {
|
|
4534
|
+
for (const extension of parser.extensions) {
|
|
4607
4535
|
extensions.add(normalizeExtension2(extension));
|
|
4608
4536
|
}
|
|
4609
4537
|
}
|
|
@@ -4669,8 +4597,8 @@ function writeAtomic(path, content) {
|
|
|
4669
4597
|
renameSync5(tmp, path);
|
|
4670
4598
|
}
|
|
4671
4599
|
function removeFile(path) {
|
|
4672
|
-
if (
|
|
4673
|
-
|
|
4600
|
+
if (existsSync11(path))
|
|
4601
|
+
rmSync4(path, { force: true });
|
|
4674
4602
|
}
|
|
4675
4603
|
var DEFAULT_MAX_SOURCE_BYTES, MIRROR_CHECKPOINT_EVERY = 25, SKIP_DIR_NAMES;
|
|
4676
4604
|
var init_sync = __esm(() => {
|
|
@@ -4838,8 +4766,6 @@ class ParallelRetriever {
|
|
|
4838
4766
|
function methodFailureCode(name) {
|
|
4839
4767
|
if (name === "minsync")
|
|
4840
4768
|
return "minsync-unavailable";
|
|
4841
|
-
if (name === "bm25")
|
|
4842
|
-
return "bm25-unavailable";
|
|
4843
4769
|
return "retrieval-method-failed";
|
|
4844
4770
|
}
|
|
4845
4771
|
var MAX_AGGREGATION_BONUS = 0.3, MAX_SECONDARY_HITS = 3, SECONDARY_HIT_WEIGHT = 0.15;
|
|
@@ -4994,9 +4920,9 @@ var init_cloud_placeholder = __esm(() => {
|
|
|
4994
4920
|
import { spawn as spawn5 } from "node:child_process";
|
|
4995
4921
|
import { Type as Type2 } from "typebox";
|
|
4996
4922
|
function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
|
|
4997
|
-
return new Promise((
|
|
4923
|
+
return new Promise((resolve) => {
|
|
4998
4924
|
if (signal?.aborted) {
|
|
4999
|
-
|
|
4925
|
+
resolve({
|
|
5000
4926
|
output: "",
|
|
5001
4927
|
exitCode: undefined,
|
|
5002
4928
|
aborted: true,
|
|
@@ -5053,7 +4979,7 @@ function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
|
|
|
5053
4979
|
if (killGraceTimer !== undefined)
|
|
5054
4980
|
clearTimeout(killGraceTimer);
|
|
5055
4981
|
signal?.removeEventListener("abort", onAbort);
|
|
5056
|
-
|
|
4982
|
+
resolve({
|
|
5057
4983
|
output: Buffer.concat(chunks).toString("utf8"),
|
|
5058
4984
|
exitCode,
|
|
5059
4985
|
aborted,
|
|
@@ -5073,7 +4999,7 @@ function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
|
|
|
5073
4999
|
if (child.pid === undefined)
|
|
5074
5000
|
return true;
|
|
5075
5001
|
if (process.platform === "win32") {
|
|
5076
|
-
return await new Promise((
|
|
5002
|
+
return await new Promise((resolve) => {
|
|
5077
5003
|
const killer = spawn5("taskkill", ["/pid", String(child.pid), "/t", "/f"], { stdio: "ignore" });
|
|
5078
5004
|
let completed = false;
|
|
5079
5005
|
const graceTimer = setTimeout(() => {
|
|
@@ -5081,7 +5007,7 @@ function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
|
|
|
5081
5007
|
return;
|
|
5082
5008
|
completed = true;
|
|
5083
5009
|
killer.kill();
|
|
5084
|
-
|
|
5010
|
+
resolve(killDirectChild());
|
|
5085
5011
|
}, KILL_EXIT_GRACE_MS);
|
|
5086
5012
|
const finishKill = (success) => {
|
|
5087
5013
|
if (completed)
|
|
@@ -5090,7 +5016,7 @@ function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
|
|
|
5090
5016
|
clearTimeout(graceTimer);
|
|
5091
5017
|
if (!success)
|
|
5092
5018
|
killDirectChild();
|
|
5093
|
-
|
|
5019
|
+
resolve(success);
|
|
5094
5020
|
};
|
|
5095
5021
|
killer.on("error", () => finishKill(false));
|
|
5096
5022
|
killer.on("close", (code) => finishKill(code === 0));
|
|
@@ -5616,7 +5542,7 @@ function createSearchAllDocumentsTool(provider) {
|
|
|
5616
5542
|
return {
|
|
5617
5543
|
name: SEARCH_ALL_DOCUMENTS_TOOL_NAME,
|
|
5618
5544
|
label: "Search All Documents",
|
|
5619
|
-
description: "Search across all configured retrieval methods (posix,
|
|
5545
|
+
description: "Search across all configured retrieval methods (posix, MinSync, datasources) and return merged, deduplicated results. Authority is server-configured; tool arguments can only provide query, topK, and an optional narrowing scope.",
|
|
5620
5546
|
parameters: searchAllSchema,
|
|
5621
5547
|
async execute(_toolCallId, params) {
|
|
5622
5548
|
const query = params.query.trim();
|
|
@@ -5669,92 +5595,8 @@ var init_search_all_tool = __esm(() => {
|
|
|
5669
5595
|
});
|
|
5670
5596
|
});
|
|
5671
5597
|
|
|
5672
|
-
// src/agent/search-bm25-tool.ts
|
|
5673
|
-
import { Type as Type8 } from "typebox";
|
|
5674
|
-
function createSearchBM25DocumentsTool(getMethod, resolveScope = (scope) => scope) {
|
|
5675
|
-
return {
|
|
5676
|
-
name: SEARCH_BM25_DOCUMENTS_TOOL_NAME,
|
|
5677
|
-
label: "Search BM25 Documents",
|
|
5678
|
-
description: "Search parsed document mirrors with MinSync lexical BM25 ranking. Use for exact terms, repeated terms, headings, identifiers, and folder-scoped document search.",
|
|
5679
|
-
parameters: searchBM25Schema,
|
|
5680
|
-
async execute(_toolCallId, params) {
|
|
5681
|
-
const method = getMethod();
|
|
5682
|
-
if (!method) {
|
|
5683
|
-
return unavailableResult("BM25 is not configured for this AutoRAG agent", {
|
|
5684
|
-
readiness: "disabled",
|
|
5685
|
-
engine: "none"
|
|
5686
|
-
});
|
|
5687
|
-
}
|
|
5688
|
-
const status = method.getStatus();
|
|
5689
|
-
if (params.query.trim().length === 0) {
|
|
5690
|
-
return {
|
|
5691
|
-
content: [{ type: "text", text: "BM25 query was empty; no documents searched." }],
|
|
5692
|
-
details: {
|
|
5693
|
-
method: "bm25",
|
|
5694
|
-
resultCount: 0,
|
|
5695
|
-
sources: [],
|
|
5696
|
-
readiness: status.readiness,
|
|
5697
|
-
engine: status.engine
|
|
5698
|
-
}
|
|
5699
|
-
};
|
|
5700
|
-
}
|
|
5701
|
-
const scope = resolveScope(params.scope);
|
|
5702
|
-
try {
|
|
5703
|
-
const results = await method.retrieve(params.query, {
|
|
5704
|
-
topK: params.topK,
|
|
5705
|
-
scope
|
|
5706
|
-
});
|
|
5707
|
-
return {
|
|
5708
|
-
content: [{ type: "text", text: formatResults2(results, method.getStatus()) }],
|
|
5709
|
-
details: {
|
|
5710
|
-
method: "bm25",
|
|
5711
|
-
resultCount: results.length,
|
|
5712
|
-
sources: [...new Set(results.map((result) => result.source))],
|
|
5713
|
-
readiness: method.getStatus().readiness,
|
|
5714
|
-
engine: method.getStatus().engine
|
|
5715
|
-
}
|
|
5716
|
-
};
|
|
5717
|
-
} catch (error) {
|
|
5718
|
-
const nextStatus = method.getStatus();
|
|
5719
|
-
const message = error instanceof BM25UnavailableError ? error.message : `BM25 search failed: ${String(error)}`;
|
|
5720
|
-
return unavailableResult(message, nextStatus);
|
|
5721
|
-
}
|
|
5722
|
-
}
|
|
5723
|
-
};
|
|
5724
|
-
}
|
|
5725
|
-
function unavailableResult(message, status) {
|
|
5726
|
-
return {
|
|
5727
|
-
content: [{ type: "text", text: `${message}
|
|
5728
|
-
readiness=${status.readiness}; engine=${status.engine}` }],
|
|
5729
|
-
details: { method: "bm25", resultCount: 0, sources: [], readiness: status.readiness, engine: status.engine }
|
|
5730
|
-
};
|
|
5731
|
-
}
|
|
5732
|
-
function formatResults2(results, status) {
|
|
5733
|
-
if (results.length === 0)
|
|
5734
|
-
return `No BM25 results. readiness=${status.readiness}; engine=${status.engine}`;
|
|
5735
|
-
const rows = results.map((result, index) => {
|
|
5736
|
-
const line = result.content.replace(/\s+/gu, " ").slice(0, 500);
|
|
5737
|
-
return `[${index + 1}] ${result.source} score=${result.score.toFixed(4)}
|
|
5738
|
-
${line}`;
|
|
5739
|
-
});
|
|
5740
|
-
return `BM25 results (readiness=${status.readiness}; engine=${status.engine}):
|
|
5741
|
-
|
|
5742
|
-
${rows.join(`
|
|
5743
|
-
|
|
5744
|
-
`)}`;
|
|
5745
|
-
}
|
|
5746
|
-
var SEARCH_BM25_DOCUMENTS_TOOL_NAME = "lexical_search_local_docs", searchBM25Schema;
|
|
5747
|
-
var init_search_bm25_tool = __esm(() => {
|
|
5748
|
-
init_bm25();
|
|
5749
|
-
searchBM25Schema = Type8.Object({
|
|
5750
|
-
query: Type8.String({ description: "Lexical query to search in parsed document mirrors with BM25." }),
|
|
5751
|
-
topK: Type8.Optional(Type8.Integer({ description: "Maximum number of BM25 chunks to return." })),
|
|
5752
|
-
scope: Type8.Optional(Type8.String({ description: "Optional opaque virtual-path scope, e.g. /docs or /docs/**." }))
|
|
5753
|
-
});
|
|
5754
|
-
});
|
|
5755
|
-
|
|
5756
5598
|
// src/agent/search-datasource-tool.ts
|
|
5757
|
-
import { Type as
|
|
5599
|
+
import { Type as Type8 } from "typebox";
|
|
5758
5600
|
function createSearchDatasourceDocumentsTool(provider) {
|
|
5759
5601
|
return {
|
|
5760
5602
|
name: SEARCH_DATASOURCE_DOCUMENTS_TOOL_NAME,
|
|
@@ -5774,7 +5616,7 @@ function createSearchDatasourceDocumentsTool(provider) {
|
|
|
5774
5616
|
scope: params.scope
|
|
5775
5617
|
});
|
|
5776
5618
|
return {
|
|
5777
|
-
content: [{ type: "text", text:
|
|
5619
|
+
content: [{ type: "text", text: formatResults2(results, diagnostics) }],
|
|
5778
5620
|
details: {
|
|
5779
5621
|
method: "datasource",
|
|
5780
5622
|
resultCount: results.length,
|
|
@@ -5785,7 +5627,7 @@ function createSearchDatasourceDocumentsTool(provider) {
|
|
|
5785
5627
|
}
|
|
5786
5628
|
};
|
|
5787
5629
|
}
|
|
5788
|
-
function
|
|
5630
|
+
function formatResults2(results, diagnostics) {
|
|
5789
5631
|
const diagnosticSummary = diagnostics.length > 0 ? `
|
|
5790
5632
|
|
|
5791
5633
|
Diagnostics: ${diagnostics.map((d) => `${d.source ?? "datasource"}:${d.code}`).join(", ")}` : "";
|
|
@@ -5804,10 +5646,10 @@ ${rows.join(`
|
|
|
5804
5646
|
}
|
|
5805
5647
|
var SEARCH_DATASOURCE_DOCUMENTS_TOOL_NAME = "search_datasource_documents", searchDatasourceSchema;
|
|
5806
5648
|
var init_search_datasource_tool = __esm(() => {
|
|
5807
|
-
searchDatasourceSchema =
|
|
5808
|
-
query:
|
|
5809
|
-
topK:
|
|
5810
|
-
scope:
|
|
5649
|
+
searchDatasourceSchema = Type8.Object({
|
|
5650
|
+
query: Type8.String({ description: "Query to search across configured external datasource skills." }),
|
|
5651
|
+
topK: Type8.Optional(Type8.Integer({ description: "Maximum number of datasource chunks to return." })),
|
|
5652
|
+
scope: Type8.Optional(Type8.String({ description: "Optional opaque datasource scope, e.g. /kakao/account or /kakao/account/**." }))
|
|
5811
5653
|
});
|
|
5812
5654
|
});
|
|
5813
5655
|
|
|
@@ -5947,7 +5789,7 @@ var init_search_documents = __esm(() => {
|
|
|
5947
5789
|
});
|
|
5948
5790
|
|
|
5949
5791
|
// src/agent/search-minsync-tool.ts
|
|
5950
|
-
import { Type as
|
|
5792
|
+
import { Type as Type9 } from "typebox";
|
|
5951
5793
|
function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => scope) {
|
|
5952
5794
|
return {
|
|
5953
5795
|
name: SEARCH_MINSYNC_DOCUMENTS_TOOL_NAME,
|
|
@@ -5957,10 +5799,10 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
|
|
|
5957
5799
|
async execute(_toolCallId, params) {
|
|
5958
5800
|
const method = getMethod();
|
|
5959
5801
|
if (!method) {
|
|
5960
|
-
return
|
|
5802
|
+
return unavailableResult("MinSync semantic search is not configured for this AutoRAG agent");
|
|
5961
5803
|
}
|
|
5962
5804
|
if (method.isBinaryMissing()) {
|
|
5963
|
-
return
|
|
5805
|
+
return unavailableResult("MinSync semantic search is unavailable; the configured binary is missing.");
|
|
5964
5806
|
}
|
|
5965
5807
|
if (params.query.trim().length === 0) {
|
|
5966
5808
|
return {
|
|
@@ -5975,7 +5817,7 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
|
|
|
5975
5817
|
scope
|
|
5976
5818
|
});
|
|
5977
5819
|
return {
|
|
5978
|
-
content: [{ type: "text", text:
|
|
5820
|
+
content: [{ type: "text", text: formatResults3(results) }],
|
|
5979
5821
|
details: {
|
|
5980
5822
|
method: "semantic_search_local_docs",
|
|
5981
5823
|
resultCount: results.length,
|
|
@@ -5984,12 +5826,12 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
|
|
|
5984
5826
|
}
|
|
5985
5827
|
};
|
|
5986
5828
|
} catch {
|
|
5987
|
-
return
|
|
5829
|
+
return unavailableResult("MinSync semantic search is unavailable; the query could not be completed.");
|
|
5988
5830
|
}
|
|
5989
5831
|
}
|
|
5990
5832
|
};
|
|
5991
5833
|
}
|
|
5992
|
-
function
|
|
5834
|
+
function unavailableResult(message) {
|
|
5993
5835
|
return {
|
|
5994
5836
|
content: [{ type: "text", text: message }],
|
|
5995
5837
|
details: {
|
|
@@ -6000,7 +5842,7 @@ function unavailableResult2(message) {
|
|
|
6000
5842
|
}
|
|
6001
5843
|
};
|
|
6002
5844
|
}
|
|
6003
|
-
function
|
|
5845
|
+
function formatResults3(results) {
|
|
6004
5846
|
if (results.length === 0)
|
|
6005
5847
|
return "No MinSync results.";
|
|
6006
5848
|
const rows = results.map((result, index) => {
|
|
@@ -6016,12 +5858,12 @@ ${rows.join(`
|
|
|
6016
5858
|
}
|
|
6017
5859
|
var SEARCH_MINSYNC_DOCUMENTS_TOOL_NAME = "semantic_search_local_docs", searchMinSyncSchema;
|
|
6018
5860
|
var init_search_minsync_tool = __esm(() => {
|
|
6019
|
-
searchMinSyncSchema =
|
|
6020
|
-
query:
|
|
5861
|
+
searchMinSyncSchema = Type9.Object({
|
|
5862
|
+
query: Type9.String({
|
|
6021
5863
|
description: "Semantic query to search parsed document mirrors with MinSync vector retrieval."
|
|
6022
5864
|
}),
|
|
6023
|
-
topK:
|
|
6024
|
-
scope:
|
|
5865
|
+
topK: Type9.Optional(Type9.Integer({ description: "Maximum number of MinSync semantic chunks to return." })),
|
|
5866
|
+
scope: Type9.Optional(Type9.String({ description: "Optional opaque virtual-path scope, e.g. /docs or /docs/**." }))
|
|
6025
5867
|
});
|
|
6026
5868
|
});
|
|
6027
5869
|
|
|
@@ -6038,7 +5880,6 @@ function buildSystemPrompt(config) {
|
|
|
6038
5880
|
toolLine(config, "bash", "read and inspect configured document collections with ls, find, grep, cat, and similar tools"),
|
|
6039
5881
|
toolLine(config, "jikji_find", "local discovery through Jikji answer packs"),
|
|
6040
5882
|
toolLine(config, "search_all_documents", "fan out across every configured retrieval method and merge results"),
|
|
6041
|
-
toolLine(config, "lexical_search_local_docs", "MinSync-backed lexical BM25 search over parsed document mirrors"),
|
|
6042
5883
|
toolLine(config, "semantic_search_local_docs", "semantic MinSync search over parsed document mirrors"),
|
|
6043
5884
|
toolLine(config, "search_datasource_documents", "search authorized external datasources; scope may only narrow access"),
|
|
6044
5885
|
toolLine(config, "load_datasource_skill", "load instructions for an authorized datasource"),
|
|
@@ -6049,7 +5890,6 @@ function buildSystemPrompt(config) {
|
|
|
6049
5890
|
"bash",
|
|
6050
5891
|
"jikji_find",
|
|
6051
5892
|
"search_all_documents",
|
|
6052
|
-
"lexical_search_local_docs",
|
|
6053
5893
|
"semantic_search_local_docs",
|
|
6054
5894
|
"search_datasource_documents",
|
|
6055
5895
|
"load_datasource_skill",
|
|
@@ -6082,8 +5922,8 @@ Your job is to retrieve candidates, read the relevant source material directly,
|
|
|
6082
5922
|
|
|
6083
5923
|
## Workflow
|
|
6084
5924
|
|
|
6085
|
-
1. **PLAN** —
|
|
6086
|
-
2. **RETRIEVE** — Use MinSync
|
|
5925
|
+
1. **PLAN** — Decide whether the query is answerable from stable general knowledge, memory, or intrinsic model knowledge. If so, answer immediately without retrieval.
|
|
5926
|
+
2. **RETRIEVE** — Use MinSync lexical/vector/hybrid retrieval, combined retrieval, Jikji, datasource search, or direct filesystem discovery as appropriate.
|
|
6087
5927
|
3. **READ** — Use \`bash\` to open and verify relevant local files. Do not curate from search snippets alone when source files are available.
|
|
6088
5928
|
4. **JUDGE** — Evaluate relevance, sufficiency, conflicts, uncertainty, and temporal context.
|
|
6089
5929
|
5. **CURATE** — Produce concise numbered knowledge units grounded in source evidence.
|
|
@@ -6096,14 +5936,17 @@ ${toolLines.join(`
|
|
|
6096
5936
|
${noSearchTools}
|
|
6097
5937
|
## Search Strategy
|
|
6098
5938
|
|
|
6099
|
-
-
|
|
5939
|
+
- For generic, stable questions answer directly from knowledge or memory without searching. For source-dependent questions, baseline MinSync and Jikji retrieval starts before the model's search decision and supplies up to 50 candidates per method; inspect it first and treat it as unverified evidence.
|
|
6100
5940
|
- Start with the most specific exact term, identifier, filename glob, or regex that preserves the query intent.
|
|
6101
5941
|
- Use \`search_all_documents\` when multiple configured retrieval methods can help.
|
|
6102
|
-
- Use MinSync
|
|
5942
|
+
- Use MinSync lexical mode for exact terminology, MinSync vector search for semantic similarity, and \`search_all_documents\` when hybrid ranking over the same MinSync chunks can help.
|
|
6103
5943
|
- Use \`bash\` to read already-retrieved local files with cat/head/sed. find/grep/rg must be small and bounded: one already-known directory from retrieval, a tight pattern, and a cap (head, maxdepth, or file types). Never recursively scan a whole search root (Downloads, Documents, Desktop, or /); those calls miss the bash timeout and stall the search loop.
|
|
6104
5944
|
- If retrieval is empty, retry a simpler query or synonyms through retrieval tools first. Do not widen filesystem discovery to compensate.
|
|
6105
5945
|
- Slash-prefixed datasource IDs such as /kakao/..., /gmail/..., /slack/..., /discord/..., and /github/... are virtual, not OS paths. Do not bash them. Search them with \`search_datasource_documents\`.
|
|
6106
5946
|
- Cross-check important claims against the original source and preserve real source paths.
|
|
5947
|
+
- When more searching is needed, first emit a brief, query-specific 1–2 line progress update describing the best current hypothesis and what is being checked next; baseline retrieval is already running in parallel. Never repeat a generic status message.
|
|
5948
|
+
- Do not use broad grep/find or recursive filesystem scans. Only inspect a narrow neighborhood around a retrieved candidate when the evidence clearly points there.
|
|
5949
|
+
- Do not query the same datasource more than three times. After three attempts, stop searching it and conclude from the evidence available.
|
|
6107
5950
|
|
|
6108
5951
|
${duplicateManagement}
|
|
6109
5952
|
## External Datasource Skills
|
|
@@ -6228,6 +6071,10 @@ class AutoRAGAgent {
|
|
|
6228
6071
|
resultCapture;
|
|
6229
6072
|
autoRefreshTimer;
|
|
6230
6073
|
refreshing = false;
|
|
6074
|
+
jikjiPrepareInFlight;
|
|
6075
|
+
jikjiReady = false;
|
|
6076
|
+
minSyncPrepareInFlight;
|
|
6077
|
+
minSyncReady = false;
|
|
6231
6078
|
refreshState = {
|
|
6232
6079
|
inFlight: false,
|
|
6233
6080
|
lastOutcome: "never",
|
|
@@ -6247,7 +6094,6 @@ class AutoRAGAgent {
|
|
|
6247
6094
|
merger = new ResultMerger;
|
|
6248
6095
|
datasourceFilter = new DatasourceResultFilter;
|
|
6249
6096
|
minSyncMethod;
|
|
6250
|
-
bm25Method;
|
|
6251
6097
|
jikjiClient;
|
|
6252
6098
|
datasourceSkills;
|
|
6253
6099
|
datasourceAccessOptions;
|
|
@@ -6261,6 +6107,7 @@ class AutoRAGAgent {
|
|
|
6261
6107
|
searchTimeoutMs;
|
|
6262
6108
|
maxSearchToolCalls;
|
|
6263
6109
|
searchToolCallCount = 0;
|
|
6110
|
+
sourceSearchCallCounts = new Map;
|
|
6264
6111
|
constructor(options) {
|
|
6265
6112
|
const { manifestDir, memoryPath } = options;
|
|
6266
6113
|
this.configuredModel = options.model;
|
|
@@ -6293,16 +6140,9 @@ class AutoRAGAgent {
|
|
|
6293
6140
|
...minSyncOpts,
|
|
6294
6141
|
root: this.workspaceProjectRoot
|
|
6295
6142
|
});
|
|
6143
|
+
this.minSyncReady = this.minSyncMethod.isReady();
|
|
6296
6144
|
this.methodRegistry.register(this.minSyncMethod);
|
|
6297
6145
|
this.methodRegistry.register(new MinSyncHybridMethod({ ...minSyncOpts, root: this.workspaceProjectRoot }));
|
|
6298
|
-
if (options.bm25 !== false) {
|
|
6299
|
-
const bm25Opts = { ...minSyncOpts, autoInstall: true, ...options.bm25 ?? {} };
|
|
6300
|
-
this.bm25Method = new MinSyncBM25Method({
|
|
6301
|
-
...bm25Opts,
|
|
6302
|
-
root: this.workspaceProjectRoot
|
|
6303
|
-
});
|
|
6304
|
-
this.methodRegistry.register(this.bm25Method);
|
|
6305
|
-
}
|
|
6306
6146
|
}
|
|
6307
6147
|
for (const skill of this.datasourceSkills) {
|
|
6308
6148
|
for (const method of skill.retrievalMethods())
|
|
@@ -6313,13 +6153,13 @@ class AutoRAGAgent {
|
|
|
6313
6153
|
...options.jikji ?? {},
|
|
6314
6154
|
root: this.workspaceProjectRoot
|
|
6315
6155
|
});
|
|
6156
|
+
this.jikjiReady = this.searchPaths.every((searchPath) => this.jikjiClient?.isPrepared(searchPath) === true);
|
|
6316
6157
|
}
|
|
6317
6158
|
const memPath = memoryPath ?? join16(resolveAutoRAGHome(), "memory.json");
|
|
6318
6159
|
this.memory = new RetrievalMemory({ storagePath: memPath });
|
|
6319
6160
|
this.memory.load();
|
|
6320
6161
|
this.runLogger = new AutoRAGRunLogger(join16(dirname9(memPath), "logs", "runs.jsonl"));
|
|
6321
6162
|
const checkMemoryTool = createCheckMemoryTool(this.memory);
|
|
6322
|
-
const searchBM25Tool = createSearchBM25DocumentsTool(() => this.bm25Method, (scope) => this.resolveRetrievalScope(scope));
|
|
6323
6163
|
const searchDatasourceTool = createSearchDatasourceDocumentsTool(this);
|
|
6324
6164
|
const searchMinSyncTool = createSearchMinSyncDocumentsTool(() => this.minSyncMethod, (scope) => this.resolveRetrievalScope(scope));
|
|
6325
6165
|
const searchAllTool = createSearchAllDocumentsTool(this);
|
|
@@ -6333,7 +6173,6 @@ class AutoRAGAgent {
|
|
|
6333
6173
|
const reservedNames = new Set([
|
|
6334
6174
|
BASH_TOOL_NAME,
|
|
6335
6175
|
"check_memory",
|
|
6336
|
-
SEARCH_BM25_DOCUMENTS_TOOL_NAME,
|
|
6337
6176
|
SEARCH_DATASOURCE_DOCUMENTS_TOOL_NAME,
|
|
6338
6177
|
LOAD_DATASOURCE_SKILL_TOOL_NAME,
|
|
6339
6178
|
EMIT_AUTORAG_RESULTS_TOOL_NAME,
|
|
@@ -6355,7 +6194,6 @@ class AutoRAGAgent {
|
|
|
6355
6194
|
bashTool,
|
|
6356
6195
|
...callerTools,
|
|
6357
6196
|
checkMemoryTool,
|
|
6358
|
-
searchBM25Tool,
|
|
6359
6197
|
searchMinSyncTool,
|
|
6360
6198
|
searchAllTool,
|
|
6361
6199
|
searchDatasourceTool,
|
|
@@ -6370,7 +6208,7 @@ class AutoRAGAgent {
|
|
|
6370
6208
|
return false;
|
|
6371
6209
|
seenToolNames.add(tool.name);
|
|
6372
6210
|
return true;
|
|
6373
|
-
});
|
|
6211
|
+
}).map((tool) => SEARCH_TOOLS.includes(tool.name) ? this.withPerSourceSearchLimit(tool) : tool);
|
|
6374
6212
|
this.tools = tools;
|
|
6375
6213
|
const toolNames = tools.map((tool) => tool.name);
|
|
6376
6214
|
this.baseSystemPromptConfig = {
|
|
@@ -6479,6 +6317,27 @@ ${summary}
|
|
|
6479
6317
|
}));
|
|
6480
6318
|
return unsubscribers;
|
|
6481
6319
|
}
|
|
6320
|
+
withPerSourceSearchLimit(tool) {
|
|
6321
|
+
return {
|
|
6322
|
+
...tool,
|
|
6323
|
+
execute: async (...args) => {
|
|
6324
|
+
const count = this.sourceSearchCallCounts.get(tool.name) ?? 0;
|
|
6325
|
+
if (count >= 3) {
|
|
6326
|
+
return {
|
|
6327
|
+
content: [
|
|
6328
|
+
{
|
|
6329
|
+
type: "text",
|
|
6330
|
+
text: `${tool.name} has reached its three-query limit. Do not call it again; conclude from the evidence already gathered.`
|
|
6331
|
+
}
|
|
6332
|
+
],
|
|
6333
|
+
details: { method: tool.name, resultCount: 0, sources: [], limitReached: true }
|
|
6334
|
+
};
|
|
6335
|
+
}
|
|
6336
|
+
this.sourceSearchCallCounts.set(tool.name, count + 1);
|
|
6337
|
+
return tool.execute(...args);
|
|
6338
|
+
}
|
|
6339
|
+
};
|
|
6340
|
+
}
|
|
6482
6341
|
recordSearchToolEvent(event) {
|
|
6483
6342
|
if (event.type !== "tool_execution_end" || !this.lastQuery)
|
|
6484
6343
|
return;
|
|
@@ -6577,8 +6436,11 @@ ${summary}
|
|
|
6577
6436
|
options = this.normalizeRetrievalOptions(options);
|
|
6578
6437
|
this.activeRun = true;
|
|
6579
6438
|
this.searchToolCallCount = 0;
|
|
6439
|
+
this.sourceSearchCallCounts.clear();
|
|
6580
6440
|
this.lastQuery = trimmedQuery;
|
|
6581
6441
|
this.lastSessionId = sessionId;
|
|
6442
|
+
this.scheduleJikjiPrepare();
|
|
6443
|
+
this.scheduleMinSyncPrepare();
|
|
6582
6444
|
let captured;
|
|
6583
6445
|
let session;
|
|
6584
6446
|
let unsubscribers = [];
|
|
@@ -6603,8 +6465,14 @@ ${summary}
|
|
|
6603
6465
|
try {
|
|
6604
6466
|
await Promise.race([
|
|
6605
6467
|
(async () => {
|
|
6606
|
-
const
|
|
6607
|
-
await session.prompt(this.buildSearchPrompt(trimmedQuery, options
|
|
6468
|
+
const retrievalPromise = this.prefetchInitialRetrievalContext(trimmedQuery, options);
|
|
6469
|
+
await session.prompt(this.buildSearchPrompt(trimmedQuery, options));
|
|
6470
|
+
if (captured === undefined) {
|
|
6471
|
+
const initialRetrievalContext = await retrievalPromise;
|
|
6472
|
+
await session.prompt(`Baseline retrieval is complete. Use this evidence before deciding whether additional search is needed:
|
|
6473
|
+
|
|
6474
|
+
${initialRetrievalContext}`);
|
|
6475
|
+
}
|
|
6608
6476
|
})(),
|
|
6609
6477
|
new Promise((_, reject) => {
|
|
6610
6478
|
timeout = setTimeout(() => {
|
|
@@ -6663,6 +6531,73 @@ ${summary}
|
|
|
6663
6531
|
this.activeRun = false;
|
|
6664
6532
|
}
|
|
6665
6533
|
}
|
|
6534
|
+
async* searchDocumentsStream(query, options = {}) {
|
|
6535
|
+
const queue = [];
|
|
6536
|
+
let progressBuffer = "";
|
|
6537
|
+
let wake;
|
|
6538
|
+
let settled = false;
|
|
6539
|
+
const unsubscribe = this.subscribe((event) => {
|
|
6540
|
+
if (event.type !== "message_update" || event.assistantMessageEvent.type !== "text_delta")
|
|
6541
|
+
return;
|
|
6542
|
+
const text = event.assistantMessageEvent.delta;
|
|
6543
|
+
if (text.trim().length === 0)
|
|
6544
|
+
return;
|
|
6545
|
+
progressBuffer += text;
|
|
6546
|
+
if (!/[.!?。!?]\s*$/u.test(progressBuffer))
|
|
6547
|
+
return;
|
|
6548
|
+
queue.push({
|
|
6549
|
+
type: "progress",
|
|
6550
|
+
sessionId: this.lastSessionId ?? "",
|
|
6551
|
+
query: query.trim(),
|
|
6552
|
+
text: progressBuffer
|
|
6553
|
+
});
|
|
6554
|
+
progressBuffer = "";
|
|
6555
|
+
wake?.();
|
|
6556
|
+
wake = undefined;
|
|
6557
|
+
});
|
|
6558
|
+
queue.push({
|
|
6559
|
+
type: "progress",
|
|
6560
|
+
sessionId: "",
|
|
6561
|
+
query: query.trim(),
|
|
6562
|
+
text: "Reviewing the query."
|
|
6563
|
+
});
|
|
6564
|
+
const run = this.searchDocuments(query, options).then((response) => {
|
|
6565
|
+
if (progressBuffer.trim() !== "") {
|
|
6566
|
+
queue.push({
|
|
6567
|
+
type: "progress",
|
|
6568
|
+
sessionId: this.lastSessionId ?? "",
|
|
6569
|
+
query: query.trim(),
|
|
6570
|
+
text: progressBuffer
|
|
6571
|
+
});
|
|
6572
|
+
progressBuffer = "";
|
|
6573
|
+
}
|
|
6574
|
+
queue.push({ type: "complete", response });
|
|
6575
|
+
}).catch((error) => {
|
|
6576
|
+
settled = true;
|
|
6577
|
+
wake?.();
|
|
6578
|
+
throw error;
|
|
6579
|
+
}).finally(() => {
|
|
6580
|
+
settled = true;
|
|
6581
|
+
wake?.();
|
|
6582
|
+
});
|
|
6583
|
+
try {
|
|
6584
|
+
while (!settled || queue.length > 0) {
|
|
6585
|
+
if (queue.length === 0) {
|
|
6586
|
+
await new Promise((resolve) => {
|
|
6587
|
+
wake = resolve;
|
|
6588
|
+
});
|
|
6589
|
+
continue;
|
|
6590
|
+
}
|
|
6591
|
+
yield queue.shift();
|
|
6592
|
+
}
|
|
6593
|
+
await run;
|
|
6594
|
+
} finally {
|
|
6595
|
+
unsubscribe();
|
|
6596
|
+
await run.catch(() => {
|
|
6597
|
+
return;
|
|
6598
|
+
});
|
|
6599
|
+
}
|
|
6600
|
+
}
|
|
6666
6601
|
datasourceAccessContext(options = {}) {
|
|
6667
6602
|
return new DatasourceAccessContext({
|
|
6668
6603
|
allowedTags: options.allowedTags ?? this.datasourceAccessOptions.allowedTags,
|
|
@@ -6721,17 +6656,6 @@ ${summary}
|
|
|
6721
6656
|
source: "tools"
|
|
6722
6657
|
});
|
|
6723
6658
|
}
|
|
6724
|
-
const bm25 = this.bm25Method?.getStatus();
|
|
6725
|
-
if (bm25 !== undefined) {
|
|
6726
|
-
if (bm25.readiness === "dependency_unavailable" || bm25.readiness === "index_missing" || bm25.readiness === "error") {
|
|
6727
|
-
diagnostics.push({
|
|
6728
|
-
code: "bm25-unavailable",
|
|
6729
|
-
severity: "warning",
|
|
6730
|
-
message: "BM25 lexical search is unavailable; results rely on other retrieval paths.",
|
|
6731
|
-
source: "bm25"
|
|
6732
|
-
});
|
|
6733
|
-
}
|
|
6734
|
-
}
|
|
6735
6659
|
if (this.minSyncMethod?.isBinaryMissing()) {
|
|
6736
6660
|
diagnostics.push({
|
|
6737
6661
|
code: "minsync-unavailable",
|
|
@@ -6746,51 +6670,84 @@ ${summary}
|
|
|
6746
6670
|
return diagnostics;
|
|
6747
6671
|
}
|
|
6748
6672
|
async prefetchInitialRetrievalContext(query, options) {
|
|
6749
|
-
const retrieveOptions = { topK:
|
|
6750
|
-
const
|
|
6751
|
-
|
|
6673
|
+
const retrieveOptions = { topK: 50, scope: options.scope };
|
|
6674
|
+
const vectorReady = this.minSyncMethod?.isReady() === true;
|
|
6675
|
+
const [jikji, vector] = await Promise.all([
|
|
6676
|
+
this.jikjiClient === undefined ? Promise.resolve(undefined) : this.findJikji(query, { topK: 50 }).catch(() => {
|
|
6752
6677
|
return;
|
|
6753
6678
|
}),
|
|
6754
|
-
this.minSyncMethod?.retrieve(query, retrieveOptions).catch(() => [])
|
|
6755
|
-
this.bm25Method?.retrieve(query, retrieveOptions).catch(() => [])
|
|
6679
|
+
vectorReady ? this.minSyncMethod?.retrieve(query, retrieveOptions).catch(() => []) : Promise.resolve([])
|
|
6756
6680
|
]);
|
|
6757
6681
|
const sections = [];
|
|
6758
6682
|
if (jikji?.answerPack !== undefined) {
|
|
6759
6683
|
sections.push(`Jikji initial candidates (preserve order when agent_should_not_rerank=true):
|
|
6760
|
-
${jikji.answerPack.answerPaths.slice(0,
|
|
6684
|
+
${jikji.answerPack.answerPaths.slice(0, 50).map((path, index) => `[${index + 1}] ${path}`).join(`
|
|
6761
6685
|
`)}`);
|
|
6762
6686
|
}
|
|
6763
|
-
const
|
|
6687
|
+
const formatResults = (label, results) => {
|
|
6764
6688
|
if (!results || results.length === 0)
|
|
6765
6689
|
return;
|
|
6690
|
+
const seen = new Set;
|
|
6766
6691
|
sections.push(`${label} initial candidates:
|
|
6767
|
-
${results.
|
|
6692
|
+
${results.filter((result) => {
|
|
6693
|
+
const key = `${result.source}\x00${result.content}`;
|
|
6694
|
+
if (seen.has(key))
|
|
6695
|
+
return false;
|
|
6696
|
+
seen.add(key);
|
|
6697
|
+
return true;
|
|
6698
|
+
}).slice(0, 50).map((result, index) => `[${index + 1}] ${result.source}
|
|
6768
6699
|
${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
6769
6700
|
`)}`);
|
|
6770
6701
|
};
|
|
6771
|
-
|
|
6772
|
-
formatResults5("MinSync lexical BM25", bm25);
|
|
6702
|
+
formatResults("MinSync semantic", vector);
|
|
6773
6703
|
return sections.length === 0 ? "No initial retrieval candidates were available; use the configured tools and report degradation honestly." : sections.join(`
|
|
6774
6704
|
|
|
6775
6705
|
`);
|
|
6776
6706
|
}
|
|
6707
|
+
scheduleMinSyncPrepareForTest() {
|
|
6708
|
+
return this.scheduleMinSyncPrepare();
|
|
6709
|
+
}
|
|
6710
|
+
scheduleMinSyncPrepare() {
|
|
6711
|
+
if (this.minSyncMethod === undefined || this.minSyncReady || this.minSyncPrepareInFlight !== undefined) {
|
|
6712
|
+
return this.minSyncPrepareInFlight;
|
|
6713
|
+
}
|
|
6714
|
+
this.minSyncPrepareInFlight = Promise.resolve().then(async () => {
|
|
6715
|
+
await this.syncParsedMirrors(false);
|
|
6716
|
+
if (!this.minSyncMethod)
|
|
6717
|
+
return;
|
|
6718
|
+
this.minSyncReady = this.minSyncMethod.isReady();
|
|
6719
|
+
if (this.minSyncReady)
|
|
6720
|
+
return;
|
|
6721
|
+
const result = await this.minSyncMethod.sync();
|
|
6722
|
+
this.minSyncReady = result?.ok === true;
|
|
6723
|
+
this.refreshState = { ...this.refreshState, minsync: result };
|
|
6724
|
+
return result;
|
|
6725
|
+
}).catch(() => {
|
|
6726
|
+
return;
|
|
6727
|
+
}).finally(() => {
|
|
6728
|
+
this.minSyncPrepareInFlight = undefined;
|
|
6729
|
+
});
|
|
6730
|
+
return this.minSyncPrepareInFlight;
|
|
6731
|
+
}
|
|
6777
6732
|
buildSearchPrompt(query, options, initialRetrievalContext) {
|
|
6778
6733
|
const limit = typeof options.topK === "number" ? ` Return at most ${options.topK} curated results.` : "";
|
|
6779
6734
|
const scope = options.scope ? ` Restrict search to virtual path scope ${options.scope}.` : "";
|
|
6780
6735
|
return `Find and curate information for this original query: ${query}${limit}${scope}
|
|
6781
6736
|
|
|
6782
|
-
` + `
|
|
6783
|
-
|
|
6737
|
+
` + `Start by deciding whether this is answerable from general knowledge or memory. If it is a generic, stable question, answer it immediately without retrieval and emit the structured result. ` + `Otherwise, baseline MinSync and Jikji retrieval is already running in parallel; do not emit final results until its next message arrives.
|
|
6738
|
+
|
|
6739
|
+
` + `Baseline retrieval context:
|
|
6740
|
+
${initialRetrievalContext ?? "Pending; continue only with a brief progress statement."}
|
|
6784
6741
|
|
|
6785
|
-
` + `Treat
|
|
6742
|
+
` + `Treat candidates as unverified evidence, verify important claims against source files, and use additional tools when needed. ` + `Use the available retrieval tools to find candidates, then use bash to read and verify the relevant source files directly. ` + `Judge relevance, conflicts, freshness, and sufficiency in this agent loop. Preserve real source paths and evidence excerpts in the result mapping.
|
|
6786
6743
|
|
|
6787
|
-
` + `When finished, call ${EMIT_AUTORAG_RESULTS_TOOL_NAME} exactly once as your final action with the curated ` + `results and the internal number-to-source mapping.`;
|
|
6744
|
+
` + `If more search is needed, first write a brief 1–2 line progress update stating the best current hypothesis and what you are checking next, then call retrieval tools. ` + `Never repeat a generic status message. Do not use broad grep/find or recursive filesystem scans: only inspect a path or narrow neighborhood surfaced by retrieval, and only when evidence clearly points there. ` + `Do not query the same datasource more than three times. After three attempts, stop searching that datasource and conclude from the evidence available. ` + `When finished, call ${EMIT_AUTORAG_RESULTS_TOOL_NAME} exactly once as your final action with the curated ` + `results and the internal number-to-source mapping.`;
|
|
6788
6745
|
}
|
|
6789
6746
|
async refresh(force = false, opts) {
|
|
6790
6747
|
const methods = opts?.methods;
|
|
6791
6748
|
const allMethods = methods === undefined;
|
|
6792
6749
|
const wants = (m) => allMethods || methods.includes(m);
|
|
6793
|
-
const needsParsed = allMethods || wants("parsed") || wants("
|
|
6750
|
+
const needsParsed = allMethods || wants("parsed") || wants("minsync");
|
|
6794
6751
|
this.refreshState = {
|
|
6795
6752
|
...this.refreshState,
|
|
6796
6753
|
inFlight: true,
|
|
@@ -6798,13 +6755,9 @@ ${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
|
6798
6755
|
};
|
|
6799
6756
|
try {
|
|
6800
6757
|
const summary = needsParsed ? await this.syncParsedMirrors(force) : await this.scanMirrorStaleness();
|
|
6801
|
-
if (wants("bm25") || wants("minsync") || allMethods) {
|
|
6802
|
-
removeLegacyBm25Artifacts(this.workspaceProjectRoot);
|
|
6803
|
-
}
|
|
6804
6758
|
const minsync = wants("minsync") ? await this.syncMinSync() : undefined;
|
|
6805
|
-
const bm25 = wants("bm25") ? await this.syncBM25(minsync) : undefined;
|
|
6806
6759
|
const datasources = wants("datasources") ? await this.indexDatasources() : [];
|
|
6807
|
-
const jikji = wants("jikji") ? await this.executeJikjiPrepare() : undefined;
|
|
6760
|
+
const jikji = allMethods || wants("jikji") || wants("minsync") ? await this.executeJikjiPrepare() : undefined;
|
|
6808
6761
|
this.retrievalScopeBindings = buildRetrievalScopeBindings(this.workspaceProjectRoot, this.searchPaths, this.configuredSearchPaths);
|
|
6809
6762
|
const jikjiDiagnostics = (jikji ?? []).map((result) => jikjiPrepareDiagnostic(result)).filter((diag) => diag !== undefined);
|
|
6810
6763
|
this.refreshState = {
|
|
@@ -6828,9 +6781,8 @@ ${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
|
6828
6781
|
...minsync.reason !== undefined ? { reason: minsync.reason } : {}
|
|
6829
6782
|
} : undefined;
|
|
6830
6783
|
return {
|
|
6831
|
-
...
|
|
6784
|
+
...summary,
|
|
6832
6785
|
diagnostics: [...this.startupDiagnostics, ...summary.diagnostics],
|
|
6833
|
-
...bm25 ? { bm25 } : {},
|
|
6834
6786
|
minsync: publicMinsync,
|
|
6835
6787
|
datasources
|
|
6836
6788
|
};
|
|
@@ -6900,11 +6852,8 @@ ${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
|
6900
6852
|
}
|
|
6901
6853
|
refreshComponentStatus() {
|
|
6902
6854
|
const status = {};
|
|
6903
|
-
const bm25 = this.bm25Method?.getStatus();
|
|
6904
|
-
if (bm25 !== undefined)
|
|
6905
|
-
status.bm25 = bm25.readiness;
|
|
6906
6855
|
if (this.minSyncMethod !== undefined) {
|
|
6907
|
-
status.minsync = this.minSyncMethod.isBinaryMissing() ? "unavailable" : this.refreshState.minsync?.ok === false ? "degraded" : this.
|
|
6856
|
+
status.minsync = this.minSyncMethod.isBinaryMissing() ? "unavailable" : this.refreshState.minsync?.ok === false ? "degraded" : this.minSyncMethod.isReady() ? "ready" : "configured";
|
|
6908
6857
|
}
|
|
6909
6858
|
if (this.jikjiClient !== undefined) {
|
|
6910
6859
|
status.jikji = this.refreshState.jikjiDiagnostics.length > 0 ? "degraded" : "configured";
|
|
@@ -6991,13 +6940,11 @@ ${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
|
6991
6940
|
}
|
|
6992
6941
|
return { excluded };
|
|
6993
6942
|
}
|
|
6994
|
-
async syncBM25(minsync) {
|
|
6995
|
-
if (this.bm25Method === undefined)
|
|
6996
|
-
return;
|
|
6997
|
-
return this.bm25Method.syncFromMinSync(minsync ?? await this.bm25Method.sync());
|
|
6998
|
-
}
|
|
6999
6943
|
async syncMinSync() {
|
|
7000
|
-
|
|
6944
|
+
const result = await this.minSyncMethod?.sync();
|
|
6945
|
+
this.minSyncReady = result?.ok === true;
|
|
6946
|
+
this.refreshState = { ...this.refreshState, minsync: result };
|
|
6947
|
+
return result;
|
|
7001
6948
|
}
|
|
7002
6949
|
async prepareJikji() {
|
|
7003
6950
|
const results = await this.executeJikjiPrepare();
|
|
@@ -7010,8 +6957,20 @@ ${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
|
7010
6957
|
for (const sourcePath of this.searchPaths) {
|
|
7011
6958
|
results.push(await this.jikjiClient.prepare(sourcePath));
|
|
7012
6959
|
}
|
|
6960
|
+
this.jikjiReady = results.length === this.searchPaths.length && results.every((result) => result.ok);
|
|
7013
6961
|
return results;
|
|
7014
6962
|
}
|
|
6963
|
+
scheduleJikjiPrepare() {
|
|
6964
|
+
if (this.jikjiClient === undefined || this.jikjiPrepareInFlight !== undefined)
|
|
6965
|
+
return;
|
|
6966
|
+
this.jikjiPrepareInFlight = Promise.resolve().then(async () => {
|
|
6967
|
+
const results = await this.executeJikjiPrepare();
|
|
6968
|
+
const diagnostics = (results ?? []).map((result) => jikjiPrepareDiagnostic(result)).filter((diag) => diag !== undefined);
|
|
6969
|
+
this.refreshState = { ...this.refreshState, jikjiDiagnostics: diagnostics };
|
|
6970
|
+
}).catch(() => {}).finally(() => {
|
|
6971
|
+
this.jikjiPrepareInFlight = undefined;
|
|
6972
|
+
});
|
|
6973
|
+
}
|
|
7015
6974
|
sanitizeJikjiPrepareResult(result) {
|
|
7016
6975
|
if (result.ok) {
|
|
7017
6976
|
return {
|
|
@@ -7031,6 +6990,23 @@ ${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
|
7031
6990
|
if (this.jikjiClient === undefined) {
|
|
7032
6991
|
return { answerPack: undefined, policy: undefined, diagnostics: [], roots: [], perRoot: [] };
|
|
7033
6992
|
}
|
|
6993
|
+
if (!this.jikjiReady) {
|
|
6994
|
+
this.scheduleJikjiPrepare();
|
|
6995
|
+
return {
|
|
6996
|
+
answerPack: undefined,
|
|
6997
|
+
policy: undefined,
|
|
6998
|
+
diagnostics: [
|
|
6999
|
+
{
|
|
7000
|
+
code: "jikji-unavailable",
|
|
7001
|
+
severity: "warning",
|
|
7002
|
+
message: "Jikji is not prepared yet; background preparation started and this turn is falling back.",
|
|
7003
|
+
source: "jikji"
|
|
7004
|
+
}
|
|
7005
|
+
],
|
|
7006
|
+
roots: this.searchPaths,
|
|
7007
|
+
perRoot: []
|
|
7008
|
+
};
|
|
7009
|
+
}
|
|
7034
7010
|
const sourceRoots = planJikjiSourceRoots(this.searchPaths);
|
|
7035
7011
|
const findOpts = {
|
|
7036
7012
|
topK: opts?.topK,
|
|
@@ -7305,7 +7281,6 @@ var init_agent = __esm(() => {
|
|
|
7305
7281
|
init_paths();
|
|
7306
7282
|
init_sync();
|
|
7307
7283
|
init_run_log();
|
|
7308
|
-
init_bm25();
|
|
7309
7284
|
init_scope();
|
|
7310
7285
|
init_bash_tool();
|
|
7311
7286
|
init_datasource_skill();
|
|
@@ -7314,7 +7289,6 @@ var init_agent = __esm(() => {
|
|
|
7314
7289
|
init_jikji_find_tool();
|
|
7315
7290
|
init_local_model();
|
|
7316
7291
|
init_search_all_tool();
|
|
7317
|
-
init_search_bm25_tool();
|
|
7318
7292
|
init_search_datasource_tool();
|
|
7319
7293
|
init_search_documents();
|
|
7320
7294
|
init_search_minsync_tool();
|
|
@@ -7324,7 +7298,6 @@ var init_agent = __esm(() => {
|
|
|
7324
7298
|
BASH_TOOL_NAME,
|
|
7325
7299
|
SEARCH_MINSYNC_DOCUMENTS_TOOL_NAME,
|
|
7326
7300
|
SEARCH_ALL_DOCUMENTS_TOOL_NAME,
|
|
7327
|
-
SEARCH_BM25_DOCUMENTS_TOOL_NAME,
|
|
7328
7301
|
SEARCH_DATASOURCE_DOCUMENTS_TOOL_NAME,
|
|
7329
7302
|
JIKJI_FIND_TOOL_NAME
|
|
7330
7303
|
];
|
|
@@ -7486,7 +7459,7 @@ function sanitizeIdSegment(value) {
|
|
|
7486
7459
|
var init_connector = () => {};
|
|
7487
7460
|
|
|
7488
7461
|
// src/datasource/chunk-store.ts
|
|
7489
|
-
import { existsSync as
|
|
7462
|
+
import { existsSync as existsSync12, mkdirSync as mkdirSync8, readFileSync as readFileSync10, writeFileSync as writeFileSync7 } from "node:fs";
|
|
7490
7463
|
import { dirname as dirname10, join as join17 } from "node:path";
|
|
7491
7464
|
function datasourceChunkStorePath(workspaceRoot, skillName, instanceId) {
|
|
7492
7465
|
return join17(workspaceRoot, ".autorag", "datasources", sanitizeIdSegment(skillName), sanitizeIdSegment(instanceId), "chunks.json");
|
|
@@ -7526,7 +7499,7 @@ class DatasourceChunkStore {
|
|
|
7526
7499
|
if (this.storePath === undefined)
|
|
7527
7500
|
return false;
|
|
7528
7501
|
try {
|
|
7529
|
-
if (!
|
|
7502
|
+
if (!existsSync12(this.storePath))
|
|
7530
7503
|
return false;
|
|
7531
7504
|
const parsed = JSON.parse(readFileSync10(this.storePath, "utf8"));
|
|
7532
7505
|
if (parsed.version !== STORE_VERSION || !Array.isArray(parsed.chunks))
|
|
@@ -8082,7 +8055,7 @@ var DEFAULT_TIMEOUT_MS3 = 15000;
|
|
|
8082
8055
|
|
|
8083
8056
|
// src/datasource/skills/cloud-drive/rclone-connector.ts
|
|
8084
8057
|
import { spawn as spawn6 } from "node:child_process";
|
|
8085
|
-
import { mkdirSync as mkdirSync9, readFileSync as readFileSync11, renameSync as renameSync6, rmSync as
|
|
8058
|
+
import { mkdirSync as mkdirSync9, readFileSync as readFileSync11, renameSync as renameSync6, rmSync as rmSync5, writeFileSync as writeFileSync8 } from "node:fs";
|
|
8086
8059
|
import { readFile as readFile4 } from "node:fs/promises";
|
|
8087
8060
|
import { dirname as dirname11, join as join18, normalize as normalize2 } from "node:path";
|
|
8088
8061
|
|
|
@@ -8177,12 +8150,12 @@ class RcloneConnector {
|
|
|
8177
8150
|
copied = await this.runner(copyArgs(remote, entry.path, temp, exportFormats, this.options.concurrency, this.options.bandwidthLimit), timeoutMs);
|
|
8178
8151
|
} catch {
|
|
8179
8152
|
copyFailure = { reason: "unavailable", message: "rclone copy failed unexpectedly" };
|
|
8180
|
-
|
|
8153
|
+
rmSync5(temp, { force: true });
|
|
8181
8154
|
return;
|
|
8182
8155
|
}
|
|
8183
8156
|
if (!copied.ok) {
|
|
8184
8157
|
copyFailure = { reason: classifyFailure(copied.stderr), message: shortFailure(copied) };
|
|
8185
|
-
|
|
8158
|
+
rmSync5(temp, { force: true });
|
|
8186
8159
|
return;
|
|
8187
8160
|
}
|
|
8188
8161
|
staged.push({ temp, destination });
|
|
@@ -8192,13 +8165,13 @@ class RcloneConnector {
|
|
|
8192
8165
|
await Promise.all(Array.from({ length: concurrency }, () => copyOne()));
|
|
8193
8166
|
if (copyFailure !== undefined) {
|
|
8194
8167
|
for (const file of staged)
|
|
8195
|
-
|
|
8168
|
+
rmSync5(file.temp, { force: true });
|
|
8196
8169
|
return { ok: false, reason: copyFailure.reason, message: copyFailure.message };
|
|
8197
8170
|
}
|
|
8198
8171
|
for (const file of staged)
|
|
8199
8172
|
renameSync6(file.temp, file.destination);
|
|
8200
8173
|
for (const entry of deletedEntries)
|
|
8201
|
-
|
|
8174
|
+
rmSync5(mirrorFilePath(paths.mirrorRoot, entry.path), { force: true });
|
|
8202
8175
|
saveManifest(paths.manifestPath, { version: 1, remoteName: remoteName(remote), entries: inventory });
|
|
8203
8176
|
const mirrorWarnings = await loadMirroredDocuments(paths.mirrorRoot, changedEntries, skillName, instanceId, remote, this.options.parserOptions);
|
|
8204
8177
|
const warnings = [
|
|
@@ -8321,11 +8294,11 @@ async function loadMirroredDocuments(root, entries, skillName, instanceId, remot
|
|
|
8321
8294
|
for (const entry of entries) {
|
|
8322
8295
|
try {
|
|
8323
8296
|
const sourcePath = mirrorFilePath(root, entry.path);
|
|
8324
|
-
const
|
|
8325
|
-
if (
|
|
8297
|
+
const parser = registry.getForVirtualPath(entry.path);
|
|
8298
|
+
if (parser === undefined)
|
|
8326
8299
|
continue;
|
|
8327
8300
|
const bytes = await readFile4(sourcePath);
|
|
8328
|
-
const parsed = await
|
|
8301
|
+
const parsed = await parser.parse({ virtualPath: entry.path, sourcePath, bytes });
|
|
8329
8302
|
for (const diagnostic of parsed.diagnostics ?? []) {
|
|
8330
8303
|
warnings.push(`${diagnostic.code}: ${diagnostic.message}`);
|
|
8331
8304
|
}
|
|
@@ -8342,8 +8315,8 @@ async function loadMirroredDocuments(root, entries, skillName, instanceId, remot
|
|
|
8342
8315
|
}
|
|
8343
8316
|
function defaultParserExtensions() {
|
|
8344
8317
|
const extensions = new Set;
|
|
8345
|
-
for (const
|
|
8346
|
-
for (const extension of
|
|
8318
|
+
for (const parser of createDefaultParserRegistry().list()) {
|
|
8319
|
+
for (const extension of parser.extensions) {
|
|
8347
8320
|
extensions.add(extension.startsWith(".") ? extension.toLowerCase() : `.${extension.toLowerCase()}`);
|
|
8348
8321
|
}
|
|
8349
8322
|
}
|
|
@@ -8575,7 +8548,7 @@ function commonArgs(options) {
|
|
|
8575
8548
|
return options.configPath === undefined ? [] : ["--config", options.configPath];
|
|
8576
8549
|
}
|
|
8577
8550
|
function spawnDiscrawl(request) {
|
|
8578
|
-
return new Promise((
|
|
8551
|
+
return new Promise((resolve) => {
|
|
8579
8552
|
const { options, args, env, signal } = request;
|
|
8580
8553
|
const portable = portableSpawnCommand(options.binaryPath ?? DEFAULT_DISCRAWL_BINARY, args);
|
|
8581
8554
|
const child = spawn7(portable.command, [...portable.args], {
|
|
@@ -8623,7 +8596,7 @@ function spawnDiscrawl(request) {
|
|
|
8623
8596
|
clearTimeout(timeout);
|
|
8624
8597
|
signal?.removeEventListener("abort", abortHandler);
|
|
8625
8598
|
const reason = error.code === "ENOENT" ? "binary-missing" : "spawn-error";
|
|
8626
|
-
|
|
8599
|
+
resolve({ ok: false, reason, stdout: stdout.text, stderr: describeSpawnFailure(reason), code: null });
|
|
8627
8600
|
});
|
|
8628
8601
|
child.on("close", (code) => {
|
|
8629
8602
|
if (settled)
|
|
@@ -8632,10 +8605,10 @@ function spawnDiscrawl(request) {
|
|
|
8632
8605
|
clearTimeout(timeout);
|
|
8633
8606
|
signal?.removeEventListener("abort", abortHandler);
|
|
8634
8607
|
if (finalReason !== undefined) {
|
|
8635
|
-
|
|
8608
|
+
resolve({ ok: false, reason: finalReason, stdout: stdout.text, stderr: stderr.text, code });
|
|
8636
8609
|
return;
|
|
8637
8610
|
}
|
|
8638
|
-
|
|
8611
|
+
resolve({
|
|
8639
8612
|
ok: code === 0,
|
|
8640
8613
|
reason: code === 0 ? undefined : "nonzero-exit",
|
|
8641
8614
|
stdout: stdout.text,
|
|
@@ -9332,7 +9305,7 @@ ${this.description}`
|
|
|
9332
9305
|
|
|
9333
9306
|
// src/datasource/skills/clawgallery/client.ts
|
|
9334
9307
|
import { spawn as spawn8 } from "node:child_process";
|
|
9335
|
-
import { existsSync as
|
|
9308
|
+
import { existsSync as existsSync13 } from "node:fs";
|
|
9336
9309
|
|
|
9337
9310
|
class ClawGalleryClient {
|
|
9338
9311
|
options;
|
|
@@ -9340,7 +9313,7 @@ class ClawGalleryClient {
|
|
|
9340
9313
|
this.options = options;
|
|
9341
9314
|
}
|
|
9342
9315
|
async bootstrap(signal) {
|
|
9343
|
-
if (this.options.path !== undefined && !
|
|
9316
|
+
if (this.options.path !== undefined && !existsSync13(this.options.path)) {
|
|
9344
9317
|
return {
|
|
9345
9318
|
ok: false,
|
|
9346
9319
|
reason: "nonzero-exit",
|
|
@@ -9434,7 +9407,7 @@ class ClawGalleryClient {
|
|
|
9434
9407
|
}
|
|
9435
9408
|
}
|
|
9436
9409
|
function spawnCli(binary, args, env, options, signal) {
|
|
9437
|
-
return new Promise((
|
|
9410
|
+
return new Promise((resolve) => {
|
|
9438
9411
|
const portable = portableSpawnCommand(binary, args);
|
|
9439
9412
|
const child = spawn8(portable.command, [...portable.args], { env, stdio: ["ignore", "pipe", "pipe"] });
|
|
9440
9413
|
let stdout = "";
|
|
@@ -9475,7 +9448,7 @@ function spawnCli(binary, args, env, options, signal) {
|
|
|
9475
9448
|
settled = true;
|
|
9476
9449
|
clearTimeout(timeout);
|
|
9477
9450
|
signal?.removeEventListener("abort", abort);
|
|
9478
|
-
|
|
9451
|
+
resolve({
|
|
9479
9452
|
ok: false,
|
|
9480
9453
|
stdout,
|
|
9481
9454
|
stderr: error.code === "ENOENT" ? "clawgallery binary not found" : "clawgallery could not start",
|
|
@@ -9489,7 +9462,7 @@ function spawnCli(binary, args, env, options, signal) {
|
|
|
9489
9462
|
settled = true;
|
|
9490
9463
|
clearTimeout(timeout);
|
|
9491
9464
|
signal?.removeEventListener("abort", abort);
|
|
9492
|
-
|
|
9465
|
+
resolve({
|
|
9493
9466
|
ok: reason === undefined && code === 0,
|
|
9494
9467
|
stdout,
|
|
9495
9468
|
stderr: stderr.slice(0, max),
|
|
@@ -9923,228 +9896,6 @@ var init_github = __esm(() => {
|
|
|
9923
9896
|
init_skill4();
|
|
9924
9897
|
});
|
|
9925
9898
|
|
|
9926
|
-
// src/datasource/skills/gmail/himalaya-connector.ts
|
|
9927
|
-
import { spawn as spawn9 } from "node:child_process";
|
|
9928
|
-
import { mkdirSync as mkdirSync10, readFileSync as readFileSync12, writeFileSync as writeFileSync9 } from "node:fs";
|
|
9929
|
-
import { dirname as dirname12, join as join19 } from "node:path";
|
|
9930
|
-
|
|
9931
|
-
class HimalayaConnector {
|
|
9932
|
-
options;
|
|
9933
|
-
runner;
|
|
9934
|
-
constructor(options = {}) {
|
|
9935
|
-
this.options = options;
|
|
9936
|
-
this.runner = options.runner ?? ((args, timeoutMs) => runBinary2(options.binaryPath ?? DEFAULT_BINARY4, args, timeoutMs, {
|
|
9937
|
-
...options.configPath === undefined ? {} : { HIMALAYA_CONFIG: options.configPath }
|
|
9938
|
-
}));
|
|
9939
|
-
}
|
|
9940
|
-
async fetch() {
|
|
9941
|
-
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS6;
|
|
9942
|
-
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS3;
|
|
9943
|
-
const accountArgs = this.options.account !== undefined ? ["--account", this.options.account] : [];
|
|
9944
|
-
const mailboxArgs = this.options.folder !== undefined ? ["--mailbox", this.options.folder] : [];
|
|
9945
|
-
const state = this.loadState();
|
|
9946
|
-
const listArgs = [
|
|
9947
|
-
...accountArgs,
|
|
9948
|
-
"--json",
|
|
9949
|
-
"envelope",
|
|
9950
|
-
"list",
|
|
9951
|
-
...mailboxArgs,
|
|
9952
|
-
"--page-size",
|
|
9953
|
-
String(this.options.pageSize ?? DEFAULT_PAGE_SIZE)
|
|
9954
|
-
];
|
|
9955
|
-
let listResult;
|
|
9956
|
-
try {
|
|
9957
|
-
listResult = await this.runner(listArgs, timeoutMs);
|
|
9958
|
-
} catch {
|
|
9959
|
-
return { ok: false, reason: "unavailable", message: "himalaya binary not found or failed to spawn" };
|
|
9960
|
-
}
|
|
9961
|
-
if (!listResult.ok) {
|
|
9962
|
-
return {
|
|
9963
|
-
ok: false,
|
|
9964
|
-
reason: classifyFailure2(`${listResult.stderr}
|
|
9965
|
-
${listResult.stdout}`),
|
|
9966
|
-
message: diagnosticFailure(listResult)
|
|
9967
|
-
};
|
|
9968
|
-
}
|
|
9969
|
-
let envelopes;
|
|
9970
|
-
try {
|
|
9971
|
-
envelopes = parseEnvelopes(listResult.stdout);
|
|
9972
|
-
} catch {
|
|
9973
|
-
return { ok: false, reason: "invalid-data", message: "envelope listing was not valid JSON" };
|
|
9974
|
-
}
|
|
9975
|
-
const account = this.options.account ?? "default";
|
|
9976
|
-
const folder = this.options.folder ?? "INBOX";
|
|
9977
|
-
const documents = [];
|
|
9978
|
-
let readFailures = 0;
|
|
9979
|
-
const nextFingerprints = { ...state.fingerprints };
|
|
9980
|
-
const listedDocIds = new Set;
|
|
9981
|
-
for (const raw of envelopes.slice(0, maxDocuments)) {
|
|
9982
|
-
const envelope = asRecord(raw);
|
|
9983
|
-
const id = asString(envelope?.id);
|
|
9984
|
-
if (envelope === undefined || id === undefined)
|
|
9985
|
-
continue;
|
|
9986
|
-
const docId = `${account}-${folder}-${id}`;
|
|
9987
|
-
listedDocIds.add(docId);
|
|
9988
|
-
const fingerprint = envelopeFingerprint(envelope);
|
|
9989
|
-
if (state.fingerprints[docId] === fingerprint)
|
|
9990
|
-
continue;
|
|
9991
|
-
const subject = asString(envelope.subject) ?? "(no subject)";
|
|
9992
|
-
const from = asRecord(envelope.from);
|
|
9993
|
-
const fromText = [asString(from?.name), asString(from?.addr)].filter(Boolean).join(" ").trim();
|
|
9994
|
-
let body = "";
|
|
9995
|
-
let bodyRead = false;
|
|
9996
|
-
try {
|
|
9997
|
-
const readResult = await this.runner([...accountArgs, "--json", "message", "read", ...mailboxArgs, id], timeoutMs);
|
|
9998
|
-
if (readResult.ok) {
|
|
9999
|
-
body = readResult.stdout.trim().slice(0, MAX_BODY_CHARS);
|
|
10000
|
-
bodyRead = true;
|
|
10001
|
-
} else
|
|
10002
|
-
readFailures += 1;
|
|
10003
|
-
} catch {
|
|
10004
|
-
readFailures += 1;
|
|
10005
|
-
}
|
|
10006
|
-
const headerBlock = [
|
|
10007
|
-
`Subject: ${subject}`,
|
|
10008
|
-
...fromText.length > 0 ? [`From: ${fromText}`] : [],
|
|
10009
|
-
...asString(envelope.date) !== undefined ? [`Date: ${asString(envelope.date)}`] : []
|
|
10010
|
-
].join(`
|
|
10011
|
-
`);
|
|
10012
|
-
documents.push({
|
|
10013
|
-
docId,
|
|
10014
|
-
hierarchy: ["accounts", account, folder],
|
|
10015
|
-
title: subject,
|
|
10016
|
-
content: body.length > 0 ? `${headerBlock}
|
|
10017
|
-
|
|
10018
|
-
${body}` : headerBlock,
|
|
10019
|
-
publishedAt: parseHimalayaDate(asString(envelope.date)),
|
|
10020
|
-
metadata: {
|
|
10021
|
-
account,
|
|
10022
|
-
folder,
|
|
10023
|
-
messageId: id,
|
|
10024
|
-
...asArray(envelope.flags).length > 0 ? {
|
|
10025
|
-
flags: asArray(envelope.flags).map((flag) => asString(flag)).filter(Boolean)
|
|
10026
|
-
} : {}
|
|
10027
|
-
}
|
|
10028
|
-
});
|
|
10029
|
-
if (bodyRead)
|
|
10030
|
-
nextFingerprints[docId] = fingerprint;
|
|
10031
|
-
}
|
|
10032
|
-
if (envelopes.length <= maxDocuments) {
|
|
10033
|
-
for (const docId of Object.keys(state.fingerprints)) {
|
|
10034
|
-
if (!listedDocIds.has(docId))
|
|
10035
|
-
delete nextFingerprints[docId];
|
|
10036
|
-
}
|
|
10037
|
-
}
|
|
10038
|
-
const warnings = readFailures > 0 ? [`${readFailures} message(s) failed to read`] : undefined;
|
|
10039
|
-
this.saveState({ version: STATE_VERSION, fingerprints: nextFingerprints });
|
|
10040
|
-
const deletedDocIds = envelopes.length <= maxDocuments ? Object.keys(state.fingerprints).filter((docId) => !listedDocIds.has(docId)) : [];
|
|
10041
|
-
return {
|
|
10042
|
-
ok: true,
|
|
10043
|
-
documents,
|
|
10044
|
-
changed: Object.keys(nextFingerprints).length !== Object.keys(state.fingerprints).length || documents.length > 0,
|
|
10045
|
-
...deletedDocIds.length > 0 ? { deletedDocIds } : {},
|
|
10046
|
-
...warnings !== undefined ? { warnings } : {}
|
|
10047
|
-
};
|
|
10048
|
-
}
|
|
10049
|
-
loadState() {
|
|
10050
|
-
const path = this.statePath();
|
|
10051
|
-
if (path === undefined)
|
|
10052
|
-
return { version: STATE_VERSION, fingerprints: {} };
|
|
10053
|
-
try {
|
|
10054
|
-
const parsed = JSON.parse(readFileSync12(path, "utf8"));
|
|
10055
|
-
if (parsed.version !== STATE_VERSION || parsed.fingerprints === undefined) {
|
|
10056
|
-
return { version: STATE_VERSION, fingerprints: {} };
|
|
10057
|
-
}
|
|
10058
|
-
return { version: STATE_VERSION, fingerprints: parsed.fingerprints };
|
|
10059
|
-
} catch {
|
|
10060
|
-
return { version: STATE_VERSION, fingerprints: {} };
|
|
10061
|
-
}
|
|
10062
|
-
}
|
|
10063
|
-
saveState(state) {
|
|
10064
|
-
const path = this.statePath();
|
|
10065
|
-
if (path === undefined)
|
|
10066
|
-
return;
|
|
10067
|
-
try {
|
|
10068
|
-
mkdirSync10(dirname12(path), { recursive: true });
|
|
10069
|
-
writeFileSync9(path, `${JSON.stringify(state)}
|
|
10070
|
-
`, "utf8");
|
|
10071
|
-
} catch {}
|
|
10072
|
-
}
|
|
10073
|
-
statePath() {
|
|
10074
|
-
if (this.options.workspaceRoot === undefined)
|
|
10075
|
-
return;
|
|
10076
|
-
return join19(this.options.workspaceRoot, ".autorag", "datasources", sanitizeIdSegment("gmail"), sanitizeIdSegment(this.options.account ?? "default"), sanitizeIdSegment(this.options.folder ?? "INBOX"), "himalaya-state.json");
|
|
10077
|
-
}
|
|
10078
|
-
}
|
|
10079
|
-
function parseHimalayaDate(value) {
|
|
10080
|
-
if (value === undefined || value.length === 0)
|
|
10081
|
-
return;
|
|
10082
|
-
const parsed = Date.parse(value.replace(" ", "T"));
|
|
10083
|
-
return Number.isNaN(parsed) ? undefined : parsed;
|
|
10084
|
-
}
|
|
10085
|
-
function classifyFailure2(diagnostic) {
|
|
10086
|
-
const lower = diagnostic.toLowerCase();
|
|
10087
|
-
if (lower.includes("cannot find") && lower.includes("account"))
|
|
10088
|
-
return "not-configured";
|
|
10089
|
-
if (lower.includes("auth") || lower.includes("login") || lower.includes("credential"))
|
|
10090
|
-
return "auth";
|
|
10091
|
-
return "api-error";
|
|
10092
|
-
}
|
|
10093
|
-
function diagnosticFailure(result) {
|
|
10094
|
-
const raw = result.stderr.trim() || result.stdout.trim();
|
|
10095
|
-
if (raw.length === 0)
|
|
10096
|
-
return `himalaya exited with code ${result.code ?? "unknown"}`;
|
|
10097
|
-
return boundDiagnosticText(raw.replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu, "<redacted-email>").replace(/(?:\/Users|\/home|[A-Z]:\\Users)[^\s"']+/gu, "<redacted-path>").replace(/(?:password|passwd|token|secret)\s*[:=]\s*\S+/giu, "$1=<redacted>"));
|
|
10098
|
-
}
|
|
10099
|
-
function parseEnvelopes(stdout) {
|
|
10100
|
-
const parsed = JSON.parse(stdout);
|
|
10101
|
-
if (Array.isArray(parsed))
|
|
10102
|
-
return asArray(parsed);
|
|
10103
|
-
const record = asRecord(parsed);
|
|
10104
|
-
return asArray(record?.envelopes ?? record?.data);
|
|
10105
|
-
}
|
|
10106
|
-
function envelopeFingerprint(envelope) {
|
|
10107
|
-
return JSON.stringify({
|
|
10108
|
-
subject: envelope.subject,
|
|
10109
|
-
date: envelope.date,
|
|
10110
|
-
from: envelope.from,
|
|
10111
|
-
to: envelope.to,
|
|
10112
|
-
flags: envelope.flags
|
|
10113
|
-
});
|
|
10114
|
-
}
|
|
10115
|
-
function runBinary2(binary, args, timeoutMs, env, cwd) {
|
|
10116
|
-
return new Promise((resolvePromise) => {
|
|
10117
|
-
const command = portableSpawnCommand(binary, args);
|
|
10118
|
-
const child = spawn9(command.command, command.args, {
|
|
10119
|
-
...env === undefined ? {} : { env: { ...process.env, ...env } },
|
|
10120
|
-
...cwd === undefined ? {} : { cwd },
|
|
10121
|
-
stdio: ["ignore", "pipe", "pipe"]
|
|
10122
|
-
});
|
|
10123
|
-
let stdout = "";
|
|
10124
|
-
let stderr = "";
|
|
10125
|
-
const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
|
|
10126
|
-
child.stdout.on("data", (chunk) => {
|
|
10127
|
-
stdout += chunk.toString("utf8");
|
|
10128
|
-
});
|
|
10129
|
-
child.stderr.on("data", (chunk) => {
|
|
10130
|
-
stderr += chunk.toString("utf8");
|
|
10131
|
-
});
|
|
10132
|
-
child.on("error", () => {
|
|
10133
|
-
clearTimeout(timer);
|
|
10134
|
-
resolvePromise({ ok: false, stdout, stderr, code: null });
|
|
10135
|
-
});
|
|
10136
|
-
child.on("close", (code) => {
|
|
10137
|
-
clearTimeout(timer);
|
|
10138
|
-
resolvePromise({ ok: code === 0, stdout, stderr, code });
|
|
10139
|
-
});
|
|
10140
|
-
});
|
|
10141
|
-
}
|
|
10142
|
-
var DEFAULT_BINARY4 = "himalaya", DEFAULT_PAGE_SIZE = 100, DEFAULT_MAX_DOCUMENTS3 = 50, DEFAULT_TIMEOUT_MS6 = 30000, MAX_BODY_CHARS = 50000, STATE_VERSION = 1;
|
|
10143
|
-
var init_himalaya_connector = __esm(() => {
|
|
10144
|
-
init_portable_spawn();
|
|
10145
|
-
init_connector();
|
|
10146
|
-
});
|
|
10147
|
-
|
|
10148
9899
|
// src/datasource/skills/gmail/connector.ts
|
|
10149
9900
|
class GmailConnector {
|
|
10150
9901
|
options;
|
|
@@ -10163,7 +9914,7 @@ class GmailConnector {
|
|
|
10163
9914
|
signal
|
|
10164
9915
|
};
|
|
10165
9916
|
const maxPages = this.options.maxPages ?? DEFAULT_MAX_PAGES2;
|
|
10166
|
-
const maxDocuments = this.options.maxDocuments ??
|
|
9917
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS3;
|
|
10167
9918
|
const messageIds = [];
|
|
10168
9919
|
let pageToken;
|
|
10169
9920
|
for (let page = 0;page < maxPages && messageIds.length < maxDocuments; page += 1) {
|
|
@@ -10211,7 +9962,7 @@ class GmailConnector {
|
|
|
10211
9962
|
const payload = asRecord(message.payload);
|
|
10212
9963
|
const headers = headerMap(payload);
|
|
10213
9964
|
const subject = headers.get("subject") ?? "(no subject)";
|
|
10214
|
-
const body = (extractBody(payload) ?? asString(message.snippet) ?? "").slice(0,
|
|
9965
|
+
const body = (extractBody(payload) ?? asString(message.snippet) ?? "").slice(0, MAX_BODY_CHARS);
|
|
10215
9966
|
const labelIds = asArray(message.labelIds).map((label) => asString(label)).filter((label) => label !== undefined);
|
|
10216
9967
|
const internalDate = asString(message.internalDate);
|
|
10217
9968
|
const publishedAt = internalDate !== undefined && /^\d+$/u.test(internalDate) ? Number(internalDate) : undefined;
|
|
@@ -10277,7 +10028,7 @@ function decodeBase64Url(data) {
|
|
|
10277
10028
|
return "";
|
|
10278
10029
|
}
|
|
10279
10030
|
}
|
|
10280
|
-
var DEFAULT_BASE_URL2 = "https://gmail.googleapis.com/gmail/v1", DEFAULT_TOKEN_ENV2 = "GMAIL_ACCESS_TOKEN", DEFAULT_MAX_PAGES2 = 10,
|
|
10031
|
+
var DEFAULT_BASE_URL2 = "https://gmail.googleapis.com/gmail/v1", DEFAULT_TOKEN_ENV2 = "GMAIL_ACCESS_TOKEN", DEFAULT_MAX_PAGES2 = 10, DEFAULT_MAX_DOCUMENTS3 = 500, MAX_BODY_CHARS = 50000;
|
|
10281
10032
|
var init_connector3 = () => {};
|
|
10282
10033
|
|
|
10283
10034
|
// src/datasource/skills/gmail/skill.ts
|
|
@@ -10289,14 +10040,11 @@ var init_skill5 = __esm(() => {
|
|
|
10289
10040
|
skillName: "gmail",
|
|
10290
10041
|
skillType: "gmail-account",
|
|
10291
10042
|
description: "Gmail datasource",
|
|
10292
|
-
capabilities: ["email", "api", "
|
|
10043
|
+
capabilities: ["email", "api", "polling", "bm25", "incremental"],
|
|
10293
10044
|
defaultTags: ["gmail", "email", "pii"],
|
|
10294
10045
|
contentType: "email",
|
|
10295
10046
|
manifestDescription: "Search indexed Gmail messages from the authorized account and labels. Use for questions about email conversations, decisions, or attachments discussed over email.",
|
|
10296
|
-
manifestNotes: [
|
|
10297
|
-
"Label and folder visibility is bounded by the server-configured OAuth token.",
|
|
10298
|
-
"When the Himalaya backend is selected, indexing is incremental and lexical BM25 retrieval is local to AutoRAG; Himalaya itself does not provide semantic embeddings."
|
|
10299
|
-
]
|
|
10047
|
+
manifestNotes: ["Label visibility is bounded by the server-configured OAuth token."]
|
|
10300
10048
|
};
|
|
10301
10049
|
GmailSkill = class GmailSkill extends ConnectorDatasourceSkill {
|
|
10302
10050
|
constructor(options = {}) {
|
|
@@ -10312,7 +10060,6 @@ var init_skill5 = __esm(() => {
|
|
|
10312
10060
|
// src/datasource/skills/gmail/index.ts
|
|
10313
10061
|
var init_gmail = __esm(() => {
|
|
10314
10062
|
init_connector3();
|
|
10315
|
-
init_himalaya_connector();
|
|
10316
10063
|
init_skill5();
|
|
10317
10064
|
});
|
|
10318
10065
|
|
|
@@ -10321,7 +10068,7 @@ var DEFAULT_KATOK_BINARY = "katok", DEFAULT_KATOK_TIMEOUT_MS = 60000, DEFAULT_KA
|
|
|
10321
10068
|
var init_types3 = () => {};
|
|
10322
10069
|
|
|
10323
10070
|
// src/datasource/skills/katok/client.ts
|
|
10324
|
-
import { spawn as
|
|
10071
|
+
import { spawn as spawn9 } from "node:child_process";
|
|
10325
10072
|
|
|
10326
10073
|
class KatokClient {
|
|
10327
10074
|
options;
|
|
@@ -10423,10 +10170,10 @@ class KatokClient {
|
|
|
10423
10170
|
}
|
|
10424
10171
|
}
|
|
10425
10172
|
function spawnKatok(request) {
|
|
10426
|
-
return new Promise((
|
|
10173
|
+
return new Promise((resolve) => {
|
|
10427
10174
|
const { options, args, env, signal } = request;
|
|
10428
10175
|
const portable = portableSpawnCommand(commandFor2(options.binaryPath), args);
|
|
10429
|
-
const child =
|
|
10176
|
+
const child = spawn9(portable.command, [...portable.args], {
|
|
10430
10177
|
env,
|
|
10431
10178
|
...request.cwd === undefined ? {} : { cwd: request.cwd },
|
|
10432
10179
|
stdio: ["ignore", "pipe", "pipe"]
|
|
@@ -10470,7 +10217,7 @@ function spawnKatok(request) {
|
|
|
10470
10217
|
clearTimeout(timeout);
|
|
10471
10218
|
signal?.removeEventListener("abort", abortHandler);
|
|
10472
10219
|
const reason = error.code === "ENOENT" ? "binary-missing" : "spawn-error";
|
|
10473
|
-
|
|
10220
|
+
resolve({ ok: false, reason, stdout: stdout.text, stderr: describeSpawnFailure2(reason), code: null });
|
|
10474
10221
|
});
|
|
10475
10222
|
child.on("close", (code) => {
|
|
10476
10223
|
if (settled)
|
|
@@ -10479,10 +10226,10 @@ function spawnKatok(request) {
|
|
|
10479
10226
|
clearTimeout(timeout);
|
|
10480
10227
|
signal?.removeEventListener("abort", abortHandler);
|
|
10481
10228
|
if (finalReason !== undefined) {
|
|
10482
|
-
|
|
10229
|
+
resolve({ ok: false, reason: finalReason, stdout: stdout.text, stderr: stderr.text, code });
|
|
10483
10230
|
return;
|
|
10484
10231
|
}
|
|
10485
|
-
|
|
10232
|
+
resolve({
|
|
10486
10233
|
ok: code === 0,
|
|
10487
10234
|
reason: code === 0 ? undefined : "nonzero-exit",
|
|
10488
10235
|
stdout: stdout.text,
|
|
@@ -10981,7 +10728,7 @@ var init_katok = __esm(() => {
|
|
|
10981
10728
|
|
|
10982
10729
|
// src/datasource/skills/mail-export/connector.ts
|
|
10983
10730
|
import { readdir, readFile as readFile5, stat as stat3 } from "node:fs/promises";
|
|
10984
|
-
import { basename as basename6, dirname as
|
|
10731
|
+
import { basename as basename6, dirname as dirname12, extname as extname7, join as join19 } from "node:path";
|
|
10985
10732
|
import { simpleParser as simpleParser2 } from "mailparser";
|
|
10986
10733
|
|
|
10987
10734
|
class MailExportConnector {
|
|
@@ -10993,7 +10740,7 @@ class MailExportConnector {
|
|
|
10993
10740
|
const paths = this.options.paths ?? [];
|
|
10994
10741
|
if (paths.length === 0)
|
|
10995
10742
|
return { ok: false, reason: "not-configured", message: "no mail export paths configured" };
|
|
10996
|
-
const maxDocuments = this.options.maxDocuments ??
|
|
10743
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS4;
|
|
10997
10744
|
const maxBytes = this.options.maxBytesPerFile ?? DEFAULT_MAX_BYTES_PER_FILE2;
|
|
10998
10745
|
const files = [];
|
|
10999
10746
|
let missingPaths = 0;
|
|
@@ -11007,7 +10754,7 @@ class MailExportConnector {
|
|
|
11007
10754
|
continue;
|
|
11008
10755
|
const extension = extname7(entry.name).toLowerCase();
|
|
11009
10756
|
if (extension === ".eml" || extension === ".mbox") {
|
|
11010
|
-
files.push(
|
|
10757
|
+
files.push(join19(entry.parentPath, entry.name));
|
|
11011
10758
|
}
|
|
11012
10759
|
}
|
|
11013
10760
|
} else if (info.isFile()) {
|
|
@@ -11042,7 +10789,7 @@ class MailExportConnector {
|
|
|
11042
10789
|
continue;
|
|
11043
10790
|
}
|
|
11044
10791
|
const stem = basename6(file, extname7(file));
|
|
11045
|
-
const mailboxDir = basename6(
|
|
10792
|
+
const mailboxDir = basename6(dirname12(file));
|
|
11046
10793
|
const hierarchy = ["mailboxes", mailboxDir, stem];
|
|
11047
10794
|
const extension = extname7(file).toLowerCase();
|
|
11048
10795
|
const rawMessages = extension === ".mbox" ? splitMbox(bytes.toString("utf8")) : [bytes.toString("utf8")];
|
|
@@ -11054,7 +10801,7 @@ class MailExportConnector {
|
|
|
11054
10801
|
const subject = parsed.subject ?? "(no subject)";
|
|
11055
10802
|
const from = Array.isArray(parsed.from) ? parsed.from[0]?.text : parsed.from?.text;
|
|
11056
10803
|
const to = Array.isArray(parsed.to) ? parsed.to[0]?.text : parsed.to?.text;
|
|
11057
|
-
const body = (parsed.text ?? "").trim().slice(0,
|
|
10804
|
+
const body = (parsed.text ?? "").trim().slice(0, MAX_BODY_CHARS2);
|
|
11058
10805
|
const headerBlock = [
|
|
11059
10806
|
`Subject: ${subject}`,
|
|
11060
10807
|
...from !== undefined ? [`From: ${from}`] : [],
|
|
@@ -11111,7 +10858,7 @@ function splitMbox(text) {
|
|
|
11111
10858
|
`));
|
|
11112
10859
|
return messages.filter((message) => message.trim().length > 0);
|
|
11113
10860
|
}
|
|
11114
|
-
var
|
|
10861
|
+
var DEFAULT_MAX_DOCUMENTS4 = 500, DEFAULT_MAX_BYTES_PER_FILE2, MAX_BODY_CHARS2 = 50000, MBOX_SEPARATOR;
|
|
11115
10862
|
var init_connector4 = __esm(() => {
|
|
11116
10863
|
DEFAULT_MAX_BYTES_PER_FILE2 = 50 * 1024 * 1024;
|
|
11117
10864
|
MBOX_SEPARATOR = /^From .*\d{4}$/mu;
|
|
@@ -11152,21 +10899,21 @@ var init_mail_export = __esm(() => {
|
|
|
11152
10899
|
});
|
|
11153
10900
|
|
|
11154
10901
|
// src/datasource/skills/mailcrawl/config.ts
|
|
11155
|
-
import { chmodSync, mkdirSync as
|
|
10902
|
+
import { chmodSync, mkdirSync as mkdirSync10 } from "node:fs";
|
|
11156
10903
|
function validateMailcrawlInstanceId(instanceId) {
|
|
11157
10904
|
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u.test(instanceId)) {
|
|
11158
10905
|
throw new Error("mailcrawl instanceId must be a safe single path segment");
|
|
11159
10906
|
}
|
|
11160
10907
|
}
|
|
11161
10908
|
function ensurePrivateMailcrawlDataDir(path) {
|
|
11162
|
-
|
|
10909
|
+
mkdirSync10(path, { recursive: true, mode: 448 });
|
|
11163
10910
|
if (process.platform !== "win32")
|
|
11164
10911
|
chmodSync(path, 448);
|
|
11165
10912
|
}
|
|
11166
10913
|
var init_config = () => {};
|
|
11167
10914
|
|
|
11168
10915
|
// src/datasource/skills/mailcrawl/client.ts
|
|
11169
|
-
import { spawn as
|
|
10916
|
+
import { spawn as spawn10 } from "node:child_process";
|
|
11170
10917
|
|
|
11171
10918
|
class MailcrawlClient {
|
|
11172
10919
|
options;
|
|
@@ -11260,7 +11007,7 @@ class MailcrawlClient {
|
|
|
11260
11007
|
}
|
|
11261
11008
|
if (dataDir !== undefined)
|
|
11262
11009
|
env.MAILCRAWL_DATA_DIR = dataDir;
|
|
11263
|
-
return spawnMailcrawl(this.options.binaryPath ??
|
|
11010
|
+
return spawnMailcrawl(this.options.binaryPath ?? DEFAULT_BINARY4, args, env, this.options, signal);
|
|
11264
11011
|
}
|
|
11265
11012
|
async index(signal) {
|
|
11266
11013
|
const result = await this.run(["index", "--json"], signal);
|
|
@@ -11280,7 +11027,7 @@ class MailcrawlClient {
|
|
|
11280
11027
|
function spawnMailcrawl(binary, args, env, options, signal) {
|
|
11281
11028
|
return new Promise((resolveResult) => {
|
|
11282
11029
|
const portable = portableSpawnCommand(binary, args);
|
|
11283
|
-
const child =
|
|
11030
|
+
const child = spawn10(portable.command, portable.args, {
|
|
11284
11031
|
env,
|
|
11285
11032
|
detached: process.platform !== "win32",
|
|
11286
11033
|
stdio: ["ignore", "pipe", "pipe"]
|
|
@@ -11290,7 +11037,7 @@ function spawnMailcrawl(binary, args, env, options, signal) {
|
|
|
11290
11037
|
const timer = setTimeout(() => {
|
|
11291
11038
|
reason = "timeout";
|
|
11292
11039
|
terminate5(child);
|
|
11293
|
-
}, options.timeoutMs ??
|
|
11040
|
+
}, options.timeoutMs ?? DEFAULT_TIMEOUT_MS6);
|
|
11294
11041
|
const abort = () => {
|
|
11295
11042
|
reason = "aborted";
|
|
11296
11043
|
terminate5(child);
|
|
@@ -11463,7 +11210,7 @@ function truncateUtf8(value, maxBytes) {
|
|
|
11463
11210
|
}
|
|
11464
11211
|
return "";
|
|
11465
11212
|
}
|
|
11466
|
-
var
|
|
11213
|
+
var DEFAULT_BINARY4 = "mailcrawl", DEFAULT_TIMEOUT_MS6 = 60000, DEFAULT_MAX_BUFFER_BYTES3 = 1048576, SAFE_ENV_KEYS, SAFE_MAILCRAWL_ENV_KEYS, REMOTE_EMBEDDING_ENV_KEYS;
|
|
11467
11214
|
var init_client6 = __esm(() => {
|
|
11468
11215
|
init_portable_spawn();
|
|
11469
11216
|
init_config();
|
|
@@ -11710,7 +11457,7 @@ var init_mailcrawl = __esm(() => {
|
|
|
11710
11457
|
});
|
|
11711
11458
|
|
|
11712
11459
|
// src/datasource/crawler-client.ts
|
|
11713
|
-
import { spawn as
|
|
11460
|
+
import { spawn as spawn11 } from "node:child_process";
|
|
11714
11461
|
|
|
11715
11462
|
class CrawlerCliClient {
|
|
11716
11463
|
profile;
|
|
@@ -11745,9 +11492,9 @@ class CrawlerCliClient {
|
|
|
11745
11492
|
}
|
|
11746
11493
|
}
|
|
11747
11494
|
function spawnCrawler(command, args, env, signal, options, cwd) {
|
|
11748
|
-
return new Promise((
|
|
11495
|
+
return new Promise((resolve) => {
|
|
11749
11496
|
const portable = portableSpawnCommand(command, args);
|
|
11750
|
-
const child =
|
|
11497
|
+
const child = spawn11(portable.command, [...portable.args], {
|
|
11751
11498
|
env,
|
|
11752
11499
|
...cwd === undefined ? {} : { cwd },
|
|
11753
11500
|
stdio: ["ignore", "pipe", "pipe"]
|
|
@@ -11760,7 +11507,7 @@ function spawnCrawler(command, args, env, signal, options, cwd) {
|
|
|
11760
11507
|
const timeout = setTimeout(() => {
|
|
11761
11508
|
finalReason = "timeout";
|
|
11762
11509
|
terminate6(child);
|
|
11763
|
-
}, options.timeoutMs ??
|
|
11510
|
+
}, options.timeoutMs ?? DEFAULT_TIMEOUT_MS7);
|
|
11764
11511
|
const abortHandler = () => {
|
|
11765
11512
|
finalReason = "aborted";
|
|
11766
11513
|
terminate6(child);
|
|
@@ -11790,7 +11537,7 @@ function spawnCrawler(command, args, env, signal, options, cwd) {
|
|
|
11790
11537
|
settled = true;
|
|
11791
11538
|
clearTimeout(timeout);
|
|
11792
11539
|
signal?.removeEventListener("abort", abortHandler);
|
|
11793
|
-
|
|
11540
|
+
resolve({
|
|
11794
11541
|
ok: false,
|
|
11795
11542
|
reason: error.code === "ENOENT" ? "binary-missing" : "spawn-error",
|
|
11796
11543
|
stdout: stdout.text,
|
|
@@ -11804,7 +11551,7 @@ function spawnCrawler(command, args, env, signal, options, cwd) {
|
|
|
11804
11551
|
settled = true;
|
|
11805
11552
|
clearTimeout(timeout);
|
|
11806
11553
|
signal?.removeEventListener("abort", abortHandler);
|
|
11807
|
-
|
|
11554
|
+
resolve({
|
|
11808
11555
|
ok: code === 0 && finalReason === undefined,
|
|
11809
11556
|
...finalReason !== undefined ? { reason: finalReason } : code === 0 ? {} : { reason: "nonzero-exit" },
|
|
11810
11557
|
stdout: stdout.text,
|
|
@@ -11852,7 +11599,7 @@ function toFailure3(binaryName, result, reason) {
|
|
|
11852
11599
|
code: result.code
|
|
11853
11600
|
};
|
|
11854
11601
|
}
|
|
11855
|
-
var
|
|
11602
|
+
var DEFAULT_TIMEOUT_MS7 = 60000, DEFAULT_MAX_BUFFER_BYTES4 = 1048576, SAFE_ENV_KEYS2;
|
|
11856
11603
|
var init_crawler_client = __esm(() => {
|
|
11857
11604
|
init_portable_spawn();
|
|
11858
11605
|
SAFE_ENV_KEYS2 = new Set([
|
|
@@ -12205,15 +11952,15 @@ var init_notion = __esm(() => {
|
|
|
12205
11952
|
});
|
|
12206
11953
|
|
|
12207
11954
|
// src/datasource/skills/obsidian/paths.ts
|
|
12208
|
-
import { join as
|
|
11955
|
+
import { join as join20 } from "node:path";
|
|
12209
11956
|
function obsidianDatasourceRoot(workspaceRoot, instanceId) {
|
|
12210
|
-
return
|
|
11957
|
+
return join20(workspaceRoot, AUTORAG_DIRNAME, DATASOURCES_DIRNAME, OBSIDIAN_DATASOURCE_DIRNAME, instanceId);
|
|
12211
11958
|
}
|
|
12212
11959
|
function obsidianQmdConfigDir(workspaceRoot, instanceId) {
|
|
12213
|
-
return
|
|
11960
|
+
return join20(obsidianDatasourceRoot(workspaceRoot, instanceId), "config");
|
|
12214
11961
|
}
|
|
12215
11962
|
function obsidianQmdCacheDir(workspaceRoot, instanceId) {
|
|
12216
|
-
return
|
|
11963
|
+
return join20(obsidianDatasourceRoot(workspaceRoot, instanceId), "cache");
|
|
12217
11964
|
}
|
|
12218
11965
|
function stripEdgeDashes(value) {
|
|
12219
11966
|
let start = 0;
|
|
@@ -12235,8 +11982,8 @@ var init_paths4 = () => {};
|
|
|
12235
11982
|
var DEFAULT_QMD_BINARY = "qmd", DEFAULT_QMD_TIMEOUT_MS = 60000, DEFAULT_QMD_MAX_BUFFER_BYTES = 1048576;
|
|
12236
11983
|
|
|
12237
11984
|
// src/datasource/skills/obsidian/client.ts
|
|
12238
|
-
import { spawn as
|
|
12239
|
-
import { mkdirSync as
|
|
11985
|
+
import { spawn as spawn12 } from "node:child_process";
|
|
11986
|
+
import { mkdirSync as mkdirSync11, writeFileSync as writeFileSync9 } from "node:fs";
|
|
12240
11987
|
import { isAbsolute as isAbsolute6, resolve as resolve10 } from "node:path";
|
|
12241
11988
|
|
|
12242
11989
|
class QmdClient {
|
|
@@ -12256,10 +12003,10 @@ class QmdClient {
|
|
|
12256
12003
|
const cacheDir = obsidianQmdCacheDir(workspaceRoot, instanceId);
|
|
12257
12004
|
try {
|
|
12258
12005
|
if (this.options.configPath === undefined) {
|
|
12259
|
-
|
|
12260
|
-
|
|
12006
|
+
mkdirSync11(configDir, { recursive: true });
|
|
12007
|
+
writeFileSync9(resolve10(configDir, "index.yml"), renderIndexYaml(collectionName, vaultPath), "utf8");
|
|
12261
12008
|
}
|
|
12262
|
-
|
|
12009
|
+
mkdirSync11(cacheDir, { recursive: true });
|
|
12263
12010
|
} catch (error) {
|
|
12264
12011
|
return failure3("spawn-error", error instanceof Error ? error.message : "failed to write qmd config");
|
|
12265
12012
|
}
|
|
@@ -12360,7 +12107,7 @@ function isAllowedEnvKey2(key) {
|
|
|
12360
12107
|
function spawnQmd(request) {
|
|
12361
12108
|
return new Promise((resolveResult) => {
|
|
12362
12109
|
const portable = portableSpawnCommand(request.binaryPath ?? DEFAULT_QMD_BINARY, request.args);
|
|
12363
|
-
const child =
|
|
12110
|
+
const child = spawn12(portable.command, [...portable.args], {
|
|
12364
12111
|
env: request.env,
|
|
12365
12112
|
...request.cwd === undefined ? {} : { cwd: request.cwd },
|
|
12366
12113
|
stdio: ["ignore", "pipe", "pipe"]
|
|
@@ -12906,7 +12653,7 @@ class RssConnector {
|
|
|
12906
12653
|
const feeds = this.options.feeds ?? [];
|
|
12907
12654
|
if (feeds.length === 0)
|
|
12908
12655
|
return { ok: false, reason: "not-configured", message: "no feeds configured" };
|
|
12909
|
-
const maxDocuments = this.options.maxDocuments ??
|
|
12656
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS5;
|
|
12910
12657
|
const maxItemsPerFeed = this.options.maxItemsPerFeed ?? DEFAULT_MAX_ITEMS_PER_FEED;
|
|
12911
12658
|
const documents = [];
|
|
12912
12659
|
const warnings = [];
|
|
@@ -13039,7 +12786,7 @@ function stripHtml(text) {
|
|
|
13039
12786
|
function present(item) {
|
|
13040
12787
|
return item !== undefined;
|
|
13041
12788
|
}
|
|
13042
|
-
var
|
|
12789
|
+
var DEFAULT_MAX_DOCUMENTS5 = 500, DEFAULT_MAX_ITEMS_PER_FEED = 100, DEFAULT_FEED_TIMEOUT_MS = 30000, MAX_CONTENT_CHARS2 = 20000, ACCEPT_HEADER = "application/rss+xml, application/atom+xml, application/xml, text/xml";
|
|
13043
12790
|
var init_connector5 = () => {};
|
|
13044
12791
|
|
|
13045
12792
|
// src/datasource/skills/rss/skill.ts
|
|
@@ -13211,7 +12958,7 @@ var init_slack = __esm(() => {
|
|
|
13211
12958
|
});
|
|
13212
12959
|
|
|
13213
12960
|
// src/datasource/skills/spotlight/connector.ts
|
|
13214
|
-
import { spawn as
|
|
12961
|
+
import { spawn as spawn13 } from "node:child_process";
|
|
13215
12962
|
import { createHash as createHash6 } from "node:crypto";
|
|
13216
12963
|
import { readFile as fsReadFile, stat as stat4 } from "node:fs/promises";
|
|
13217
12964
|
import { basename as basename7, extname as extname8 } from "node:path";
|
|
@@ -13235,7 +12982,7 @@ class SpotlightConnector {
|
|
|
13235
12982
|
if (queries.length === 0) {
|
|
13236
12983
|
return { ok: false, reason: "not-configured", message: "no spotlight queries configured" };
|
|
13237
12984
|
}
|
|
13238
|
-
const timeoutMs = this.options.timeoutMs ??
|
|
12985
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
13239
12986
|
const status = await this.run("mdutil", ["-s", "/"], timeoutMs);
|
|
13240
12987
|
if (status.ok && /indexing disabled/iu.test(status.stdout)) {
|
|
13241
12988
|
return {
|
|
@@ -13245,7 +12992,7 @@ class SpotlightConnector {
|
|
|
13245
12992
|
};
|
|
13246
12993
|
}
|
|
13247
12994
|
const maxPerQuery = this.options.maxResultsPerQuery ?? DEFAULT_MAX_RESULTS_PER_QUERY;
|
|
13248
|
-
const maxDocuments = this.options.maxDocuments ??
|
|
12995
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS6;
|
|
13249
12996
|
const seen = new Set;
|
|
13250
12997
|
const paths = [];
|
|
13251
12998
|
let mdfindMissing = false;
|
|
@@ -13329,8 +13076,8 @@ class SpotlightConnector {
|
|
|
13329
13076
|
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
13330
13077
|
}
|
|
13331
13078
|
}
|
|
13332
|
-
var DEFAULT_MAX_RESULTS_PER_QUERY = 100,
|
|
13333
|
-
const child =
|
|
13079
|
+
var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS6 = 300, DEFAULT_MAX_BYTES_PER_FILE3, DEFAULT_TIMEOUT_MS8 = 15000, MAX_CONTENT_CHARS3 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve) => {
|
|
13080
|
+
const child = spawn13(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
|
|
13334
13081
|
let stdout = "";
|
|
13335
13082
|
let stderr = "";
|
|
13336
13083
|
let settled = false;
|
|
@@ -13339,7 +13086,7 @@ var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS7 = 300, DEFAULT_M
|
|
|
13339
13086
|
return;
|
|
13340
13087
|
settled = true;
|
|
13341
13088
|
clearTimeout(timer);
|
|
13342
|
-
|
|
13089
|
+
resolve(result);
|
|
13343
13090
|
};
|
|
13344
13091
|
const timer = setTimeout(() => {
|
|
13345
13092
|
child.kill("SIGKILL");
|
|
@@ -13681,12 +13428,19 @@ function buildDatasourceSkills(config, workspaceRoot) {
|
|
|
13681
13428
|
}
|
|
13682
13429
|
const normalizedEntry = entry.instanceId === undefined && entry.type !== undefined ? { ...entry, instanceId: name } : entry;
|
|
13683
13430
|
const skill = builder(normalizedEntry, workspaceRoot, name);
|
|
13431
|
+
if (skill === undefined) {
|
|
13432
|
+
unknown.push(name);
|
|
13433
|
+
continue;
|
|
13434
|
+
}
|
|
13684
13435
|
const hasChannelFilter = (entry.channels?.ids?.length ?? 0) > 0 || (entry.channels?.names?.length ?? 0) > 0;
|
|
13685
|
-
|
|
13686
|
-
|
|
13687
|
-
|
|
13688
|
-
|
|
13689
|
-
|
|
13436
|
+
let aliased = skill;
|
|
13437
|
+
if (name !== templateName || hasChannelFilter) {
|
|
13438
|
+
aliased = new AliasedDatasourceSkill(skill, {
|
|
13439
|
+
alias: name,
|
|
13440
|
+
...entry.channels?.ids !== undefined ? { channelIds: entry.channels.ids } : {},
|
|
13441
|
+
...entry.channels?.names !== undefined ? { channelNames: entry.channels.names } : {}
|
|
13442
|
+
});
|
|
13443
|
+
}
|
|
13690
13444
|
skills.push(typeof entry.description === "string" && entry.description.trim().length > 0 ? new DescribedDatasourceSkill(aliased, entry.description.trim()) : aliased);
|
|
13691
13445
|
}
|
|
13692
13446
|
return { skills, unknown };
|
|
@@ -13697,7 +13451,6 @@ var init_factory = __esm(() => {
|
|
|
13697
13451
|
init_cloud_drive();
|
|
13698
13452
|
init_discrawl();
|
|
13699
13453
|
init_github();
|
|
13700
|
-
init_himalaya_connector();
|
|
13701
13454
|
init_gmail();
|
|
13702
13455
|
init_katok();
|
|
13703
13456
|
init_mail_export();
|
|
@@ -13816,17 +13569,8 @@ var init_factory = __esm(() => {
|
|
|
13816
13569
|
}),
|
|
13817
13570
|
gmail: (config, workspaceRoot, registrationName) => {
|
|
13818
13571
|
const connector = config.connector;
|
|
13819
|
-
if (connector?.backend === "himalaya")
|
|
13820
|
-
|
|
13821
|
-
return new GmailSkill({
|
|
13822
|
-
...common(config, workspaceRoot),
|
|
13823
|
-
skillName: registrationName,
|
|
13824
|
-
connector: new HimalayaConnector({
|
|
13825
|
-
...himalayaOptions,
|
|
13826
|
-
...workspaceRoot !== undefined ? { workspaceRoot } : {}
|
|
13827
|
-
})
|
|
13828
|
-
});
|
|
13829
|
-
}
|
|
13572
|
+
if (connector?.backend === "himalaya")
|
|
13573
|
+
return;
|
|
13830
13574
|
return new GmailSkill({
|
|
13831
13575
|
...common(config, workspaceRoot),
|
|
13832
13576
|
skillName: registrationName,
|
|
@@ -14104,13 +13848,10 @@ init_opendataloader_pdf();
|
|
|
14104
13848
|
init_plain_text();
|
|
14105
13849
|
init_registry();
|
|
14106
13850
|
// src/retrieval/index.ts
|
|
14107
|
-
init_bm25();
|
|
14108
13851
|
init_scope();
|
|
14109
13852
|
export {
|
|
14110
13853
|
AliasedDatasourceSkill,
|
|
14111
13854
|
AutoRAGAgent,
|
|
14112
|
-
BM25UnavailableError,
|
|
14113
|
-
BM25_SUBDIR,
|
|
14114
13855
|
BUILTIN_DATASOURCE_SKILL_NAMES,
|
|
14115
13856
|
CLOUD_DRIVE_SKILL_DEFINITION,
|
|
14116
13857
|
CloudDriveSkill,
|
|
@@ -14160,7 +13901,6 @@ export {
|
|
|
14160
13901
|
MailcrawlClient,
|
|
14161
13902
|
MailcrawlMethod,
|
|
14162
13903
|
MailcrawlSkill,
|
|
14163
|
-
MinSyncBM25Method,
|
|
14164
13904
|
MinSyncClient,
|
|
14165
13905
|
MinSyncHybridMethod,
|
|
14166
13906
|
MinSyncReleaseError,
|
|
@@ -14213,7 +13953,6 @@ export {
|
|
|
14213
13953
|
ensureMinSyncBinary,
|
|
14214
13954
|
executableName,
|
|
14215
13955
|
fetchLatestMinSyncRelease,
|
|
14216
|
-
hasLegacyBm25Artifacts,
|
|
14217
13956
|
isDatasourceSource,
|
|
14218
13957
|
isDue,
|
|
14219
13958
|
jikjiExecutableName,
|
|
@@ -14241,7 +13980,6 @@ export {
|
|
|
14241
13980
|
parsedMirrorRoot,
|
|
14242
13981
|
parsedOutputPath,
|
|
14243
13982
|
planJikjiSourceRoots,
|
|
14244
|
-
removeLegacyBm25Artifacts,
|
|
14245
13983
|
renderMemoryContext,
|
|
14246
13984
|
resolveReturnedPath,
|
|
14247
13985
|
rewriteEmbedderConfig,
|