@autorag/librarian 2.4.0 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -2
- package/dist/agent/agent.d.ts +31 -11
- package/dist/agent/index.d.ts +1 -1
- package/dist/agent/search-documents.d.ts +11 -2
- package/dist/cli/commands/evidence.d.ts +3 -0
- package/dist/cli/commands/index.d.ts +1 -1
- package/dist/cli/commands/refresh.d.ts +3 -4
- package/dist/cli/commands/search.d.ts +2 -2
- package/dist/cli/commands/tui.d.ts +1 -1
- package/dist/cli/commands/watch.d.ts +1 -1
- package/dist/cli/config.d.ts +0 -11
- package/dist/cli/index.js +771 -1044
- package/dist/datasource/skills/gmail/index.d.ts +0 -1
- package/dist/datasource/skills/gmail/skill.d.ts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +402 -664
- package/dist/jikji/client.d.ts +2 -0
- package/dist/minsync/index.d.ts +1 -1
- package/dist/minsync/method.d.ts +6 -18
- package/dist/retrieval/index.d.ts +0 -2
- package/dist/retrieval/registry.d.ts +2 -2
- package/dist/retrieval/types.d.ts +1 -1
- package/dist/ui/discover.d.ts +0 -2
- package/package.json +1 -1
- package/skills/autorag/SKILL.md +19 -6
- package/skills/autorag-setup/SKILL.md +10 -4
- package/dist/agent/search-bm25-tool.d.ts +0 -26
- package/dist/datasource/skills/gmail/himalaya-connector.d.ts +0 -51
- package/dist/retrieval/methods/bm25.d.ts +0 -23
package/dist/cli/index.js
CHANGED
|
@@ -1,21 +1,15 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
enumerable: true,
|
|
13
|
-
configurable: true,
|
|
14
|
-
set: __exportSetter.bind(all, name)
|
|
15
|
-
});
|
|
2
|
+
var __esm = (fn, res, err) => () => {
|
|
3
|
+
if (fn)
|
|
4
|
+
try {
|
|
5
|
+
res = fn(fn = 0);
|
|
6
|
+
} catch (e) {
|
|
7
|
+
err = [e];
|
|
8
|
+
}
|
|
9
|
+
if (err)
|
|
10
|
+
throw err[0];
|
|
11
|
+
return res;
|
|
16
12
|
};
|
|
17
|
-
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
18
|
-
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
19
13
|
|
|
20
14
|
// src/cli/output.ts
|
|
21
15
|
function diagnosticProjection(d) {
|
|
@@ -39,13 +33,6 @@ function refreshEnvelope(result) {
|
|
|
39
33
|
},
|
|
40
34
|
diagnostics: (result.diagnostics ?? []).map(diagnosticProjection)
|
|
41
35
|
};
|
|
42
|
-
if (result.bm25) {
|
|
43
|
-
envelope.bm25 = {
|
|
44
|
-
indexedChunks: result.bm25.indexedChunks,
|
|
45
|
-
readiness: result.bm25.readiness,
|
|
46
|
-
engine: result.bm25.engine
|
|
47
|
-
};
|
|
48
|
-
}
|
|
49
36
|
if (result.datasources && result.datasources.length > 0) {
|
|
50
37
|
envelope.datasources = result.datasources.map((ds) => ({
|
|
51
38
|
ok: ds.ok,
|
|
@@ -66,9 +53,6 @@ function renderRefreshHuman(result, debug) {
|
|
|
66
53
|
const lines = [];
|
|
67
54
|
lines.push("refresh: ok");
|
|
68
55
|
lines.push(` counts: scanned=${result.scanned} written=${result.written} deleted=${result.deleted} skipped=${result.skipped}`);
|
|
69
|
-
if (result.bm25) {
|
|
70
|
-
lines.push(` bm25: indexedChunks=${result.bm25.indexedChunks} readiness=${result.bm25.readiness} engine=${result.bm25.engine}`);
|
|
71
|
-
}
|
|
72
56
|
if (result.datasources && result.datasources.length > 0) {
|
|
73
57
|
for (const ds of result.datasources) {
|
|
74
58
|
lines.push(` datasource: ok=${ds.ok} skill=${ds.skill} instanceId=${ds.instanceId} indexedAt=${ds.indexedAt}`);
|
|
@@ -102,8 +86,6 @@ function renderStatusHuman(status, debug) {
|
|
|
102
86
|
}
|
|
103
87
|
const comps = status.components;
|
|
104
88
|
const compParts = [];
|
|
105
|
-
if (comps.bm25)
|
|
106
|
-
compParts.push(`bm25=${comps.bm25}`);
|
|
107
89
|
if (comps.minsync)
|
|
108
90
|
compParts.push(`minsync=${comps.minsync}`);
|
|
109
91
|
if (comps.jikji)
|
|
@@ -691,7 +673,7 @@ class ClawGalleryClient {
|
|
|
691
673
|
}
|
|
692
674
|
}
|
|
693
675
|
function spawnCli(binary, args, env, options, signal) {
|
|
694
|
-
return new Promise((
|
|
676
|
+
return new Promise((resolve) => {
|
|
695
677
|
const portable = portableSpawnCommand(binary, args);
|
|
696
678
|
const child = spawn(portable.command, [...portable.args], { env, stdio: ["ignore", "pipe", "pipe"] });
|
|
697
679
|
let stdout = "";
|
|
@@ -732,7 +714,7 @@ function spawnCli(binary, args, env, options, signal) {
|
|
|
732
714
|
settled = true;
|
|
733
715
|
clearTimeout(timeout);
|
|
734
716
|
signal?.removeEventListener("abort", abort);
|
|
735
|
-
|
|
717
|
+
resolve({
|
|
736
718
|
ok: false,
|
|
737
719
|
stdout,
|
|
738
720
|
stderr: error.code === "ENOENT" ? "clawgallery binary not found" : "clawgallery could not start",
|
|
@@ -746,7 +728,7 @@ function spawnCli(binary, args, env, options, signal) {
|
|
|
746
728
|
settled = true;
|
|
747
729
|
clearTimeout(timeout);
|
|
748
730
|
signal?.removeEventListener("abort", abort);
|
|
749
|
-
|
|
731
|
+
resolve({
|
|
750
732
|
ok: reason === undefined && code === 0,
|
|
751
733
|
stdout,
|
|
752
734
|
stderr: stderr.slice(0, max),
|
|
@@ -1531,7 +1513,7 @@ function renderTable(table) {
|
|
|
1531
1513
|
return renderMergedTable(cells);
|
|
1532
1514
|
const cellsByPosition = new Map(cells.map((cell) => [`${cell.row}:${cell.column}`, cell.text]));
|
|
1533
1515
|
const header = Array.from({ length: table.columnCount }, (_, column) => `Column ${column + 1}`);
|
|
1534
|
-
const rows = Array.from({ length: table.rowCount }, (_, row) => Array.from({ length: table.columnCount }, (
|
|
1516
|
+
const rows = Array.from({ length: table.rowCount }, (_, row) => Array.from({ length: table.columnCount }, (_, column) => cellsByPosition.get(`${row}:${column}`) ?? ""));
|
|
1535
1517
|
return [
|
|
1536
1518
|
`| ${header.join(" | ")} |`,
|
|
1537
1519
|
`| ${header.map(() => "---").join(" | ")} |`,
|
|
@@ -1992,7 +1974,7 @@ async function readZipXmlText(bytes, pathPattern) {
|
|
|
1992
1974
|
}
|
|
1993
1975
|
async function readXmlFile(file, remainingBytes) {
|
|
1994
1976
|
const stream = file.nodeStream("nodebuffer");
|
|
1995
|
-
return new Promise((
|
|
1977
|
+
return new Promise((resolve, reject) => {
|
|
1996
1978
|
const chunks = [];
|
|
1997
1979
|
let totalBytes = 0;
|
|
1998
1980
|
stream.on("data", (chunk) => {
|
|
@@ -2006,7 +1988,7 @@ async function readXmlFile(file, remainingBytes) {
|
|
|
2006
1988
|
chunks.push(buffer);
|
|
2007
1989
|
});
|
|
2008
1990
|
stream.on("error", reject);
|
|
2009
|
-
stream.on("end", () =>
|
|
1991
|
+
stream.on("end", () => resolve(Buffer2.concat(chunks).toString("utf8")));
|
|
2010
1992
|
});
|
|
2011
1993
|
}
|
|
2012
1994
|
function uncompressedSize(file) {
|
|
@@ -2112,8 +2094,8 @@ function tesseractOcr(input) {
|
|
|
2112
2094
|
let cleanupReject = () => {
|
|
2113
2095
|
return;
|
|
2114
2096
|
};
|
|
2115
|
-
const cleanup = new Promise((
|
|
2116
|
-
cleanupResolve =
|
|
2097
|
+
const cleanup = new Promise((resolve, reject) => {
|
|
2098
|
+
cleanupResolve = resolve;
|
|
2117
2099
|
cleanupReject = reject;
|
|
2118
2100
|
});
|
|
2119
2101
|
const result = runTesseractOcr(input, cleanupResolve, cleanupReject);
|
|
@@ -2122,7 +2104,7 @@ function tesseractOcr(input) {
|
|
|
2122
2104
|
async function runTesseractOcr(input, cleanupResolve, cleanupReject) {
|
|
2123
2105
|
let worker;
|
|
2124
2106
|
let termination;
|
|
2125
|
-
const
|
|
2107
|
+
const terminate = async () => {
|
|
2126
2108
|
if (worker === undefined)
|
|
2127
2109
|
return;
|
|
2128
2110
|
termination ??= worker.terminate().then(() => {
|
|
@@ -2132,7 +2114,7 @@ async function runTesseractOcr(input, cleanupResolve, cleanupReject) {
|
|
|
2132
2114
|
};
|
|
2133
2115
|
const abort = () => {
|
|
2134
2116
|
if (worker !== undefined) {
|
|
2135
|
-
|
|
2117
|
+
terminate().then(cleanupResolve, cleanupReject);
|
|
2136
2118
|
}
|
|
2137
2119
|
};
|
|
2138
2120
|
input.signal.addEventListener("abort", abort, { once: true });
|
|
@@ -2144,18 +2126,18 @@ async function runTesseractOcr(input, cleanupResolve, cleanupReject) {
|
|
|
2144
2126
|
return result.data.text;
|
|
2145
2127
|
} finally {
|
|
2146
2128
|
input.signal.removeEventListener("abort", abort);
|
|
2147
|
-
await
|
|
2129
|
+
await terminate().then(cleanupResolve, cleanupReject);
|
|
2148
2130
|
}
|
|
2149
2131
|
}
|
|
2150
2132
|
function withTimeout(operation, timeoutMs, onTimeout) {
|
|
2151
|
-
return new Promise((
|
|
2133
|
+
return new Promise((resolve, reject) => {
|
|
2152
2134
|
const timeout = setTimeout(() => {
|
|
2153
2135
|
onTimeout();
|
|
2154
2136
|
operation.cleanup.finally(() => reject(new Error(`OCR timed out after ${timeoutMs}ms`)));
|
|
2155
2137
|
}, timeoutMs);
|
|
2156
2138
|
operation.result.then((value) => {
|
|
2157
2139
|
clearTimeout(timeout);
|
|
2158
|
-
|
|
2140
|
+
resolve(value);
|
|
2159
2141
|
}, (error) => {
|
|
2160
2142
|
clearTimeout(timeout);
|
|
2161
2143
|
operation.cleanup.then(() => reject(error), (cleanupError) => reject(cleanupError));
|
|
@@ -2438,17 +2420,17 @@ import { extname as extname4 } from "node:path";
|
|
|
2438
2420
|
class ParserRegistry {
|
|
2439
2421
|
parsersByExtension = new Map;
|
|
2440
2422
|
constructor(parsers = []) {
|
|
2441
|
-
for (const
|
|
2442
|
-
this.register(
|
|
2423
|
+
for (const parser of parsers) {
|
|
2424
|
+
this.register(parser);
|
|
2443
2425
|
}
|
|
2444
2426
|
}
|
|
2445
|
-
register(
|
|
2446
|
-
for (const extension of
|
|
2427
|
+
register(parser) {
|
|
2428
|
+
for (const extension of parser.extensions) {
|
|
2447
2429
|
const normalized = normalizeExtension(extension);
|
|
2448
2430
|
if (this.parsersByExtension.has(normalized)) {
|
|
2449
2431
|
throw new Error(`Parser extension "${normalized}" is already registered`);
|
|
2450
2432
|
}
|
|
2451
|
-
this.parsersByExtension.set(normalized,
|
|
2433
|
+
this.parsersByExtension.set(normalized, parser);
|
|
2452
2434
|
}
|
|
2453
2435
|
}
|
|
2454
2436
|
getForVirtualPath(virtualPath) {
|
|
@@ -2812,11 +2794,11 @@ async function loadMirroredDocuments(root, entries, skillName, instanceId, remot
|
|
|
2812
2794
|
for (const entry of entries) {
|
|
2813
2795
|
try {
|
|
2814
2796
|
const sourcePath = mirrorFilePath(root, entry.path);
|
|
2815
|
-
const
|
|
2816
|
-
if (
|
|
2797
|
+
const parser = registry.getForVirtualPath(entry.path);
|
|
2798
|
+
if (parser === undefined)
|
|
2817
2799
|
continue;
|
|
2818
2800
|
const bytes = await readFile3(sourcePath);
|
|
2819
|
-
const parsed = await
|
|
2801
|
+
const parsed = await parser.parse({ virtualPath: entry.path, sourcePath, bytes });
|
|
2820
2802
|
for (const diagnostic of parsed.diagnostics ?? []) {
|
|
2821
2803
|
warnings.push(`${diagnostic.code}: ${diagnostic.message}`);
|
|
2822
2804
|
}
|
|
@@ -2833,8 +2815,8 @@ async function loadMirroredDocuments(root, entries, skillName, instanceId, remot
|
|
|
2833
2815
|
}
|
|
2834
2816
|
function defaultParserExtensions() {
|
|
2835
2817
|
const extensions = new Set;
|
|
2836
|
-
for (const
|
|
2837
|
-
for (const extension of
|
|
2818
|
+
for (const parser of createDefaultParserRegistry().list()) {
|
|
2819
|
+
for (const extension of parser.extensions) {
|
|
2838
2820
|
extensions.add(extension.startsWith(".") ? extension.toLowerCase() : `.${extension.toLowerCase()}`);
|
|
2839
2821
|
}
|
|
2840
2822
|
}
|
|
@@ -3588,7 +3570,7 @@ function commonArgs(options) {
|
|
|
3588
3570
|
return options.configPath === undefined ? [] : ["--config", options.configPath];
|
|
3589
3571
|
}
|
|
3590
3572
|
function spawnDiscrawl(request) {
|
|
3591
|
-
return new Promise((
|
|
3573
|
+
return new Promise((resolve) => {
|
|
3592
3574
|
const { options, args, env, signal } = request;
|
|
3593
3575
|
const portable = portableSpawnCommand(options.binaryPath ?? DEFAULT_DISCRAWL_BINARY, args);
|
|
3594
3576
|
const child = spawn3(portable.command, [...portable.args], {
|
|
@@ -3636,7 +3618,7 @@ function spawnDiscrawl(request) {
|
|
|
3636
3618
|
clearTimeout(timeout);
|
|
3637
3619
|
signal?.removeEventListener("abort", abortHandler);
|
|
3638
3620
|
const reason = error.code === "ENOENT" ? "binary-missing" : "spawn-error";
|
|
3639
|
-
|
|
3621
|
+
resolve({ ok: false, reason, stdout: stdout.text, stderr: describeSpawnFailure(reason), code: null });
|
|
3640
3622
|
});
|
|
3641
3623
|
child.on("close", (code) => {
|
|
3642
3624
|
if (settled)
|
|
@@ -3645,10 +3627,10 @@ function spawnDiscrawl(request) {
|
|
|
3645
3627
|
clearTimeout(timeout);
|
|
3646
3628
|
signal?.removeEventListener("abort", abortHandler);
|
|
3647
3629
|
if (finalReason !== undefined) {
|
|
3648
|
-
|
|
3630
|
+
resolve({ ok: false, reason: finalReason, stdout: stdout.text, stderr: stderr.text, code });
|
|
3649
3631
|
return;
|
|
3650
3632
|
}
|
|
3651
|
-
|
|
3633
|
+
resolve({
|
|
3652
3634
|
ok: code === 0,
|
|
3653
3635
|
reason: code === 0 ? undefined : "nonzero-exit",
|
|
3654
3636
|
stdout: stdout.text,
|
|
@@ -4271,8 +4253,8 @@ class DiscrawlSkill {
|
|
|
4271
4253
|
};
|
|
4272
4254
|
}
|
|
4273
4255
|
}
|
|
4274
|
-
function failureCode(
|
|
4275
|
-
switch (
|
|
4256
|
+
function failureCode(failure, fallback) {
|
|
4257
|
+
switch (failure.reason) {
|
|
4276
4258
|
case "binary-missing":
|
|
4277
4259
|
case "spawn-error":
|
|
4278
4260
|
case "timeout":
|
|
@@ -4284,9 +4266,9 @@ function failureCode(failure2, fallback) {
|
|
|
4284
4266
|
return fallback;
|
|
4285
4267
|
}
|
|
4286
4268
|
}
|
|
4287
|
-
function describe(
|
|
4288
|
-
const detail =
|
|
4289
|
-
return `${
|
|
4269
|
+
function describe(failure) {
|
|
4270
|
+
const detail = failure.stderr.length > 0 ? `: ${failure.stderr}` : "";
|
|
4271
|
+
return `${failure.reason}${detail}`;
|
|
4290
4272
|
}
|
|
4291
4273
|
var DISCORD_DATASOURCE_ID2 = "discord", DISCORD_SKILL_TYPE = "discord-archive", DEFAULT_INSTANCE_ID2 = "default", DEFAULT_POLLING_INTERVAL_MS2, DEFAULT_DISCORD_TAGS2;
|
|
4292
4274
|
var init_skill3 = __esm(() => {
|
|
@@ -4425,228 +4407,6 @@ var init_github = __esm(() => {
|
|
|
4425
4407
|
init_skill4();
|
|
4426
4408
|
});
|
|
4427
4409
|
|
|
4428
|
-
// src/datasource/skills/gmail/himalaya-connector.ts
|
|
4429
|
-
import { spawn as spawn4 } from "node:child_process";
|
|
4430
|
-
import { mkdirSync as mkdirSync4, readFileSync as readFileSync6, writeFileSync as writeFileSync4 } from "node:fs";
|
|
4431
|
-
import { dirname as dirname5, join as join7 } from "node:path";
|
|
4432
|
-
|
|
4433
|
-
class HimalayaConnector {
|
|
4434
|
-
options;
|
|
4435
|
-
runner;
|
|
4436
|
-
constructor(options = {}) {
|
|
4437
|
-
this.options = options;
|
|
4438
|
-
this.runner = options.runner ?? ((args, timeoutMs) => runBinary2(options.binaryPath ?? DEFAULT_BINARY3, args, timeoutMs, {
|
|
4439
|
-
...options.configPath === undefined ? {} : { HIMALAYA_CONFIG: options.configPath }
|
|
4440
|
-
}));
|
|
4441
|
-
}
|
|
4442
|
-
async fetch() {
|
|
4443
|
-
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS4;
|
|
4444
|
-
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS3;
|
|
4445
|
-
const accountArgs = this.options.account !== undefined ? ["--account", this.options.account] : [];
|
|
4446
|
-
const mailboxArgs = this.options.folder !== undefined ? ["--mailbox", this.options.folder] : [];
|
|
4447
|
-
const state = this.loadState();
|
|
4448
|
-
const listArgs = [
|
|
4449
|
-
...accountArgs,
|
|
4450
|
-
"--json",
|
|
4451
|
-
"envelope",
|
|
4452
|
-
"list",
|
|
4453
|
-
...mailboxArgs,
|
|
4454
|
-
"--page-size",
|
|
4455
|
-
String(this.options.pageSize ?? DEFAULT_PAGE_SIZE)
|
|
4456
|
-
];
|
|
4457
|
-
let listResult;
|
|
4458
|
-
try {
|
|
4459
|
-
listResult = await this.runner(listArgs, timeoutMs);
|
|
4460
|
-
} catch {
|
|
4461
|
-
return { ok: false, reason: "unavailable", message: "himalaya binary not found or failed to spawn" };
|
|
4462
|
-
}
|
|
4463
|
-
if (!listResult.ok) {
|
|
4464
|
-
return {
|
|
4465
|
-
ok: false,
|
|
4466
|
-
reason: classifyFailure2(`${listResult.stderr}
|
|
4467
|
-
${listResult.stdout}`),
|
|
4468
|
-
message: diagnosticFailure(listResult)
|
|
4469
|
-
};
|
|
4470
|
-
}
|
|
4471
|
-
let envelopes;
|
|
4472
|
-
try {
|
|
4473
|
-
envelopes = parseEnvelopes(listResult.stdout);
|
|
4474
|
-
} catch {
|
|
4475
|
-
return { ok: false, reason: "invalid-data", message: "envelope listing was not valid JSON" };
|
|
4476
|
-
}
|
|
4477
|
-
const account = this.options.account ?? "default";
|
|
4478
|
-
const folder = this.options.folder ?? "INBOX";
|
|
4479
|
-
const documents = [];
|
|
4480
|
-
let readFailures = 0;
|
|
4481
|
-
const nextFingerprints = { ...state.fingerprints };
|
|
4482
|
-
const listedDocIds = new Set;
|
|
4483
|
-
for (const raw of envelopes.slice(0, maxDocuments)) {
|
|
4484
|
-
const envelope = asRecord(raw);
|
|
4485
|
-
const id = asString(envelope?.id);
|
|
4486
|
-
if (envelope === undefined || id === undefined)
|
|
4487
|
-
continue;
|
|
4488
|
-
const docId = `${account}-${folder}-${id}`;
|
|
4489
|
-
listedDocIds.add(docId);
|
|
4490
|
-
const fingerprint = envelopeFingerprint(envelope);
|
|
4491
|
-
if (state.fingerprints[docId] === fingerprint)
|
|
4492
|
-
continue;
|
|
4493
|
-
const subject = asString(envelope.subject) ?? "(no subject)";
|
|
4494
|
-
const from = asRecord(envelope.from);
|
|
4495
|
-
const fromText = [asString(from?.name), asString(from?.addr)].filter(Boolean).join(" ").trim();
|
|
4496
|
-
let body = "";
|
|
4497
|
-
let bodyRead = false;
|
|
4498
|
-
try {
|
|
4499
|
-
const readResult = await this.runner([...accountArgs, "--json", "message", "read", ...mailboxArgs, id], timeoutMs);
|
|
4500
|
-
if (readResult.ok) {
|
|
4501
|
-
body = readResult.stdout.trim().slice(0, MAX_BODY_CHARS);
|
|
4502
|
-
bodyRead = true;
|
|
4503
|
-
} else
|
|
4504
|
-
readFailures += 1;
|
|
4505
|
-
} catch {
|
|
4506
|
-
readFailures += 1;
|
|
4507
|
-
}
|
|
4508
|
-
const headerBlock = [
|
|
4509
|
-
`Subject: ${subject}`,
|
|
4510
|
-
...fromText.length > 0 ? [`From: ${fromText}`] : [],
|
|
4511
|
-
...asString(envelope.date) !== undefined ? [`Date: ${asString(envelope.date)}`] : []
|
|
4512
|
-
].join(`
|
|
4513
|
-
`);
|
|
4514
|
-
documents.push({
|
|
4515
|
-
docId,
|
|
4516
|
-
hierarchy: ["accounts", account, folder],
|
|
4517
|
-
title: subject,
|
|
4518
|
-
content: body.length > 0 ? `${headerBlock}
|
|
4519
|
-
|
|
4520
|
-
${body}` : headerBlock,
|
|
4521
|
-
publishedAt: parseHimalayaDate(asString(envelope.date)),
|
|
4522
|
-
metadata: {
|
|
4523
|
-
account,
|
|
4524
|
-
folder,
|
|
4525
|
-
messageId: id,
|
|
4526
|
-
...asArray(envelope.flags).length > 0 ? {
|
|
4527
|
-
flags: asArray(envelope.flags).map((flag) => asString(flag)).filter(Boolean)
|
|
4528
|
-
} : {}
|
|
4529
|
-
}
|
|
4530
|
-
});
|
|
4531
|
-
if (bodyRead)
|
|
4532
|
-
nextFingerprints[docId] = fingerprint;
|
|
4533
|
-
}
|
|
4534
|
-
if (envelopes.length <= maxDocuments) {
|
|
4535
|
-
for (const docId of Object.keys(state.fingerprints)) {
|
|
4536
|
-
if (!listedDocIds.has(docId))
|
|
4537
|
-
delete nextFingerprints[docId];
|
|
4538
|
-
}
|
|
4539
|
-
}
|
|
4540
|
-
const warnings = readFailures > 0 ? [`${readFailures} message(s) failed to read`] : undefined;
|
|
4541
|
-
this.saveState({ version: STATE_VERSION, fingerprints: nextFingerprints });
|
|
4542
|
-
const deletedDocIds = envelopes.length <= maxDocuments ? Object.keys(state.fingerprints).filter((docId) => !listedDocIds.has(docId)) : [];
|
|
4543
|
-
return {
|
|
4544
|
-
ok: true,
|
|
4545
|
-
documents,
|
|
4546
|
-
changed: Object.keys(nextFingerprints).length !== Object.keys(state.fingerprints).length || documents.length > 0,
|
|
4547
|
-
...deletedDocIds.length > 0 ? { deletedDocIds } : {},
|
|
4548
|
-
...warnings !== undefined ? { warnings } : {}
|
|
4549
|
-
};
|
|
4550
|
-
}
|
|
4551
|
-
loadState() {
|
|
4552
|
-
const path = this.statePath();
|
|
4553
|
-
if (path === undefined)
|
|
4554
|
-
return { version: STATE_VERSION, fingerprints: {} };
|
|
4555
|
-
try {
|
|
4556
|
-
const parsed = JSON.parse(readFileSync6(path, "utf8"));
|
|
4557
|
-
if (parsed.version !== STATE_VERSION || parsed.fingerprints === undefined) {
|
|
4558
|
-
return { version: STATE_VERSION, fingerprints: {} };
|
|
4559
|
-
}
|
|
4560
|
-
return { version: STATE_VERSION, fingerprints: parsed.fingerprints };
|
|
4561
|
-
} catch {
|
|
4562
|
-
return { version: STATE_VERSION, fingerprints: {} };
|
|
4563
|
-
}
|
|
4564
|
-
}
|
|
4565
|
-
saveState(state) {
|
|
4566
|
-
const path = this.statePath();
|
|
4567
|
-
if (path === undefined)
|
|
4568
|
-
return;
|
|
4569
|
-
try {
|
|
4570
|
-
mkdirSync4(dirname5(path), { recursive: true });
|
|
4571
|
-
writeFileSync4(path, `${JSON.stringify(state)}
|
|
4572
|
-
`, "utf8");
|
|
4573
|
-
} catch {}
|
|
4574
|
-
}
|
|
4575
|
-
statePath() {
|
|
4576
|
-
if (this.options.workspaceRoot === undefined)
|
|
4577
|
-
return;
|
|
4578
|
-
return join7(this.options.workspaceRoot, ".autorag", "datasources", sanitizeIdSegment("gmail"), sanitizeIdSegment(this.options.account ?? "default"), sanitizeIdSegment(this.options.folder ?? "INBOX"), "himalaya-state.json");
|
|
4579
|
-
}
|
|
4580
|
-
}
|
|
4581
|
-
function parseHimalayaDate(value) {
|
|
4582
|
-
if (value === undefined || value.length === 0)
|
|
4583
|
-
return;
|
|
4584
|
-
const parsed = Date.parse(value.replace(" ", "T"));
|
|
4585
|
-
return Number.isNaN(parsed) ? undefined : parsed;
|
|
4586
|
-
}
|
|
4587
|
-
function classifyFailure2(diagnostic) {
|
|
4588
|
-
const lower = diagnostic.toLowerCase();
|
|
4589
|
-
if (lower.includes("cannot find") && lower.includes("account"))
|
|
4590
|
-
return "not-configured";
|
|
4591
|
-
if (lower.includes("auth") || lower.includes("login") || lower.includes("credential"))
|
|
4592
|
-
return "auth";
|
|
4593
|
-
return "api-error";
|
|
4594
|
-
}
|
|
4595
|
-
function diagnosticFailure(result) {
|
|
4596
|
-
const raw = result.stderr.trim() || result.stdout.trim();
|
|
4597
|
-
if (raw.length === 0)
|
|
4598
|
-
return `himalaya exited with code ${result.code ?? "unknown"}`;
|
|
4599
|
-
return boundDiagnosticText(raw.replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu, "<redacted-email>").replace(/(?:\/Users|\/home|[A-Z]:\\Users)[^\s"']+/gu, "<redacted-path>").replace(/(?:password|passwd|token|secret)\s*[:=]\s*\S+/giu, "$1=<redacted>"));
|
|
4600
|
-
}
|
|
4601
|
-
function parseEnvelopes(stdout) {
|
|
4602
|
-
const parsed = JSON.parse(stdout);
|
|
4603
|
-
if (Array.isArray(parsed))
|
|
4604
|
-
return asArray(parsed);
|
|
4605
|
-
const record = asRecord(parsed);
|
|
4606
|
-
return asArray(record?.envelopes ?? record?.data);
|
|
4607
|
-
}
|
|
4608
|
-
function envelopeFingerprint(envelope) {
|
|
4609
|
-
return JSON.stringify({
|
|
4610
|
-
subject: envelope.subject,
|
|
4611
|
-
date: envelope.date,
|
|
4612
|
-
from: envelope.from,
|
|
4613
|
-
to: envelope.to,
|
|
4614
|
-
flags: envelope.flags
|
|
4615
|
-
});
|
|
4616
|
-
}
|
|
4617
|
-
function runBinary2(binary, args, timeoutMs, env, cwd) {
|
|
4618
|
-
return new Promise((resolvePromise) => {
|
|
4619
|
-
const command = portableSpawnCommand(binary, args);
|
|
4620
|
-
const child = spawn4(command.command, command.args, {
|
|
4621
|
-
...env === undefined ? {} : { env: { ...process.env, ...env } },
|
|
4622
|
-
...cwd === undefined ? {} : { cwd },
|
|
4623
|
-
stdio: ["ignore", "pipe", "pipe"]
|
|
4624
|
-
});
|
|
4625
|
-
let stdout = "";
|
|
4626
|
-
let stderr = "";
|
|
4627
|
-
const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
|
|
4628
|
-
child.stdout.on("data", (chunk) => {
|
|
4629
|
-
stdout += chunk.toString("utf8");
|
|
4630
|
-
});
|
|
4631
|
-
child.stderr.on("data", (chunk) => {
|
|
4632
|
-
stderr += chunk.toString("utf8");
|
|
4633
|
-
});
|
|
4634
|
-
child.on("error", () => {
|
|
4635
|
-
clearTimeout(timer);
|
|
4636
|
-
resolvePromise({ ok: false, stdout, stderr, code: null });
|
|
4637
|
-
});
|
|
4638
|
-
child.on("close", (code) => {
|
|
4639
|
-
clearTimeout(timer);
|
|
4640
|
-
resolvePromise({ ok: code === 0, stdout, stderr, code });
|
|
4641
|
-
});
|
|
4642
|
-
});
|
|
4643
|
-
}
|
|
4644
|
-
var DEFAULT_BINARY3 = "himalaya", DEFAULT_PAGE_SIZE = 100, DEFAULT_MAX_DOCUMENTS3 = 50, DEFAULT_TIMEOUT_MS4 = 30000, MAX_BODY_CHARS = 50000, STATE_VERSION = 1;
|
|
4645
|
-
var init_himalaya_connector = __esm(() => {
|
|
4646
|
-
init_portable_spawn();
|
|
4647
|
-
init_connector();
|
|
4648
|
-
});
|
|
4649
|
-
|
|
4650
4410
|
// src/datasource/skills/gmail/connector.ts
|
|
4651
4411
|
class GmailConnector {
|
|
4652
4412
|
options;
|
|
@@ -4665,7 +4425,7 @@ class GmailConnector {
|
|
|
4665
4425
|
signal
|
|
4666
4426
|
};
|
|
4667
4427
|
const maxPages = this.options.maxPages ?? DEFAULT_MAX_PAGES2;
|
|
4668
|
-
const maxDocuments = this.options.maxDocuments ??
|
|
4428
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS3;
|
|
4669
4429
|
const messageIds = [];
|
|
4670
4430
|
let pageToken;
|
|
4671
4431
|
for (let page = 0;page < maxPages && messageIds.length < maxDocuments; page += 1) {
|
|
@@ -4713,7 +4473,7 @@ class GmailConnector {
|
|
|
4713
4473
|
const payload = asRecord(message.payload);
|
|
4714
4474
|
const headers = headerMap(payload);
|
|
4715
4475
|
const subject = headers.get("subject") ?? "(no subject)";
|
|
4716
|
-
const body = (extractBody(payload) ?? asString(message.snippet) ?? "").slice(0,
|
|
4476
|
+
const body = (extractBody(payload) ?? asString(message.snippet) ?? "").slice(0, MAX_BODY_CHARS);
|
|
4717
4477
|
const labelIds = asArray(message.labelIds).map((label) => asString(label)).filter((label) => label !== undefined);
|
|
4718
4478
|
const internalDate = asString(message.internalDate);
|
|
4719
4479
|
const publishedAt = internalDate !== undefined && /^\d+$/u.test(internalDate) ? Number(internalDate) : undefined;
|
|
@@ -4779,7 +4539,7 @@ function decodeBase64Url(data) {
|
|
|
4779
4539
|
return "";
|
|
4780
4540
|
}
|
|
4781
4541
|
}
|
|
4782
|
-
var DEFAULT_BASE_URL2 = "https://gmail.googleapis.com/gmail/v1", DEFAULT_TOKEN_ENV2 = "GMAIL_ACCESS_TOKEN", DEFAULT_MAX_PAGES2 = 10,
|
|
4542
|
+
var DEFAULT_BASE_URL2 = "https://gmail.googleapis.com/gmail/v1", DEFAULT_TOKEN_ENV2 = "GMAIL_ACCESS_TOKEN", DEFAULT_MAX_PAGES2 = 10, DEFAULT_MAX_DOCUMENTS3 = 500, MAX_BODY_CHARS = 50000;
|
|
4783
4543
|
var init_connector3 = () => {};
|
|
4784
4544
|
|
|
4785
4545
|
// src/datasource/skills/gmail/skill.ts
|
|
@@ -4791,14 +4551,11 @@ var init_skill5 = __esm(() => {
|
|
|
4791
4551
|
skillName: "gmail",
|
|
4792
4552
|
skillType: "gmail-account",
|
|
4793
4553
|
description: "Gmail datasource",
|
|
4794
|
-
capabilities: ["email", "api", "
|
|
4554
|
+
capabilities: ["email", "api", "polling", "bm25", "incremental"],
|
|
4795
4555
|
defaultTags: ["gmail", "email", "pii"],
|
|
4796
4556
|
contentType: "email",
|
|
4797
4557
|
manifestDescription: "Search indexed Gmail messages from the authorized account and labels. Use for questions about email conversations, decisions, or attachments discussed over email.",
|
|
4798
|
-
manifestNotes: [
|
|
4799
|
-
"Label and folder visibility is bounded by the server-configured OAuth token.",
|
|
4800
|
-
"When the Himalaya backend is selected, indexing is incremental and lexical BM25 retrieval is local to AutoRAG; Himalaya itself does not provide semantic embeddings."
|
|
4801
|
-
]
|
|
4558
|
+
manifestNotes: ["Label visibility is bounded by the server-configured OAuth token."]
|
|
4802
4559
|
};
|
|
4803
4560
|
GmailSkill = class GmailSkill extends ConnectorDatasourceSkill {
|
|
4804
4561
|
constructor(options = {}) {
|
|
@@ -4814,7 +4571,6 @@ var init_skill5 = __esm(() => {
|
|
|
4814
4571
|
// src/datasource/skills/gmail/index.ts
|
|
4815
4572
|
var init_gmail = __esm(() => {
|
|
4816
4573
|
init_connector3();
|
|
4817
|
-
init_himalaya_connector();
|
|
4818
4574
|
init_skill5();
|
|
4819
4575
|
});
|
|
4820
4576
|
|
|
@@ -4823,7 +4579,7 @@ var DEFAULT_KATOK_BINARY = "katok", DEFAULT_KATOK_TIMEOUT_MS = 60000, DEFAULT_KA
|
|
|
4823
4579
|
var init_types2 = () => {};
|
|
4824
4580
|
|
|
4825
4581
|
// src/datasource/skills/katok/client.ts
|
|
4826
|
-
import { spawn as
|
|
4582
|
+
import { spawn as spawn4 } from "node:child_process";
|
|
4827
4583
|
|
|
4828
4584
|
class KatokClient {
|
|
4829
4585
|
options;
|
|
@@ -4925,10 +4681,10 @@ class KatokClient {
|
|
|
4925
4681
|
}
|
|
4926
4682
|
}
|
|
4927
4683
|
function spawnKatok(request) {
|
|
4928
|
-
return new Promise((
|
|
4684
|
+
return new Promise((resolve) => {
|
|
4929
4685
|
const { options, args, env, signal } = request;
|
|
4930
4686
|
const portable = portableSpawnCommand(commandFor(options.binaryPath), args);
|
|
4931
|
-
const child =
|
|
4687
|
+
const child = spawn4(portable.command, [...portable.args], {
|
|
4932
4688
|
env,
|
|
4933
4689
|
...request.cwd === undefined ? {} : { cwd: request.cwd },
|
|
4934
4690
|
stdio: ["ignore", "pipe", "pipe"]
|
|
@@ -4972,7 +4728,7 @@ function spawnKatok(request) {
|
|
|
4972
4728
|
clearTimeout(timeout);
|
|
4973
4729
|
signal?.removeEventListener("abort", abortHandler);
|
|
4974
4730
|
const reason = error.code === "ENOENT" ? "binary-missing" : "spawn-error";
|
|
4975
|
-
|
|
4731
|
+
resolve({ ok: false, reason, stdout: stdout.text, stderr: describeSpawnFailure2(reason), code: null });
|
|
4976
4732
|
});
|
|
4977
4733
|
child.on("close", (code) => {
|
|
4978
4734
|
if (settled)
|
|
@@ -4981,10 +4737,10 @@ function spawnKatok(request) {
|
|
|
4981
4737
|
clearTimeout(timeout);
|
|
4982
4738
|
signal?.removeEventListener("abort", abortHandler);
|
|
4983
4739
|
if (finalReason !== undefined) {
|
|
4984
|
-
|
|
4740
|
+
resolve({ ok: false, reason: finalReason, stdout: stdout.text, stderr: stderr.text, code });
|
|
4985
4741
|
return;
|
|
4986
4742
|
}
|
|
4987
|
-
|
|
4743
|
+
resolve({
|
|
4988
4744
|
ok: code === 0,
|
|
4989
4745
|
reason: code === 0 ? undefined : "nonzero-exit",
|
|
4990
4746
|
stdout: stdout.text,
|
|
@@ -5483,7 +5239,7 @@ var init_katok = __esm(() => {
|
|
|
5483
5239
|
|
|
5484
5240
|
// src/datasource/skills/mail-export/connector.ts
|
|
5485
5241
|
import { readdir, readFile as readFile4, stat } from "node:fs/promises";
|
|
5486
|
-
import { basename as basename3, dirname as
|
|
5242
|
+
import { basename as basename3, dirname as dirname5, extname as extname5, join as join7 } from "node:path";
|
|
5487
5243
|
import { simpleParser as simpleParser2 } from "mailparser";
|
|
5488
5244
|
|
|
5489
5245
|
class MailExportConnector {
|
|
@@ -5495,7 +5251,7 @@ class MailExportConnector {
|
|
|
5495
5251
|
const paths = this.options.paths ?? [];
|
|
5496
5252
|
if (paths.length === 0)
|
|
5497
5253
|
return { ok: false, reason: "not-configured", message: "no mail export paths configured" };
|
|
5498
|
-
const maxDocuments = this.options.maxDocuments ??
|
|
5254
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS4;
|
|
5499
5255
|
const maxBytes = this.options.maxBytesPerFile ?? DEFAULT_MAX_BYTES_PER_FILE2;
|
|
5500
5256
|
const files = [];
|
|
5501
5257
|
let missingPaths = 0;
|
|
@@ -5509,7 +5265,7 @@ class MailExportConnector {
|
|
|
5509
5265
|
continue;
|
|
5510
5266
|
const extension = extname5(entry.name).toLowerCase();
|
|
5511
5267
|
if (extension === ".eml" || extension === ".mbox") {
|
|
5512
|
-
files.push(
|
|
5268
|
+
files.push(join7(entry.parentPath, entry.name));
|
|
5513
5269
|
}
|
|
5514
5270
|
}
|
|
5515
5271
|
} else if (info.isFile()) {
|
|
@@ -5544,7 +5300,7 @@ class MailExportConnector {
|
|
|
5544
5300
|
continue;
|
|
5545
5301
|
}
|
|
5546
5302
|
const stem = basename3(file, extname5(file));
|
|
5547
|
-
const mailboxDir = basename3(
|
|
5303
|
+
const mailboxDir = basename3(dirname5(file));
|
|
5548
5304
|
const hierarchy = ["mailboxes", mailboxDir, stem];
|
|
5549
5305
|
const extension = extname5(file).toLowerCase();
|
|
5550
5306
|
const rawMessages = extension === ".mbox" ? splitMbox(bytes.toString("utf8")) : [bytes.toString("utf8")];
|
|
@@ -5556,7 +5312,7 @@ class MailExportConnector {
|
|
|
5556
5312
|
const subject = parsed.subject ?? "(no subject)";
|
|
5557
5313
|
const from = Array.isArray(parsed.from) ? parsed.from[0]?.text : parsed.from?.text;
|
|
5558
5314
|
const to = Array.isArray(parsed.to) ? parsed.to[0]?.text : parsed.to?.text;
|
|
5559
|
-
const body = (parsed.text ?? "").trim().slice(0,
|
|
5315
|
+
const body = (parsed.text ?? "").trim().slice(0, MAX_BODY_CHARS2);
|
|
5560
5316
|
const headerBlock = [
|
|
5561
5317
|
`Subject: ${subject}`,
|
|
5562
5318
|
...from !== undefined ? [`From: ${from}`] : [],
|
|
@@ -5613,7 +5369,7 @@ function splitMbox(text) {
|
|
|
5613
5369
|
`));
|
|
5614
5370
|
return messages.filter((message) => message.trim().length > 0);
|
|
5615
5371
|
}
|
|
5616
|
-
var
|
|
5372
|
+
var DEFAULT_MAX_DOCUMENTS4 = 500, DEFAULT_MAX_BYTES_PER_FILE2, MAX_BODY_CHARS2 = 50000, MBOX_SEPARATOR;
|
|
5617
5373
|
var init_connector4 = __esm(() => {
|
|
5618
5374
|
DEFAULT_MAX_BYTES_PER_FILE2 = 50 * 1024 * 1024;
|
|
5619
5375
|
MBOX_SEPARATOR = /^From .*\d{4}$/mu;
|
|
@@ -5654,21 +5410,21 @@ var init_mail_export = __esm(() => {
|
|
|
5654
5410
|
});
|
|
5655
5411
|
|
|
5656
5412
|
// src/datasource/skills/mailcrawl/config.ts
|
|
5657
|
-
import { chmodSync, mkdirSync as
|
|
5413
|
+
import { chmodSync, mkdirSync as mkdirSync4 } from "node:fs";
|
|
5658
5414
|
function validateMailcrawlInstanceId(instanceId) {
|
|
5659
5415
|
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u.test(instanceId)) {
|
|
5660
5416
|
throw new Error("mailcrawl instanceId must be a safe single path segment");
|
|
5661
5417
|
}
|
|
5662
5418
|
}
|
|
5663
5419
|
function ensurePrivateMailcrawlDataDir(path) {
|
|
5664
|
-
|
|
5420
|
+
mkdirSync4(path, { recursive: true, mode: 448 });
|
|
5665
5421
|
if (process.platform !== "win32")
|
|
5666
5422
|
chmodSync(path, 448);
|
|
5667
5423
|
}
|
|
5668
5424
|
var init_config = () => {};
|
|
5669
5425
|
|
|
5670
5426
|
// src/datasource/skills/mailcrawl/client.ts
|
|
5671
|
-
import { spawn as
|
|
5427
|
+
import { spawn as spawn5 } from "node:child_process";
|
|
5672
5428
|
|
|
5673
5429
|
class MailcrawlClient {
|
|
5674
5430
|
options;
|
|
@@ -5762,7 +5518,7 @@ class MailcrawlClient {
|
|
|
5762
5518
|
}
|
|
5763
5519
|
if (dataDir !== undefined)
|
|
5764
5520
|
env.MAILCRAWL_DATA_DIR = dataDir;
|
|
5765
|
-
return spawnMailcrawl(this.options.binaryPath ??
|
|
5521
|
+
return spawnMailcrawl(this.options.binaryPath ?? DEFAULT_BINARY3, args, env, this.options, signal);
|
|
5766
5522
|
}
|
|
5767
5523
|
async index(signal) {
|
|
5768
5524
|
const result = await this.run(["index", "--json"], signal);
|
|
@@ -5782,7 +5538,7 @@ class MailcrawlClient {
|
|
|
5782
5538
|
function spawnMailcrawl(binary, args, env, options, signal) {
|
|
5783
5539
|
return new Promise((resolveResult) => {
|
|
5784
5540
|
const portable = portableSpawnCommand(binary, args);
|
|
5785
|
-
const child =
|
|
5541
|
+
const child = spawn5(portable.command, portable.args, {
|
|
5786
5542
|
env,
|
|
5787
5543
|
detached: process.platform !== "win32",
|
|
5788
5544
|
stdio: ["ignore", "pipe", "pipe"]
|
|
@@ -5792,7 +5548,7 @@ function spawnMailcrawl(binary, args, env, options, signal) {
|
|
|
5792
5548
|
const timer = setTimeout(() => {
|
|
5793
5549
|
reason = "timeout";
|
|
5794
5550
|
terminate4(child);
|
|
5795
|
-
}, options.timeoutMs ??
|
|
5551
|
+
}, options.timeoutMs ?? DEFAULT_TIMEOUT_MS4);
|
|
5796
5552
|
const abort = () => {
|
|
5797
5553
|
reason = "aborted";
|
|
5798
5554
|
terminate4(child);
|
|
@@ -5965,7 +5721,7 @@ function truncateUtf8(value, maxBytes) {
|
|
|
5965
5721
|
}
|
|
5966
5722
|
return "";
|
|
5967
5723
|
}
|
|
5968
|
-
var
|
|
5724
|
+
var DEFAULT_BINARY3 = "mailcrawl", DEFAULT_TIMEOUT_MS4 = 60000, DEFAULT_MAX_BUFFER_BYTES2 = 1048576, SAFE_ENV_KEYS, SAFE_MAILCRAWL_ENV_KEYS, REMOTE_EMBEDDING_ENV_KEYS;
|
|
5969
5725
|
var init_client4 = __esm(() => {
|
|
5970
5726
|
init_portable_spawn();
|
|
5971
5727
|
init_config();
|
|
@@ -6212,7 +5968,7 @@ var init_mailcrawl = __esm(() => {
|
|
|
6212
5968
|
});
|
|
6213
5969
|
|
|
6214
5970
|
// src/datasource/crawler-client.ts
|
|
6215
|
-
import { spawn as
|
|
5971
|
+
import { spawn as spawn6 } from "node:child_process";
|
|
6216
5972
|
|
|
6217
5973
|
class CrawlerCliClient {
|
|
6218
5974
|
profile;
|
|
@@ -6247,9 +6003,9 @@ class CrawlerCliClient {
|
|
|
6247
6003
|
}
|
|
6248
6004
|
}
|
|
6249
6005
|
function spawnCrawler(command, args, env, signal, options, cwd) {
|
|
6250
|
-
return new Promise((
|
|
6006
|
+
return new Promise((resolve) => {
|
|
6251
6007
|
const portable = portableSpawnCommand(command, args);
|
|
6252
|
-
const child =
|
|
6008
|
+
const child = spawn6(portable.command, [...portable.args], {
|
|
6253
6009
|
env,
|
|
6254
6010
|
...cwd === undefined ? {} : { cwd },
|
|
6255
6011
|
stdio: ["ignore", "pipe", "pipe"]
|
|
@@ -6262,7 +6018,7 @@ function spawnCrawler(command, args, env, signal, options, cwd) {
|
|
|
6262
6018
|
const timeout = setTimeout(() => {
|
|
6263
6019
|
finalReason = "timeout";
|
|
6264
6020
|
terminate5(child);
|
|
6265
|
-
}, options.timeoutMs ??
|
|
6021
|
+
}, options.timeoutMs ?? DEFAULT_TIMEOUT_MS5);
|
|
6266
6022
|
const abortHandler = () => {
|
|
6267
6023
|
finalReason = "aborted";
|
|
6268
6024
|
terminate5(child);
|
|
@@ -6292,7 +6048,7 @@ function spawnCrawler(command, args, env, signal, options, cwd) {
|
|
|
6292
6048
|
settled = true;
|
|
6293
6049
|
clearTimeout(timeout);
|
|
6294
6050
|
signal?.removeEventListener("abort", abortHandler);
|
|
6295
|
-
|
|
6051
|
+
resolve({
|
|
6296
6052
|
ok: false,
|
|
6297
6053
|
reason: error.code === "ENOENT" ? "binary-missing" : "spawn-error",
|
|
6298
6054
|
stdout: stdout.text,
|
|
@@ -6306,7 +6062,7 @@ function spawnCrawler(command, args, env, signal, options, cwd) {
|
|
|
6306
6062
|
settled = true;
|
|
6307
6063
|
clearTimeout(timeout);
|
|
6308
6064
|
signal?.removeEventListener("abort", abortHandler);
|
|
6309
|
-
|
|
6065
|
+
resolve({
|
|
6310
6066
|
ok: code === 0 && finalReason === undefined,
|
|
6311
6067
|
...finalReason !== undefined ? { reason: finalReason } : code === 0 ? {} : { reason: "nonzero-exit" },
|
|
6312
6068
|
stdout: stdout.text,
|
|
@@ -6354,7 +6110,7 @@ function toFailure3(binaryName, result, reason) {
|
|
|
6354
6110
|
code: result.code
|
|
6355
6111
|
};
|
|
6356
6112
|
}
|
|
6357
|
-
var
|
|
6113
|
+
var DEFAULT_TIMEOUT_MS5 = 60000, DEFAULT_MAX_BUFFER_BYTES3 = 1048576, SAFE_ENV_KEYS2;
|
|
6358
6114
|
var init_crawler_client = __esm(() => {
|
|
6359
6115
|
init_portable_spawn();
|
|
6360
6116
|
SAFE_ENV_KEYS2 = new Set([
|
|
@@ -6707,15 +6463,15 @@ var init_notion = __esm(() => {
|
|
|
6707
6463
|
});
|
|
6708
6464
|
|
|
6709
6465
|
// src/datasource/skills/obsidian/paths.ts
|
|
6710
|
-
import { join as
|
|
6466
|
+
import { join as join8 } from "node:path";
|
|
6711
6467
|
function obsidianDatasourceRoot(workspaceRoot, instanceId) {
|
|
6712
|
-
return
|
|
6468
|
+
return join8(workspaceRoot, AUTORAG_DIRNAME, DATASOURCES_DIRNAME, OBSIDIAN_DATASOURCE_DIRNAME, instanceId);
|
|
6713
6469
|
}
|
|
6714
6470
|
function obsidianQmdConfigDir(workspaceRoot, instanceId) {
|
|
6715
|
-
return
|
|
6471
|
+
return join8(obsidianDatasourceRoot(workspaceRoot, instanceId), "config");
|
|
6716
6472
|
}
|
|
6717
6473
|
function obsidianQmdCacheDir(workspaceRoot, instanceId) {
|
|
6718
|
-
return
|
|
6474
|
+
return join8(obsidianDatasourceRoot(workspaceRoot, instanceId), "cache");
|
|
6719
6475
|
}
|
|
6720
6476
|
function stripEdgeDashes(value) {
|
|
6721
6477
|
let start = 0;
|
|
@@ -6737,8 +6493,8 @@ var init_paths3 = () => {};
|
|
|
6737
6493
|
var DEFAULT_QMD_BINARY = "qmd", DEFAULT_QMD_TIMEOUT_MS = 60000, DEFAULT_QMD_MAX_BUFFER_BYTES = 1048576;
|
|
6738
6494
|
|
|
6739
6495
|
// src/datasource/skills/obsidian/client.ts
|
|
6740
|
-
import { spawn as
|
|
6741
|
-
import { mkdirSync as
|
|
6496
|
+
import { spawn as spawn7 } from "node:child_process";
|
|
6497
|
+
import { mkdirSync as mkdirSync5, writeFileSync as writeFileSync4 } from "node:fs";
|
|
6742
6498
|
import { isAbsolute as isAbsolute3, resolve as resolve4 } from "node:path";
|
|
6743
6499
|
|
|
6744
6500
|
class QmdClient {
|
|
@@ -6758,10 +6514,10 @@ class QmdClient {
|
|
|
6758
6514
|
const cacheDir = obsidianQmdCacheDir(workspaceRoot, instanceId);
|
|
6759
6515
|
try {
|
|
6760
6516
|
if (this.options.configPath === undefined) {
|
|
6761
|
-
|
|
6762
|
-
|
|
6517
|
+
mkdirSync5(configDir, { recursive: true });
|
|
6518
|
+
writeFileSync4(resolve4(configDir, "index.yml"), renderIndexYaml(collectionName, vaultPath), "utf8");
|
|
6763
6519
|
}
|
|
6764
|
-
|
|
6520
|
+
mkdirSync5(cacheDir, { recursive: true });
|
|
6765
6521
|
} catch (error) {
|
|
6766
6522
|
return failure3("spawn-error", error instanceof Error ? error.message : "failed to write qmd config");
|
|
6767
6523
|
}
|
|
@@ -6862,7 +6618,7 @@ function isAllowedEnvKey2(key) {
|
|
|
6862
6618
|
function spawnQmd(request) {
|
|
6863
6619
|
return new Promise((resolveResult) => {
|
|
6864
6620
|
const portable = portableSpawnCommand(request.binaryPath ?? DEFAULT_QMD_BINARY, request.args);
|
|
6865
|
-
const child =
|
|
6621
|
+
const child = spawn7(portable.command, [...portable.args], {
|
|
6866
6622
|
env: request.env,
|
|
6867
6623
|
...request.cwd === undefined ? {} : { cwd: request.cwd },
|
|
6868
6624
|
stdio: ["ignore", "pipe", "pipe"]
|
|
@@ -7408,7 +7164,7 @@ class RssConnector {
|
|
|
7408
7164
|
const feeds = this.options.feeds ?? [];
|
|
7409
7165
|
if (feeds.length === 0)
|
|
7410
7166
|
return { ok: false, reason: "not-configured", message: "no feeds configured" };
|
|
7411
|
-
const maxDocuments = this.options.maxDocuments ??
|
|
7167
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS5;
|
|
7412
7168
|
const maxItemsPerFeed = this.options.maxItemsPerFeed ?? DEFAULT_MAX_ITEMS_PER_FEED;
|
|
7413
7169
|
const documents = [];
|
|
7414
7170
|
const warnings = [];
|
|
@@ -7541,7 +7297,7 @@ function stripHtml(text) {
|
|
|
7541
7297
|
function present(item) {
|
|
7542
7298
|
return item !== undefined;
|
|
7543
7299
|
}
|
|
7544
|
-
var
|
|
7300
|
+
var DEFAULT_MAX_DOCUMENTS5 = 500, DEFAULT_MAX_ITEMS_PER_FEED = 100, DEFAULT_FEED_TIMEOUT_MS = 30000, MAX_CONTENT_CHARS2 = 20000, ACCEPT_HEADER = "application/rss+xml, application/atom+xml, application/xml, text/xml";
|
|
7545
7301
|
var init_connector5 = () => {};
|
|
7546
7302
|
|
|
7547
7303
|
// src/datasource/skills/rss/skill.ts
|
|
@@ -7713,7 +7469,7 @@ var init_slack = __esm(() => {
|
|
|
7713
7469
|
});
|
|
7714
7470
|
|
|
7715
7471
|
// src/datasource/skills/spotlight/connector.ts
|
|
7716
|
-
import { spawn as
|
|
7472
|
+
import { spawn as spawn8 } from "node:child_process";
|
|
7717
7473
|
import { createHash as createHash3 } from "node:crypto";
|
|
7718
7474
|
import { readFile as fsReadFile, stat as stat2 } from "node:fs/promises";
|
|
7719
7475
|
import { basename as basename4, extname as extname6 } from "node:path";
|
|
@@ -7737,7 +7493,7 @@ class SpotlightConnector {
|
|
|
7737
7493
|
if (queries.length === 0) {
|
|
7738
7494
|
return { ok: false, reason: "not-configured", message: "no spotlight queries configured" };
|
|
7739
7495
|
}
|
|
7740
|
-
const timeoutMs = this.options.timeoutMs ??
|
|
7496
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS6;
|
|
7741
7497
|
const status = await this.run("mdutil", ["-s", "/"], timeoutMs);
|
|
7742
7498
|
if (status.ok && /indexing disabled/iu.test(status.stdout)) {
|
|
7743
7499
|
return {
|
|
@@ -7747,7 +7503,7 @@ class SpotlightConnector {
|
|
|
7747
7503
|
};
|
|
7748
7504
|
}
|
|
7749
7505
|
const maxPerQuery = this.options.maxResultsPerQuery ?? DEFAULT_MAX_RESULTS_PER_QUERY;
|
|
7750
|
-
const maxDocuments = this.options.maxDocuments ??
|
|
7506
|
+
const maxDocuments = this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS6;
|
|
7751
7507
|
const seen = new Set;
|
|
7752
7508
|
const paths = [];
|
|
7753
7509
|
let mdfindMissing = false;
|
|
@@ -7831,8 +7587,8 @@ class SpotlightConnector {
|
|
|
7831
7587
|
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
7832
7588
|
}
|
|
7833
7589
|
}
|
|
7834
|
-
var DEFAULT_MAX_RESULTS_PER_QUERY = 100,
|
|
7835
|
-
const child =
|
|
7590
|
+
var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS6 = 300, DEFAULT_MAX_BYTES_PER_FILE3, DEFAULT_TIMEOUT_MS6 = 15000, MAX_CONTENT_CHARS3 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve) => {
|
|
7591
|
+
const child = spawn8(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
|
|
7836
7592
|
let stdout = "";
|
|
7837
7593
|
let stderr = "";
|
|
7838
7594
|
let settled = false;
|
|
@@ -7841,7 +7597,7 @@ var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS7 = 300, DEFAULT_M
|
|
|
7841
7597
|
return;
|
|
7842
7598
|
settled = true;
|
|
7843
7599
|
clearTimeout(timer);
|
|
7844
|
-
|
|
7600
|
+
resolve(result);
|
|
7845
7601
|
};
|
|
7846
7602
|
const timer = setTimeout(() => {
|
|
7847
7603
|
child.kill("SIGKILL");
|
|
@@ -8183,12 +7939,19 @@ function buildDatasourceSkills(config, workspaceRoot) {
|
|
|
8183
7939
|
}
|
|
8184
7940
|
const normalizedEntry = entry.instanceId === undefined && entry.type !== undefined ? { ...entry, instanceId: name } : entry;
|
|
8185
7941
|
const skill = builder(normalizedEntry, workspaceRoot, name);
|
|
7942
|
+
if (skill === undefined) {
|
|
7943
|
+
unknown.push(name);
|
|
7944
|
+
continue;
|
|
7945
|
+
}
|
|
8186
7946
|
const hasChannelFilter = (entry.channels?.ids?.length ?? 0) > 0 || (entry.channels?.names?.length ?? 0) > 0;
|
|
8187
|
-
|
|
8188
|
-
|
|
8189
|
-
|
|
8190
|
-
|
|
8191
|
-
|
|
7947
|
+
let aliased = skill;
|
|
7948
|
+
if (name !== templateName || hasChannelFilter) {
|
|
7949
|
+
aliased = new AliasedDatasourceSkill(skill, {
|
|
7950
|
+
alias: name,
|
|
7951
|
+
...entry.channels?.ids !== undefined ? { channelIds: entry.channels.ids } : {},
|
|
7952
|
+
...entry.channels?.names !== undefined ? { channelNames: entry.channels.names } : {}
|
|
7953
|
+
});
|
|
7954
|
+
}
|
|
8192
7955
|
skills.push(typeof entry.description === "string" && entry.description.trim().length > 0 ? new DescribedDatasourceSkill(aliased, entry.description.trim()) : aliased);
|
|
8193
7956
|
}
|
|
8194
7957
|
return { skills, unknown };
|
|
@@ -8199,7 +7962,6 @@ var init_factory = __esm(() => {
|
|
|
8199
7962
|
init_cloud_drive();
|
|
8200
7963
|
init_discrawl();
|
|
8201
7964
|
init_github();
|
|
8202
|
-
init_himalaya_connector();
|
|
8203
7965
|
init_gmail();
|
|
8204
7966
|
init_katok();
|
|
8205
7967
|
init_mail_export();
|
|
@@ -8318,17 +8080,8 @@ var init_factory = __esm(() => {
|
|
|
8318
8080
|
}),
|
|
8319
8081
|
gmail: (config, workspaceRoot, registrationName) => {
|
|
8320
8082
|
const connector = config.connector;
|
|
8321
|
-
if (connector?.backend === "himalaya")
|
|
8322
|
-
|
|
8323
|
-
return new GmailSkill({
|
|
8324
|
-
...common(config, workspaceRoot),
|
|
8325
|
-
skillName: registrationName,
|
|
8326
|
-
connector: new HimalayaConnector({
|
|
8327
|
-
...himalayaOptions,
|
|
8328
|
-
...workspaceRoot !== undefined ? { workspaceRoot } : {}
|
|
8329
|
-
})
|
|
8330
|
-
});
|
|
8331
|
-
}
|
|
8083
|
+
if (connector?.backend === "himalaya")
|
|
8084
|
+
return;
|
|
8332
8085
|
return new GmailSkill({
|
|
8333
8086
|
...common(config, workspaceRoot),
|
|
8334
8087
|
skillName: registrationName,
|
|
@@ -8375,15 +8128,15 @@ import { randomUUID as randomUUID2 } from "node:crypto";
|
|
|
8375
8128
|
import {
|
|
8376
8129
|
closeSync,
|
|
8377
8130
|
lstatSync,
|
|
8378
|
-
mkdirSync as
|
|
8131
|
+
mkdirSync as mkdirSync6,
|
|
8379
8132
|
openSync,
|
|
8380
8133
|
readdirSync,
|
|
8381
|
-
readFileSync as
|
|
8134
|
+
readFileSync as readFileSync6,
|
|
8382
8135
|
rmdirSync,
|
|
8383
8136
|
unlinkSync,
|
|
8384
8137
|
writeSync
|
|
8385
8138
|
} from "node:fs";
|
|
8386
|
-
import { join as
|
|
8139
|
+
import { join as join9 } from "node:path";
|
|
8387
8140
|
function hasErrorCode(error, code) {
|
|
8388
8141
|
return error instanceof Error && "code" in error && error.code === code;
|
|
8389
8142
|
}
|
|
@@ -8427,7 +8180,7 @@ function inspectRegularFile(path) {
|
|
|
8427
8180
|
return { kind: "other" };
|
|
8428
8181
|
let contents;
|
|
8429
8182
|
try {
|
|
8430
|
-
contents =
|
|
8183
|
+
contents = readFileSync6(path, "utf8");
|
|
8431
8184
|
} catch (error) {
|
|
8432
8185
|
if (hasErrorCode(error, "ENOENT"))
|
|
8433
8186
|
return { kind: "missing" };
|
|
@@ -8532,7 +8285,7 @@ function reapStaleLockDirectory(lockPath, staleMs) {
|
|
|
8532
8285
|
return removeEmptyLockDirectory(lockPath);
|
|
8533
8286
|
const staleSnapshots = [];
|
|
8534
8287
|
for (const name of names) {
|
|
8535
|
-
const inspection = inspectRegularFile(
|
|
8288
|
+
const inspection = inspectRegularFile(join9(lockPath, name));
|
|
8536
8289
|
if (inspection.kind === "missing")
|
|
8537
8290
|
continue;
|
|
8538
8291
|
if (inspection.kind !== "file" || !isStaleSnapshot(inspection, staleMs))
|
|
@@ -8587,14 +8340,14 @@ function createLockFile(lockPath, contents) {
|
|
|
8587
8340
|
}
|
|
8588
8341
|
function tryCreateOwnerMarker(lockPath, owner, contents) {
|
|
8589
8342
|
try {
|
|
8590
|
-
|
|
8343
|
+
mkdirSync6(lockPath, { mode: 448 });
|
|
8591
8344
|
} catch (error) {
|
|
8592
8345
|
if (hasErrorCode(error, "EEXIST"))
|
|
8593
8346
|
return;
|
|
8594
8347
|
throw error;
|
|
8595
8348
|
}
|
|
8596
8349
|
const markerName = `owner-${owner.token}.json`;
|
|
8597
|
-
const markerPath =
|
|
8350
|
+
const markerPath = join9(lockPath, markerName);
|
|
8598
8351
|
let acquired = false;
|
|
8599
8352
|
try {
|
|
8600
8353
|
try {
|
|
@@ -8680,8 +8433,8 @@ var init_file_lock = __esm(() => {
|
|
|
8680
8433
|
|
|
8681
8434
|
// src/cli/config.ts
|
|
8682
8435
|
import { randomUUID as randomUUID3 } from "node:crypto";
|
|
8683
|
-
import { existsSync as existsSync5, mkdirSync as
|
|
8684
|
-
import { basename as basename5, dirname as
|
|
8436
|
+
import { existsSync as existsSync5, mkdirSync as mkdirSync7, readFileSync as readFileSync7, renameSync as renameSync3, unlinkSync as unlinkSync2, writeFileSync as writeFileSync5 } from "node:fs";
|
|
8437
|
+
import { basename as basename5, dirname as dirname6, isAbsolute as isAbsolute4, join as join10, resolve as resolve5 } from "node:path";
|
|
8685
8438
|
import { findEnvKeys, getEnvApiKey, getModel, getProviders } from "@earendil-works/pi-ai/compat";
|
|
8686
8439
|
function resolveConfigPath(input) {
|
|
8687
8440
|
const flags = input.flags;
|
|
@@ -8696,9 +8449,9 @@ function resolveConfigPath(input) {
|
|
|
8696
8449
|
return { configPath: envConfig, explicit: true };
|
|
8697
8450
|
}
|
|
8698
8451
|
return {
|
|
8699
|
-
configPath:
|
|
8452
|
+
configPath: join10(resolveAutoRAGHome(env), DEFAULT_CONFIG_FILENAME),
|
|
8700
8453
|
explicit: false,
|
|
8701
|
-
legacyPath:
|
|
8454
|
+
legacyPath: join10(cwd, LEGACY_CONFIG_FILENAME)
|
|
8702
8455
|
};
|
|
8703
8456
|
}
|
|
8704
8457
|
function readConfigFile(configPath, explicit) {
|
|
@@ -8716,7 +8469,7 @@ function readConfigFile(configPath, explicit) {
|
|
|
8716
8469
|
}
|
|
8717
8470
|
let text;
|
|
8718
8471
|
try {
|
|
8719
|
-
text =
|
|
8472
|
+
text = readFileSync7(configPath, "utf8");
|
|
8720
8473
|
} catch (err) {
|
|
8721
8474
|
throw new ConfigError(`Failed to read config file: ${err.message}`);
|
|
8722
8475
|
}
|
|
@@ -8754,9 +8507,9 @@ function acquireConfigWriteLock(configPath) {
|
|
|
8754
8507
|
});
|
|
8755
8508
|
}
|
|
8756
8509
|
function replaceFileAtomically(path, contents, assertCommitAllowed) {
|
|
8757
|
-
const temporaryPath =
|
|
8510
|
+
const temporaryPath = join10(dirname6(path), `.${basename5(path)}.${process.pid}.${randomUUID3()}.tmp`);
|
|
8758
8511
|
try {
|
|
8759
|
-
|
|
8512
|
+
writeFileSync5(temporaryPath, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
|
|
8760
8513
|
assertCommitAllowed?.();
|
|
8761
8514
|
renameSync3(temporaryPath, path);
|
|
8762
8515
|
} finally {
|
|
@@ -8767,12 +8520,12 @@ function migrateLegacyConfig(configPath, legacyPath) {
|
|
|
8767
8520
|
if (existsSync5(configPath) || !existsSync5(legacyPath))
|
|
8768
8521
|
return;
|
|
8769
8522
|
const legacy = readConfigFile(legacyPath, true);
|
|
8770
|
-
const legacyBytes =
|
|
8771
|
-
const migrated = normalizeLegacyConfigPaths(legacy ?? {},
|
|
8523
|
+
const legacyBytes = readFileSync7(legacyPath);
|
|
8524
|
+
const migrated = normalizeLegacyConfigPaths(legacy ?? {}, dirname6(legacyPath));
|
|
8772
8525
|
const migratedPathsUnchanged = legacy?.workspacePath === migrated.workspacePath && legacy?.memoryPath === migrated.memoryPath && JSON.stringify(legacy?.searchPaths) === JSON.stringify(migrated.searchPaths);
|
|
8773
8526
|
const migratedBytes = migratedPathsUnchanged ? legacyBytes : `${JSON.stringify(migrated, null, 2)}
|
|
8774
8527
|
`;
|
|
8775
|
-
|
|
8528
|
+
mkdirSync7(dirname6(configPath), { recursive: true });
|
|
8776
8529
|
const lock = acquireConfigWriteLock(configPath);
|
|
8777
8530
|
try {
|
|
8778
8531
|
const winner = readConfigFile(configPath, false);
|
|
@@ -8802,7 +8555,7 @@ function resolveConfigFileReadOnly(configPath, explicit, legacyPath) {
|
|
|
8802
8555
|
const legacy = readConfigFile(legacyPath, false);
|
|
8803
8556
|
if (legacy === undefined)
|
|
8804
8557
|
return {};
|
|
8805
|
-
return normalizeLegacyConfigPaths(legacy,
|
|
8558
|
+
return normalizeLegacyConfigPaths(legacy, dirname6(legacyPath));
|
|
8806
8559
|
}
|
|
8807
8560
|
function resolveSearchPaths(searchPaths, origin) {
|
|
8808
8561
|
return searchPaths.map((searchPath) => resolvePersistedPath(searchPath, origin));
|
|
@@ -9070,24 +8823,6 @@ function normalizeUiConfig(raw) {
|
|
|
9070
8823
|
}
|
|
9071
8824
|
return out;
|
|
9072
8825
|
}
|
|
9073
|
-
function normalizeBm25Method(raw) {
|
|
9074
|
-
if (raw === false)
|
|
9075
|
-
return { enabled: false };
|
|
9076
|
-
if (raw === undefined || raw === null)
|
|
9077
|
-
return { enabled: true };
|
|
9078
|
-
if (typeof raw !== "object" || Array.isArray(raw)) {
|
|
9079
|
-
throw new ConfigError("bm25 must be an object or false");
|
|
9080
|
-
}
|
|
9081
|
-
const record = raw;
|
|
9082
|
-
for (const key of Object.keys(record)) {
|
|
9083
|
-
if (BM25_LEGACY_IGNORED.has(key))
|
|
9084
|
-
continue;
|
|
9085
|
-
if (!BM25_ALLOWLIST.has(key)) {
|
|
9086
|
-
throw new ConfigError(`bm25.${key} is not a recognized field`);
|
|
9087
|
-
}
|
|
9088
|
-
}
|
|
9089
|
-
return { enabled: record.enabled !== false };
|
|
9090
|
-
}
|
|
9091
8826
|
function normalizeMinSyncMethod(raw) {
|
|
9092
8827
|
if (raw === false)
|
|
9093
8828
|
return { enabled: false };
|
|
@@ -9098,14 +8833,14 @@ function normalizeMinSyncMethod(raw) {
|
|
|
9098
8833
|
}
|
|
9099
8834
|
const record = raw;
|
|
9100
8835
|
for (const key of Object.keys(record)) {
|
|
8836
|
+
if (MINSYNC_LEGACY_IGNORED.has(key))
|
|
8837
|
+
continue;
|
|
9101
8838
|
if (!MINSYNC_ALLOWLIST.has(key)) {
|
|
9102
8839
|
throw new ConfigError(`minSync.${key} is not a recognized field`);
|
|
9103
8840
|
}
|
|
9104
8841
|
}
|
|
9105
8842
|
const enabled = record.enabled !== false;
|
|
9106
8843
|
const out = { enabled, autoInstall: record.autoInstall !== false };
|
|
9107
|
-
if (typeof record.binaryPath === "string" && record.binaryPath.length > 0)
|
|
9108
|
-
out.binaryPath = record.binaryPath;
|
|
9109
8844
|
if (typeof record.workspacePath === "string" && record.workspacePath.length > 0) {
|
|
9110
8845
|
out.workspacePath = record.workspacePath;
|
|
9111
8846
|
}
|
|
@@ -9128,7 +8863,6 @@ function normalizeMinSyncMethod(raw) {
|
|
|
9128
8863
|
}
|
|
9129
8864
|
function normalizeIndexingConfig(raw) {
|
|
9130
8865
|
return {
|
|
9131
|
-
bm25: normalizeBm25Method(raw.bm25),
|
|
9132
8866
|
minSync: normalizeMinSyncMethod(raw.minSync)
|
|
9133
8867
|
};
|
|
9134
8868
|
}
|
|
@@ -9141,10 +8875,10 @@ function resolveConfig(input) {
|
|
|
9141
8875
|
const file = readOnly ? resolveConfigFileReadOnly(configPath, explicit, legacyPath) : !explicit && legacyPath ? migrateLegacyConfig(configPath, legacyPath) ?? readConfigFile(configPath, explicit) ?? {} : readConfigFile(configPath, explicit) ?? {};
|
|
9142
8876
|
const defaultSearchPaths = ["."];
|
|
9143
8877
|
const defaultWorkspacePath = cwd;
|
|
9144
|
-
const defaultMemoryPath =
|
|
8878
|
+
const defaultMemoryPath = join10(resolveAutoRAGHome(env), "memory.json");
|
|
9145
8879
|
const flagSearchPaths = parseCsv(flagString(flags, "search-paths"));
|
|
9146
8880
|
const envSearchPaths = parseCsv(envString(env, "AUTORAG_SEARCH_PATHS"));
|
|
9147
|
-
const configOrigin =
|
|
8881
|
+
const configOrigin = dirname6(resolve5(configPath));
|
|
9148
8882
|
const fileWorkspacePath = typeof file.workspacePath === "string" ? resolvePersistedPath(file.workspacePath, configOrigin) : undefined;
|
|
9149
8883
|
const fileSearchPaths = file.searchPaths ? resolveSearchPaths(file.searchPaths, fileWorkspacePath ?? configOrigin) : undefined;
|
|
9150
8884
|
const searchPaths = flagSearchPaths ?? envSearchPaths ?? fileSearchPaths ?? defaultSearchPaths;
|
|
@@ -9167,10 +8901,8 @@ function resolveConfig(input) {
|
|
|
9167
8901
|
if (model)
|
|
9168
8902
|
config.model = model;
|
|
9169
8903
|
const normalized = normalizeIndexingConfig({
|
|
9170
|
-
bm25: file.bm25,
|
|
9171
8904
|
minSync: file.minSync
|
|
9172
8905
|
});
|
|
9173
|
-
config.bm25 = normalized.bm25;
|
|
9174
8906
|
config.minSync = normalized.minSync;
|
|
9175
8907
|
config.jikji = file.jikji === false ? false : file.jikji ?? {};
|
|
9176
8908
|
if (file.parserOptions)
|
|
@@ -9218,12 +8950,6 @@ function buildAgentOptions(config) {
|
|
|
9218
8950
|
} else {
|
|
9219
8951
|
opts.minSync = false;
|
|
9220
8952
|
}
|
|
9221
|
-
if (config.bm25 && config.bm25.enabled !== false) {
|
|
9222
|
-
const { enabled: _omitBm25Enabled, ...bm25Fields } = config.bm25;
|
|
9223
|
-
opts.bm25 = bm25Fields;
|
|
9224
|
-
} else {
|
|
9225
|
-
opts.bm25 = false;
|
|
9226
|
-
}
|
|
9227
8953
|
opts.jikji = config.jikji === false ? false : config.jikji ?? {};
|
|
9228
8954
|
if (config.parserOptions)
|
|
9229
8955
|
opts.parserOptions = config.parserOptions;
|
|
@@ -9448,7 +9174,7 @@ function writeConfigObject(path, config) {
|
|
|
9448
9174
|
if (config === null || typeof config !== "object" || Array.isArray(config)) {
|
|
9449
9175
|
throw new ConfigError("Config file must be a JSON object");
|
|
9450
9176
|
}
|
|
9451
|
-
|
|
9177
|
+
mkdirSync7(dirname6(path), { recursive: true });
|
|
9452
9178
|
const contents = `${JSON.stringify(config, null, 2)}
|
|
9453
9179
|
`;
|
|
9454
9180
|
const lock = acquireConfigWriteLock(path);
|
|
@@ -9461,7 +9187,7 @@ function writeConfigObject(path, config) {
|
|
|
9461
9187
|
function writeDefaultConfig(path, partial, opts = {}) {
|
|
9462
9188
|
const cwd = resolve5(opts.cwd ?? process.cwd());
|
|
9463
9189
|
const workspacePath = resolvePersistedPath(partial.workspacePath ?? ".", cwd);
|
|
9464
|
-
const memoryPath = partial.memoryPath === undefined ?
|
|
9190
|
+
const memoryPath = partial.memoryPath === undefined ? join10(resolveAutoRAGHome(opts.env), "memory.json") : resolvePersistedPath(partial.memoryPath, workspacePath);
|
|
9465
9191
|
const full = {
|
|
9466
9192
|
searchPaths: resolveSearchPaths(partial.searchPaths ?? ["."], cwd),
|
|
9467
9193
|
workspacePath,
|
|
@@ -9470,10 +9196,8 @@ function writeDefaultConfig(path, partial, opts = {}) {
|
|
|
9470
9196
|
if (partial.model)
|
|
9471
9197
|
full.model = partial.model;
|
|
9472
9198
|
const normalizedMethods = normalizeIndexingConfig({
|
|
9473
|
-
bm25: partial.bm25,
|
|
9474
9199
|
minSync: partial.minSync
|
|
9475
9200
|
});
|
|
9476
|
-
full.bm25 = normalizedMethods.bm25;
|
|
9477
9201
|
full.minSync = normalizedMethods.minSync;
|
|
9478
9202
|
full.jikji = partial.jikji ?? {};
|
|
9479
9203
|
full.dupey = partial.dupey ?? { enabled: true };
|
|
@@ -9482,7 +9206,7 @@ function writeDefaultConfig(path, partial, opts = {}) {
|
|
|
9482
9206
|
full.parserOptions = partial.parserOptions;
|
|
9483
9207
|
if (partial.ui !== undefined)
|
|
9484
9208
|
full.ui = normalizeUiConfig(partial.ui);
|
|
9485
|
-
|
|
9209
|
+
mkdirSync7(dirname6(path), { recursive: true });
|
|
9486
9210
|
const contents = `${JSON.stringify(full, null, 2)}
|
|
9487
9211
|
`;
|
|
9488
9212
|
const lock = acquireConfigWriteLock(path);
|
|
@@ -9494,7 +9218,7 @@ function writeDefaultConfig(path, partial, opts = {}) {
|
|
|
9494
9218
|
replaceFileAtomically(path, contents, lock.assertOwned);
|
|
9495
9219
|
else {
|
|
9496
9220
|
lock.assertOwned();
|
|
9497
|
-
|
|
9221
|
+
writeFileSync5(path, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
|
|
9498
9222
|
}
|
|
9499
9223
|
} catch (error) {
|
|
9500
9224
|
if (!opts.force && isEexistError(error)) {
|
|
@@ -9505,7 +9229,7 @@ function writeDefaultConfig(path, partial, opts = {}) {
|
|
|
9505
9229
|
lock.release();
|
|
9506
9230
|
}
|
|
9507
9231
|
}
|
|
9508
|
-
var DEFAULT_CONFIG_FILENAME = "config.json", LEGACY_CONFIG_FILENAME = "autorag.config.json", CONFIG_LOCK_RETRY_MS = 10, CONFIG_LOCK_TIMEOUT_MS = 1e4, CONFIG_LOCK_STALE_MS = 30000, ConfigError, AGENT_MODEL_APIS, AGENT_MODEL_INPUTS, API_KEY_ENV_PATTERN, POSITIVE_INT_FIELDS, EMBEDDER_ALLOWLIST,
|
|
9232
|
+
var DEFAULT_CONFIG_FILENAME = "config.json", LEGACY_CONFIG_FILENAME = "autorag.config.json", CONFIG_LOCK_RETRY_MS = 10, CONFIG_LOCK_TIMEOUT_MS = 1e4, CONFIG_LOCK_STALE_MS = 30000, ConfigError, AGENT_MODEL_APIS, AGENT_MODEL_INPUTS, API_KEY_ENV_PATTERN, POSITIVE_INT_FIELDS, EMBEDDER_ALLOWLIST, MINSYNC_ALLOWLIST, MINSYNC_LEGACY_IGNORED, UI_TOKEN_ENV_PATTERN;
|
|
9509
9233
|
var init_config2 = __esm(() => {
|
|
9510
9234
|
init_local_model();
|
|
9511
9235
|
init_home();
|
|
@@ -9540,27 +9264,21 @@ var init_config2 = __esm(() => {
|
|
|
9540
9264
|
"maxRetries",
|
|
9541
9265
|
"maxConcurrent"
|
|
9542
9266
|
]);
|
|
9543
|
-
BM25_ALLOWLIST = new Set(["enabled"]);
|
|
9544
|
-
BM25_LEGACY_IGNORED = new Set(["indexPath", "fallback", "forceEngine", "importBinding"]);
|
|
9545
9267
|
MINSYNC_ALLOWLIST = new Set([
|
|
9546
9268
|
"enabled",
|
|
9547
9269
|
"autoInstall",
|
|
9548
|
-
"binaryPath",
|
|
9549
9270
|
"workspacePath",
|
|
9550
9271
|
"maxChunkSize",
|
|
9551
9272
|
"installer",
|
|
9552
9273
|
"embedder"
|
|
9553
9274
|
]);
|
|
9275
|
+
MINSYNC_LEGACY_IGNORED = new Set(["binaryPath"]);
|
|
9554
9276
|
UI_TOKEN_ENV_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
9555
9277
|
});
|
|
9556
9278
|
|
|
9557
9279
|
// src/cli/commands/init.ts
|
|
9558
|
-
|
|
9559
|
-
|
|
9560
|
-
runInit: () => runInit
|
|
9561
|
-
});
|
|
9562
|
-
import { existsSync as existsSync6, readFileSync as readFileSync9 } from "node:fs";
|
|
9563
|
-
import { dirname as dirname8 } from "node:path";
|
|
9280
|
+
import { existsSync as existsSync6, readFileSync as readFileSync8 } from "node:fs";
|
|
9281
|
+
import { dirname as dirname7 } from "node:path";
|
|
9564
9282
|
async function runInit(ctx) {
|
|
9565
9283
|
const flags = ctx.flags;
|
|
9566
9284
|
const partial = {};
|
|
@@ -9617,14 +9335,13 @@ async function runInit(ctx) {
|
|
|
9617
9335
|
if (migratingLegacy) {
|
|
9618
9336
|
try {
|
|
9619
9337
|
const legacyPath = resolvedPath.legacyPath;
|
|
9620
|
-
const legacy = JSON.parse(
|
|
9621
|
-
const normalizedLegacy = normalizeLegacyConfigPaths(legacy,
|
|
9338
|
+
const legacy = JSON.parse(readFileSync8(legacyPath, "utf8"));
|
|
9339
|
+
const normalizedLegacy = normalizeLegacyConfigPaths(legacy, dirname7(legacyPath));
|
|
9622
9340
|
partial.searchPaths ??= normalizedLegacy.searchPaths;
|
|
9623
9341
|
partial.workspacePath ??= normalizedLegacy.workspacePath;
|
|
9624
9342
|
partial.memoryPath ??= normalizedLegacy.memoryPath;
|
|
9625
9343
|
partial.model ??= legacy.model;
|
|
9626
9344
|
partial.minSync ??= legacy.minSync;
|
|
9627
|
-
partial.bm25 ??= legacy.bm25;
|
|
9628
9345
|
partial.jikji ??= legacy.jikji;
|
|
9629
9346
|
partial.parserOptions ??= legacy.parserOptions;
|
|
9630
9347
|
partial.dupey ??= legacy.dupey;
|
|
@@ -9806,7 +9523,7 @@ class DatasourceResultFilter {
|
|
|
9806
9523
|
}
|
|
9807
9524
|
|
|
9808
9525
|
// src/dupey/cli.ts
|
|
9809
|
-
import { spawn as
|
|
9526
|
+
import { spawn as spawn9 } from "node:child_process";
|
|
9810
9527
|
import { resolve as resolve6 } from "node:path";
|
|
9811
9528
|
async function scanWithDupey(dir, options = {}) {
|
|
9812
9529
|
const absoluteDir = resolve6(dir);
|
|
@@ -9829,7 +9546,7 @@ function isScanResult(value) {
|
|
|
9829
9546
|
}
|
|
9830
9547
|
function spawnDupey(executable, args, options) {
|
|
9831
9548
|
return new Promise((resolveOutput, reject) => {
|
|
9832
|
-
const child =
|
|
9549
|
+
const child = spawn9(executable, [...args], {
|
|
9833
9550
|
cwd: options.cwd,
|
|
9834
9551
|
stdio: ["ignore", "pipe", "pipe"]
|
|
9835
9552
|
});
|
|
@@ -9971,7 +9688,7 @@ var init_diagnostics = __esm(() => {
|
|
|
9971
9688
|
|
|
9972
9689
|
// src/jikji/answer-pack.ts
|
|
9973
9690
|
import { realpathSync as realpathSync2 } from "node:fs";
|
|
9974
|
-
import { isAbsolute as isAbsolute5, join as
|
|
9691
|
+
import { isAbsolute as isAbsolute5, join as join11, relative as relative2, resolve as resolve8 } from "node:path";
|
|
9975
9692
|
function isRecord8(value) {
|
|
9976
9693
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
9977
9694
|
}
|
|
@@ -10100,7 +9817,7 @@ function normalizeJikjiAnswerPath(rawPath, sourceRoots) {
|
|
|
10100
9817
|
const rel = relative2(realRoot, realCandidate);
|
|
10101
9818
|
if (rel.startsWith("..") || isAbsolute5(rel))
|
|
10102
9819
|
continue;
|
|
10103
|
-
return
|
|
9820
|
+
return join11(root.rootPath, rel);
|
|
10104
9821
|
}
|
|
10105
9822
|
return;
|
|
10106
9823
|
}
|
|
@@ -10114,9 +9831,9 @@ var init_answer_pack = __esm(() => {
|
|
|
10114
9831
|
});
|
|
10115
9832
|
|
|
10116
9833
|
// src/jikji/installer.ts
|
|
10117
|
-
import { spawn as
|
|
9834
|
+
import { spawn as spawn10 } from "node:child_process";
|
|
10118
9835
|
import { existsSync as existsSync7 } from "node:fs";
|
|
10119
|
-
import { delimiter, join as
|
|
9836
|
+
import { delimiter, join as join12 } from "node:path";
|
|
10120
9837
|
function jikjiExecutableName(platform = process.platform) {
|
|
10121
9838
|
return platform === "win32" ? "jikji.exe" : "jikji";
|
|
10122
9839
|
}
|
|
@@ -10124,7 +9841,7 @@ function cargoExecutableName(platform = process.platform) {
|
|
|
10124
9841
|
return platform === "win32" ? "cargo.exe" : "cargo";
|
|
10125
9842
|
}
|
|
10126
9843
|
function cachedJikjiBinaryPath(root, platform = process.platform) {
|
|
10127
|
-
return
|
|
9844
|
+
return join12(root, ".autorag", "bin", jikjiExecutableName(platform));
|
|
10128
9845
|
}
|
|
10129
9846
|
function lookupExecutableInPath(executable, env) {
|
|
10130
9847
|
const pathEnv = env.PATH;
|
|
@@ -10133,7 +9850,7 @@ function lookupExecutableInPath(executable, env) {
|
|
|
10133
9850
|
for (const dir of pathEnv.split(delimiter)) {
|
|
10134
9851
|
if (dir.length === 0)
|
|
10135
9852
|
continue;
|
|
10136
|
-
const candidate =
|
|
9853
|
+
const candidate = join12(dir, executable);
|
|
10137
9854
|
if (existsSync7(candidate))
|
|
10138
9855
|
return candidate;
|
|
10139
9856
|
}
|
|
@@ -10141,7 +9858,7 @@ function lookupExecutableInPath(executable, env) {
|
|
|
10141
9858
|
}
|
|
10142
9859
|
function defaultRunner(cargo, env) {
|
|
10143
9860
|
return (args, timeoutMs) => new Promise((resolvePromise) => {
|
|
10144
|
-
const child =
|
|
9861
|
+
const child = spawn10(cargo, args, { env, stdio: ["ignore", "ignore", "pipe"] });
|
|
10145
9862
|
let stderr = "";
|
|
10146
9863
|
const timeout = setTimeout(() => {
|
|
10147
9864
|
if (!child.killed)
|
|
@@ -10175,7 +9892,7 @@ async function ensureJikjiBinary(options) {
|
|
|
10175
9892
|
message: "Jikji auto-install needs the Rust toolchain (cargo) on PATH — install via https://rustup.rs, or install jikji manually (`cargo install jikji-cli`)."
|
|
10176
9893
|
};
|
|
10177
9894
|
}
|
|
10178
|
-
const args = ["install", JIKJI_CRATE_NAME, "--locked", "--root",
|
|
9895
|
+
const args = ["install", JIKJI_CRATE_NAME, "--locked", "--root", join12(options.root, ".autorag")];
|
|
10179
9896
|
if (options.version !== undefined)
|
|
10180
9897
|
args.push("--version", options.version);
|
|
10181
9898
|
const runner = options.runner ?? defaultRunner(cargo, env);
|
|
@@ -10195,8 +9912,9 @@ var init_installer = __esm(() => {
|
|
|
10195
9912
|
});
|
|
10196
9913
|
|
|
10197
9914
|
// src/jikji/client.ts
|
|
10198
|
-
import { spawn as
|
|
9915
|
+
import { spawn as spawn11 } from "node:child_process";
|
|
10199
9916
|
import { existsSync as existsSync8 } from "node:fs";
|
|
9917
|
+
import { join as join13 } from "node:path";
|
|
10200
9918
|
|
|
10201
9919
|
class JikjiClient {
|
|
10202
9920
|
options;
|
|
@@ -10204,13 +9922,16 @@ class JikjiClient {
|
|
|
10204
9922
|
constructor(options = {}) {
|
|
10205
9923
|
this.options = options;
|
|
10206
9924
|
}
|
|
9925
|
+
isPrepared(root) {
|
|
9926
|
+
return existsSync8(join13(root, ".jikji", "doc_meta")) || existsSync8(join13(root, ".jikji", "doc_text"));
|
|
9927
|
+
}
|
|
10207
9928
|
async resolveCommand() {
|
|
10208
|
-
if (this.options.binaryPath !== undefined && this.options.binaryPath !==
|
|
9929
|
+
if (this.options.binaryPath !== undefined && this.options.binaryPath !== DEFAULT_BINARY4) {
|
|
10209
9930
|
return commandFor2(this.options.binaryPath);
|
|
10210
9931
|
}
|
|
10211
9932
|
if (this.resolvedCommand !== undefined)
|
|
10212
9933
|
return this.resolvedCommand;
|
|
10213
|
-
const pathCommand = lookupExecutableInPath(
|
|
9934
|
+
const pathCommand = lookupExecutableInPath(DEFAULT_BINARY4, controlledEnv5(this.options.env));
|
|
10214
9935
|
if (pathCommand !== undefined) {
|
|
10215
9936
|
this.resolvedCommand = pathCommand;
|
|
10216
9937
|
return this.resolvedCommand;
|
|
@@ -10229,7 +9950,7 @@ class JikjiClient {
|
|
|
10229
9950
|
}
|
|
10230
9951
|
}
|
|
10231
9952
|
}
|
|
10232
|
-
this.resolvedCommand =
|
|
9953
|
+
this.resolvedCommand = DEFAULT_BINARY4;
|
|
10233
9954
|
return this.resolvedCommand;
|
|
10234
9955
|
}
|
|
10235
9956
|
async prepare(root, options = {}) {
|
|
@@ -10295,10 +10016,10 @@ class JikjiClient {
|
|
|
10295
10016
|
}
|
|
10296
10017
|
}
|
|
10297
10018
|
function spawnJikji(request) {
|
|
10298
|
-
return new Promise((
|
|
10019
|
+
return new Promise((resolve) => {
|
|
10299
10020
|
const options = request.options;
|
|
10300
10021
|
const portable = portableSpawnCommand(request.command, request.args);
|
|
10301
|
-
const child =
|
|
10022
|
+
const child = spawn11(portable.command, [...portable.args], {
|
|
10302
10023
|
env: controlledEnv5(options.env),
|
|
10303
10024
|
...request.cwd === undefined ? {} : { cwd: request.cwd },
|
|
10304
10025
|
stdio: ["ignore", "pipe", "pipe"]
|
|
@@ -10310,7 +10031,7 @@ function spawnJikji(request) {
|
|
|
10310
10031
|
const timeout = setTimeout(() => {
|
|
10311
10032
|
finalReason = "timeout";
|
|
10312
10033
|
terminate7(child);
|
|
10313
|
-
}, options.timeoutMs ??
|
|
10034
|
+
}, options.timeoutMs ?? DEFAULT_TIMEOUT_MS7);
|
|
10314
10035
|
const abortHandler = () => {
|
|
10315
10036
|
finalReason = "aborted";
|
|
10316
10037
|
terminate7(child);
|
|
@@ -10340,7 +10061,7 @@ function spawnJikji(request) {
|
|
|
10340
10061
|
settled = true;
|
|
10341
10062
|
clearTimeout(timeout);
|
|
10342
10063
|
request.signal?.removeEventListener("abort", abortHandler);
|
|
10343
|
-
|
|
10064
|
+
resolve({ ok: false, reason: "spawn-error", stdout: stdout.text, stderr: error.message, code: null });
|
|
10344
10065
|
});
|
|
10345
10066
|
child.on("close", (code) => {
|
|
10346
10067
|
if (settled)
|
|
@@ -10349,15 +10070,15 @@ function spawnJikji(request) {
|
|
|
10349
10070
|
clearTimeout(timeout);
|
|
10350
10071
|
request.signal?.removeEventListener("abort", abortHandler);
|
|
10351
10072
|
if (finalReason !== undefined) {
|
|
10352
|
-
|
|
10073
|
+
resolve({ ok: false, reason: finalReason, stdout: stdout.text, stderr: stderr.text, code });
|
|
10353
10074
|
return;
|
|
10354
10075
|
}
|
|
10355
|
-
|
|
10076
|
+
resolve({ ok: code === 0, stdout: stdout.text, stderr: stderr.text, code });
|
|
10356
10077
|
});
|
|
10357
10078
|
});
|
|
10358
10079
|
}
|
|
10359
10080
|
function commandFor2(binaryPath) {
|
|
10360
|
-
return binaryPath === undefined || binaryPath ===
|
|
10081
|
+
return binaryPath === undefined || binaryPath === DEFAULT_BINARY4 ? DEFAULT_BINARY4 : binaryPath;
|
|
10361
10082
|
}
|
|
10362
10083
|
function buildPrepareArgs(options, root) {
|
|
10363
10084
|
const args = ["prepare", root, "--json"];
|
|
@@ -10424,7 +10145,7 @@ function appendBounded5(state, chunk, maxBytes) {
|
|
|
10424
10145
|
const remainingBytes = Math.max(maxBytes - state.bytes, 0);
|
|
10425
10146
|
return { text: state.text + chunk.slice(0, remainingBytes), bytes: maxBytes, capped: true };
|
|
10426
10147
|
}
|
|
10427
|
-
var
|
|
10148
|
+
var DEFAULT_BINARY4 = "jikji", DEFAULT_TIMEOUT_MS7 = 1e4, DEFAULT_MAX_BUFFER_BYTES4 = 1048576;
|
|
10428
10149
|
var init_client6 = __esm(() => {
|
|
10429
10150
|
init_portable_spawn();
|
|
10430
10151
|
init_answer_pack();
|
|
@@ -10493,7 +10214,7 @@ var init_jikji = __esm(() => {
|
|
|
10493
10214
|
});
|
|
10494
10215
|
|
|
10495
10216
|
// src/manifest/loader.ts
|
|
10496
|
-
import { existsSync as existsSync9, readdirSync as readdirSync2, readFileSync as
|
|
10217
|
+
import { existsSync as existsSync9, readdirSync as readdirSync2, readFileSync as readFileSync9 } from "node:fs";
|
|
10497
10218
|
import { extname as extname7, join as join14 } from "node:path";
|
|
10498
10219
|
import { parse as parseYaml } from "yaml";
|
|
10499
10220
|
function isValidManifest(data) {
|
|
@@ -10503,7 +10224,7 @@ function isValidManifest(data) {
|
|
|
10503
10224
|
return typeof obj.name === "string" && obj.name.length > 0 && typeof obj.description === "string" && obj.description.length > 0 && typeof obj.type === "string" && ["vector", "bm25", "hybrid", "visual"].includes(obj.type);
|
|
10504
10225
|
}
|
|
10505
10226
|
function loadManifest2(filePath) {
|
|
10506
|
-
const content =
|
|
10227
|
+
const content = readFileSync9(filePath, "utf-8");
|
|
10507
10228
|
const ext = extname7(filePath).toLowerCase();
|
|
10508
10229
|
let data;
|
|
10509
10230
|
if (ext === ".yaml" || ext === ".yml") {
|
|
@@ -10700,8 +10421,8 @@ var init_evidence_id = () => {};
|
|
|
10700
10421
|
|
|
10701
10422
|
// src/memory/memory.ts
|
|
10702
10423
|
import { createHash as createHash5, randomUUID as randomUUID4 } from "node:crypto";
|
|
10703
|
-
import { existsSync as existsSync10, mkdirSync as
|
|
10704
|
-
import { dirname as
|
|
10424
|
+
import { existsSync as existsSync10, mkdirSync as mkdirSync8, readFileSync as readFileSync10, renameSync as renameSync4, unlinkSync as unlinkSync3, writeFileSync as writeFileSync6 } from "node:fs";
|
|
10425
|
+
import { dirname as dirname8 } from "node:path";
|
|
10705
10426
|
function emptyMemoryV4() {
|
|
10706
10427
|
return {
|
|
10707
10428
|
version: 4,
|
|
@@ -10830,8 +10551,8 @@ function migrateV3(data) {
|
|
|
10830
10551
|
function hashText(value) {
|
|
10831
10552
|
return createHash5("sha256").update(value).digest("hex");
|
|
10832
10553
|
}
|
|
10833
|
-
function resultId(sessionId,
|
|
10834
|
-
return `${sessionId}:${
|
|
10554
|
+
function resultId(sessionId, number) {
|
|
10555
|
+
return `${sessionId}:${number}`;
|
|
10835
10556
|
}
|
|
10836
10557
|
function resultHash(query, title, summary, evidenceIds) {
|
|
10837
10558
|
return hashText([query, title, summary, ...evidenceIds].join("\x00"));
|
|
@@ -10938,9 +10659,9 @@ class RetrievalMemory {
|
|
|
10938
10659
|
this.persistedData = cloneMemory(this.data);
|
|
10939
10660
|
}
|
|
10940
10661
|
save() {
|
|
10941
|
-
const dir =
|
|
10662
|
+
const dir = dirname8(this.storagePath);
|
|
10942
10663
|
if (!existsSync10(dir))
|
|
10943
|
-
|
|
10664
|
+
mkdirSync8(dir, { recursive: true });
|
|
10944
10665
|
const lock = this.acquireLock();
|
|
10945
10666
|
const localData = this.data;
|
|
10946
10667
|
let tmpPath;
|
|
@@ -10948,7 +10669,7 @@ class RetrievalMemory {
|
|
|
10948
10669
|
this.data = this.mergeWithPersisted(this.readPersistedData());
|
|
10949
10670
|
this.capData();
|
|
10950
10671
|
tmpPath = `${this.storagePath}.${randomUUID4()}.tmp`;
|
|
10951
|
-
|
|
10672
|
+
writeFileSync6(tmpPath, `${JSON.stringify(this.data, null, 2)}
|
|
10952
10673
|
`, "utf-8");
|
|
10953
10674
|
lock.assertOwned();
|
|
10954
10675
|
renameSync4(tmpPath, this.storagePath);
|
|
@@ -11026,11 +10747,11 @@ class RetrievalMemory {
|
|
|
11026
10747
|
timestamp: Date.now()
|
|
11027
10748
|
});
|
|
11028
10749
|
const evidenceWeight = curated.evidenceIds.length > 0 ? explicitWeight / curated.evidenceIds.length : 0;
|
|
11029
|
-
for (const
|
|
11030
|
-
const evidence = this.data.evidenceChunks.find((chunk) => chunk.stableEvidenceId ===
|
|
10750
|
+
for (const stableEvidenceId of curated.evidenceIds) {
|
|
10751
|
+
const evidence = this.data.evidenceChunks.find((chunk) => chunk.stableEvidenceId === stableEvidenceId);
|
|
11031
10752
|
this.data.feedbackSignals.push({
|
|
11032
10753
|
id: randomUUID4(),
|
|
11033
|
-
target: { type: "evidence_chunk", stableEvidenceId
|
|
10754
|
+
target: { type: "evidence_chunk", stableEvidenceId },
|
|
11034
10755
|
query: curated.query,
|
|
11035
10756
|
method: evidence?.method,
|
|
11036
10757
|
sentiment,
|
|
@@ -11116,8 +10837,8 @@ class RetrievalMemory {
|
|
|
11116
10837
|
for (const signal of this.data.feedbackSignals) {
|
|
11117
10838
|
if (signal.target.type !== "evidence_chunk")
|
|
11118
10839
|
continue;
|
|
11119
|
-
const
|
|
11120
|
-
const evidence = this.data.evidenceChunks.find((chunk) => chunk.stableEvidenceId ===
|
|
10840
|
+
const stableEvidenceId = signal.target.stableEvidenceId;
|
|
10841
|
+
const evidence = this.data.evidenceChunks.find((chunk) => chunk.stableEvidenceId === stableEvidenceId);
|
|
11121
10842
|
if (!evidence)
|
|
11122
10843
|
continue;
|
|
11123
10844
|
add(documentAreas, evidence.documentArea, signal);
|
|
@@ -11214,7 +10935,7 @@ class RetrievalMemory {
|
|
|
11214
10935
|
if (!existsSync10(this.storagePath))
|
|
11215
10936
|
return emptyMemoryV4();
|
|
11216
10937
|
try {
|
|
11217
|
-
const parsed = JSON.parse(
|
|
10938
|
+
const parsed = JSON.parse(readFileSync10(this.storagePath, "utf-8"));
|
|
11218
10939
|
if (isV4(parsed))
|
|
11219
10940
|
return normalizeV4(parsed);
|
|
11220
10941
|
if (isV3(parsed))
|
|
@@ -11478,7 +11199,7 @@ var init_memory = __esm(() => {
|
|
|
11478
11199
|
});
|
|
11479
11200
|
|
|
11480
11201
|
// src/minsync/embedder-config.ts
|
|
11481
|
-
import { readFileSync as
|
|
11202
|
+
import { readFileSync as readFileSync11, writeFileSync as writeFileSync7 } from "node:fs";
|
|
11482
11203
|
import { join as join15 } from "node:path";
|
|
11483
11204
|
import { parse as parse3, stringify } from "smol-toml";
|
|
11484
11205
|
function minSyncConfigPath(workspacePath) {
|
|
@@ -11487,7 +11208,7 @@ function minSyncConfigPath(workspacePath) {
|
|
|
11487
11208
|
function configuredMaxChunkSize(workspacePath) {
|
|
11488
11209
|
let raw;
|
|
11489
11210
|
try {
|
|
11490
|
-
raw =
|
|
11211
|
+
raw = readFileSync11(minSyncConfigPath(workspacePath), "utf8");
|
|
11491
11212
|
} catch {
|
|
11492
11213
|
return;
|
|
11493
11214
|
}
|
|
@@ -11505,7 +11226,7 @@ function rewriteEmbedderConfig(workspacePath, embedder, chunkerConfig = {}) {
|
|
|
11505
11226
|
const configPath = minSyncConfigPath(workspacePath);
|
|
11506
11227
|
let raw;
|
|
11507
11228
|
try {
|
|
11508
|
-
raw =
|
|
11229
|
+
raw = readFileSync11(configPath, "utf8");
|
|
11509
11230
|
} catch {
|
|
11510
11231
|
return false;
|
|
11511
11232
|
}
|
|
@@ -11541,18 +11262,18 @@ function rewriteEmbedderConfig(workspacePath, embedder, chunkerConfig = {}) {
|
|
|
11541
11262
|
options.dimension = embedder.dimension;
|
|
11542
11263
|
if (chunkerConfig.maxChunkSize !== undefined)
|
|
11543
11264
|
chunkerOptions.max_chunk_size = chunkerConfig.maxChunkSize;
|
|
11544
|
-
|
|
11265
|
+
writeFileSync7(configPath, stringify(parsed));
|
|
11545
11266
|
return true;
|
|
11546
11267
|
}
|
|
11547
11268
|
var MINSYNC_CONFIG_DIR = ".minsync", MINSYNC_CONFIG_FILE = "config.toml";
|
|
11548
11269
|
var init_embedder_config = () => {};
|
|
11549
11270
|
|
|
11550
11271
|
// src/minsync/process.ts
|
|
11551
|
-
import { spawn as
|
|
11272
|
+
import { spawn as spawn12 } from "node:child_process";
|
|
11552
11273
|
function spawnProcess(command, args, cwd, options = {}) {
|
|
11553
|
-
return new Promise((
|
|
11274
|
+
return new Promise((resolve) => {
|
|
11554
11275
|
const portable = portableSpawnCommand(command, args);
|
|
11555
|
-
const child =
|
|
11276
|
+
const child = spawn12(portable.command, [...portable.args], { cwd, stdio: ["ignore", "pipe", "pipe"] });
|
|
11556
11277
|
let stdout = "";
|
|
11557
11278
|
let stderr = "";
|
|
11558
11279
|
let timedOut = false;
|
|
@@ -11571,15 +11292,15 @@ function spawnProcess(command, args, cwd, options = {}) {
|
|
|
11571
11292
|
child.on("error", (error) => {
|
|
11572
11293
|
if (timer)
|
|
11573
11294
|
clearTimeout(timer);
|
|
11574
|
-
|
|
11295
|
+
resolve({ ok: false, stdout, stderr: error.message, code: null });
|
|
11575
11296
|
});
|
|
11576
11297
|
child.on("close", (code) => {
|
|
11577
11298
|
if (timer)
|
|
11578
11299
|
clearTimeout(timer);
|
|
11579
11300
|
if (timedOut) {
|
|
11580
|
-
|
|
11301
|
+
resolve({ ok: false, stdout, stderr: "process timed out", code: null, timedOut: true });
|
|
11581
11302
|
} else {
|
|
11582
|
-
|
|
11303
|
+
resolve({ ok: code === 0, stdout, stderr, code });
|
|
11583
11304
|
}
|
|
11584
11305
|
});
|
|
11585
11306
|
});
|
|
@@ -11589,7 +11310,7 @@ var init_process = __esm(() => {
|
|
|
11589
11310
|
});
|
|
11590
11311
|
|
|
11591
11312
|
// src/minsync/client.ts
|
|
11592
|
-
import { existsSync as existsSync11, readFileSync as
|
|
11313
|
+
import { existsSync as existsSync11, readFileSync as readFileSync12, rmSync as rmSync2, writeFileSync as writeFileSync8 } from "node:fs";
|
|
11593
11314
|
import { join as join16 } from "node:path";
|
|
11594
11315
|
|
|
11595
11316
|
class MinSyncClient {
|
|
@@ -11647,7 +11368,7 @@ class MinSyncClient {
|
|
|
11647
11368
|
const configRewritten = shouldRewriteConfig && rewriteEmbedderConfig(this.workspacePath, this.embedder ?? {}, { maxChunkSize: this.maxChunkSize });
|
|
11648
11369
|
const restoreConfig = () => {
|
|
11649
11370
|
if (configRewritten && originalConfig !== undefined)
|
|
11650
|
-
|
|
11371
|
+
writeFileSync8(configPath, originalConfig);
|
|
11651
11372
|
};
|
|
11652
11373
|
const check = await this.spawn(["check", "--format", "json"], spawnOpts);
|
|
11653
11374
|
if (!check.ok) {
|
|
@@ -11702,7 +11423,7 @@ class MinSyncClient {
|
|
|
11702
11423
|
}
|
|
11703
11424
|
function readConfigSnapshot(configPath) {
|
|
11704
11425
|
try {
|
|
11705
|
-
return
|
|
11426
|
+
return readFileSync12(configPath, "utf8");
|
|
11706
11427
|
} catch (error) {
|
|
11707
11428
|
if (error.code === "ENOENT")
|
|
11708
11429
|
return;
|
|
@@ -11765,11 +11486,11 @@ var init_client7 = __esm(() => {
|
|
|
11765
11486
|
|
|
11766
11487
|
// src/minsync/installer.ts
|
|
11767
11488
|
import { createHash as createHash6 } from "node:crypto";
|
|
11768
|
-
import { copyFileSync, createWriteStream, existsSync as existsSync12, mkdirSync as
|
|
11489
|
+
import { copyFileSync, createWriteStream, existsSync as existsSync12, mkdirSync as mkdirSync9, renameSync as renameSync5, rmSync as rmSync3 } from "node:fs";
|
|
11769
11490
|
import { chmod, mkdtemp as mkdtemp2 } from "node:fs/promises";
|
|
11770
11491
|
import { get } from "node:https";
|
|
11771
11492
|
import { tmpdir as tmpdir2 } from "node:os";
|
|
11772
|
-
import { basename as basename6, dirname as
|
|
11493
|
+
import { basename as basename6, dirname as dirname9, join as join17 } from "node:path";
|
|
11773
11494
|
import { pipeline } from "node:stream/promises";
|
|
11774
11495
|
async function ensureMinSyncBinary(options) {
|
|
11775
11496
|
const binaryPath = join17(options.root, ".autorag", "bin", executableName(options.platform ?? process.platform));
|
|
@@ -11787,15 +11508,15 @@ async function ensureMinSyncBinary(options) {
|
|
|
11787
11508
|
}
|
|
11788
11509
|
requireSha256(asset);
|
|
11789
11510
|
const assetInstaller = options.assetInstaller ?? installReleaseAsset;
|
|
11790
|
-
|
|
11511
|
+
mkdirSync9(dirname9(binaryPath), { recursive: true });
|
|
11791
11512
|
await assetInstaller(asset, binaryPath);
|
|
11792
11513
|
await chmod(binaryPath, 493);
|
|
11793
11514
|
return { binaryPath, version: release.tagName };
|
|
11794
11515
|
}
|
|
11795
11516
|
async function installMinSyncFromCargo(options, destination) {
|
|
11796
11517
|
const cargoRoot = join17(options.root, ".autorag", "minsync-cargo");
|
|
11797
|
-
|
|
11798
|
-
|
|
11518
|
+
mkdirSync9(dirname9(destination), { recursive: true });
|
|
11519
|
+
mkdirSync9(cargoRoot, { recursive: true });
|
|
11799
11520
|
const result = await spawnProcess("cargo", ["install", "minsync", "--version", MINSYNC_VERSION, "--locked", "--root", cargoRoot], options.root);
|
|
11800
11521
|
if (!result.ok) {
|
|
11801
11522
|
throw new MinSyncReleaseError(result.stderr || `Could not install MinSync ${MINSYNC_VERSION} from crates.io`);
|
|
@@ -11859,11 +11580,11 @@ async function verifySha256(path, expected) {
|
|
|
11859
11580
|
throw new MinSyncReleaseError(`MinSync asset sha256 mismatch: expected ${expected}, got ${actual}`);
|
|
11860
11581
|
}
|
|
11861
11582
|
function readHttpsText(url, redirectCount = 0) {
|
|
11862
|
-
return new Promise((
|
|
11583
|
+
return new Promise((resolve, reject) => {
|
|
11863
11584
|
get(url, { headers: { "User-Agent": "AutoRAG-MinSync" } }, (response) => {
|
|
11864
11585
|
if (isRedirect(response.statusCode) && response.headers.location) {
|
|
11865
11586
|
response.resume();
|
|
11866
|
-
readHttpsText(new URL(response.headers.location, url).toString(), redirectCount + 1).then(
|
|
11587
|
+
readHttpsText(new URL(response.headers.location, url).toString(), redirectCount + 1).then(resolve, reject);
|
|
11867
11588
|
return;
|
|
11868
11589
|
}
|
|
11869
11590
|
if (redirectCount > 5) {
|
|
@@ -11882,17 +11603,17 @@ function readHttpsText(url, redirectCount = 0) {
|
|
|
11882
11603
|
body += chunk;
|
|
11883
11604
|
});
|
|
11884
11605
|
response.on("end", () => {
|
|
11885
|
-
|
|
11606
|
+
resolve(body);
|
|
11886
11607
|
});
|
|
11887
11608
|
}).on("error", reject);
|
|
11888
11609
|
});
|
|
11889
11610
|
}
|
|
11890
11611
|
function openHttps(url, redirectCount = 0) {
|
|
11891
|
-
return new Promise((
|
|
11612
|
+
return new Promise((resolve, reject) => {
|
|
11892
11613
|
get(url, { headers: { "User-Agent": "AutoRAG-MinSync" } }, (response) => {
|
|
11893
11614
|
if (isRedirect(response.statusCode) && response.headers.location) {
|
|
11894
11615
|
response.resume();
|
|
11895
|
-
openHttps(new URL(response.headers.location, url).toString(), redirectCount + 1).then(
|
|
11616
|
+
openHttps(new URL(response.headers.location, url).toString(), redirectCount + 1).then(resolve, reject);
|
|
11896
11617
|
return;
|
|
11897
11618
|
}
|
|
11898
11619
|
if (redirectCount > 5) {
|
|
@@ -11905,7 +11626,7 @@ function openHttps(url, redirectCount = 0) {
|
|
|
11905
11626
|
response.resume();
|
|
11906
11627
|
return;
|
|
11907
11628
|
}
|
|
11908
|
-
|
|
11629
|
+
resolve(response);
|
|
11909
11630
|
}).on("error", reject);
|
|
11910
11631
|
});
|
|
11911
11632
|
}
|
|
@@ -11952,61 +11673,39 @@ var init_installer2 = __esm(() => {
|
|
|
11952
11673
|
};
|
|
11953
11674
|
});
|
|
11954
11675
|
|
|
11955
|
-
// src/retrieval/methods/bm25.ts
|
|
11956
|
-
import { existsSync as existsSync13, rmSync as rmSync4 } from "node:fs";
|
|
11957
|
-
import { join as join18 } from "node:path";
|
|
11958
|
-
function removeLegacyBm25Artifacts(root) {
|
|
11959
|
-
rmSync4(join18(root, BM25_SUBDIR), { recursive: true, force: true });
|
|
11960
|
-
}
|
|
11961
|
-
function hasLegacyBm25Artifacts(root) {
|
|
11962
|
-
return existsSync13(join18(root, BM25_SUBDIR));
|
|
11963
|
-
}
|
|
11964
|
-
var BM25_SUBDIR, BM25UnavailableError;
|
|
11965
|
-
var init_bm25 = __esm(() => {
|
|
11966
|
-
BM25_SUBDIR = join18(".autorag", "bm25");
|
|
11967
|
-
BM25UnavailableError = class BM25UnavailableError extends Error {
|
|
11968
|
-
readiness;
|
|
11969
|
-
constructor(readiness, message) {
|
|
11970
|
-
super(message);
|
|
11971
|
-
this.name = "BM25UnavailableError";
|
|
11972
|
-
this.readiness = readiness;
|
|
11973
|
-
}
|
|
11974
|
-
};
|
|
11975
|
-
});
|
|
11976
|
-
|
|
11977
11676
|
// src/minsync/paths.ts
|
|
11978
|
-
import { join as
|
|
11677
|
+
import { join as join18 } from "node:path";
|
|
11979
11678
|
function minSyncWorkspaceRoot(root) {
|
|
11980
|
-
return
|
|
11679
|
+
return join18(root, MINSYNC_SUBDIR);
|
|
11981
11680
|
}
|
|
11982
11681
|
function minSyncDocumentPath(workspacePath, virtualPath) {
|
|
11983
11682
|
const safePath = virtualPath.replace(/^\/+/, "");
|
|
11984
|
-
return
|
|
11683
|
+
return join18(workspacePath, MINSYNC_FILES_SUBDIR, `${safePath}.md`);
|
|
11985
11684
|
}
|
|
11986
11685
|
var MINSYNC_SUBDIR, MINSYNC_FILES_SUBDIR = "files";
|
|
11987
11686
|
var init_paths4 = __esm(() => {
|
|
11988
|
-
MINSYNC_SUBDIR =
|
|
11687
|
+
MINSYNC_SUBDIR = join18(".autorag", "minsync");
|
|
11989
11688
|
});
|
|
11990
11689
|
|
|
11991
11690
|
// src/minsync/workspace.ts
|
|
11992
11691
|
import { randomUUID as randomUUID5 } from "node:crypto";
|
|
11993
11692
|
import {
|
|
11994
11693
|
copyFileSync as copyFileSync2,
|
|
11995
|
-
existsSync as
|
|
11694
|
+
existsSync as existsSync13,
|
|
11996
11695
|
lstatSync as lstatSync2,
|
|
11997
|
-
mkdirSync as
|
|
11696
|
+
mkdirSync as mkdirSync10,
|
|
11998
11697
|
readdirSync as readdirSync3,
|
|
11999
|
-
readFileSync as
|
|
11698
|
+
readFileSync as readFileSync13,
|
|
12000
11699
|
renameSync as renameSync6,
|
|
12001
|
-
rmSync as
|
|
12002
|
-
writeFileSync as
|
|
11700
|
+
rmSync as rmSync4,
|
|
11701
|
+
writeFileSync as writeFileSync9
|
|
12003
11702
|
} from "node:fs";
|
|
12004
|
-
import { basename as basename7, dirname as
|
|
11703
|
+
import { basename as basename7, dirname as dirname10, isAbsolute as isAbsolute7, join as join19, relative as relative4, sep as sep3 } from "node:path";
|
|
12005
11704
|
function syncMinSyncWorkspace(root, options = {}) {
|
|
12006
11705
|
const workspacePath = options.workspacePath ?? minSyncWorkspaceRoot(root);
|
|
12007
|
-
const filesRoot =
|
|
11706
|
+
const filesRoot = join19(workspacePath, "files");
|
|
12008
11707
|
const index = loadMirrorIndex(root);
|
|
12009
|
-
const entries = Object.values(index.entries).sort((a, b) => a.virtualPath.localeCompare(b.virtualPath)).filter((entry) => normalizeVirtualPath(entry.virtualPath) === entry.virtualPath &&
|
|
11708
|
+
const entries = Object.values(index.entries).sort((a, b) => a.virtualPath.localeCompare(b.virtualPath)).filter((entry) => normalizeVirtualPath(entry.virtualPath) === entry.virtualPath && existsSync13(entry.outputPath)).map((entry) => {
|
|
12010
11709
|
const minSyncPath = minSyncDocumentPath(workspacePath, entry.virtualPath);
|
|
12011
11710
|
return {
|
|
12012
11711
|
virtualPath: entry.virtualPath,
|
|
@@ -12026,7 +11725,7 @@ function syncMinSyncWorkspace(root, options = {}) {
|
|
|
12026
11725
|
const previousEntry = previousState?.entries[entry.virtualPath];
|
|
12027
11726
|
const unchanged = previousEntry?.outputPath === mirrorEntry.outputPath && previousEntry.updatedAt === mirrorEntry.updatedAt && isRegularFile(entry.minSyncPath);
|
|
12028
11727
|
if (!unchanged) {
|
|
12029
|
-
ensureManagedDirectory(filesRoot,
|
|
11728
|
+
ensureManagedDirectory(filesRoot, dirname10(entry.minSyncPath));
|
|
12030
11729
|
copyFileAtomically(entry.parsedOutputPath, entry.minSyncPath);
|
|
12031
11730
|
}
|
|
12032
11731
|
currentState[entry.virtualPath] = {
|
|
@@ -12040,14 +11739,14 @@ function syncMinSyncWorkspace(root, options = {}) {
|
|
|
12040
11739
|
return { workspacePath, entries };
|
|
12041
11740
|
}
|
|
12042
11741
|
function stagingStatePath(workspacePath) {
|
|
12043
|
-
return
|
|
11742
|
+
return join19(dirname10(workspacePath), `.${basename7(workspacePath)}-autorag-staging.json`);
|
|
12044
11743
|
}
|
|
12045
11744
|
function loadStagingState(workspacePath) {
|
|
12046
11745
|
const path = stagingStatePath(workspacePath);
|
|
12047
11746
|
if (!isRegularFile(path))
|
|
12048
11747
|
return;
|
|
12049
11748
|
try {
|
|
12050
|
-
const parsed = JSON.parse(
|
|
11749
|
+
const parsed = JSON.parse(readFileSync13(path, "utf8"));
|
|
12051
11750
|
return isMinSyncStagingState(parsed) ? parsed : undefined;
|
|
12052
11751
|
} catch {
|
|
12053
11752
|
return;
|
|
@@ -12056,7 +11755,7 @@ function loadStagingState(workspacePath) {
|
|
|
12056
11755
|
function saveStagingState(workspacePath, state) {
|
|
12057
11756
|
const path = stagingStatePath(workspacePath);
|
|
12058
11757
|
const temporaryPath = `${path}.${randomUUID5()}.tmp`;
|
|
12059
|
-
|
|
11758
|
+
writeFileSync9(temporaryPath, `${JSON.stringify(state, null, 2)}
|
|
12060
11759
|
`);
|
|
12061
11760
|
renameSync6(temporaryPath, path);
|
|
12062
11761
|
}
|
|
@@ -12068,9 +11767,9 @@ function isMinSyncStagingState(value) {
|
|
|
12068
11767
|
function ensureManagedFilesRoot(filesRoot, rebuild) {
|
|
12069
11768
|
const status = lstatSync2(filesRoot, { throwIfNoEntry: false });
|
|
12070
11769
|
if (status !== undefined && (rebuild || !status.isDirectory())) {
|
|
12071
|
-
|
|
11770
|
+
rmSync4(filesRoot, { recursive: status.isDirectory(), force: true });
|
|
12072
11771
|
}
|
|
12073
|
-
|
|
11772
|
+
mkdirSync10(filesRoot, { recursive: true });
|
|
12074
11773
|
}
|
|
12075
11774
|
function ensureManagedDirectory(filesRoot, directory) {
|
|
12076
11775
|
const relativeDirectory = relative4(filesRoot, directory);
|
|
@@ -12080,23 +11779,23 @@ function ensureManagedDirectory(filesRoot, directory) {
|
|
|
12080
11779
|
const segments = relativeDirectory.split(sep3).filter(Boolean);
|
|
12081
11780
|
let current = filesRoot;
|
|
12082
11781
|
for (const segment of segments) {
|
|
12083
|
-
current =
|
|
11782
|
+
current = join19(current, segment);
|
|
12084
11783
|
const status = lstatSync2(current, { throwIfNoEntry: false });
|
|
12085
11784
|
if (status?.isDirectory())
|
|
12086
11785
|
continue;
|
|
12087
11786
|
if (status !== undefined) {
|
|
12088
|
-
|
|
11787
|
+
rmSync4(current, { force: true });
|
|
12089
11788
|
}
|
|
12090
|
-
|
|
11789
|
+
mkdirSync10(current);
|
|
12091
11790
|
}
|
|
12092
11791
|
}
|
|
12093
11792
|
function copyFileAtomically(source, destination) {
|
|
12094
|
-
const temporaryPath =
|
|
11793
|
+
const temporaryPath = join19(dirname10(destination), `.${basename7(destination)}.${randomUUID5()}.tmp`);
|
|
12095
11794
|
try {
|
|
12096
11795
|
copyFileSync2(source, temporaryPath);
|
|
12097
11796
|
renameSync6(temporaryPath, destination);
|
|
12098
11797
|
} finally {
|
|
12099
|
-
|
|
11798
|
+
rmSync4(temporaryPath, { force: true });
|
|
12100
11799
|
}
|
|
12101
11800
|
}
|
|
12102
11801
|
function isRegularFile(path) {
|
|
@@ -12104,11 +11803,11 @@ function isRegularFile(path) {
|
|
|
12104
11803
|
}
|
|
12105
11804
|
function removeUnexpectedStagedFiles(directory, desiredPaths) {
|
|
12106
11805
|
for (const entry of readdirSync3(directory, { withFileTypes: true })) {
|
|
12107
|
-
const path =
|
|
11806
|
+
const path = join19(directory, entry.name);
|
|
12108
11807
|
if (entry.isDirectory()) {
|
|
12109
11808
|
removeUnexpectedStagedFiles(path, desiredPaths);
|
|
12110
11809
|
} else if (!desiredPaths.has(path)) {
|
|
12111
|
-
|
|
11810
|
+
rmSync4(path, { force: true });
|
|
12112
11811
|
}
|
|
12113
11812
|
}
|
|
12114
11813
|
}
|
|
@@ -12134,7 +11833,7 @@ function buildMinSyncPathMap(root, workspacePath) {
|
|
|
12134
11833
|
return byPath;
|
|
12135
11834
|
}
|
|
12136
11835
|
function legacyParsedVirtualPath(root, virtualPath) {
|
|
12137
|
-
return
|
|
11836
|
+
return join19(root, ".autorag", "parsed", "files", `${virtualPath.replace(/^\/+/, "")}.md`);
|
|
12138
11837
|
}
|
|
12139
11838
|
var init_workspace = __esm(() => {
|
|
12140
11839
|
init_source_paths();
|
|
@@ -12143,8 +11842,8 @@ var init_workspace = __esm(() => {
|
|
|
12143
11842
|
});
|
|
12144
11843
|
|
|
12145
11844
|
// src/minsync/method.ts
|
|
12146
|
-
import { accessSync, constants, existsSync as
|
|
12147
|
-
import { basename as basename8, delimiter as delimiter2, join as
|
|
11845
|
+
import { accessSync, constants, existsSync as existsSync14, realpathSync as realpathSync3 } from "node:fs";
|
|
11846
|
+
import { basename as basename8, delimiter as delimiter2, join as join20, normalize as normalize2 } from "node:path";
|
|
12148
11847
|
function degrade(workspacePath, reason) {
|
|
12149
11848
|
return { ok: false, synced: 0, workspacePath, reason };
|
|
12150
11849
|
}
|
|
@@ -12156,7 +11855,7 @@ function lookupInPath(env) {
|
|
|
12156
11855
|
for (const dir of pathEnv.split(delimiter2)) {
|
|
12157
11856
|
if (dir.length === 0)
|
|
12158
11857
|
continue;
|
|
12159
|
-
const candidate =
|
|
11858
|
+
const candidate = join20(dir, execName);
|
|
12160
11859
|
try {
|
|
12161
11860
|
accessSync(candidate, constants.X_OK);
|
|
12162
11861
|
return candidate;
|
|
@@ -12216,7 +11915,10 @@ class MinSyncVectorMethod {
|
|
|
12216
11915
|
return binaryResult;
|
|
12217
11916
|
}
|
|
12218
11917
|
isBinaryMissing() {
|
|
12219
|
-
return this.binaryPath !== undefined && !
|
|
11918
|
+
return this.binaryPath !== undefined && !existsSync14(this.binaryPath);
|
|
11919
|
+
}
|
|
11920
|
+
isReady() {
|
|
11921
|
+
return existsSync14(join20(this.workspacePath, ".minsync", "cursor.json"));
|
|
12220
11922
|
}
|
|
12221
11923
|
async retrieve(query, options) {
|
|
12222
11924
|
const topK = options.topK ?? 20;
|
|
@@ -12256,16 +11958,15 @@ class MinSyncVectorMethod {
|
|
|
12256
11958
|
return results;
|
|
12257
11959
|
}
|
|
12258
11960
|
async resolveBinary() {
|
|
12259
|
-
if (this.binaryPath &&
|
|
11961
|
+
if (this.binaryPath && existsSync14(this.binaryPath))
|
|
12260
11962
|
return this.binaryPath;
|
|
12261
|
-
}
|
|
12262
11963
|
if (this.binaryPath !== undefined)
|
|
12263
11964
|
return;
|
|
12264
11965
|
const pathBinary = lookupInPath(process.env);
|
|
12265
11966
|
if (pathBinary)
|
|
12266
11967
|
return pathBinary;
|
|
12267
|
-
const cachedBinary =
|
|
12268
|
-
if (
|
|
11968
|
+
const cachedBinary = join20(this.root, ".autorag", "bin", executableName(process.platform));
|
|
11969
|
+
if (existsSync14(cachedBinary))
|
|
12269
11970
|
return cachedBinary;
|
|
12270
11971
|
if (this.autoInstall) {
|
|
12271
11972
|
try {
|
|
@@ -12278,63 +11979,13 @@ class MinSyncVectorMethod {
|
|
|
12278
11979
|
return;
|
|
12279
11980
|
}
|
|
12280
11981
|
}
|
|
12281
|
-
var
|
|
11982
|
+
var MinSyncHybridMethod;
|
|
12282
11983
|
var init_method = __esm(() => {
|
|
12283
|
-
init_bm25();
|
|
12284
11984
|
init_scope();
|
|
12285
11985
|
init_client7();
|
|
12286
11986
|
init_installer2();
|
|
12287
11987
|
init_paths4();
|
|
12288
11988
|
init_workspace();
|
|
12289
|
-
MinSyncBM25Method = class MinSyncBM25Method extends MinSyncVectorMethod {
|
|
12290
|
-
status = { readiness: "index_missing", engine: "minsync" };
|
|
12291
|
-
constructor(options) {
|
|
12292
|
-
super({ ...options, mode: "bm25" });
|
|
12293
|
-
}
|
|
12294
|
-
describe() {
|
|
12295
|
-
return {
|
|
12296
|
-
name: "bm25",
|
|
12297
|
-
type: "bm25",
|
|
12298
|
-
description: "MinSync-backed BM25 lexical retrieval over parsed markdown mirror chunks",
|
|
12299
|
-
status: this.status.readiness === "ready" ? "active" : "stub",
|
|
12300
|
-
capabilities: [
|
|
12301
|
-
"lexical",
|
|
12302
|
-
"incremental",
|
|
12303
|
-
"parsed-mirrors",
|
|
12304
|
-
"virtual-paths",
|
|
12305
|
-
`readiness:${this.status.readiness}`
|
|
12306
|
-
]
|
|
12307
|
-
};
|
|
12308
|
-
}
|
|
12309
|
-
async retrieve(query, options) {
|
|
12310
|
-
const binaryResult = await this.resolveBinaryForRetrieve();
|
|
12311
|
-
if (binaryResult === undefined) {
|
|
12312
|
-
const readiness = this.status.readiness === "ready" ? "dependency_unavailable" : this.status.readiness;
|
|
12313
|
-
throw new BM25UnavailableError(readiness === "index_missing" ? "dependency_unavailable" : readiness, this.status.message ?? "MinSync BM25 is unavailable");
|
|
12314
|
-
}
|
|
12315
|
-
return super.retrieve(query, options);
|
|
12316
|
-
}
|
|
12317
|
-
syncFromMinSync(result) {
|
|
12318
|
-
this.status = result.ok ? { readiness: "ready", engine: "minsync" } : {
|
|
12319
|
-
readiness: result.reason === "missing-binary" ? "dependency_unavailable" : "error",
|
|
12320
|
-
engine: "minsync",
|
|
12321
|
-
message: result.reason
|
|
12322
|
-
};
|
|
12323
|
-
return {
|
|
12324
|
-
indexPath: result.workspacePath,
|
|
12325
|
-
indexedChunks: result.synced,
|
|
12326
|
-
readiness: this.status.readiness,
|
|
12327
|
-
engine: this.status.engine
|
|
12328
|
-
};
|
|
12329
|
-
}
|
|
12330
|
-
getStatus() {
|
|
12331
|
-
return this.status;
|
|
12332
|
-
}
|
|
12333
|
-
async resolveBinaryForRetrieve() {
|
|
12334
|
-
const binaryResult = await this.resolveBinary();
|
|
12335
|
-
return typeof binaryResult === "string" ? binaryResult : undefined;
|
|
12336
|
-
}
|
|
12337
|
-
};
|
|
12338
11989
|
MinSyncHybridMethod = class MinSyncHybridMethod extends MinSyncVectorMethod {
|
|
12339
11990
|
constructor(options) {
|
|
12340
11991
|
super({ ...options, mode: "hybrid" });
|
|
@@ -12363,14 +12014,14 @@ var init_minsync = __esm(() => {
|
|
|
12363
12014
|
|
|
12364
12015
|
// src/mirror/sync.ts
|
|
12365
12016
|
import { randomUUID as randomUUID6 } from "node:crypto";
|
|
12366
|
-
import { existsSync as
|
|
12017
|
+
import { existsSync as existsSync15, mkdirSync as mkdirSync11, renameSync as renameSync7, rmSync as rmSync5, writeFileSync as writeFileSync10 } from "node:fs";
|
|
12367
12018
|
import { opendir, readFile as readFile5, stat as stat4 } from "node:fs/promises";
|
|
12368
|
-
import { dirname as
|
|
12369
|
-
function parserFailureCode(
|
|
12370
|
-
return
|
|
12019
|
+
import { dirname as dirname11, extname as extname8, resolve as resolve10 } from "node:path";
|
|
12020
|
+
function parserFailureCode(parser, error) {
|
|
12021
|
+
return parser.name === "opendataloader-pdf" && /UnsupportedClassVersionError/.test(error.message) ? "pdf-java-version" : "parser-failed";
|
|
12371
12022
|
}
|
|
12372
|
-
function parserFailureMessage(
|
|
12373
|
-
if (parserFailureCode(
|
|
12023
|
+
function parserFailureMessage(parser, error) {
|
|
12024
|
+
if (parserFailureCode(parser, error) === "pdf-java-version") {
|
|
12374
12025
|
return "PDF parsing requires Java 11 or newer; the Java runtime selected from PATH is too old.";
|
|
12375
12026
|
}
|
|
12376
12027
|
return "The registered parser failed on this file; it was skipped during indexing.";
|
|
@@ -12410,8 +12061,8 @@ async function syncParsedMirrors(options) {
|
|
|
12410
12061
|
});
|
|
12411
12062
|
continue;
|
|
12412
12063
|
}
|
|
12413
|
-
const
|
|
12414
|
-
if (!
|
|
12064
|
+
const parser = registry.getForVirtualPath(entry.virtualPath);
|
|
12065
|
+
if (!parser) {
|
|
12415
12066
|
deleted += removePrevious(options.root, previous, entry.virtualPath);
|
|
12416
12067
|
handledPrevious.add(entry.virtualPath);
|
|
12417
12068
|
skipped += 1;
|
|
@@ -12434,12 +12085,12 @@ async function syncParsedMirrors(options) {
|
|
|
12434
12085
|
}
|
|
12435
12086
|
const previousEntry = previous.entries[entry.virtualPath];
|
|
12436
12087
|
const outputPath = parsedOutputPath(options.root, entry.virtualPath);
|
|
12437
|
-
const unchanged = !options.force && previousEntry?.sourceMtimeNs === entry.mtimeNs && previousEntry.sourceSizeBytes === entry.sizeBytes && previousEntry.parserName ===
|
|
12088
|
+
const unchanged = !options.force && previousEntry?.sourceMtimeNs === entry.mtimeNs && previousEntry.sourceSizeBytes === entry.sizeBytes && previousEntry.parserName === parser.name && existsSync15(outputPath);
|
|
12438
12089
|
if (!unchanged) {
|
|
12439
12090
|
let parsed;
|
|
12440
12091
|
try {
|
|
12441
12092
|
const bytes = await readFile5(entry.sourcePath);
|
|
12442
|
-
parsed = await
|
|
12093
|
+
parsed = await parser.parse({ virtualPath: entry.virtualPath, sourcePath: entry.sourcePath, bytes });
|
|
12443
12094
|
} catch (error) {
|
|
12444
12095
|
if (!(error instanceof ParseError))
|
|
12445
12096
|
throw error;
|
|
@@ -12447,9 +12098,9 @@ async function syncParsedMirrors(options) {
|
|
|
12447
12098
|
handledPrevious.add(entry.virtualPath);
|
|
12448
12099
|
skipped += 1;
|
|
12449
12100
|
diagnostics.push({
|
|
12450
|
-
code: parserFailureCode(
|
|
12101
|
+
code: parserFailureCode(parser, error),
|
|
12451
12102
|
severity: "warning",
|
|
12452
|
-
message: parserFailureMessage(
|
|
12103
|
+
message: parserFailureMessage(parser, error),
|
|
12453
12104
|
source: entry.virtualPath
|
|
12454
12105
|
});
|
|
12455
12106
|
sinceCheckpoint += 1;
|
|
@@ -12473,7 +12124,7 @@ async function syncParsedMirrors(options) {
|
|
|
12473
12124
|
virtualPath: entry.virtualPath,
|
|
12474
12125
|
sourcePath: entry.sourcePath,
|
|
12475
12126
|
outputPath,
|
|
12476
|
-
parserName:
|
|
12127
|
+
parserName: parser.name,
|
|
12477
12128
|
sourceMtimeNs: entry.mtimeNs,
|
|
12478
12129
|
sourceSizeBytes: entry.sizeBytes,
|
|
12479
12130
|
updatedAt: unchanged ? previousEntry?.updatedAt ?? new Date().toISOString() : new Date().toISOString()
|
|
@@ -12532,8 +12183,8 @@ async function detectMirrorStaleness(options) {
|
|
|
12532
12183
|
}
|
|
12533
12184
|
function supportedExtensionSet(registry) {
|
|
12534
12185
|
const extensions = new Set;
|
|
12535
|
-
for (const
|
|
12536
|
-
for (const extension of
|
|
12186
|
+
for (const parser of registry.list()) {
|
|
12187
|
+
for (const extension of parser.extensions) {
|
|
12537
12188
|
extensions.add(normalizeExtension2(extension));
|
|
12538
12189
|
}
|
|
12539
12190
|
}
|
|
@@ -12593,14 +12244,14 @@ function removePrevious(root, index, virtualPath) {
|
|
|
12593
12244
|
return 1;
|
|
12594
12245
|
}
|
|
12595
12246
|
function writeAtomic(path, content) {
|
|
12596
|
-
|
|
12247
|
+
mkdirSync11(dirname11(path), { recursive: true });
|
|
12597
12248
|
const tmp = `${path}.${randomUUID6()}.tmp`;
|
|
12598
|
-
|
|
12249
|
+
writeFileSync10(tmp, content);
|
|
12599
12250
|
renameSync7(tmp, path);
|
|
12600
12251
|
}
|
|
12601
12252
|
function removeFile(path) {
|
|
12602
|
-
if (
|
|
12603
|
-
|
|
12253
|
+
if (existsSync15(path))
|
|
12254
|
+
rmSync5(path, { force: true });
|
|
12604
12255
|
}
|
|
12605
12256
|
var DEFAULT_MAX_SOURCE_BYTES, MIRROR_CHECKPOINT_EVERY = 25, SKIP_DIR_NAMES;
|
|
12606
12257
|
var init_sync = __esm(() => {
|
|
@@ -12636,8 +12287,8 @@ var init_sync = __esm(() => {
|
|
|
12636
12287
|
});
|
|
12637
12288
|
|
|
12638
12289
|
// src/observability/run-log.ts
|
|
12639
|
-
import { appendFileSync, mkdirSync as
|
|
12640
|
-
import { dirname as
|
|
12290
|
+
import { appendFileSync, mkdirSync as mkdirSync12 } from "node:fs";
|
|
12291
|
+
import { dirname as dirname12 } from "node:path";
|
|
12641
12292
|
|
|
12642
12293
|
class AutoRAGRunLogger {
|
|
12643
12294
|
path;
|
|
@@ -12646,7 +12297,7 @@ class AutoRAGRunLogger {
|
|
|
12646
12297
|
}
|
|
12647
12298
|
write(event) {
|
|
12648
12299
|
try {
|
|
12649
|
-
|
|
12300
|
+
mkdirSync12(dirname12(this.path), { recursive: true });
|
|
12650
12301
|
appendFileSync(this.path, `${JSON.stringify(event)}
|
|
12651
12302
|
`, "utf8");
|
|
12652
12303
|
} catch (error) {}
|
|
@@ -12768,8 +12419,6 @@ class ParallelRetriever {
|
|
|
12768
12419
|
function methodFailureCode(name) {
|
|
12769
12420
|
if (name === "minsync")
|
|
12770
12421
|
return "minsync-unavailable";
|
|
12771
|
-
if (name === "bm25")
|
|
12772
|
-
return "bm25-unavailable";
|
|
12773
12422
|
return "retrieval-method-failed";
|
|
12774
12423
|
}
|
|
12775
12424
|
var MAX_AGGREGATION_BONUS = 0.3, MAX_SECONDARY_HITS = 3, SECONDARY_HIT_WEIGHT = 0.15;
|
|
@@ -12798,7 +12447,7 @@ class RetrievalMethodRegistry {
|
|
|
12798
12447
|
// src/filesystem/cloud-placeholder.ts
|
|
12799
12448
|
import { execFile } from "node:child_process";
|
|
12800
12449
|
import { lstat } from "node:fs/promises";
|
|
12801
|
-
import { dirname as
|
|
12450
|
+
import { dirname as dirname13, relative as relative5, sep as sep4 } from "node:path";
|
|
12802
12451
|
import { promisify } from "node:util";
|
|
12803
12452
|
function pathLooksLikeCloudRoot(target) {
|
|
12804
12453
|
const normalized = target.replaceAll("\\", "/");
|
|
@@ -12836,7 +12485,7 @@ async function classifyDarwinRoot(target) {
|
|
|
12836
12485
|
const provider = await readFileProviderDomain(cursor);
|
|
12837
12486
|
if (provider !== undefined)
|
|
12838
12487
|
return { kind: "file-provider", reason: "file-provider-xattr", provider };
|
|
12839
|
-
const parent =
|
|
12488
|
+
const parent = dirname13(cursor);
|
|
12840
12489
|
if (parent === cursor)
|
|
12841
12490
|
break;
|
|
12842
12491
|
cursor = parent;
|
|
@@ -12921,12 +12570,12 @@ var init_cloud_placeholder = __esm(() => {
|
|
|
12921
12570
|
});
|
|
12922
12571
|
|
|
12923
12572
|
// src/agent/bash-tool.ts
|
|
12924
|
-
import { spawn as
|
|
12573
|
+
import { spawn as spawn13 } from "node:child_process";
|
|
12925
12574
|
import { Type as Type2 } from "typebox";
|
|
12926
12575
|
function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
|
|
12927
|
-
return new Promise((
|
|
12576
|
+
return new Promise((resolve) => {
|
|
12928
12577
|
if (signal?.aborted) {
|
|
12929
|
-
|
|
12578
|
+
resolve({
|
|
12930
12579
|
output: "",
|
|
12931
12580
|
exitCode: undefined,
|
|
12932
12581
|
aborted: true,
|
|
@@ -12937,7 +12586,7 @@ function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
|
|
|
12937
12586
|
return;
|
|
12938
12587
|
}
|
|
12939
12588
|
const shell = process.platform === "win32" ? "bash.exe" : "/bin/bash";
|
|
12940
|
-
const child =
|
|
12589
|
+
const child = spawn13(shell, ["-c", command], {
|
|
12941
12590
|
cwd,
|
|
12942
12591
|
detached: process.platform !== "win32",
|
|
12943
12592
|
windowsHide: true
|
|
@@ -12983,7 +12632,7 @@ function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
|
|
|
12983
12632
|
if (killGraceTimer !== undefined)
|
|
12984
12633
|
clearTimeout(killGraceTimer);
|
|
12985
12634
|
signal?.removeEventListener("abort", onAbort);
|
|
12986
|
-
|
|
12635
|
+
resolve({
|
|
12987
12636
|
output: Buffer.concat(chunks).toString("utf8"),
|
|
12988
12637
|
exitCode,
|
|
12989
12638
|
aborted,
|
|
@@ -13003,15 +12652,15 @@ function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
|
|
|
13003
12652
|
if (child.pid === undefined)
|
|
13004
12653
|
return true;
|
|
13005
12654
|
if (process.platform === "win32") {
|
|
13006
|
-
return await new Promise((
|
|
13007
|
-
const killer =
|
|
12655
|
+
return await new Promise((resolve) => {
|
|
12656
|
+
const killer = spawn13("taskkill", ["/pid", String(child.pid), "/t", "/f"], { stdio: "ignore" });
|
|
13008
12657
|
let completed = false;
|
|
13009
12658
|
const graceTimer = setTimeout(() => {
|
|
13010
12659
|
if (completed)
|
|
13011
12660
|
return;
|
|
13012
12661
|
completed = true;
|
|
13013
12662
|
killer.kill();
|
|
13014
|
-
|
|
12663
|
+
resolve(killDirectChild());
|
|
13015
12664
|
}, KILL_EXIT_GRACE_MS);
|
|
13016
12665
|
const finishKill = (success) => {
|
|
13017
12666
|
if (completed)
|
|
@@ -13020,7 +12669,7 @@ function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
|
|
|
13020
12669
|
clearTimeout(graceTimer);
|
|
13021
12670
|
if (!success)
|
|
13022
12671
|
killDirectChild();
|
|
13023
|
-
|
|
12672
|
+
resolve(success);
|
|
13024
12673
|
};
|
|
13025
12674
|
killer.on("error", () => finishKill(false));
|
|
13026
12675
|
killer.on("close", (code) => finishKill(code === 0));
|
|
@@ -13068,7 +12717,7 @@ function runCommand(command, cwd, timeoutMs, maxOutputBytes, signal) {
|
|
|
13068
12717
|
});
|
|
13069
12718
|
}
|
|
13070
12719
|
function createBashTool(options) {
|
|
13071
|
-
const defaultTimeout = options.timeoutMs ??
|
|
12720
|
+
const defaultTimeout = options.timeoutMs ?? DEFAULT_TIMEOUT_MS8;
|
|
13072
12721
|
const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
|
13073
12722
|
return {
|
|
13074
12723
|
name: BASH_TOOL_NAME,
|
|
@@ -13165,7 +12814,7 @@ function createBashTool(options) {
|
|
|
13165
12814
|
}
|
|
13166
12815
|
};
|
|
13167
12816
|
}
|
|
13168
|
-
var BASH_TOOL_NAME = "bash",
|
|
12817
|
+
var BASH_TOOL_NAME = "bash", DEFAULT_TIMEOUT_MS8 = 120000, DEFAULT_MAX_OUTPUT_BYTES = 131072, KILL_EXIT_GRACE_MS = 5000, bashSchema, CLOUD_CONTENT_COMMAND;
|
|
13169
12818
|
var init_bash_tool = __esm(() => {
|
|
13170
12819
|
init_cloud_placeholder();
|
|
13171
12820
|
bashSchema = Type2.Object({
|
|
@@ -13499,7 +13148,7 @@ function createSearchAllDocumentsTool(provider) {
|
|
|
13499
13148
|
return {
|
|
13500
13149
|
name: SEARCH_ALL_DOCUMENTS_TOOL_NAME,
|
|
13501
13150
|
label: "Search All Documents",
|
|
13502
|
-
description: "Search across all configured retrieval methods (posix,
|
|
13151
|
+
description: "Search across all configured retrieval methods (posix, MinSync, datasources) and return merged, deduplicated results. Authority is server-configured; tool arguments can only provide query, topK, and an optional narrowing scope.",
|
|
13503
13152
|
parameters: searchAllSchema,
|
|
13504
13153
|
async execute(_toolCallId, params) {
|
|
13505
13154
|
const query = params.query.trim();
|
|
@@ -13552,92 +13201,8 @@ var init_search_all_tool = __esm(() => {
|
|
|
13552
13201
|
});
|
|
13553
13202
|
});
|
|
13554
13203
|
|
|
13555
|
-
// src/agent/search-bm25-tool.ts
|
|
13556
|
-
import { Type as Type8 } from "typebox";
|
|
13557
|
-
function createSearchBM25DocumentsTool(getMethod, resolveScope = (scope) => scope) {
|
|
13558
|
-
return {
|
|
13559
|
-
name: SEARCH_BM25_DOCUMENTS_TOOL_NAME,
|
|
13560
|
-
label: "Search BM25 Documents",
|
|
13561
|
-
description: "Search parsed document mirrors with MinSync lexical BM25 ranking. Use for exact terms, repeated terms, headings, identifiers, and folder-scoped document search.",
|
|
13562
|
-
parameters: searchBM25Schema,
|
|
13563
|
-
async execute(_toolCallId, params) {
|
|
13564
|
-
const method = getMethod();
|
|
13565
|
-
if (!method) {
|
|
13566
|
-
return unavailableResult("BM25 is not configured for this AutoRAG agent", {
|
|
13567
|
-
readiness: "disabled",
|
|
13568
|
-
engine: "none"
|
|
13569
|
-
});
|
|
13570
|
-
}
|
|
13571
|
-
const status = method.getStatus();
|
|
13572
|
-
if (params.query.trim().length === 0) {
|
|
13573
|
-
return {
|
|
13574
|
-
content: [{ type: "text", text: "BM25 query was empty; no documents searched." }],
|
|
13575
|
-
details: {
|
|
13576
|
-
method: "bm25",
|
|
13577
|
-
resultCount: 0,
|
|
13578
|
-
sources: [],
|
|
13579
|
-
readiness: status.readiness,
|
|
13580
|
-
engine: status.engine
|
|
13581
|
-
}
|
|
13582
|
-
};
|
|
13583
|
-
}
|
|
13584
|
-
const scope = resolveScope(params.scope);
|
|
13585
|
-
try {
|
|
13586
|
-
const results = await method.retrieve(params.query, {
|
|
13587
|
-
topK: params.topK,
|
|
13588
|
-
scope
|
|
13589
|
-
});
|
|
13590
|
-
return {
|
|
13591
|
-
content: [{ type: "text", text: formatResults2(results, method.getStatus()) }],
|
|
13592
|
-
details: {
|
|
13593
|
-
method: "bm25",
|
|
13594
|
-
resultCount: results.length,
|
|
13595
|
-
sources: [...new Set(results.map((result) => result.source))],
|
|
13596
|
-
readiness: method.getStatus().readiness,
|
|
13597
|
-
engine: method.getStatus().engine
|
|
13598
|
-
}
|
|
13599
|
-
};
|
|
13600
|
-
} catch (error) {
|
|
13601
|
-
const nextStatus = method.getStatus();
|
|
13602
|
-
const message = error instanceof BM25UnavailableError ? error.message : `BM25 search failed: ${String(error)}`;
|
|
13603
|
-
return unavailableResult(message, nextStatus);
|
|
13604
|
-
}
|
|
13605
|
-
}
|
|
13606
|
-
};
|
|
13607
|
-
}
|
|
13608
|
-
function unavailableResult(message, status) {
|
|
13609
|
-
return {
|
|
13610
|
-
content: [{ type: "text", text: `${message}
|
|
13611
|
-
readiness=${status.readiness}; engine=${status.engine}` }],
|
|
13612
|
-
details: { method: "bm25", resultCount: 0, sources: [], readiness: status.readiness, engine: status.engine }
|
|
13613
|
-
};
|
|
13614
|
-
}
|
|
13615
|
-
function formatResults2(results, status) {
|
|
13616
|
-
if (results.length === 0)
|
|
13617
|
-
return `No BM25 results. readiness=${status.readiness}; engine=${status.engine}`;
|
|
13618
|
-
const rows = results.map((result, index) => {
|
|
13619
|
-
const line = result.content.replace(/\s+/gu, " ").slice(0, 500);
|
|
13620
|
-
return `[${index + 1}] ${result.source} score=${result.score.toFixed(4)}
|
|
13621
|
-
${line}`;
|
|
13622
|
-
});
|
|
13623
|
-
return `BM25 results (readiness=${status.readiness}; engine=${status.engine}):
|
|
13624
|
-
|
|
13625
|
-
${rows.join(`
|
|
13626
|
-
|
|
13627
|
-
`)}`;
|
|
13628
|
-
}
|
|
13629
|
-
var SEARCH_BM25_DOCUMENTS_TOOL_NAME = "lexical_search_local_docs", searchBM25Schema;
|
|
13630
|
-
var init_search_bm25_tool = __esm(() => {
|
|
13631
|
-
init_bm25();
|
|
13632
|
-
searchBM25Schema = Type8.Object({
|
|
13633
|
-
query: Type8.String({ description: "Lexical query to search in parsed document mirrors with BM25." }),
|
|
13634
|
-
topK: Type8.Optional(Type8.Integer({ description: "Maximum number of BM25 chunks to return." })),
|
|
13635
|
-
scope: Type8.Optional(Type8.String({ description: "Optional opaque virtual-path scope, e.g. /docs or /docs/**." }))
|
|
13636
|
-
});
|
|
13637
|
-
});
|
|
13638
|
-
|
|
13639
13204
|
// src/agent/search-datasource-tool.ts
|
|
13640
|
-
import { Type as
|
|
13205
|
+
import { Type as Type8 } from "typebox";
|
|
13641
13206
|
function createSearchDatasourceDocumentsTool(provider) {
|
|
13642
13207
|
return {
|
|
13643
13208
|
name: SEARCH_DATASOURCE_DOCUMENTS_TOOL_NAME,
|
|
@@ -13657,7 +13222,7 @@ function createSearchDatasourceDocumentsTool(provider) {
|
|
|
13657
13222
|
scope: params.scope
|
|
13658
13223
|
});
|
|
13659
13224
|
return {
|
|
13660
|
-
content: [{ type: "text", text:
|
|
13225
|
+
content: [{ type: "text", text: formatResults2(results, diagnostics) }],
|
|
13661
13226
|
details: {
|
|
13662
13227
|
method: "datasource",
|
|
13663
13228
|
resultCount: results.length,
|
|
@@ -13668,7 +13233,7 @@ function createSearchDatasourceDocumentsTool(provider) {
|
|
|
13668
13233
|
}
|
|
13669
13234
|
};
|
|
13670
13235
|
}
|
|
13671
|
-
function
|
|
13236
|
+
function formatResults2(results, diagnostics) {
|
|
13672
13237
|
const diagnosticSummary = diagnostics.length > 0 ? `
|
|
13673
13238
|
|
|
13674
13239
|
Diagnostics: ${diagnostics.map((d) => `${d.source ?? "datasource"}:${d.code}`).join(", ")}` : "";
|
|
@@ -13687,10 +13252,10 @@ ${rows.join(`
|
|
|
13687
13252
|
}
|
|
13688
13253
|
var SEARCH_DATASOURCE_DOCUMENTS_TOOL_NAME = "search_datasource_documents", searchDatasourceSchema;
|
|
13689
13254
|
var init_search_datasource_tool = __esm(() => {
|
|
13690
|
-
searchDatasourceSchema =
|
|
13691
|
-
query:
|
|
13692
|
-
topK:
|
|
13693
|
-
scope:
|
|
13255
|
+
searchDatasourceSchema = Type8.Object({
|
|
13256
|
+
query: Type8.String({ description: "Query to search across configured external datasource skills." }),
|
|
13257
|
+
topK: Type8.Optional(Type8.Integer({ description: "Maximum number of datasource chunks to return." })),
|
|
13258
|
+
scope: Type8.Optional(Type8.String({ description: "Optional opaque datasource scope, e.g. /kakao/account or /kakao/account/**." }))
|
|
13694
13259
|
});
|
|
13695
13260
|
});
|
|
13696
13261
|
|
|
@@ -13744,7 +13309,7 @@ function normalizeEntryEvidenceRefs(entry) {
|
|
|
13744
13309
|
function recordStructuredResultsSession(sessionId, query, details, sessions, memory, componentDiagnostics = []) {
|
|
13745
13310
|
const resultNumbers = details.results.map((result) => result.number).sort((a, b) => a - b);
|
|
13746
13311
|
const mappingNumbers = details.mapping.map((entry) => entry.number).sort((a, b) => a - b);
|
|
13747
|
-
const oneToOne = resultNumbers.length === mappingNumbers.length && resultNumbers.every((
|
|
13312
|
+
const oneToOne = resultNumbers.length === mappingNumbers.length && resultNumbers.every((number, index) => number === mappingNumbers[index]);
|
|
13748
13313
|
if (!oneToOne) {
|
|
13749
13314
|
throw new Error("emit_autorag_results: result numbers and mapping numbers must be one-to-one");
|
|
13750
13315
|
}
|
|
@@ -13830,7 +13395,7 @@ var init_search_documents = __esm(() => {
|
|
|
13830
13395
|
});
|
|
13831
13396
|
|
|
13832
13397
|
// src/agent/search-minsync-tool.ts
|
|
13833
|
-
import { Type as
|
|
13398
|
+
import { Type as Type9 } from "typebox";
|
|
13834
13399
|
function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => scope) {
|
|
13835
13400
|
return {
|
|
13836
13401
|
name: SEARCH_MINSYNC_DOCUMENTS_TOOL_NAME,
|
|
@@ -13840,10 +13405,10 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
|
|
|
13840
13405
|
async execute(_toolCallId, params) {
|
|
13841
13406
|
const method = getMethod();
|
|
13842
13407
|
if (!method) {
|
|
13843
|
-
return
|
|
13408
|
+
return unavailableResult("MinSync semantic search is not configured for this AutoRAG agent");
|
|
13844
13409
|
}
|
|
13845
13410
|
if (method.isBinaryMissing()) {
|
|
13846
|
-
return
|
|
13411
|
+
return unavailableResult("MinSync semantic search is unavailable; the configured binary is missing.");
|
|
13847
13412
|
}
|
|
13848
13413
|
if (params.query.trim().length === 0) {
|
|
13849
13414
|
return {
|
|
@@ -13858,7 +13423,7 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
|
|
|
13858
13423
|
scope
|
|
13859
13424
|
});
|
|
13860
13425
|
return {
|
|
13861
|
-
content: [{ type: "text", text:
|
|
13426
|
+
content: [{ type: "text", text: formatResults3(results) }],
|
|
13862
13427
|
details: {
|
|
13863
13428
|
method: "semantic_search_local_docs",
|
|
13864
13429
|
resultCount: results.length,
|
|
@@ -13867,12 +13432,12 @@ function createSearchMinSyncDocumentsTool(getMethod, resolveScope = (scope) => s
|
|
|
13867
13432
|
}
|
|
13868
13433
|
};
|
|
13869
13434
|
} catch {
|
|
13870
|
-
return
|
|
13435
|
+
return unavailableResult("MinSync semantic search is unavailable; the query could not be completed.");
|
|
13871
13436
|
}
|
|
13872
13437
|
}
|
|
13873
13438
|
};
|
|
13874
13439
|
}
|
|
13875
|
-
function
|
|
13440
|
+
function unavailableResult(message) {
|
|
13876
13441
|
return {
|
|
13877
13442
|
content: [{ type: "text", text: message }],
|
|
13878
13443
|
details: {
|
|
@@ -13883,7 +13448,7 @@ function unavailableResult2(message) {
|
|
|
13883
13448
|
}
|
|
13884
13449
|
};
|
|
13885
13450
|
}
|
|
13886
|
-
function
|
|
13451
|
+
function formatResults3(results) {
|
|
13887
13452
|
if (results.length === 0)
|
|
13888
13453
|
return "No MinSync results.";
|
|
13889
13454
|
const rows = results.map((result, index) => {
|
|
@@ -13899,12 +13464,12 @@ ${rows.join(`
|
|
|
13899
13464
|
}
|
|
13900
13465
|
var SEARCH_MINSYNC_DOCUMENTS_TOOL_NAME = "semantic_search_local_docs", searchMinSyncSchema;
|
|
13901
13466
|
var init_search_minsync_tool = __esm(() => {
|
|
13902
|
-
searchMinSyncSchema =
|
|
13903
|
-
query:
|
|
13467
|
+
searchMinSyncSchema = Type9.Object({
|
|
13468
|
+
query: Type9.String({
|
|
13904
13469
|
description: "Semantic query to search parsed document mirrors with MinSync vector retrieval."
|
|
13905
13470
|
}),
|
|
13906
|
-
topK:
|
|
13907
|
-
scope:
|
|
13471
|
+
topK: Type9.Optional(Type9.Integer({ description: "Maximum number of MinSync semantic chunks to return." })),
|
|
13472
|
+
scope: Type9.Optional(Type9.String({ description: "Optional opaque virtual-path scope, e.g. /docs or /docs/**." }))
|
|
13908
13473
|
});
|
|
13909
13474
|
});
|
|
13910
13475
|
|
|
@@ -13921,7 +13486,6 @@ function buildSystemPrompt(config) {
|
|
|
13921
13486
|
toolLine(config, "bash", "read and inspect configured document collections with ls, find, grep, cat, and similar tools"),
|
|
13922
13487
|
toolLine(config, "jikji_find", "local discovery through Jikji answer packs"),
|
|
13923
13488
|
toolLine(config, "search_all_documents", "fan out across every configured retrieval method and merge results"),
|
|
13924
|
-
toolLine(config, "lexical_search_local_docs", "MinSync-backed lexical BM25 search over parsed document mirrors"),
|
|
13925
13489
|
toolLine(config, "semantic_search_local_docs", "semantic MinSync search over parsed document mirrors"),
|
|
13926
13490
|
toolLine(config, "search_datasource_documents", "search authorized external datasources; scope may only narrow access"),
|
|
13927
13491
|
toolLine(config, "load_datasource_skill", "load instructions for an authorized datasource"),
|
|
@@ -13932,7 +13496,6 @@ function buildSystemPrompt(config) {
|
|
|
13932
13496
|
"bash",
|
|
13933
13497
|
"jikji_find",
|
|
13934
13498
|
"search_all_documents",
|
|
13935
|
-
"lexical_search_local_docs",
|
|
13936
13499
|
"semantic_search_local_docs",
|
|
13937
13500
|
"search_datasource_documents",
|
|
13938
13501
|
"load_datasource_skill",
|
|
@@ -13965,8 +13528,8 @@ Your job is to retrieve candidates, read the relevant source material directly,
|
|
|
13965
13528
|
|
|
13966
13529
|
## Workflow
|
|
13967
13530
|
|
|
13968
|
-
1. **PLAN** —
|
|
13969
|
-
2. **RETRIEVE** — Use MinSync
|
|
13531
|
+
1. **PLAN** — Decide whether the query is answerable from stable general knowledge, memory, or intrinsic model knowledge. If so, answer immediately without retrieval.
|
|
13532
|
+
2. **RETRIEVE** — Use MinSync lexical/vector/hybrid retrieval, combined retrieval, Jikji, datasource search, or direct filesystem discovery as appropriate.
|
|
13970
13533
|
3. **READ** — Use \`bash\` to open and verify relevant local files. Do not curate from search snippets alone when source files are available.
|
|
13971
13534
|
4. **JUDGE** — Evaluate relevance, sufficiency, conflicts, uncertainty, and temporal context.
|
|
13972
13535
|
5. **CURATE** — Produce concise numbered knowledge units grounded in source evidence.
|
|
@@ -13979,14 +13542,17 @@ ${toolLines.join(`
|
|
|
13979
13542
|
${noSearchTools}
|
|
13980
13543
|
## Search Strategy
|
|
13981
13544
|
|
|
13982
|
-
-
|
|
13545
|
+
- For generic, stable questions answer directly from knowledge or memory without searching. For source-dependent questions, baseline MinSync and Jikji retrieval starts before the model's search decision and supplies up to 50 candidates per method; inspect it first and treat it as unverified evidence.
|
|
13983
13546
|
- Start with the most specific exact term, identifier, filename glob, or regex that preserves the query intent.
|
|
13984
13547
|
- Use \`search_all_documents\` when multiple configured retrieval methods can help.
|
|
13985
|
-
- Use MinSync
|
|
13548
|
+
- Use MinSync lexical mode for exact terminology, MinSync vector search for semantic similarity, and \`search_all_documents\` when hybrid ranking over the same MinSync chunks can help.
|
|
13986
13549
|
- Use \`bash\` to read already-retrieved local files with cat/head/sed. find/grep/rg must be small and bounded: one already-known directory from retrieval, a tight pattern, and a cap (head, maxdepth, or file types). Never recursively scan a whole search root (Downloads, Documents, Desktop, or /); those calls miss the bash timeout and stall the search loop.
|
|
13987
13550
|
- If retrieval is empty, retry a simpler query or synonyms through retrieval tools first. Do not widen filesystem discovery to compensate.
|
|
13988
13551
|
- Slash-prefixed datasource IDs such as /kakao/..., /gmail/..., /slack/..., /discord/..., and /github/... are virtual, not OS paths. Do not bash them. Search them with \`search_datasource_documents\`.
|
|
13989
13552
|
- Cross-check important claims against the original source and preserve real source paths.
|
|
13553
|
+
- When more searching is needed, first emit a brief, query-specific 1–2 line progress update describing the best current hypothesis and what is being checked next; baseline retrieval is already running in parallel. Never repeat a generic status message.
|
|
13554
|
+
- Do not use broad grep/find or recursive filesystem scans. Only inspect a narrow neighborhood around a retrieved candidate when the evidence clearly points there.
|
|
13555
|
+
- Do not query the same datasource more than three times. After three attempts, stop searching it and conclude from the evidence available.
|
|
13990
13556
|
|
|
13991
13557
|
${duplicateManagement}
|
|
13992
13558
|
## External Datasource Skills
|
|
@@ -14090,7 +13656,7 @@ var init_watch_refresh = __esm(() => {
|
|
|
14090
13656
|
// src/agent/agent.ts
|
|
14091
13657
|
import { randomUUID as randomUUID7 } from "node:crypto";
|
|
14092
13658
|
import { watch as fsWatch, realpathSync as realpathSync4, statSync } from "node:fs";
|
|
14093
|
-
import { dirname as
|
|
13659
|
+
import { dirname as dirname14, join as join21, resolve as resolve11 } from "node:path";
|
|
14094
13660
|
import { Agent } from "@earendil-works/pi-agent-core";
|
|
14095
13661
|
import { streamSimple } from "@earendil-works/pi-ai/compat";
|
|
14096
13662
|
|
|
@@ -14111,6 +13677,10 @@ class AutoRAGAgent {
|
|
|
14111
13677
|
resultCapture;
|
|
14112
13678
|
autoRefreshTimer;
|
|
14113
13679
|
refreshing = false;
|
|
13680
|
+
jikjiPrepareInFlight;
|
|
13681
|
+
jikjiReady = false;
|
|
13682
|
+
minSyncPrepareInFlight;
|
|
13683
|
+
minSyncReady = false;
|
|
14114
13684
|
refreshState = {
|
|
14115
13685
|
inFlight: false,
|
|
14116
13686
|
lastOutcome: "never",
|
|
@@ -14130,7 +13700,6 @@ class AutoRAGAgent {
|
|
|
14130
13700
|
merger = new ResultMerger;
|
|
14131
13701
|
datasourceFilter = new DatasourceResultFilter;
|
|
14132
13702
|
minSyncMethod;
|
|
14133
|
-
bm25Method;
|
|
14134
13703
|
jikjiClient;
|
|
14135
13704
|
datasourceSkills;
|
|
14136
13705
|
datasourceAccessOptions;
|
|
@@ -14144,6 +13713,7 @@ class AutoRAGAgent {
|
|
|
14144
13713
|
searchTimeoutMs;
|
|
14145
13714
|
maxSearchToolCalls;
|
|
14146
13715
|
searchToolCallCount = 0;
|
|
13716
|
+
sourceSearchCallCounts = new Map;
|
|
14147
13717
|
constructor(options) {
|
|
14148
13718
|
const { manifestDir, memoryPath } = options;
|
|
14149
13719
|
this.configuredModel = options.model;
|
|
@@ -14176,16 +13746,9 @@ class AutoRAGAgent {
|
|
|
14176
13746
|
...minSyncOpts,
|
|
14177
13747
|
root: this.workspaceProjectRoot
|
|
14178
13748
|
});
|
|
13749
|
+
this.minSyncReady = this.minSyncMethod.isReady();
|
|
14179
13750
|
this.methodRegistry.register(this.minSyncMethod);
|
|
14180
13751
|
this.methodRegistry.register(new MinSyncHybridMethod({ ...minSyncOpts, root: this.workspaceProjectRoot }));
|
|
14181
|
-
if (options.bm25 !== false) {
|
|
14182
|
-
const bm25Opts = { ...minSyncOpts, autoInstall: true, ...options.bm25 ?? {} };
|
|
14183
|
-
this.bm25Method = new MinSyncBM25Method({
|
|
14184
|
-
...bm25Opts,
|
|
14185
|
-
root: this.workspaceProjectRoot
|
|
14186
|
-
});
|
|
14187
|
-
this.methodRegistry.register(this.bm25Method);
|
|
14188
|
-
}
|
|
14189
13752
|
}
|
|
14190
13753
|
for (const skill of this.datasourceSkills) {
|
|
14191
13754
|
for (const method of skill.retrievalMethods())
|
|
@@ -14196,13 +13759,13 @@ class AutoRAGAgent {
|
|
|
14196
13759
|
...options.jikji ?? {},
|
|
14197
13760
|
root: this.workspaceProjectRoot
|
|
14198
13761
|
});
|
|
13762
|
+
this.jikjiReady = this.searchPaths.every((searchPath) => this.jikjiClient?.isPrepared(searchPath) === true);
|
|
14199
13763
|
}
|
|
14200
|
-
const memPath = memoryPath ??
|
|
13764
|
+
const memPath = memoryPath ?? join21(resolveAutoRAGHome(), "memory.json");
|
|
14201
13765
|
this.memory = new RetrievalMemory({ storagePath: memPath });
|
|
14202
13766
|
this.memory.load();
|
|
14203
|
-
this.runLogger = new AutoRAGRunLogger(
|
|
13767
|
+
this.runLogger = new AutoRAGRunLogger(join21(dirname14(memPath), "logs", "runs.jsonl"));
|
|
14204
13768
|
const checkMemoryTool = createCheckMemoryTool(this.memory);
|
|
14205
|
-
const searchBM25Tool = createSearchBM25DocumentsTool(() => this.bm25Method, (scope) => this.resolveRetrievalScope(scope));
|
|
14206
13769
|
const searchDatasourceTool = createSearchDatasourceDocumentsTool(this);
|
|
14207
13770
|
const searchMinSyncTool = createSearchMinSyncDocumentsTool(() => this.minSyncMethod, (scope) => this.resolveRetrievalScope(scope));
|
|
14208
13771
|
const searchAllTool = createSearchAllDocumentsTool(this);
|
|
@@ -14216,7 +13779,6 @@ class AutoRAGAgent {
|
|
|
14216
13779
|
const reservedNames = new Set([
|
|
14217
13780
|
BASH_TOOL_NAME,
|
|
14218
13781
|
"check_memory",
|
|
14219
|
-
SEARCH_BM25_DOCUMENTS_TOOL_NAME,
|
|
14220
13782
|
SEARCH_DATASOURCE_DOCUMENTS_TOOL_NAME,
|
|
14221
13783
|
LOAD_DATASOURCE_SKILL_TOOL_NAME,
|
|
14222
13784
|
EMIT_AUTORAG_RESULTS_TOOL_NAME,
|
|
@@ -14238,7 +13800,6 @@ class AutoRAGAgent {
|
|
|
14238
13800
|
bashTool,
|
|
14239
13801
|
...callerTools,
|
|
14240
13802
|
checkMemoryTool,
|
|
14241
|
-
searchBM25Tool,
|
|
14242
13803
|
searchMinSyncTool,
|
|
14243
13804
|
searchAllTool,
|
|
14244
13805
|
searchDatasourceTool,
|
|
@@ -14253,7 +13814,7 @@ class AutoRAGAgent {
|
|
|
14253
13814
|
return false;
|
|
14254
13815
|
seenToolNames.add(tool.name);
|
|
14255
13816
|
return true;
|
|
14256
|
-
});
|
|
13817
|
+
}).map((tool) => SEARCH_TOOLS.includes(tool.name) ? this.withPerSourceSearchLimit(tool) : tool);
|
|
14257
13818
|
this.tools = tools;
|
|
14258
13819
|
const toolNames = tools.map((tool) => tool.name);
|
|
14259
13820
|
this.baseSystemPromptConfig = {
|
|
@@ -14362,6 +13923,27 @@ ${summary}
|
|
|
14362
13923
|
}));
|
|
14363
13924
|
return unsubscribers;
|
|
14364
13925
|
}
|
|
13926
|
+
withPerSourceSearchLimit(tool) {
|
|
13927
|
+
return {
|
|
13928
|
+
...tool,
|
|
13929
|
+
execute: async (...args) => {
|
|
13930
|
+
const count = this.sourceSearchCallCounts.get(tool.name) ?? 0;
|
|
13931
|
+
if (count >= 3) {
|
|
13932
|
+
return {
|
|
13933
|
+
content: [
|
|
13934
|
+
{
|
|
13935
|
+
type: "text",
|
|
13936
|
+
text: `${tool.name} has reached its three-query limit. Do not call it again; conclude from the evidence already gathered.`
|
|
13937
|
+
}
|
|
13938
|
+
],
|
|
13939
|
+
details: { method: tool.name, resultCount: 0, sources: [], limitReached: true }
|
|
13940
|
+
};
|
|
13941
|
+
}
|
|
13942
|
+
this.sourceSearchCallCounts.set(tool.name, count + 1);
|
|
13943
|
+
return tool.execute(...args);
|
|
13944
|
+
}
|
|
13945
|
+
};
|
|
13946
|
+
}
|
|
14365
13947
|
recordSearchToolEvent(event) {
|
|
14366
13948
|
if (event.type !== "tool_execution_end" || !this.lastQuery)
|
|
14367
13949
|
return;
|
|
@@ -14460,8 +14042,11 @@ ${summary}
|
|
|
14460
14042
|
options = this.normalizeRetrievalOptions(options);
|
|
14461
14043
|
this.activeRun = true;
|
|
14462
14044
|
this.searchToolCallCount = 0;
|
|
14045
|
+
this.sourceSearchCallCounts.clear();
|
|
14463
14046
|
this.lastQuery = trimmedQuery;
|
|
14464
14047
|
this.lastSessionId = sessionId;
|
|
14048
|
+
this.scheduleJikjiPrepare();
|
|
14049
|
+
this.scheduleMinSyncPrepare();
|
|
14465
14050
|
let captured;
|
|
14466
14051
|
let session;
|
|
14467
14052
|
let unsubscribers = [];
|
|
@@ -14486,8 +14071,14 @@ ${summary}
|
|
|
14486
14071
|
try {
|
|
14487
14072
|
await Promise.race([
|
|
14488
14073
|
(async () => {
|
|
14489
|
-
const
|
|
14490
|
-
await session.prompt(this.buildSearchPrompt(trimmedQuery, options
|
|
14074
|
+
const retrievalPromise = this.prefetchInitialRetrievalContext(trimmedQuery, options);
|
|
14075
|
+
await session.prompt(this.buildSearchPrompt(trimmedQuery, options));
|
|
14076
|
+
if (captured === undefined) {
|
|
14077
|
+
const initialRetrievalContext = await retrievalPromise;
|
|
14078
|
+
await session.prompt(`Baseline retrieval is complete. Use this evidence before deciding whether additional search is needed:
|
|
14079
|
+
|
|
14080
|
+
${initialRetrievalContext}`);
|
|
14081
|
+
}
|
|
14491
14082
|
})(),
|
|
14492
14083
|
new Promise((_, reject) => {
|
|
14493
14084
|
timeout = setTimeout(() => {
|
|
@@ -14546,6 +14137,73 @@ ${summary}
|
|
|
14546
14137
|
this.activeRun = false;
|
|
14547
14138
|
}
|
|
14548
14139
|
}
|
|
14140
|
+
async* searchDocumentsStream(query, options = {}) {
|
|
14141
|
+
const queue = [];
|
|
14142
|
+
let progressBuffer = "";
|
|
14143
|
+
let wake;
|
|
14144
|
+
let settled = false;
|
|
14145
|
+
const unsubscribe = this.subscribe((event) => {
|
|
14146
|
+
if (event.type !== "message_update" || event.assistantMessageEvent.type !== "text_delta")
|
|
14147
|
+
return;
|
|
14148
|
+
const text = event.assistantMessageEvent.delta;
|
|
14149
|
+
if (text.trim().length === 0)
|
|
14150
|
+
return;
|
|
14151
|
+
progressBuffer += text;
|
|
14152
|
+
if (!/[.!?。!?]\s*$/u.test(progressBuffer))
|
|
14153
|
+
return;
|
|
14154
|
+
queue.push({
|
|
14155
|
+
type: "progress",
|
|
14156
|
+
sessionId: this.lastSessionId ?? "",
|
|
14157
|
+
query: query.trim(),
|
|
14158
|
+
text: progressBuffer
|
|
14159
|
+
});
|
|
14160
|
+
progressBuffer = "";
|
|
14161
|
+
wake?.();
|
|
14162
|
+
wake = undefined;
|
|
14163
|
+
});
|
|
14164
|
+
queue.push({
|
|
14165
|
+
type: "progress",
|
|
14166
|
+
sessionId: "",
|
|
14167
|
+
query: query.trim(),
|
|
14168
|
+
text: "Reviewing the query."
|
|
14169
|
+
});
|
|
14170
|
+
const run = this.searchDocuments(query, options).then((response) => {
|
|
14171
|
+
if (progressBuffer.trim() !== "") {
|
|
14172
|
+
queue.push({
|
|
14173
|
+
type: "progress",
|
|
14174
|
+
sessionId: this.lastSessionId ?? "",
|
|
14175
|
+
query: query.trim(),
|
|
14176
|
+
text: progressBuffer
|
|
14177
|
+
});
|
|
14178
|
+
progressBuffer = "";
|
|
14179
|
+
}
|
|
14180
|
+
queue.push({ type: "complete", response });
|
|
14181
|
+
}).catch((error) => {
|
|
14182
|
+
settled = true;
|
|
14183
|
+
wake?.();
|
|
14184
|
+
throw error;
|
|
14185
|
+
}).finally(() => {
|
|
14186
|
+
settled = true;
|
|
14187
|
+
wake?.();
|
|
14188
|
+
});
|
|
14189
|
+
try {
|
|
14190
|
+
while (!settled || queue.length > 0) {
|
|
14191
|
+
if (queue.length === 0) {
|
|
14192
|
+
await new Promise((resolve) => {
|
|
14193
|
+
wake = resolve;
|
|
14194
|
+
});
|
|
14195
|
+
continue;
|
|
14196
|
+
}
|
|
14197
|
+
yield queue.shift();
|
|
14198
|
+
}
|
|
14199
|
+
await run;
|
|
14200
|
+
} finally {
|
|
14201
|
+
unsubscribe();
|
|
14202
|
+
await run.catch(() => {
|
|
14203
|
+
return;
|
|
14204
|
+
});
|
|
14205
|
+
}
|
|
14206
|
+
}
|
|
14549
14207
|
datasourceAccessContext(options = {}) {
|
|
14550
14208
|
return new DatasourceAccessContext({
|
|
14551
14209
|
allowedTags: options.allowedTags ?? this.datasourceAccessOptions.allowedTags,
|
|
@@ -14604,17 +14262,6 @@ ${summary}
|
|
|
14604
14262
|
source: "tools"
|
|
14605
14263
|
});
|
|
14606
14264
|
}
|
|
14607
|
-
const bm25 = this.bm25Method?.getStatus();
|
|
14608
|
-
if (bm25 !== undefined) {
|
|
14609
|
-
if (bm25.readiness === "dependency_unavailable" || bm25.readiness === "index_missing" || bm25.readiness === "error") {
|
|
14610
|
-
diagnostics.push({
|
|
14611
|
-
code: "bm25-unavailable",
|
|
14612
|
-
severity: "warning",
|
|
14613
|
-
message: "BM25 lexical search is unavailable; results rely on other retrieval paths.",
|
|
14614
|
-
source: "bm25"
|
|
14615
|
-
});
|
|
14616
|
-
}
|
|
14617
|
-
}
|
|
14618
14265
|
if (this.minSyncMethod?.isBinaryMissing()) {
|
|
14619
14266
|
diagnostics.push({
|
|
14620
14267
|
code: "minsync-unavailable",
|
|
@@ -14629,51 +14276,84 @@ ${summary}
|
|
|
14629
14276
|
return diagnostics;
|
|
14630
14277
|
}
|
|
14631
14278
|
async prefetchInitialRetrievalContext(query, options) {
|
|
14632
|
-
const retrieveOptions = { topK:
|
|
14633
|
-
const
|
|
14634
|
-
|
|
14279
|
+
const retrieveOptions = { topK: 50, scope: options.scope };
|
|
14280
|
+
const vectorReady = this.minSyncMethod?.isReady() === true;
|
|
14281
|
+
const [jikji, vector] = await Promise.all([
|
|
14282
|
+
this.jikjiClient === undefined ? Promise.resolve(undefined) : this.findJikji(query, { topK: 50 }).catch(() => {
|
|
14635
14283
|
return;
|
|
14636
14284
|
}),
|
|
14637
|
-
this.minSyncMethod?.retrieve(query, retrieveOptions).catch(() => [])
|
|
14638
|
-
this.bm25Method?.retrieve(query, retrieveOptions).catch(() => [])
|
|
14285
|
+
vectorReady ? this.minSyncMethod?.retrieve(query, retrieveOptions).catch(() => []) : Promise.resolve([])
|
|
14639
14286
|
]);
|
|
14640
14287
|
const sections = [];
|
|
14641
14288
|
if (jikji?.answerPack !== undefined) {
|
|
14642
14289
|
sections.push(`Jikji initial candidates (preserve order when agent_should_not_rerank=true):
|
|
14643
|
-
${jikji.answerPack.answerPaths.slice(0,
|
|
14290
|
+
${jikji.answerPack.answerPaths.slice(0, 50).map((path, index) => `[${index + 1}] ${path}`).join(`
|
|
14644
14291
|
`)}`);
|
|
14645
14292
|
}
|
|
14646
|
-
const
|
|
14293
|
+
const formatResults = (label, results) => {
|
|
14647
14294
|
if (!results || results.length === 0)
|
|
14648
14295
|
return;
|
|
14296
|
+
const seen = new Set;
|
|
14649
14297
|
sections.push(`${label} initial candidates:
|
|
14650
|
-
${results.
|
|
14298
|
+
${results.filter((result) => {
|
|
14299
|
+
const key = `${result.source}\x00${result.content}`;
|
|
14300
|
+
if (seen.has(key))
|
|
14301
|
+
return false;
|
|
14302
|
+
seen.add(key);
|
|
14303
|
+
return true;
|
|
14304
|
+
}).slice(0, 50).map((result, index) => `[${index + 1}] ${result.source}
|
|
14651
14305
|
${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
14652
14306
|
`)}`);
|
|
14653
14307
|
};
|
|
14654
|
-
|
|
14655
|
-
formatResults5("MinSync lexical BM25", bm25);
|
|
14308
|
+
formatResults("MinSync semantic", vector);
|
|
14656
14309
|
return sections.length === 0 ? "No initial retrieval candidates were available; use the configured tools and report degradation honestly." : sections.join(`
|
|
14657
14310
|
|
|
14658
14311
|
`);
|
|
14659
14312
|
}
|
|
14313
|
+
scheduleMinSyncPrepareForTest() {
|
|
14314
|
+
return this.scheduleMinSyncPrepare();
|
|
14315
|
+
}
|
|
14316
|
+
scheduleMinSyncPrepare() {
|
|
14317
|
+
if (this.minSyncMethod === undefined || this.minSyncReady || this.minSyncPrepareInFlight !== undefined) {
|
|
14318
|
+
return this.minSyncPrepareInFlight;
|
|
14319
|
+
}
|
|
14320
|
+
this.minSyncPrepareInFlight = Promise.resolve().then(async () => {
|
|
14321
|
+
await this.syncParsedMirrors(false);
|
|
14322
|
+
if (!this.minSyncMethod)
|
|
14323
|
+
return;
|
|
14324
|
+
this.minSyncReady = this.minSyncMethod.isReady();
|
|
14325
|
+
if (this.minSyncReady)
|
|
14326
|
+
return;
|
|
14327
|
+
const result = await this.minSyncMethod.sync();
|
|
14328
|
+
this.minSyncReady = result?.ok === true;
|
|
14329
|
+
this.refreshState = { ...this.refreshState, minsync: result };
|
|
14330
|
+
return result;
|
|
14331
|
+
}).catch(() => {
|
|
14332
|
+
return;
|
|
14333
|
+
}).finally(() => {
|
|
14334
|
+
this.minSyncPrepareInFlight = undefined;
|
|
14335
|
+
});
|
|
14336
|
+
return this.minSyncPrepareInFlight;
|
|
14337
|
+
}
|
|
14660
14338
|
buildSearchPrompt(query, options, initialRetrievalContext) {
|
|
14661
14339
|
const limit = typeof options.topK === "number" ? ` Return at most ${options.topK} curated results.` : "";
|
|
14662
14340
|
const scope = options.scope ? ` Restrict search to virtual path scope ${options.scope}.` : "";
|
|
14663
14341
|
return `Find and curate information for this original query: ${query}${limit}${scope}
|
|
14664
14342
|
|
|
14665
|
-
` + `
|
|
14666
|
-
|
|
14343
|
+
` + `Start by deciding whether this is answerable from general knowledge or memory. If it is a generic, stable question, answer it immediately without retrieval and emit the structured result. ` + `Otherwise, baseline MinSync and Jikji retrieval is already running in parallel; do not emit final results until its next message arrives.
|
|
14344
|
+
|
|
14345
|
+
` + `Baseline retrieval context:
|
|
14346
|
+
${initialRetrievalContext ?? "Pending; continue only with a brief progress statement."}
|
|
14667
14347
|
|
|
14668
|
-
` + `Treat
|
|
14348
|
+
` + `Treat candidates as unverified evidence, verify important claims against source files, and use additional tools when needed. ` + `Use the available retrieval tools to find candidates, then use bash to read and verify the relevant source files directly. ` + `Judge relevance, conflicts, freshness, and sufficiency in this agent loop. Preserve real source paths and evidence excerpts in the result mapping.
|
|
14669
14349
|
|
|
14670
|
-
` + `When finished, call ${EMIT_AUTORAG_RESULTS_TOOL_NAME} exactly once as your final action with the curated ` + `results and the internal number-to-source mapping.`;
|
|
14350
|
+
` + `If more search is needed, first write a brief 1–2 line progress update stating the best current hypothesis and what you are checking next, then call retrieval tools. ` + `Never repeat a generic status message. Do not use broad grep/find or recursive filesystem scans: only inspect a path or narrow neighborhood surfaced by retrieval, and only when evidence clearly points there. ` + `Do not query the same datasource more than three times. After three attempts, stop searching that datasource and conclude from the evidence available. ` + `When finished, call ${EMIT_AUTORAG_RESULTS_TOOL_NAME} exactly once as your final action with the curated ` + `results and the internal number-to-source mapping.`;
|
|
14671
14351
|
}
|
|
14672
14352
|
async refresh(force = false, opts) {
|
|
14673
14353
|
const methods = opts?.methods;
|
|
14674
14354
|
const allMethods = methods === undefined;
|
|
14675
14355
|
const wants = (m) => allMethods || methods.includes(m);
|
|
14676
|
-
const needsParsed = allMethods || wants("parsed") || wants("
|
|
14356
|
+
const needsParsed = allMethods || wants("parsed") || wants("minsync");
|
|
14677
14357
|
this.refreshState = {
|
|
14678
14358
|
...this.refreshState,
|
|
14679
14359
|
inFlight: true,
|
|
@@ -14681,13 +14361,9 @@ ${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
|
14681
14361
|
};
|
|
14682
14362
|
try {
|
|
14683
14363
|
const summary = needsParsed ? await this.syncParsedMirrors(force) : await this.scanMirrorStaleness();
|
|
14684
|
-
if (wants("bm25") || wants("minsync") || allMethods) {
|
|
14685
|
-
removeLegacyBm25Artifacts(this.workspaceProjectRoot);
|
|
14686
|
-
}
|
|
14687
14364
|
const minsync = wants("minsync") ? await this.syncMinSync() : undefined;
|
|
14688
|
-
const bm25 = wants("bm25") ? await this.syncBM25(minsync) : undefined;
|
|
14689
14365
|
const datasources = wants("datasources") ? await this.indexDatasources() : [];
|
|
14690
|
-
const jikji = wants("jikji") ? await this.executeJikjiPrepare() : undefined;
|
|
14366
|
+
const jikji = allMethods || wants("jikji") || wants("minsync") ? await this.executeJikjiPrepare() : undefined;
|
|
14691
14367
|
this.retrievalScopeBindings = buildRetrievalScopeBindings(this.workspaceProjectRoot, this.searchPaths, this.configuredSearchPaths);
|
|
14692
14368
|
const jikjiDiagnostics = (jikji ?? []).map((result) => jikjiPrepareDiagnostic(result)).filter((diag) => diag !== undefined);
|
|
14693
14369
|
this.refreshState = {
|
|
@@ -14711,9 +14387,8 @@ ${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
|
14711
14387
|
...minsync.reason !== undefined ? { reason: minsync.reason } : {}
|
|
14712
14388
|
} : undefined;
|
|
14713
14389
|
return {
|
|
14714
|
-
...
|
|
14390
|
+
...summary,
|
|
14715
14391
|
diagnostics: [...this.startupDiagnostics, ...summary.diagnostics],
|
|
14716
|
-
...bm25 ? { bm25 } : {},
|
|
14717
14392
|
minsync: publicMinsync,
|
|
14718
14393
|
datasources
|
|
14719
14394
|
};
|
|
@@ -14783,11 +14458,8 @@ ${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
|
14783
14458
|
}
|
|
14784
14459
|
refreshComponentStatus() {
|
|
14785
14460
|
const status = {};
|
|
14786
|
-
const bm25 = this.bm25Method?.getStatus();
|
|
14787
|
-
if (bm25 !== undefined)
|
|
14788
|
-
status.bm25 = bm25.readiness;
|
|
14789
14461
|
if (this.minSyncMethod !== undefined) {
|
|
14790
|
-
status.minsync = this.minSyncMethod.isBinaryMissing() ? "unavailable" : this.refreshState.minsync?.ok === false ? "degraded" : this.
|
|
14462
|
+
status.minsync = this.minSyncMethod.isBinaryMissing() ? "unavailable" : this.refreshState.minsync?.ok === false ? "degraded" : this.minSyncMethod.isReady() ? "ready" : "configured";
|
|
14791
14463
|
}
|
|
14792
14464
|
if (this.jikjiClient !== undefined) {
|
|
14793
14465
|
status.jikji = this.refreshState.jikjiDiagnostics.length > 0 ? "degraded" : "configured";
|
|
@@ -14853,7 +14525,7 @@ ${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
|
14853
14525
|
written: 0,
|
|
14854
14526
|
deleted: 0,
|
|
14855
14527
|
skipped: 0,
|
|
14856
|
-
indexPath:
|
|
14528
|
+
indexPath: join21(this.workspaceProjectRoot, PARSED_MIRROR_SUBDIR),
|
|
14857
14529
|
diagnostics
|
|
14858
14530
|
};
|
|
14859
14531
|
}
|
|
@@ -14874,13 +14546,11 @@ ${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
|
14874
14546
|
}
|
|
14875
14547
|
return { excluded };
|
|
14876
14548
|
}
|
|
14877
|
-
async syncBM25(minsync) {
|
|
14878
|
-
if (this.bm25Method === undefined)
|
|
14879
|
-
return;
|
|
14880
|
-
return this.bm25Method.syncFromMinSync(minsync ?? await this.bm25Method.sync());
|
|
14881
|
-
}
|
|
14882
14549
|
async syncMinSync() {
|
|
14883
|
-
|
|
14550
|
+
const result = await this.minSyncMethod?.sync();
|
|
14551
|
+
this.minSyncReady = result?.ok === true;
|
|
14552
|
+
this.refreshState = { ...this.refreshState, minsync: result };
|
|
14553
|
+
return result;
|
|
14884
14554
|
}
|
|
14885
14555
|
async prepareJikji() {
|
|
14886
14556
|
const results = await this.executeJikjiPrepare();
|
|
@@ -14893,8 +14563,20 @@ ${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
|
14893
14563
|
for (const sourcePath of this.searchPaths) {
|
|
14894
14564
|
results.push(await this.jikjiClient.prepare(sourcePath));
|
|
14895
14565
|
}
|
|
14566
|
+
this.jikjiReady = results.length === this.searchPaths.length && results.every((result) => result.ok);
|
|
14896
14567
|
return results;
|
|
14897
14568
|
}
|
|
14569
|
+
scheduleJikjiPrepare() {
|
|
14570
|
+
if (this.jikjiClient === undefined || this.jikjiPrepareInFlight !== undefined)
|
|
14571
|
+
return;
|
|
14572
|
+
this.jikjiPrepareInFlight = Promise.resolve().then(async () => {
|
|
14573
|
+
const results = await this.executeJikjiPrepare();
|
|
14574
|
+
const diagnostics = (results ?? []).map((result) => jikjiPrepareDiagnostic(result)).filter((diag) => diag !== undefined);
|
|
14575
|
+
this.refreshState = { ...this.refreshState, jikjiDiagnostics: diagnostics };
|
|
14576
|
+
}).catch(() => {}).finally(() => {
|
|
14577
|
+
this.jikjiPrepareInFlight = undefined;
|
|
14578
|
+
});
|
|
14579
|
+
}
|
|
14898
14580
|
sanitizeJikjiPrepareResult(result) {
|
|
14899
14581
|
if (result.ok) {
|
|
14900
14582
|
return {
|
|
@@ -14914,6 +14596,23 @@ ${result.content.replace(/\s+/gu, " ").slice(0, 400)}`).join(`
|
|
|
14914
14596
|
if (this.jikjiClient === undefined) {
|
|
14915
14597
|
return { answerPack: undefined, policy: undefined, diagnostics: [], roots: [], perRoot: [] };
|
|
14916
14598
|
}
|
|
14599
|
+
if (!this.jikjiReady) {
|
|
14600
|
+
this.scheduleJikjiPrepare();
|
|
14601
|
+
return {
|
|
14602
|
+
answerPack: undefined,
|
|
14603
|
+
policy: undefined,
|
|
14604
|
+
diagnostics: [
|
|
14605
|
+
{
|
|
14606
|
+
code: "jikji-unavailable",
|
|
14607
|
+
severity: "warning",
|
|
14608
|
+
message: "Jikji is not prepared yet; background preparation started and this turn is falling back.",
|
|
14609
|
+
source: "jikji"
|
|
14610
|
+
}
|
|
14611
|
+
],
|
|
14612
|
+
roots: this.searchPaths,
|
|
14613
|
+
perRoot: []
|
|
14614
|
+
};
|
|
14615
|
+
}
|
|
14917
14616
|
const sourceRoots = planJikjiSourceRoots(this.searchPaths);
|
|
14918
14617
|
const findOpts = {
|
|
14919
14618
|
topK: opts?.topK,
|
|
@@ -15188,7 +14887,6 @@ var init_agent = __esm(() => {
|
|
|
15188
14887
|
init_paths();
|
|
15189
14888
|
init_sync();
|
|
15190
14889
|
init_run_log();
|
|
15191
|
-
init_bm25();
|
|
15192
14890
|
init_scope();
|
|
15193
14891
|
init_bash_tool();
|
|
15194
14892
|
init_datasource_skill();
|
|
@@ -15197,7 +14895,6 @@ var init_agent = __esm(() => {
|
|
|
15197
14895
|
init_jikji_find_tool();
|
|
15198
14896
|
init_local_model();
|
|
15199
14897
|
init_search_all_tool();
|
|
15200
|
-
init_search_bm25_tool();
|
|
15201
14898
|
init_search_datasource_tool();
|
|
15202
14899
|
init_search_documents();
|
|
15203
14900
|
init_search_minsync_tool();
|
|
@@ -15207,18 +14904,12 @@ var init_agent = __esm(() => {
|
|
|
15207
14904
|
BASH_TOOL_NAME,
|
|
15208
14905
|
SEARCH_MINSYNC_DOCUMENTS_TOOL_NAME,
|
|
15209
14906
|
SEARCH_ALL_DOCUMENTS_TOOL_NAME,
|
|
15210
|
-
SEARCH_BM25_DOCUMENTS_TOOL_NAME,
|
|
15211
14907
|
SEARCH_DATASOURCE_DOCUMENTS_TOOL_NAME,
|
|
15212
14908
|
JIKJI_FIND_TOOL_NAME
|
|
15213
14909
|
];
|
|
15214
14910
|
});
|
|
15215
14911
|
|
|
15216
14912
|
// src/cli/commands/refresh.ts
|
|
15217
|
-
var exports_refresh = {};
|
|
15218
|
-
__export(exports_refresh, {
|
|
15219
|
-
parseMethodFlag: () => parseMethodFlag,
|
|
15220
|
-
runRefresh: () => runRefresh
|
|
15221
|
-
});
|
|
15222
14913
|
async function runRefresh(ctx) {
|
|
15223
14914
|
try {
|
|
15224
14915
|
const config = resolveConfig({ flags: ctx.flags, cwd: ctx.cwd });
|
|
@@ -15246,9 +14937,9 @@ function parseMethodFlag(flag) {
|
|
|
15246
14937
|
}
|
|
15247
14938
|
}
|
|
15248
14939
|
if (entries.includes("all")) {
|
|
15249
|
-
return ["parsed", "
|
|
14940
|
+
return ["parsed", "minsync", "datasources", "jikji"];
|
|
15250
14941
|
}
|
|
15251
|
-
const order = ["parsed", "
|
|
14942
|
+
const order = ["parsed", "minsync", "datasources", "jikji"];
|
|
15252
14943
|
const seen = new Set;
|
|
15253
14944
|
for (const m of order) {
|
|
15254
14945
|
if (entries.includes(m))
|
|
@@ -15260,14 +14951,10 @@ var ALL_METHOD_NAMES;
|
|
|
15260
14951
|
var init_refresh = __esm(() => {
|
|
15261
14952
|
init_agent();
|
|
15262
14953
|
init_config2();
|
|
15263
|
-
ALL_METHOD_NAMES = ["all", "
|
|
14954
|
+
ALL_METHOD_NAMES = ["all", "minsync", "parsed", "datasources", "jikji"];
|
|
15264
14955
|
});
|
|
15265
14956
|
|
|
15266
14957
|
// src/cli/commands/status.ts
|
|
15267
|
-
var exports_status = {};
|
|
15268
|
-
__export(exports_status, {
|
|
15269
|
-
runStatus: () => runStatus
|
|
15270
|
-
});
|
|
15271
14958
|
async function runStatus(ctx) {
|
|
15272
14959
|
try {
|
|
15273
14960
|
const config = resolveConfig({ flags: ctx.flags, cwd: ctx.cwd });
|
|
@@ -15286,11 +14973,6 @@ var init_status = __esm(() => {
|
|
|
15286
14973
|
});
|
|
15287
14974
|
|
|
15288
14975
|
// src/cli/commands/search.ts
|
|
15289
|
-
var exports_search = {};
|
|
15290
|
-
__export(exports_search, {
|
|
15291
|
-
classifySearchHealthHint: () => classifySearchHealthHint,
|
|
15292
|
-
runSearch: () => runSearch
|
|
15293
|
-
});
|
|
15294
14976
|
function classifySearchHealthHint(error) {
|
|
15295
14977
|
const message = (error instanceof Error ? error.message : String(error)).toLowerCase();
|
|
15296
14978
|
const name = error instanceof Error ? error.name : "";
|
|
@@ -15385,8 +15067,18 @@ async function runSearch(ctx, deps = {}) {
|
|
|
15385
15067
|
}
|
|
15386
15068
|
const options = buildSearchOptions(ctx.flags);
|
|
15387
15069
|
try {
|
|
15388
|
-
const
|
|
15389
|
-
|
|
15070
|
+
for await (const event of agent.searchDocumentsStream(query, options)) {
|
|
15071
|
+
switch (event.type) {
|
|
15072
|
+
case "progress":
|
|
15073
|
+
if (event.text.trim() === "")
|
|
15074
|
+
break;
|
|
15075
|
+
ctx.stdout(ctx.json ? JSON.stringify({ type: "progress", text: event.text }) : event.text);
|
|
15076
|
+
break;
|
|
15077
|
+
case "complete":
|
|
15078
|
+
ctx.stdout(renderSearch(event.response, { json: ctx.json, debug: ctx.debug }));
|
|
15079
|
+
break;
|
|
15080
|
+
}
|
|
15081
|
+
}
|
|
15390
15082
|
return 0;
|
|
15391
15083
|
} catch (error) {
|
|
15392
15084
|
const hint = classifySearchHealthHint(error);
|
|
@@ -15419,10 +15111,6 @@ var init_search = __esm(() => {
|
|
|
15419
15111
|
});
|
|
15420
15112
|
|
|
15421
15113
|
// src/cli/commands/feedback.ts
|
|
15422
|
-
var exports_feedback = {};
|
|
15423
|
-
__export(exports_feedback, {
|
|
15424
|
-
runFeedback: () => runFeedback
|
|
15425
|
-
});
|
|
15426
15114
|
function parseCsvInts(value) {
|
|
15427
15115
|
if (typeof value !== "string" || value.trim() === "")
|
|
15428
15116
|
return [];
|
|
@@ -15443,8 +15131,8 @@ async function runFeedback(ctx) {
|
|
|
15443
15131
|
return 2;
|
|
15444
15132
|
}
|
|
15445
15133
|
const feedback = [
|
|
15446
|
-
...usefulNumbers.map((
|
|
15447
|
-
...notUsefulNumbers.map((
|
|
15134
|
+
...usefulNumbers.map((number) => ({ number, useful: true })),
|
|
15135
|
+
...notUsefulNumbers.map((number) => ({ number, useful: false }))
|
|
15448
15136
|
];
|
|
15449
15137
|
let memoryPath;
|
|
15450
15138
|
try {
|
|
@@ -15478,11 +15166,86 @@ var init_feedback = __esm(() => {
|
|
|
15478
15166
|
init_config2();
|
|
15479
15167
|
});
|
|
15480
15168
|
|
|
15481
|
-
// src/cli/commands/
|
|
15482
|
-
|
|
15483
|
-
|
|
15484
|
-
|
|
15169
|
+
// src/cli/commands/evidence.ts
|
|
15170
|
+
function parseResultNumber(value) {
|
|
15171
|
+
if (typeof value !== "string" || value.trim() === "")
|
|
15172
|
+
return;
|
|
15173
|
+
const number = Number(value);
|
|
15174
|
+
return Number.isInteger(number) && number > 0 ? number : undefined;
|
|
15175
|
+
}
|
|
15176
|
+
function evidenceFor(schema, sessionId, resultNumber) {
|
|
15177
|
+
const results = schema.curatedResults.filter((result) => result.sessionId === sessionId).filter((result) => resultNumber === undefined || result.number === resultNumber).sort((a, b) => a.number - b.number).map((result) => {
|
|
15178
|
+
const chunks = result.evidenceIds.map((id) => schema.evidenceChunks.find((chunk) => chunk.stableEvidenceId === id)).filter((chunk) => chunk !== undefined).map((chunk) => ({
|
|
15179
|
+
stableEvidenceId: chunk.stableEvidenceId,
|
|
15180
|
+
method: chunk.method,
|
|
15181
|
+
source: chunk.source,
|
|
15182
|
+
...chunk.excerpt !== undefined ? { excerpt: chunk.excerpt } : {},
|
|
15183
|
+
...chunk.content !== undefined ? { content: chunk.content } : {},
|
|
15184
|
+
...chunk.retrievalResultId !== undefined ? { retrievalResultId: chunk.retrievalResultId } : {},
|
|
15185
|
+
...chunk.chunkIndex !== undefined ? { chunkIndex: chunk.chunkIndex } : {},
|
|
15186
|
+
...chunk.lineNumber !== undefined ? { lineNumber: chunk.lineNumber } : {},
|
|
15187
|
+
...chunk.metadata !== undefined ? { metadata: chunk.metadata } : {}
|
|
15188
|
+
}));
|
|
15189
|
+
return {
|
|
15190
|
+
number: result.number,
|
|
15191
|
+
title: result.title,
|
|
15192
|
+
summary: result.summary,
|
|
15193
|
+
chunks
|
|
15194
|
+
};
|
|
15195
|
+
});
|
|
15196
|
+
return { sessionId, results };
|
|
15197
|
+
}
|
|
15198
|
+
function renderEvidence(view, json) {
|
|
15199
|
+
if (json)
|
|
15200
|
+
return JSON.stringify(view, null, 2);
|
|
15201
|
+
const lines = [`session: ${view.sessionId}`];
|
|
15202
|
+
for (const result of view.results) {
|
|
15203
|
+
lines.push(`[${result.number}] ${result.title}`);
|
|
15204
|
+
for (const chunk of result.chunks) {
|
|
15205
|
+
lines.push(` source: ${chunk.source}`);
|
|
15206
|
+
lines.push(` method: ${chunk.method}`);
|
|
15207
|
+
lines.push(` evidenceId: ${chunk.stableEvidenceId}`);
|
|
15208
|
+
if (chunk.chunkIndex !== undefined)
|
|
15209
|
+
lines.push(` chunkIndex: ${chunk.chunkIndex}`);
|
|
15210
|
+
if (chunk.lineNumber !== undefined)
|
|
15211
|
+
lines.push(` lineNumber: ${chunk.lineNumber}`);
|
|
15212
|
+
if (chunk.excerpt !== undefined)
|
|
15213
|
+
lines.push(` excerpt: ${chunk.excerpt}`);
|
|
15214
|
+
if (chunk.content !== undefined)
|
|
15215
|
+
lines.push(` content: ${chunk.content}`);
|
|
15216
|
+
}
|
|
15217
|
+
}
|
|
15218
|
+
return lines.join(`
|
|
15219
|
+
`);
|
|
15220
|
+
}
|
|
15221
|
+
async function runEvidence(ctx) {
|
|
15222
|
+
const sessionId = ctx.positionals[0]?.trim();
|
|
15223
|
+
if (!sessionId) {
|
|
15224
|
+
ctx.stderr(renderError(new Error("Usage: autorag evidence <sessionId> [--result N]"), { json: ctx.json }));
|
|
15225
|
+
return 2;
|
|
15226
|
+
}
|
|
15227
|
+
try {
|
|
15228
|
+
const config = resolveConfig({ flags: ctx.flags, cwd: ctx.cwd });
|
|
15229
|
+
const memory = new RetrievalMemory({ storagePath: config.memoryPath });
|
|
15230
|
+
memory.load();
|
|
15231
|
+
const view = evidenceFor(memory.getSchema(), sessionId, parseResultNumber(ctx.flags.result));
|
|
15232
|
+
if (view.results.length === 0) {
|
|
15233
|
+
ctx.stderr(renderError(new Error(`No evidence found for session ${sessionId}.`), { json: ctx.json }));
|
|
15234
|
+
return 2;
|
|
15235
|
+
}
|
|
15236
|
+
ctx.stdout(renderEvidence(view, ctx.json));
|
|
15237
|
+
return 0;
|
|
15238
|
+
} catch (error) {
|
|
15239
|
+
ctx.stderr(renderError(error, { json: ctx.json }));
|
|
15240
|
+
return 1;
|
|
15241
|
+
}
|
|
15242
|
+
}
|
|
15243
|
+
var init_evidence = __esm(() => {
|
|
15244
|
+
init_memory();
|
|
15245
|
+
init_config2();
|
|
15485
15246
|
});
|
|
15247
|
+
|
|
15248
|
+
// src/cli/commands/memory.ts
|
|
15486
15249
|
async function runMemory(ctx) {
|
|
15487
15250
|
const subcommand = ctx.positionals[0];
|
|
15488
15251
|
if (subcommand !== "inspect") {
|
|
@@ -15507,12 +15270,8 @@ var init_memory2 = __esm(() => {
|
|
|
15507
15270
|
});
|
|
15508
15271
|
|
|
15509
15272
|
// src/cli/commands/index.ts
|
|
15510
|
-
|
|
15511
|
-
|
|
15512
|
-
runIndex: () => runIndex
|
|
15513
|
-
});
|
|
15514
|
-
import { existsSync as existsSync17, lstatSync as lstatSync3, rmSync as rmSync7 } from "node:fs";
|
|
15515
|
-
import { join as join23, resolve as resolve12, sep as sep5 } from "node:path";
|
|
15273
|
+
import { existsSync as existsSync16, lstatSync as lstatSync3, rmSync as rmSync6 } from "node:fs";
|
|
15274
|
+
import { join as join22, resolve as resolve12, sep as sep5 } from "node:path";
|
|
15516
15275
|
async function runIndex(ctx) {
|
|
15517
15276
|
const sub = ctx.positionals[0];
|
|
15518
15277
|
if (sub !== "reset" && sub !== "rebuild") {
|
|
@@ -15531,7 +15290,7 @@ async function runIndex(ctx) {
|
|
|
15531
15290
|
const autoragDir = resolve12(config.workspacePath, ".autorag");
|
|
15532
15291
|
const targets = targetSubdirs.map((subdir) => resolve12(config.workspacePath, subdir));
|
|
15533
15292
|
for (const dir of [autoragDir, ...targets]) {
|
|
15534
|
-
if (
|
|
15293
|
+
if (existsSync16(dir) && lstatSync3(dir).isSymbolicLink()) {
|
|
15535
15294
|
ctx.stderr(renderError(new Error("Refusing to reset: an index path is a symlink, not a real directory."), {
|
|
15536
15295
|
json: ctx.json
|
|
15537
15296
|
}));
|
|
@@ -15546,8 +15305,8 @@ async function runIndex(ctx) {
|
|
|
15546
15305
|
}
|
|
15547
15306
|
if (!ctx.flags.yes) {
|
|
15548
15307
|
if (ctx.promptYesNo) {
|
|
15549
|
-
const
|
|
15550
|
-
if (!
|
|
15308
|
+
const ok = await ctx.promptYesNo(`Reset the ${targetNames.join(", ")} indexes under ${join22(config.workspacePath, ".autorag")}?`);
|
|
15309
|
+
if (!ok) {
|
|
15551
15310
|
ctx.stderr(renderError(new Error("Reset declined."), { json: ctx.json }));
|
|
15552
15311
|
return 2;
|
|
15553
15312
|
}
|
|
@@ -15557,7 +15316,7 @@ async function runIndex(ctx) {
|
|
|
15557
15316
|
}
|
|
15558
15317
|
}
|
|
15559
15318
|
for (const target of targets) {
|
|
15560
|
-
|
|
15319
|
+
rmSync6(target, { recursive: true, force: true });
|
|
15561
15320
|
}
|
|
15562
15321
|
if (sub === "reset") {
|
|
15563
15322
|
ctx.stdout(renderIndex({ action: "reset", removed: [...targetNames] }, { json: ctx.json }));
|
|
@@ -15595,11 +15354,8 @@ function resolveResetScope(methods) {
|
|
|
15595
15354
|
refresh.push(m);
|
|
15596
15355
|
if (m === "parsed") {
|
|
15597
15356
|
addTarget("parsed", PARSED_MIRROR_SUBDIR);
|
|
15598
|
-
} else if (m === "bm25") {
|
|
15599
|
-
addTarget("bm25", BM25_SUBDIR);
|
|
15600
15357
|
} else if (m === "minsync") {
|
|
15601
15358
|
addTarget("minsync", MINSYNC_SUBDIR);
|
|
15602
|
-
addTarget("bm25", BM25_SUBDIR);
|
|
15603
15359
|
}
|
|
15604
15360
|
}
|
|
15605
15361
|
return {
|
|
@@ -15617,18 +15373,13 @@ var init_commands = __esm(() => {
|
|
|
15617
15373
|
init_agent();
|
|
15618
15374
|
init_paths4();
|
|
15619
15375
|
init_paths();
|
|
15620
|
-
init_bm25();
|
|
15621
15376
|
init_config2();
|
|
15622
15377
|
init_refresh();
|
|
15623
|
-
ALL_RESET_TARGETS = [PARSED_MIRROR_SUBDIR, MINSYNC_SUBDIR
|
|
15624
|
-
ALL_RESET_TARGET_NAMES = ["parsed", "minsync"
|
|
15378
|
+
ALL_RESET_TARGETS = [PARSED_MIRROR_SUBDIR, MINSYNC_SUBDIR];
|
|
15379
|
+
ALL_RESET_TARGET_NAMES = ["parsed", "minsync"];
|
|
15625
15380
|
});
|
|
15626
15381
|
|
|
15627
15382
|
// src/cli/commands/watch.ts
|
|
15628
|
-
var exports_watch = {};
|
|
15629
|
-
__export(exports_watch, {
|
|
15630
|
-
runWatch: () => runWatch
|
|
15631
|
-
});
|
|
15632
15383
|
function parsePositiveInt(value, fallback) {
|
|
15633
15384
|
if (typeof value !== "string" || value.trim().length === 0)
|
|
15634
15385
|
return fallback;
|
|
@@ -15665,14 +15416,14 @@ async function runWatch(ctx) {
|
|
|
15665
15416
|
...started,
|
|
15666
15417
|
message: "watching for source changes; send SIGINT/SIGTERM to stop"
|
|
15667
15418
|
}) : `AutoRAG watch started (roots=${started.roots}, debounceMs=${debounceMs}). Ctrl-C to stop.`);
|
|
15668
|
-
await new Promise((
|
|
15419
|
+
await new Promise((resolve) => {
|
|
15669
15420
|
let stopped = false;
|
|
15670
15421
|
const stop = () => {
|
|
15671
15422
|
if (stopped)
|
|
15672
15423
|
return;
|
|
15673
15424
|
stopped = true;
|
|
15674
15425
|
handle.stop();
|
|
15675
|
-
|
|
15426
|
+
resolve();
|
|
15676
15427
|
};
|
|
15677
15428
|
process.once("SIGINT", stop);
|
|
15678
15429
|
process.once("SIGTERM", stop);
|
|
@@ -15694,10 +15445,6 @@ var init_watch = __esm(() => {
|
|
|
15694
15445
|
});
|
|
15695
15446
|
|
|
15696
15447
|
// src/cli/commands/health.ts
|
|
15697
|
-
var exports_health = {};
|
|
15698
|
-
__export(exports_health, {
|
|
15699
|
-
runHealth: () => runHealth
|
|
15700
|
-
});
|
|
15701
15448
|
import { completeSimple } from "@earendil-works/pi-ai/compat";
|
|
15702
15449
|
function sanitizeMessage(raw) {
|
|
15703
15450
|
let out = raw.split(/\n\s+at\s/)[0] ?? raw;
|
|
@@ -15709,7 +15456,7 @@ function sanitizeMessage(raw) {
|
|
|
15709
15456
|
function parseTimeoutMs(flags) {
|
|
15710
15457
|
const raw = flags["timeout-ms"];
|
|
15711
15458
|
if (raw === undefined || raw === true)
|
|
15712
|
-
return
|
|
15459
|
+
return DEFAULT_TIMEOUT_MS9;
|
|
15713
15460
|
const text = typeof raw === "string" ? raw.trim() : "";
|
|
15714
15461
|
if (!/^\d+$/.test(text) || Number(text) <= 0) {
|
|
15715
15462
|
return `--timeout-ms must be a positive integer (got "${text}")`;
|
|
@@ -15795,7 +15542,7 @@ async function runHealth(ctx, deps = {}) {
|
|
|
15795
15542
|
indexHealth: { separate: true, command: "autorag status", included: false }
|
|
15796
15543
|
};
|
|
15797
15544
|
if (typeof timeout === "string") {
|
|
15798
|
-
const
|
|
15545
|
+
const report = {
|
|
15799
15546
|
...base,
|
|
15800
15547
|
ok: false,
|
|
15801
15548
|
category: "config",
|
|
@@ -15803,14 +15550,14 @@ async function runHealth(ctx, deps = {}) {
|
|
|
15803
15550
|
coverage: coverage(),
|
|
15804
15551
|
config: { ok: false, source: "defaults", message: timeout }
|
|
15805
15552
|
};
|
|
15806
|
-
ctx.stdout(renderHealth(
|
|
15553
|
+
ctx.stdout(renderHealth(report, { json: ctx.json, debug: ctx.debug }));
|
|
15807
15554
|
return 2;
|
|
15808
15555
|
}
|
|
15809
15556
|
let config;
|
|
15810
15557
|
try {
|
|
15811
15558
|
config = (deps.configResolver ?? resolveConfigReadOnly)({ flags: ctx.flags, cwd: ctx.cwd });
|
|
15812
15559
|
} catch (error) {
|
|
15813
|
-
const
|
|
15560
|
+
const report = {
|
|
15814
15561
|
...base,
|
|
15815
15562
|
ok: false,
|
|
15816
15563
|
category: "config",
|
|
@@ -15818,7 +15565,7 @@ async function runHealth(ctx, deps = {}) {
|
|
|
15818
15565
|
coverage: coverage(),
|
|
15819
15566
|
config: { ok: false, source: "defaults", message: sanitizeMessage(String(error)) }
|
|
15820
15567
|
};
|
|
15821
|
-
ctx.stdout(renderHealth(
|
|
15568
|
+
ctx.stdout(renderHealth(report, { json: ctx.json, debug: ctx.debug }));
|
|
15822
15569
|
return 2;
|
|
15823
15570
|
}
|
|
15824
15571
|
let resolved;
|
|
@@ -15826,7 +15573,7 @@ async function runHealth(ctx, deps = {}) {
|
|
|
15826
15573
|
const detailed = (deps.modelResolver ?? resolveAgentModelDetailed)(config);
|
|
15827
15574
|
resolved = detailed;
|
|
15828
15575
|
} catch (error) {
|
|
15829
|
-
const
|
|
15576
|
+
const report = {
|
|
15830
15577
|
...base,
|
|
15831
15578
|
ok: false,
|
|
15832
15579
|
category: "model_resolution",
|
|
@@ -15840,12 +15587,12 @@ async function runHealth(ctx, deps = {}) {
|
|
|
15840
15587
|
message: sanitizeMessage(error instanceof Error ? error.message : String(error))
|
|
15841
15588
|
}
|
|
15842
15589
|
};
|
|
15843
|
-
ctx.stdout(renderHealth(
|
|
15590
|
+
ctx.stdout(renderHealth(report, { json: ctx.json, debug: ctx.debug }));
|
|
15844
15591
|
return 2;
|
|
15845
15592
|
}
|
|
15846
15593
|
const model = roleReport(resolved.role);
|
|
15847
15594
|
if (!resolved.role.auth.present) {
|
|
15848
|
-
const
|
|
15595
|
+
const report = {
|
|
15849
15596
|
...base,
|
|
15850
15597
|
ok: false,
|
|
15851
15598
|
category: "auth_missing",
|
|
@@ -15855,11 +15602,11 @@ async function runHealth(ctx, deps = {}) {
|
|
|
15855
15602
|
model,
|
|
15856
15603
|
probe: { skipped: true, ok: false, category: "auth_missing" }
|
|
15857
15604
|
};
|
|
15858
|
-
ctx.stdout(renderHealth(
|
|
15605
|
+
ctx.stdout(renderHealth(report, { json: ctx.json, debug: ctx.debug }));
|
|
15859
15606
|
return 1;
|
|
15860
15607
|
}
|
|
15861
15608
|
if (ctx.flags["skip-probes"] === true) {
|
|
15862
|
-
const
|
|
15609
|
+
const report = {
|
|
15863
15610
|
...base,
|
|
15864
15611
|
ok: true,
|
|
15865
15612
|
category: "ok",
|
|
@@ -15869,7 +15616,7 @@ async function runHealth(ctx, deps = {}) {
|
|
|
15869
15616
|
model,
|
|
15870
15617
|
probe: { skipped: true, ok: false, category: "ok" }
|
|
15871
15618
|
};
|
|
15872
|
-
ctx.stdout(renderHealth(
|
|
15619
|
+
ctx.stdout(renderHealth(report, { json: ctx.json, debug: ctx.debug }));
|
|
15873
15620
|
return 0;
|
|
15874
15621
|
}
|
|
15875
15622
|
const probe = await runProbe(deps.probe ?? defaultProbe, {
|
|
@@ -15892,16 +15639,12 @@ async function runHealth(ctx, deps = {}) {
|
|
|
15892
15639
|
ctx.stdout(renderHealth(report, { json: ctx.json, debug: ctx.debug }));
|
|
15893
15640
|
return exitCode(probe.category);
|
|
15894
15641
|
}
|
|
15895
|
-
var
|
|
15642
|
+
var DEFAULT_TIMEOUT_MS9 = 1e4;
|
|
15896
15643
|
var init_health = __esm(() => {
|
|
15897
15644
|
init_config2();
|
|
15898
15645
|
});
|
|
15899
15646
|
|
|
15900
15647
|
// src/cli/commands/duplicates.ts
|
|
15901
|
-
var exports_duplicates = {};
|
|
15902
|
-
__export(exports_duplicates, {
|
|
15903
|
-
runDuplicates: () => runDuplicates
|
|
15904
|
-
});
|
|
15905
15648
|
import { resolve as resolve13 } from "node:path";
|
|
15906
15649
|
async function runDuplicates(ctx, scanner = (path) => scanWithDupey(path)) {
|
|
15907
15650
|
try {
|
|
@@ -15997,14 +15740,14 @@ function renderSlashHelp() {
|
|
|
15997
15740
|
}
|
|
15998
15741
|
|
|
15999
15742
|
// src/cli/tui-session-store.ts
|
|
16000
|
-
import { existsSync as
|
|
16001
|
-
import { dirname as
|
|
15743
|
+
import { existsSync as existsSync17, mkdirSync as mkdirSync13, readFileSync as readFileSync14, renameSync as renameSync8, writeFileSync as writeFileSync11 } from "node:fs";
|
|
15744
|
+
import { dirname as dirname15 } from "node:path";
|
|
16002
15745
|
function createFileTuiSessionStore(path) {
|
|
16003
15746
|
const readSessions = () => {
|
|
16004
|
-
if (!
|
|
15747
|
+
if (!existsSync17(path))
|
|
16005
15748
|
return [];
|
|
16006
15749
|
try {
|
|
16007
|
-
const parsed = JSON.parse(
|
|
15750
|
+
const parsed = JSON.parse(readFileSync14(path, "utf8"));
|
|
16008
15751
|
if (!Array.isArray(parsed))
|
|
16009
15752
|
return [];
|
|
16010
15753
|
return parsed.filter(isTuiSessionRecord).sort((a, b) => b.updatedAt - a.updatedAt);
|
|
@@ -16018,10 +15761,10 @@ function createFileTuiSessionStore(path) {
|
|
|
16018
15761
|
save: (session) => {
|
|
16019
15762
|
const sessions = readSessions().filter((item) => item.id !== session.id);
|
|
16020
15763
|
sessions.unshift(session);
|
|
16021
|
-
const dir =
|
|
16022
|
-
|
|
15764
|
+
const dir = dirname15(path);
|
|
15765
|
+
mkdirSync13(dir, { recursive: true });
|
|
16023
15766
|
const tmpPath = `${path}.${process.pid}.tmp`;
|
|
16024
|
-
|
|
15767
|
+
writeFileSync11(tmpPath, `${JSON.stringify(sessions.slice(0, MAX_SESSIONS), null, 2)}
|
|
16025
15768
|
`, "utf8");
|
|
16026
15769
|
renameSync8(tmpPath, path);
|
|
16027
15770
|
}
|
|
@@ -16075,14 +15818,7 @@ var MAX_SESSIONS = 100;
|
|
|
16075
15818
|
var init_tui_session_store = () => {};
|
|
16076
15819
|
|
|
16077
15820
|
// src/cli/commands/tui.ts
|
|
16078
|
-
|
|
16079
|
-
__export(exports_tui, {
|
|
16080
|
-
createTuiPresenter: () => createTuiPresenter,
|
|
16081
|
-
parseSlashCommand: () => parseSlashCommand,
|
|
16082
|
-
renderTuiSessionView: () => renderTuiSessionView,
|
|
16083
|
-
runTui: () => runTui
|
|
16084
|
-
});
|
|
16085
|
-
import { join as join24 } from "node:path";
|
|
15821
|
+
import { join as join23 } from "node:path";
|
|
16086
15822
|
import {
|
|
16087
15823
|
CombinedAutocompleteProvider,
|
|
16088
15824
|
Editor,
|
|
@@ -16207,6 +15943,26 @@ function createTuiPresenter() {
|
|
|
16207
15943
|
function dimGray(text) {
|
|
16208
15944
|
return `${ANSI_DIM_GRAY}${text}${ANSI_RESET}`;
|
|
16209
15945
|
}
|
|
15946
|
+
function renderProgressEvent(event) {
|
|
15947
|
+
return event.type === "progress" && event.text.trim() !== "" ? event.text : undefined;
|
|
15948
|
+
}
|
|
15949
|
+
async function consumeSearchStream(agent, query, handlers) {
|
|
15950
|
+
for await (const event of agent.searchDocumentsStream(query)) {
|
|
15951
|
+
if (handlers.isInterrupted?.())
|
|
15952
|
+
return;
|
|
15953
|
+
switch (event.type) {
|
|
15954
|
+
case "progress": {
|
|
15955
|
+
const progress = renderProgressEvent(event);
|
|
15956
|
+
if (progress !== undefined)
|
|
15957
|
+
handlers.onProgress(progress);
|
|
15958
|
+
break;
|
|
15959
|
+
}
|
|
15960
|
+
case "complete":
|
|
15961
|
+
handlers.onComplete(event.response);
|
|
15962
|
+
break;
|
|
15963
|
+
}
|
|
15964
|
+
}
|
|
15965
|
+
}
|
|
16210
15966
|
function stripAnsi(text) {
|
|
16211
15967
|
return text.replace(/\u001b\[[0-9;]*m/g, "");
|
|
16212
15968
|
}
|
|
@@ -16241,8 +15997,8 @@ ${content}`;
|
|
|
16241
15997
|
}
|
|
16242
15998
|
function defaultSessionStore(ctx) {
|
|
16243
15999
|
const config = resolveConfig({ flags: ctx.flags, cwd: ctx.cwd });
|
|
16244
|
-
const workspace = createFileTuiSessionStore(
|
|
16245
|
-
const global = createFileTuiSessionStore(
|
|
16000
|
+
const workspace = createFileTuiSessionStore(join23(config.workspacePath, ".autorag", "tui-sessions.json"));
|
|
16001
|
+
const global = createFileTuiSessionStore(join23(resolveAutoRAGHome(), "tui-sessions.json"));
|
|
16246
16002
|
return createMergedTuiSessionStore(workspace, global);
|
|
16247
16003
|
}
|
|
16248
16004
|
function runRealTui(ctx, agent, store) {
|
|
@@ -16265,7 +16021,7 @@ ${trace.join(`
|
|
|
16265
16021
|
`));
|
|
16266
16022
|
};
|
|
16267
16023
|
const status = new Text(presenter.status());
|
|
16268
|
-
const
|
|
16024
|
+
const renderStatus = () => {
|
|
16269
16025
|
status.setText(presenter.status());
|
|
16270
16026
|
};
|
|
16271
16027
|
const editor = new Editor(tui, {
|
|
@@ -16285,18 +16041,18 @@ ${trace.join(`
|
|
|
16285
16041
|
let resumeSelection = false;
|
|
16286
16042
|
let completedResponse;
|
|
16287
16043
|
let completedAnswer = "";
|
|
16288
|
-
return new Promise((
|
|
16044
|
+
return new Promise((resolve) => {
|
|
16289
16045
|
const finish = () => {
|
|
16290
16046
|
if (settled)
|
|
16291
16047
|
return;
|
|
16292
16048
|
settled = true;
|
|
16293
16049
|
unsubscribe?.();
|
|
16294
16050
|
tui.stop();
|
|
16295
|
-
|
|
16051
|
+
resolve(0);
|
|
16296
16052
|
};
|
|
16297
16053
|
const unsubscribe = agent.subscribe?.((event) => {
|
|
16298
16054
|
presenter.handle(event);
|
|
16299
|
-
|
|
16055
|
+
renderStatus();
|
|
16300
16056
|
renderTranscript();
|
|
16301
16057
|
tui.requestRender();
|
|
16302
16058
|
});
|
|
@@ -16313,7 +16069,7 @@ interrupted`;
|
|
|
16313
16069
|
transcriptHistory = `${transcriptHistory}
|
|
16314
16070
|
paused`;
|
|
16315
16071
|
}
|
|
16316
|
-
|
|
16072
|
+
renderStatus();
|
|
16317
16073
|
renderTranscript();
|
|
16318
16074
|
tui.requestRender();
|
|
16319
16075
|
return;
|
|
@@ -16373,23 +16129,30 @@ ${renderTuiSessionList(store.list())}`;
|
|
|
16373
16129
|
presenter.beginRun();
|
|
16374
16130
|
editor.disableSubmit = true;
|
|
16375
16131
|
presenter.setWorking(true);
|
|
16376
|
-
|
|
16132
|
+
renderStatus();
|
|
16377
16133
|
finalAnswer = "";
|
|
16378
16134
|
transcriptHistory = `${transcriptHistory}
|
|
16379
16135
|
|
|
16380
|
-
> ${query}
|
|
16381
|
-
search: preparing retrieval`;
|
|
16136
|
+
> ${query}`;
|
|
16382
16137
|
renderTranscript();
|
|
16383
16138
|
tui.requestRender();
|
|
16384
|
-
agent
|
|
16385
|
-
|
|
16386
|
-
|
|
16387
|
-
|
|
16388
|
-
|
|
16389
|
-
|
|
16390
|
-
|
|
16391
|
-
|
|
16392
|
-
|
|
16139
|
+
consumeSearchStream(agent, query, {
|
|
16140
|
+
onProgress: (progress) => {
|
|
16141
|
+
transcriptHistory = `${transcriptHistory}
|
|
16142
|
+
|
|
16143
|
+
${progress}`;
|
|
16144
|
+
renderTranscript();
|
|
16145
|
+
tui.requestRender();
|
|
16146
|
+
},
|
|
16147
|
+
onComplete: (response) => {
|
|
16148
|
+
const answer = renderSearch(response, { json: false, debug: ctx.debug });
|
|
16149
|
+
completedResponse = response;
|
|
16150
|
+
completedAnswer = answer;
|
|
16151
|
+
finalAnswer = answer;
|
|
16152
|
+
renderTranscript();
|
|
16153
|
+
tui.requestRender();
|
|
16154
|
+
},
|
|
16155
|
+
isInterrupted: () => interrupted
|
|
16393
16156
|
}).catch((error) => {
|
|
16394
16157
|
if (interrupted)
|
|
16395
16158
|
return;
|
|
@@ -16403,7 +16166,7 @@ ${renderError(error, {
|
|
|
16403
16166
|
}).finally(() => {
|
|
16404
16167
|
queryRunning = false;
|
|
16405
16168
|
presenter.setWorking(false);
|
|
16406
|
-
|
|
16169
|
+
renderStatus();
|
|
16407
16170
|
if (completedResponse !== undefined) {
|
|
16408
16171
|
store.save({
|
|
16409
16172
|
id: completedResponse.sessionId,
|
|
@@ -16490,13 +16253,18 @@ async function runTui(ctx, deps = {}) {
|
|
|
16490
16253
|
interrupted = false;
|
|
16491
16254
|
presenter.beginRun();
|
|
16492
16255
|
try {
|
|
16493
|
-
|
|
16494
|
-
|
|
16495
|
-
|
|
16496
|
-
|
|
16497
|
-
|
|
16498
|
-
|
|
16499
|
-
|
|
16256
|
+
await consumeSearchStream(agent, query, {
|
|
16257
|
+
onProgress: (progress) => {
|
|
16258
|
+
tui.rendered.push(progress);
|
|
16259
|
+
},
|
|
16260
|
+
onComplete: (response) => {
|
|
16261
|
+
const answer = renderSearch(response, { json: false, debug: ctx.debug });
|
|
16262
|
+
tui.rendered.push(answer);
|
|
16263
|
+
completedResponse = response;
|
|
16264
|
+
completedAnswer = answer;
|
|
16265
|
+
},
|
|
16266
|
+
isInterrupted: () => interrupted
|
|
16267
|
+
});
|
|
16500
16268
|
} catch (error) {
|
|
16501
16269
|
if (!interrupted) {
|
|
16502
16270
|
tui.rendered.push(renderError(error, { json: false, debug: ctx.debug }));
|
|
@@ -16553,20 +16321,20 @@ var init_tui = __esm(() => {
|
|
|
16553
16321
|
});
|
|
16554
16322
|
|
|
16555
16323
|
// src/ui/browse.ts
|
|
16556
|
-
import { existsSync as
|
|
16324
|
+
import { existsSync as existsSync18, readdirSync as readdirSync4, statSync as statSync2 } from "node:fs";
|
|
16557
16325
|
import { homedir as homedir3 } from "node:os";
|
|
16558
|
-
import { dirname as
|
|
16326
|
+
import { dirname as dirname16, join as join24, resolve as resolve14 } from "node:path";
|
|
16559
16327
|
function browseDirectory(inputPath) {
|
|
16560
16328
|
const path = resolve14(inputPath && inputPath.trim().length > 0 ? inputPath : homedir3());
|
|
16561
|
-
if (!
|
|
16329
|
+
if (!existsSync18(path))
|
|
16562
16330
|
throw new ConfigError("Folder was not found.");
|
|
16563
|
-
let
|
|
16331
|
+
let stat;
|
|
16564
16332
|
try {
|
|
16565
|
-
|
|
16333
|
+
stat = statSync2(path);
|
|
16566
16334
|
} catch {
|
|
16567
16335
|
throw new ConfigError("Folder was not found.");
|
|
16568
16336
|
}
|
|
16569
|
-
if (!
|
|
16337
|
+
if (!stat.isDirectory())
|
|
16570
16338
|
throw new ConfigError("Path is not a folder.");
|
|
16571
16339
|
let names = [];
|
|
16572
16340
|
try {
|
|
@@ -16578,7 +16346,7 @@ function browseDirectory(inputPath) {
|
|
|
16578
16346
|
for (const name of names) {
|
|
16579
16347
|
if (name === "." || name === "..")
|
|
16580
16348
|
continue;
|
|
16581
|
-
const child =
|
|
16349
|
+
const child = join24(path, name);
|
|
16582
16350
|
let directory = false;
|
|
16583
16351
|
try {
|
|
16584
16352
|
directory = statSync2(child).isDirectory();
|
|
@@ -16590,7 +16358,7 @@ function browseDirectory(inputPath) {
|
|
|
16590
16358
|
break;
|
|
16591
16359
|
}
|
|
16592
16360
|
entries.sort((a, b) => Number(b.directory) - Number(a.directory) || a.name.localeCompare(b.name));
|
|
16593
|
-
const parent =
|
|
16361
|
+
const parent = dirname16(path);
|
|
16594
16362
|
return {
|
|
16595
16363
|
path,
|
|
16596
16364
|
parent: parent === path ? null : parent,
|
|
@@ -16758,21 +16526,11 @@ var init_catalog = __esm(() => {
|
|
|
16758
16526
|
},
|
|
16759
16527
|
{
|
|
16760
16528
|
type: "gmail",
|
|
16761
|
-
title: "Gmail
|
|
16762
|
-
summary: "Gmail
|
|
16529
|
+
title: "Gmail",
|
|
16530
|
+
summary: "Search Gmail through the Gmail API.",
|
|
16763
16531
|
defaultTags: ["gmail", "email", "pii"],
|
|
16764
|
-
binaryName: "himalaya",
|
|
16765
16532
|
fields: [
|
|
16766
16533
|
INSTANCE,
|
|
16767
|
-
{
|
|
16768
|
-
key: "connector.backend",
|
|
16769
|
-
label: "Backend",
|
|
16770
|
-
kind: "select",
|
|
16771
|
-
options: [
|
|
16772
|
-
{ value: "", label: "Gmail API (token env)" },
|
|
16773
|
-
{ value: "himalaya", label: "himalaya (IMAP)" }
|
|
16774
|
-
]
|
|
16775
|
-
},
|
|
16776
16534
|
{
|
|
16777
16535
|
key: "connector.tokenEnv",
|
|
16778
16536
|
label: "Access token environment variable",
|
|
@@ -16785,10 +16543,7 @@ var init_catalog = __esm(() => {
|
|
|
16785
16543
|
kind: "textarea",
|
|
16786
16544
|
placeholder: "INBOX",
|
|
16787
16545
|
help: "One label id per line."
|
|
16788
|
-
}
|
|
16789
|
-
{ key: "connector.account", label: "himalaya account", kind: "text" },
|
|
16790
|
-
{ key: "connector.folder", label: "Mail folder", kind: "text", placeholder: "INBOX" },
|
|
16791
|
-
BINARY
|
|
16546
|
+
}
|
|
16792
16547
|
]
|
|
16793
16548
|
},
|
|
16794
16549
|
{
|
|
@@ -16950,16 +16705,7 @@ var init_catalog = __esm(() => {
|
|
|
16950
16705
|
question: "Which rclone remote (Google Drive, OneDrive, …) should this connection use?"
|
|
16951
16706
|
}
|
|
16952
16707
|
],
|
|
16953
|
-
gmail: [
|
|
16954
|
-
{
|
|
16955
|
-
key: "account",
|
|
16956
|
-
label: "Mailbox",
|
|
16957
|
-
kind: "select",
|
|
16958
|
-
choices: "mail-accounts",
|
|
16959
|
-
allowOther: true,
|
|
16960
|
-
question: "Which email account should this connection use (Gmail, Outlook, iCloud, or another IMAP mailbox)?"
|
|
16961
|
-
}
|
|
16962
|
-
],
|
|
16708
|
+
gmail: [],
|
|
16963
16709
|
"mail-export": [
|
|
16964
16710
|
{
|
|
16965
16711
|
key: "paths",
|
|
@@ -17023,18 +16769,21 @@ var init_catalog = __esm(() => {
|
|
|
17023
16769
|
});
|
|
17024
16770
|
|
|
17025
16771
|
// src/ui/probe.ts
|
|
17026
|
-
import { existsSync as
|
|
17027
|
-
import { delimiter as delimiter3, join as
|
|
16772
|
+
import { existsSync as existsSync19 } from "node:fs";
|
|
16773
|
+
import { delimiter as delimiter3, join as join25 } from "node:path";
|
|
17028
16774
|
function probeConnection(input, deps = {}) {
|
|
17029
16775
|
const catalog = getDatasourceType(input.type);
|
|
17030
16776
|
if (catalog === undefined) {
|
|
17031
16777
|
return { ok: false, status: "unknown-type", detail: `Unknown datasource type: ${input.type}` };
|
|
17032
16778
|
}
|
|
17033
16779
|
const env = deps.env ?? process.env;
|
|
17034
|
-
const pathExists = deps.pathExists ??
|
|
16780
|
+
const pathExists = deps.pathExists ?? existsSync19;
|
|
17035
16781
|
const binaryExists = deps.binaryExists ?? defaultBinaryExists;
|
|
17036
16782
|
const connector = input.connector ?? {};
|
|
17037
16783
|
const fail = (status, detail) => ({ ok: false, status, detail });
|
|
16784
|
+
if (input.type === "gmail" && connector.backend === "himalaya") {
|
|
16785
|
+
return fail("not-configured", "Himalaya-backed mail must be configured through the mailcrawl datasource.");
|
|
16786
|
+
}
|
|
17038
16787
|
if (input.type === "github") {
|
|
17039
16788
|
const repos = asStringList(connector.repos);
|
|
17040
16789
|
if (repos.length === 0)
|
|
@@ -17082,12 +16831,6 @@ function probeConnection(input, deps = {}) {
|
|
|
17082
16831
|
return ok6("Queries saved.");
|
|
17083
16832
|
}
|
|
17084
16833
|
if (input.type === "gmail") {
|
|
17085
|
-
if (connector.backend === "himalaya") {
|
|
17086
|
-
if (!binaryExists("himalaya", asString4(connector.binaryPath))) {
|
|
17087
|
-
return fail("binary-missing", "himalaya is not installed.");
|
|
17088
|
-
}
|
|
17089
|
-
return ok6("himalaya will use its own account config.");
|
|
17090
|
-
}
|
|
17091
16834
|
const tokenEnv = envName(connector.tokenEnv, "GMAIL_ACCESS_TOKEN");
|
|
17092
16835
|
if (!envHas(env, tokenEnv))
|
|
17093
16836
|
return fail("auth-missing", `${tokenEnv} is not set in the environment.`);
|
|
@@ -17152,14 +16895,14 @@ function asFeedUrls(value) {
|
|
|
17152
16895
|
}
|
|
17153
16896
|
function defaultBinaryExists(binaryName, binaryPath) {
|
|
17154
16897
|
if (binaryPath !== undefined && binaryPath.length > 0)
|
|
17155
|
-
return
|
|
16898
|
+
return existsSync19(binaryPath);
|
|
17156
16899
|
const pathEnv = process.env.PATH ?? "";
|
|
17157
16900
|
const extensions = process.platform === "win32" ? [".exe", ".cmd", ".bat", ""] : [""];
|
|
17158
16901
|
for (const dir of pathEnv.split(delimiter3)) {
|
|
17159
16902
|
if (dir.length === 0)
|
|
17160
16903
|
continue;
|
|
17161
16904
|
for (const extension of extensions) {
|
|
17162
|
-
if (
|
|
16905
|
+
if (existsSync19(join25(dir, `${binaryName}${extension}`)))
|
|
17163
16906
|
return true;
|
|
17164
16907
|
}
|
|
17165
16908
|
}
|
|
@@ -17188,6 +16931,9 @@ function upsertConnection(configPath, input) {
|
|
|
17188
16931
|
const raw = readRawConfigObject(configPath);
|
|
17189
16932
|
const datasources = asObject(raw.datasources);
|
|
17190
16933
|
const connector = normalizeConnector(type, stripSecrets(input.connector ?? {}));
|
|
16934
|
+
if (isRetiredGmailHimalaya(type, connector)) {
|
|
16935
|
+
throw new ConfigError("Himalaya-backed mail must use the mailcrawl datasource.");
|
|
16936
|
+
}
|
|
17191
16937
|
const enabled = input.enabled !== false;
|
|
17192
16938
|
const entry = {
|
|
17193
16939
|
type,
|
|
@@ -17214,6 +16960,12 @@ function toggleConnection(configPath, alias, enabled) {
|
|
|
17214
16960
|
if (current === false || current === true) {
|
|
17215
16961
|
datasources[alias] = { type: alias, enabled };
|
|
17216
16962
|
} else if (typeof current === "object" && current !== null && !Array.isArray(current)) {
|
|
16963
|
+
const currentEntry = asObject(current);
|
|
16964
|
+
const currentType = typeof currentEntry.type === "string" && currentEntry.type.length > 0 ? currentEntry.type : alias;
|
|
16965
|
+
const currentConnector = asObject(currentEntry.connector);
|
|
16966
|
+
if (isRetiredGmailHimalaya(currentType, currentConnector)) {
|
|
16967
|
+
throw new ConfigError("Himalaya-backed mail must use the mailcrawl datasource.");
|
|
16968
|
+
}
|
|
17217
16969
|
datasources[alias] = { ...current, enabled };
|
|
17218
16970
|
} else {
|
|
17219
16971
|
throw new ConfigError(`Unknown connection: ${alias}`);
|
|
@@ -17281,7 +17033,7 @@ function readConnections(raw) {
|
|
|
17281
17033
|
const entry = value;
|
|
17282
17034
|
const type = typeof entry.type === "string" && entry.type.length > 0 ? entry.type : alias;
|
|
17283
17035
|
const connector = stripSecrets(asObject(entry.connector));
|
|
17284
|
-
const enabled = entry.enabled !== false;
|
|
17036
|
+
const enabled = entry.enabled !== false && !isRetiredGmailHimalaya(type, connector);
|
|
17285
17037
|
connections.push({
|
|
17286
17038
|
alias,
|
|
17287
17039
|
type,
|
|
@@ -17302,6 +17054,9 @@ function tagsFor(type, connector) {
|
|
|
17302
17054
|
}
|
|
17303
17055
|
return tags;
|
|
17304
17056
|
}
|
|
17057
|
+
function isRetiredGmailHimalaya(type, connector) {
|
|
17058
|
+
return type === "gmail" && connector.backend === "himalaya";
|
|
17059
|
+
}
|
|
17305
17060
|
function normalizeAlias(alias) {
|
|
17306
17061
|
const trimmed = alias.trim();
|
|
17307
17062
|
if (!ALIAS_PATTERN.test(trimmed)) {
|
|
@@ -17423,55 +17178,32 @@ var init_config_store = __esm(() => {
|
|
|
17423
17178
|
});
|
|
17424
17179
|
|
|
17425
17180
|
// src/ui/discover.ts
|
|
17426
|
-
import { spawn as
|
|
17181
|
+
import { spawn as spawn14 } from "node:child_process";
|
|
17427
17182
|
async function listRcloneRemotes(run = runCli) {
|
|
17428
17183
|
const result = await run("rclone", ["listremotes"]);
|
|
17429
17184
|
if (!result.ok)
|
|
17430
17185
|
return [];
|
|
17431
17186
|
return result.stdout.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0 && !line.startsWith("#")).map((value) => ({ value, label: value }));
|
|
17432
17187
|
}
|
|
17433
|
-
async function listMailAccounts(run = runCli) {
|
|
17434
|
-
const discovered = await listHimalayaAccounts(run);
|
|
17435
|
-
const seen = new Set(discovered.map((item) => item.value.toLowerCase()));
|
|
17436
|
-
const presets = MAIL_PRESETS.filter((item) => !seen.has(item.value.toLowerCase()) && !seen.has(item.label.toLowerCase()));
|
|
17437
|
-
return [...discovered, ...presets, { value: "other", label: "Other…" }];
|
|
17438
|
-
}
|
|
17439
|
-
async function listHimalayaAccounts(run = runCli) {
|
|
17440
|
-
const result = await run("himalaya", ["account", "list"]);
|
|
17441
|
-
if (!result.ok)
|
|
17442
|
-
return [];
|
|
17443
|
-
const choices = [];
|
|
17444
|
-
for (const raw of result.stdout.split(/\r?\n/)) {
|
|
17445
|
-
const line = raw.replace(/^\s*[*-]\s*/, "").trim();
|
|
17446
|
-
if (line.length === 0 || line.toLowerCase().startsWith("account") || line.startsWith("─") || line.startsWith("-")) {
|
|
17447
|
-
continue;
|
|
17448
|
-
}
|
|
17449
|
-
const name = line.split(/\s{2,}|\t/)[0]?.trim();
|
|
17450
|
-
if (name && name !== "NAME" && name !== "Name")
|
|
17451
|
-
choices.push({ value: name, label: name });
|
|
17452
|
-
}
|
|
17453
|
-
return choices;
|
|
17454
|
-
}
|
|
17455
17188
|
async function choicesForType(type, run = runCli) {
|
|
17456
17189
|
if (type === "cloud-drive") {
|
|
17457
17190
|
return { rcloneRemotes: await listRcloneRemotes(run), mailAccounts: [] };
|
|
17458
17191
|
}
|
|
17459
|
-
if (type === "gmail")
|
|
17460
|
-
return { rcloneRemotes: [], mailAccounts:
|
|
17461
|
-
}
|
|
17192
|
+
if (type === "gmail")
|
|
17193
|
+
return { rcloneRemotes: [], mailAccounts: [] };
|
|
17462
17194
|
return { rcloneRemotes: [], mailAccounts: [] };
|
|
17463
17195
|
}
|
|
17464
17196
|
function runCli(binary, args) {
|
|
17465
|
-
return new Promise((
|
|
17197
|
+
return new Promise((resolve) => {
|
|
17466
17198
|
let settled = false;
|
|
17467
|
-
const child =
|
|
17199
|
+
const child = spawn14(binary, [...args], { stdio: ["ignore", "pipe", "pipe"] });
|
|
17468
17200
|
const chunks = [];
|
|
17469
|
-
const finish = (
|
|
17201
|
+
const finish = (ok) => {
|
|
17470
17202
|
if (settled)
|
|
17471
17203
|
return;
|
|
17472
17204
|
settled = true;
|
|
17473
17205
|
clearTimeout(timer);
|
|
17474
|
-
|
|
17206
|
+
resolve({ ok, stdout: Buffer.concat(chunks).toString("utf8") });
|
|
17475
17207
|
};
|
|
17476
17208
|
const timer = setTimeout(() => {
|
|
17477
17209
|
child.kill("SIGKILL");
|
|
@@ -17484,14 +17216,7 @@ function runCli(binary, args) {
|
|
|
17484
17216
|
child.on("close", (code) => finish(code === 0));
|
|
17485
17217
|
});
|
|
17486
17218
|
}
|
|
17487
|
-
var
|
|
17488
|
-
var init_discover = __esm(() => {
|
|
17489
|
-
MAIL_PRESETS = [
|
|
17490
|
-
{ value: "gmail", label: "Gmail" },
|
|
17491
|
-
{ value: "outlook", label: "Outlook" },
|
|
17492
|
-
{ value: "icloud", label: "iCloud" }
|
|
17493
|
-
];
|
|
17494
|
-
});
|
|
17219
|
+
var init_discover = () => {};
|
|
17495
17220
|
|
|
17496
17221
|
// src/ui/html.ts
|
|
17497
17222
|
function renderUiPage(state) {
|
|
@@ -17576,7 +17301,7 @@ function renderUiPage(state) {
|
|
|
17576
17301
|
<section class="card">
|
|
17577
17302
|
<label for="folders">Folders to search</label>
|
|
17578
17303
|
<textarea id="folders">${folders}</textarea>
|
|
17579
|
-
<p class="help">
|
|
17304
|
+
<p class="help">MinSync and Jikji index these folders by default. One absolute path per line.</p>
|
|
17580
17305
|
<div class="row actions"><button type="button" class="primary" id="save-folders">Save folders</button></div>
|
|
17581
17306
|
</section>
|
|
17582
17307
|
|
|
@@ -17887,8 +17612,8 @@ async function startUiServer(options) {
|
|
|
17887
17612
|
host: boundHost === "[::1]" ? "::1" : boundHost,
|
|
17888
17613
|
port: address.port,
|
|
17889
17614
|
token,
|
|
17890
|
-
close: () => new Promise((
|
|
17891
|
-
httpServer.close((error) => error ? reject(error) :
|
|
17615
|
+
close: () => new Promise((resolve, reject) => {
|
|
17616
|
+
httpServer.close((error) => error ? reject(error) : resolve());
|
|
17892
17617
|
})
|
|
17893
17618
|
};
|
|
17894
17619
|
}
|
|
@@ -18130,7 +17855,7 @@ function parseExtras(raw) {
|
|
|
18130
17855
|
}
|
|
18131
17856
|
}
|
|
18132
17857
|
function listen(server, port, host) {
|
|
18133
|
-
return new Promise((
|
|
17858
|
+
return new Promise((resolve, reject) => {
|
|
18134
17859
|
const onError = (error) => {
|
|
18135
17860
|
server.off("error", onError);
|
|
18136
17861
|
reject(error);
|
|
@@ -18138,7 +17863,7 @@ function listen(server, port, host) {
|
|
|
18138
17863
|
server.once("error", onError);
|
|
18139
17864
|
server.listen(port, host, () => {
|
|
18140
17865
|
server.off("error", onError);
|
|
18141
|
-
|
|
17866
|
+
resolve();
|
|
18142
17867
|
});
|
|
18143
17868
|
});
|
|
18144
17869
|
}
|
|
@@ -18153,18 +17878,13 @@ var init_server = __esm(() => {
|
|
|
18153
17878
|
});
|
|
18154
17879
|
|
|
18155
17880
|
// src/cli/commands/ui.ts
|
|
18156
|
-
|
|
18157
|
-
__export(exports_ui, {
|
|
18158
|
-
openBrowser: () => openBrowser,
|
|
18159
|
-
runUi: () => runUi
|
|
18160
|
-
});
|
|
18161
|
-
import { spawn as spawn16 } from "node:child_process";
|
|
17881
|
+
import { spawn as spawn15 } from "node:child_process";
|
|
18162
17882
|
import { randomBytes } from "node:crypto";
|
|
18163
|
-
import { existsSync as
|
|
17883
|
+
import { existsSync as existsSync20 } from "node:fs";
|
|
18164
17884
|
async function runUi(ctx, deps = {}) {
|
|
18165
17885
|
const flags = ctx.flags;
|
|
18166
17886
|
const resolved = resolveConfigPath({ flags, cwd: ctx.cwd });
|
|
18167
|
-
if (!
|
|
17887
|
+
if (!existsSync20(resolved.configPath)) {
|
|
18168
17888
|
ctx.stderr(renderError(new ConfigError(`Config file not found: ${resolved.configPath}. Run autorag init first.`), {
|
|
18169
17889
|
json: ctx.json
|
|
18170
17890
|
}));
|
|
@@ -18240,13 +17960,13 @@ async function runUi(ctx, deps = {}) {
|
|
|
18240
17960
|
}
|
|
18241
17961
|
}
|
|
18242
17962
|
async function defaultWaitUntilStopped(server) {
|
|
18243
|
-
await new Promise((
|
|
17963
|
+
await new Promise((resolve) => {
|
|
18244
17964
|
let stopped = false;
|
|
18245
17965
|
const stop = () => {
|
|
18246
17966
|
if (stopped)
|
|
18247
17967
|
return;
|
|
18248
17968
|
stopped = true;
|
|
18249
|
-
server.close().finally(
|
|
17969
|
+
server.close().finally(resolve);
|
|
18250
17970
|
};
|
|
18251
17971
|
process.once("SIGINT", stop);
|
|
18252
17972
|
process.once("SIGTERM", stop);
|
|
@@ -18255,7 +17975,7 @@ async function defaultWaitUntilStopped(server) {
|
|
|
18255
17975
|
async function openBrowser(url) {
|
|
18256
17976
|
const command = process.platform === "darwin" ? "open" : process.platform === "win32" ? "cmd" : "xdg-open";
|
|
18257
17977
|
const args = process.platform === "win32" ? ["/c", "start", "", url] : [url];
|
|
18258
|
-
|
|
17978
|
+
spawn15(command, args, { detached: true, stdio: "ignore", windowsHide: true }).unref();
|
|
18259
17979
|
}
|
|
18260
17980
|
var init_ui = __esm(() => {
|
|
18261
17981
|
init_server();
|
|
@@ -18288,6 +18008,7 @@ var VALUE_FLAGS = new Set([
|
|
|
18288
18008
|
"top-k",
|
|
18289
18009
|
"scope",
|
|
18290
18010
|
"tags",
|
|
18011
|
+
"result",
|
|
18291
18012
|
"useful",
|
|
18292
18013
|
"not-useful",
|
|
18293
18014
|
"debounce-ms",
|
|
@@ -18311,6 +18032,7 @@ var COMMANDS = [
|
|
|
18311
18032
|
"status",
|
|
18312
18033
|
"search",
|
|
18313
18034
|
"feedback",
|
|
18035
|
+
"evidence",
|
|
18314
18036
|
"memory",
|
|
18315
18037
|
"index",
|
|
18316
18038
|
"watch",
|
|
@@ -18330,14 +18052,15 @@ Commands:
|
|
|
18330
18052
|
--embedder-id ID --embedder-base-url URL --embedder-api-key-env VAR
|
|
18331
18053
|
--embedder-dimension N --embedder-batch-size N
|
|
18332
18054
|
--minsync-max-chunk-size N --force)
|
|
18333
|
-
refresh
|
|
18055
|
+
refresh Refresh every configured index; --method narrows the run
|
|
18334
18056
|
watch Watch configured roots (or --once for cron/poll tick)
|
|
18335
18057
|
status Show corpus freshness and index health
|
|
18336
18058
|
search <query> Search and curate documents (requires a configured model)
|
|
18337
18059
|
feedback <session> Record numbered feedback (--useful 1,3 --not-useful 2)
|
|
18060
|
+
evidence <session> Show persisted source/chunk evidence (--result N)
|
|
18338
18061
|
memory inspect Inspect the retrieval memory snapshot
|
|
18339
|
-
index reset Remove parsed/minsync indexes
|
|
18340
|
-
index rebuild Reset then re-run a refresh (--method
|
|
18062
|
+
index reset Remove parsed/minsync indexes (--method)
|
|
18063
|
+
index rebuild Reset then re-run a refresh (--method minsync|all)
|
|
18341
18064
|
health Check model/provider auth and completion access (no index check)
|
|
18342
18065
|
duplicates [DIR] Scan exact/near duplicate document families; never deletes files
|
|
18343
18066
|
tui Open an interactive Pi-powered librarian terminal UI
|
|
@@ -18346,7 +18069,7 @@ Commands:
|
|
|
18346
18069
|
|
|
18347
18070
|
Setup:
|
|
18348
18071
|
autorag init --search-paths /path/to/docs,/path/to/notes # choose folders
|
|
18349
|
-
autorag refresh #
|
|
18072
|
+
autorag refresh # parsed + MinSync + datasources + Jikji
|
|
18350
18073
|
autorag watch --once # single index refresh tick (cron)
|
|
18351
18074
|
autorag search "your question" # curated answer
|
|
18352
18075
|
|
|
@@ -18360,7 +18083,7 @@ Global flags:
|
|
|
18360
18083
|
--once For watch: run one refresh tick and exit (for cron)
|
|
18361
18084
|
--immediate For watch: refresh once before reading fs events (default true)
|
|
18362
18085
|
--debounce-ms <n> For watch: debounce milliseconds for fs events (default 1500)
|
|
18363
|
-
--method <csv> For refresh/index:
|
|
18086
|
+
--method <csv> For refresh/index: minsync,parsed,datasources,jikji,all
|
|
18364
18087
|
--skip-probes For health: skip the network completion probe (auth checks still run)
|
|
18365
18088
|
--timeout-ms <n> For health: per-probe timeout in ms (default 10000)
|
|
18366
18089
|
--port <n> For ui: loopback port (default 8787, 0 for ephemeral)
|
|
@@ -18406,62 +18129,66 @@ function parseArgs(argv) {
|
|
|
18406
18129
|
}
|
|
18407
18130
|
function promptYesNo(question) {
|
|
18408
18131
|
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
18409
|
-
return new Promise((
|
|
18132
|
+
return new Promise((resolve) => {
|
|
18410
18133
|
rl.question(`${question} [y/N] `, (answer) => {
|
|
18411
18134
|
rl.close();
|
|
18412
|
-
|
|
18135
|
+
resolve(/^y(es)?$/i.test(answer.trim()));
|
|
18413
18136
|
});
|
|
18414
18137
|
});
|
|
18415
18138
|
}
|
|
18416
18139
|
async function dispatch(command, ctx) {
|
|
18417
18140
|
switch (command) {
|
|
18418
18141
|
case "init": {
|
|
18419
|
-
|
|
18420
|
-
return
|
|
18142
|
+
await Promise.resolve().then(() => init_init());
|
|
18143
|
+
return runInit(ctx);
|
|
18421
18144
|
}
|
|
18422
18145
|
case "refresh": {
|
|
18423
|
-
|
|
18424
|
-
return
|
|
18146
|
+
await Promise.resolve().then(() => init_refresh());
|
|
18147
|
+
return runRefresh(ctx);
|
|
18425
18148
|
}
|
|
18426
18149
|
case "status": {
|
|
18427
|
-
|
|
18428
|
-
return
|
|
18150
|
+
await Promise.resolve().then(() => init_status());
|
|
18151
|
+
return runStatus(ctx);
|
|
18429
18152
|
}
|
|
18430
18153
|
case "search": {
|
|
18431
|
-
|
|
18432
|
-
return
|
|
18154
|
+
await Promise.resolve().then(() => init_search());
|
|
18155
|
+
return runSearch(ctx);
|
|
18433
18156
|
}
|
|
18434
18157
|
case "feedback": {
|
|
18435
|
-
|
|
18436
|
-
return
|
|
18158
|
+
await Promise.resolve().then(() => init_feedback());
|
|
18159
|
+
return runFeedback(ctx);
|
|
18160
|
+
}
|
|
18161
|
+
case "evidence": {
|
|
18162
|
+
await Promise.resolve().then(() => init_evidence());
|
|
18163
|
+
return runEvidence(ctx);
|
|
18437
18164
|
}
|
|
18438
18165
|
case "memory": {
|
|
18439
|
-
|
|
18440
|
-
return
|
|
18166
|
+
await Promise.resolve().then(() => init_memory2());
|
|
18167
|
+
return runMemory(ctx);
|
|
18441
18168
|
}
|
|
18442
18169
|
case "index": {
|
|
18443
|
-
|
|
18444
|
-
return
|
|
18170
|
+
await Promise.resolve().then(() => init_commands());
|
|
18171
|
+
return runIndex(ctx);
|
|
18445
18172
|
}
|
|
18446
18173
|
case "watch": {
|
|
18447
|
-
|
|
18448
|
-
return
|
|
18174
|
+
await Promise.resolve().then(() => init_watch());
|
|
18175
|
+
return runWatch(ctx);
|
|
18449
18176
|
}
|
|
18450
18177
|
case "health": {
|
|
18451
|
-
|
|
18452
|
-
return
|
|
18178
|
+
await Promise.resolve().then(() => init_health());
|
|
18179
|
+
return runHealth(ctx);
|
|
18453
18180
|
}
|
|
18454
18181
|
case "duplicates": {
|
|
18455
|
-
|
|
18456
|
-
return
|
|
18182
|
+
await Promise.resolve().then(() => init_duplicates());
|
|
18183
|
+
return runDuplicates(ctx);
|
|
18457
18184
|
}
|
|
18458
18185
|
case "tui": {
|
|
18459
|
-
|
|
18460
|
-
return
|
|
18186
|
+
await Promise.resolve().then(() => init_tui());
|
|
18187
|
+
return runTui(ctx);
|
|
18461
18188
|
}
|
|
18462
18189
|
case "ui": {
|
|
18463
|
-
|
|
18464
|
-
return
|
|
18190
|
+
await Promise.resolve().then(() => init_ui());
|
|
18191
|
+
return runUi(ctx);
|
|
18465
18192
|
}
|
|
18466
18193
|
}
|
|
18467
18194
|
}
|