@autorag/librarian 2.13.2 → 2.13.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +3 -0
- package/README.md +0 -1
- package/dist/agent/context-token-guard.d.ts +11 -6
- package/dist/agent/token-count.d.ts +4 -0
- package/dist/cli/index.js +287 -632
- package/dist/cloud/context-overflow.d.ts +29 -0
- package/dist/core.js +203 -547
- package/dist/datasource/connector-skill.d.ts +1 -1
- package/dist/datasource/connector.d.ts +1 -1
- package/dist/datasource/index.d.ts +0 -1
- package/dist/index.js +205 -552
- package/dist/mcp/index.js +205 -549
- package/dist/profiles.js +288 -632
- package/package.json +2 -1
- package/skills/autorag-doctor/SKILL.md +1 -2
- package/skills/autorag-lite-setup/SKILL.md +1 -1
- package/skills/autorag-setup/SKILL.md +3 -4
- package/dist/datasource/skills/spotlight/connector.d.ts +0 -53
- package/dist/datasource/skills/spotlight/index.d.ts +0 -2
- package/dist/datasource/skills/spotlight/skill.d.ts +0 -33
package/dist/profiles.js
CHANGED
|
@@ -968,6 +968,51 @@ var init_credential = __esm(() => {
|
|
|
968
968
|
init_models();
|
|
969
969
|
});
|
|
970
970
|
|
|
971
|
+
// src/cloud/context-overflow.ts
|
|
972
|
+
import {
|
|
973
|
+
createAssistantMessageEventStream,
|
|
974
|
+
isContextOverflow
|
|
975
|
+
} from "@earendil-works/pi-ai";
|
|
976
|
+
import { getApiProvider } from "@earendil-works/pi-ai/compat";
|
|
977
|
+
function recordRequestEstimate(model, estimatedTokens) {
|
|
978
|
+
lastEstimates.set(modelKey(model), estimatedTokens);
|
|
979
|
+
}
|
|
980
|
+
function noteContextOverflow(model) {
|
|
981
|
+
const key = modelKey(model);
|
|
982
|
+
const failed = lastEstimates.get(key);
|
|
983
|
+
if (failed === undefined)
|
|
984
|
+
return;
|
|
985
|
+
ceilings.set(key, Math.min(Math.floor(failed / 2), ceilings.get(key) ?? Number.POSITIVE_INFINITY));
|
|
986
|
+
}
|
|
987
|
+
function contextOverflowCeiling(model) {
|
|
988
|
+
return ceilings.get(modelKey(model));
|
|
989
|
+
}
|
|
990
|
+
function streamDazziSimple(model, context, options) {
|
|
991
|
+
const api = getApiProvider(model.api);
|
|
992
|
+
if (api === undefined)
|
|
993
|
+
throw new Error(`No API provider registered for api: ${model.api}`);
|
|
994
|
+
const inner = api.streamSimple(model, context, options);
|
|
995
|
+
const stream = createAssistantMessageEventStream();
|
|
996
|
+
(async () => {
|
|
997
|
+
try {
|
|
998
|
+
for await (const event of inner) {
|
|
999
|
+
if (event.type === "error" && event.error.stopReason === "error" && isContextOverflow(event.error)) {
|
|
1000
|
+
noteContextOverflow(model);
|
|
1001
|
+
}
|
|
1002
|
+
stream.push(event);
|
|
1003
|
+
}
|
|
1004
|
+
} finally {
|
|
1005
|
+
stream.end();
|
|
1006
|
+
}
|
|
1007
|
+
})();
|
|
1008
|
+
return stream;
|
|
1009
|
+
}
|
|
1010
|
+
var lastEstimates, ceilings, modelKey = (model) => `${model.provider}/${model.id}`;
|
|
1011
|
+
var init_context_overflow = __esm(() => {
|
|
1012
|
+
lastEstimates = new Map;
|
|
1013
|
+
ceilings = new Map;
|
|
1014
|
+
});
|
|
1015
|
+
|
|
971
1016
|
// src/cloud/authorize.ts
|
|
972
1017
|
function buildAuthorizeUrl(params) {
|
|
973
1018
|
const url = new URL(`${normalizeBaseUrl(params.baseUrl)}${AUTHORIZE_PATH}`);
|
|
@@ -1323,6 +1368,7 @@ function createDazziProviderConfig(env = process.env) {
|
|
|
1323
1368
|
name: PROVIDER_NAME,
|
|
1324
1369
|
baseUrl: endpoints.apiBaseUrl,
|
|
1325
1370
|
api: "openai-responses",
|
|
1371
|
+
streamSimple: streamDazziSimple,
|
|
1326
1372
|
oauth: {
|
|
1327
1373
|
name: PROVIDER_NAME,
|
|
1328
1374
|
login: (callbacks) => loginDazzi(callbacks, { baseUrl }),
|
|
@@ -1338,6 +1384,7 @@ async function registerDazziProvider(runtime, env = process.env) {
|
|
|
1338
1384
|
}
|
|
1339
1385
|
var init_provider = __esm(() => {
|
|
1340
1386
|
init_config();
|
|
1387
|
+
init_context_overflow();
|
|
1341
1388
|
init_models();
|
|
1342
1389
|
init_oauth_login();
|
|
1343
1390
|
});
|
|
@@ -13727,413 +13774,6 @@ var init_slack = __esm(() => {
|
|
|
13727
13774
|
};
|
|
13728
13775
|
});
|
|
13729
13776
|
|
|
13730
|
-
// src/datasource/skills/spotlight/connector.ts
|
|
13731
|
-
import { spawn as spawn12 } from "node:child_process";
|
|
13732
|
-
import { createHash as createHash10 } from "node:crypto";
|
|
13733
|
-
import { readFile as fsReadFile, stat as stat2 } from "node:fs/promises";
|
|
13734
|
-
import { basename as basename5, extname as extname6 } from "node:path";
|
|
13735
|
-
|
|
13736
|
-
class SpotlightConnector {
|
|
13737
|
-
options;
|
|
13738
|
-
run;
|
|
13739
|
-
readFile;
|
|
13740
|
-
platform;
|
|
13741
|
-
constructor(options = {}) {
|
|
13742
|
-
this.options = options;
|
|
13743
|
-
this.run = options.run ?? spawnRunner;
|
|
13744
|
-
this.readFile = options.readFile ?? ((path) => fsReadFile(path, "utf8"));
|
|
13745
|
-
this.platform = options.platform ?? process.platform;
|
|
13746
|
-
}
|
|
13747
|
-
async probe() {
|
|
13748
|
-
const queries = (this.options.queries ?? []).map((query) => query.trim()).filter((query) => query.length > 0);
|
|
13749
|
-
const failure = await this.readinessFailure(queries);
|
|
13750
|
-
return failure ?? { ok: true, documents: [] };
|
|
13751
|
-
}
|
|
13752
|
-
async search(query, limit = DEFAULT_MAX_DOCUMENTS4, acceptDocument) {
|
|
13753
|
-
const trimmed = query.trim();
|
|
13754
|
-
if (trimmed.length === 0 || limit <= 0)
|
|
13755
|
-
return { ok: true, documents: [] };
|
|
13756
|
-
const queries = (this.options.queries ?? []).map((query) => query.trim()).filter((query) => query.length > 0);
|
|
13757
|
-
const failure = await this.readinessFailure(queries);
|
|
13758
|
-
if (failure !== undefined)
|
|
13759
|
-
return failure;
|
|
13760
|
-
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS6;
|
|
13761
|
-
const maxPerQuery = this.options.maxResultsPerQuery ?? DEFAULT_MAX_RESULTS_PER_QUERY;
|
|
13762
|
-
const maxDocuments = Math.min(this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS4, limit);
|
|
13763
|
-
const seen = new Set;
|
|
13764
|
-
const paths = [];
|
|
13765
|
-
const textQuery = trimmed.split(/\s+/u).map((term) => `"${term.replace(/\\/gu, "\\\\").replace(/"/gu, "\\\"")}"`).join(" AND ");
|
|
13766
|
-
let mdfindMissing = false;
|
|
13767
|
-
let failedQueries = 0;
|
|
13768
|
-
const failures = [];
|
|
13769
|
-
for (const filter of queries) {
|
|
13770
|
-
const query = `(${filter}) AND (${textQuery})`;
|
|
13771
|
-
const args = this.options.onlyIn !== undefined ? ["-onlyin", this.options.onlyIn, query] : [query];
|
|
13772
|
-
const result = await this.run("mdfind", args, timeoutMs);
|
|
13773
|
-
if (!result.ok) {
|
|
13774
|
-
if (result.error === "ENOENT") {
|
|
13775
|
-
mdfindMissing = true;
|
|
13776
|
-
break;
|
|
13777
|
-
}
|
|
13778
|
-
failedQueries += 1;
|
|
13779
|
-
failures.push(result);
|
|
13780
|
-
continue;
|
|
13781
|
-
}
|
|
13782
|
-
let kept = 0;
|
|
13783
|
-
for (const line of result.stdout.split(`
|
|
13784
|
-
`)) {
|
|
13785
|
-
const path = line.trim();
|
|
13786
|
-
if (path.length === 0 || seen.has(path))
|
|
13787
|
-
continue;
|
|
13788
|
-
if (kept >= maxPerQuery)
|
|
13789
|
-
break;
|
|
13790
|
-
seen.add(path);
|
|
13791
|
-
paths.push(path);
|
|
13792
|
-
kept += 1;
|
|
13793
|
-
}
|
|
13794
|
-
}
|
|
13795
|
-
if (mdfindMissing) {
|
|
13796
|
-
return { ok: false, reason: "unavailable", message: "mdfind binary not found; Spotlight requires macOS" };
|
|
13797
|
-
}
|
|
13798
|
-
if (failedQueries === queries.length) {
|
|
13799
|
-
return {
|
|
13800
|
-
ok: false,
|
|
13801
|
-
reason: "unavailable",
|
|
13802
|
-
message: "every configured mdfind query failed",
|
|
13803
|
-
stderr: failures[0]?.stderr,
|
|
13804
|
-
stdout: failures[0]?.stdout,
|
|
13805
|
-
code: failures[0]?.code
|
|
13806
|
-
};
|
|
13807
|
-
}
|
|
13808
|
-
const documents = [];
|
|
13809
|
-
let permissionDenied = 0;
|
|
13810
|
-
let readFailures = 0;
|
|
13811
|
-
const maxBytes = this.options.maxBytesPerFile ?? DEFAULT_MAX_BYTES_PER_FILE2;
|
|
13812
|
-
const maxContentChars = this.options.maxContentChars ?? MAX_CONTENT_CHARS3;
|
|
13813
|
-
for (const path of paths) {
|
|
13814
|
-
if (documents.length >= maxDocuments)
|
|
13815
|
-
break;
|
|
13816
|
-
const docId = createHash10("sha256").update(path).digest("hex").slice(0, 16);
|
|
13817
|
-
if (acceptDocument !== undefined && !acceptDocument(docId))
|
|
13818
|
-
continue;
|
|
13819
|
-
try {
|
|
13820
|
-
const info = await stat2(path);
|
|
13821
|
-
if (!info.isFile() || info.size > maxBytes)
|
|
13822
|
-
continue;
|
|
13823
|
-
const text = await this.readFile(path);
|
|
13824
|
-
const content = text.trim().slice(0, maxContentChars);
|
|
13825
|
-
if (content.length === 0)
|
|
13826
|
-
continue;
|
|
13827
|
-
documents.push({
|
|
13828
|
-
docId,
|
|
13829
|
-
hierarchy: ["files"],
|
|
13830
|
-
title: basename5(path),
|
|
13831
|
-
content,
|
|
13832
|
-
publishedAt: info.mtimeMs,
|
|
13833
|
-
metadata: { path, extension: extname6(path).toLowerCase() }
|
|
13834
|
-
});
|
|
13835
|
-
} catch (error) {
|
|
13836
|
-
const code = error.code;
|
|
13837
|
-
if (code === "EPERM" || code === "EACCES")
|
|
13838
|
-
permissionDenied += 1;
|
|
13839
|
-
else
|
|
13840
|
-
readFailures += 1;
|
|
13841
|
-
}
|
|
13842
|
-
}
|
|
13843
|
-
if (documents.length === 0 && permissionDenied > 0) {
|
|
13844
|
-
return {
|
|
13845
|
-
ok: false,
|
|
13846
|
-
reason: "permission",
|
|
13847
|
-
message: `macOS denied access to all Spotlight hits; ${FDA_GUIDANCE}`
|
|
13848
|
-
};
|
|
13849
|
-
}
|
|
13850
|
-
if (documents.length === 0 && failedQueries > 0) {
|
|
13851
|
-
return {
|
|
13852
|
-
ok: false,
|
|
13853
|
-
reason: "unavailable",
|
|
13854
|
-
message: "some mdfind queries failed and no readable hits remained",
|
|
13855
|
-
stderr: failures[0]?.stderr,
|
|
13856
|
-
stdout: failures[0]?.stdout,
|
|
13857
|
-
code: failures[0]?.code
|
|
13858
|
-
};
|
|
13859
|
-
}
|
|
13860
|
-
const warnings = [];
|
|
13861
|
-
if (failedQueries > 0)
|
|
13862
|
-
warnings.push(`${failedQueries} mdfind quer(y/ies) failed`);
|
|
13863
|
-
if (permissionDenied > 0) {
|
|
13864
|
-
warnings.push(`${permissionDenied} Spotlight hit(s) were denied by macOS privacy protections; ${FDA_GUIDANCE}`);
|
|
13865
|
-
}
|
|
13866
|
-
if (readFailures > 0)
|
|
13867
|
-
warnings.push(`${readFailures} Spotlight hit(s) failed to read`);
|
|
13868
|
-
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
13869
|
-
}
|
|
13870
|
-
async readinessFailure(queries) {
|
|
13871
|
-
if (this.platform !== "darwin") {
|
|
13872
|
-
return { ok: false, reason: "unavailable", message: "spotlight datasource is supported on macOS only" };
|
|
13873
|
-
}
|
|
13874
|
-
if (queries.length === 0) {
|
|
13875
|
-
return { ok: false, reason: "not-configured", message: "no spotlight queries configured" };
|
|
13876
|
-
}
|
|
13877
|
-
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS6;
|
|
13878
|
-
const status = await this.run("mdutil", ["-s", "/"], timeoutMs);
|
|
13879
|
-
if (status.ok && /indexing disabled/iu.test(status.stdout)) {
|
|
13880
|
-
return {
|
|
13881
|
-
ok: false,
|
|
13882
|
-
reason: "unavailable",
|
|
13883
|
-
message: "Spotlight indexing is disabled; enable it with `sudo mdutil -i on /`"
|
|
13884
|
-
};
|
|
13885
|
-
}
|
|
13886
|
-
return;
|
|
13887
|
-
}
|
|
13888
|
-
}
|
|
13889
|
-
var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS4 = 300, DEFAULT_MAX_BYTES_PER_FILE2, DEFAULT_TIMEOUT_MS6 = 15000, MAX_CONTENT_CHARS3 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve) => {
|
|
13890
|
-
const child = spawn12(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
|
|
13891
|
-
let stdout = "";
|
|
13892
|
-
let stderr = "";
|
|
13893
|
-
let settled = false;
|
|
13894
|
-
const finish = (result) => {
|
|
13895
|
-
if (settled)
|
|
13896
|
-
return;
|
|
13897
|
-
settled = true;
|
|
13898
|
-
clearTimeout(timer);
|
|
13899
|
-
resolve(result);
|
|
13900
|
-
};
|
|
13901
|
-
const timer = setTimeout(() => {
|
|
13902
|
-
child.kill("SIGKILL");
|
|
13903
|
-
finish({ ok: false, stdout, stderr: "process timed out", code: null });
|
|
13904
|
-
}, timeoutMs);
|
|
13905
|
-
child.stdout.setEncoding("utf8");
|
|
13906
|
-
child.stderr.setEncoding("utf8");
|
|
13907
|
-
child.stdout.on("data", (chunk) => {
|
|
13908
|
-
if (stdout.length < MAX_BUFFER_BYTES)
|
|
13909
|
-
stdout += chunk;
|
|
13910
|
-
});
|
|
13911
|
-
child.stderr.on("data", (chunk) => {
|
|
13912
|
-
if (stderr.length < MAX_BUFFER_BYTES)
|
|
13913
|
-
stderr += chunk;
|
|
13914
|
-
});
|
|
13915
|
-
child.on("error", (error) => {
|
|
13916
|
-
finish({ ok: false, stdout, stderr: error.message, code: null, error: error.code });
|
|
13917
|
-
});
|
|
13918
|
-
child.on("close", (code) => {
|
|
13919
|
-
finish({ ok: code === 0, stdout, stderr, code });
|
|
13920
|
-
});
|
|
13921
|
-
});
|
|
13922
|
-
var init_connector5 = __esm(() => {
|
|
13923
|
-
DEFAULT_MAX_BYTES_PER_FILE2 = 256 * 1024;
|
|
13924
|
-
MAX_BUFFER_BYTES = 4 * 1024 * 1024;
|
|
13925
|
-
});
|
|
13926
|
-
|
|
13927
|
-
// src/datasource/skills/spotlight/skill.ts
|
|
13928
|
-
class SpotlightSkill {
|
|
13929
|
-
connector;
|
|
13930
|
-
skillName;
|
|
13931
|
-
instanceId;
|
|
13932
|
-
instances;
|
|
13933
|
-
tags;
|
|
13934
|
-
lastIndexedAt;
|
|
13935
|
-
lastError;
|
|
13936
|
-
constructor(options = {}) {
|
|
13937
|
-
this.skillName = options.skillName ?? "spotlight";
|
|
13938
|
-
this.instanceId = options.instanceId ?? "default";
|
|
13939
|
-
this.instances = options.instances !== undefined && options.instances.length > 0 ? options.instances : [this.instanceId];
|
|
13940
|
-
this.tags = options.tags ?? DEFAULT_TAGS4;
|
|
13941
|
-
this.connector = options.connector ?? new SpotlightConnector(options.connectorOptions ?? {});
|
|
13942
|
-
}
|
|
13943
|
-
describe() {
|
|
13944
|
-
return {
|
|
13945
|
-
name: this.skillName,
|
|
13946
|
-
id: this.skillName,
|
|
13947
|
-
type: "spotlight-search",
|
|
13948
|
-
description: "Live macOS Spotlight search via the built-in mdfind CLI",
|
|
13949
|
-
capabilities: ["files", "macos-only", "local-search", "live-search", "scoped", "external-cli"],
|
|
13950
|
-
tags: this.tags,
|
|
13951
|
-
status: "active",
|
|
13952
|
-
requiresExternalCli: true,
|
|
13953
|
-
datasourceId: this.skillName,
|
|
13954
|
-
instanceId: this.instanceId,
|
|
13955
|
-
instances: this.instances
|
|
13956
|
-
};
|
|
13957
|
-
}
|
|
13958
|
-
polling() {
|
|
13959
|
-
return {
|
|
13960
|
-
mode: "none",
|
|
13961
|
-
...this.lastIndexedAt !== undefined ? { lastIndexedAt: this.lastIndexedAt } : {},
|
|
13962
|
-
...this.lastError !== undefined ? { lastError: this.lastError } : {}
|
|
13963
|
-
};
|
|
13964
|
-
}
|
|
13965
|
-
async index() {
|
|
13966
|
-
try {
|
|
13967
|
-
const result = await this.connector.probe();
|
|
13968
|
-
if (!result.ok) {
|
|
13969
|
-
const code = connectorFailureToDiagnosticCode(result.reason);
|
|
13970
|
-
const message = boundDiagnosticText(result.message ?? result.reason);
|
|
13971
|
-
this.lastError = message;
|
|
13972
|
-
return this.fail(code, message);
|
|
13973
|
-
}
|
|
13974
|
-
this.lastError = undefined;
|
|
13975
|
-
this.lastIndexedAt = Date.now();
|
|
13976
|
-
return {
|
|
13977
|
-
ok: true,
|
|
13978
|
-
instanceId: this.instanceId,
|
|
13979
|
-
skill: this.skillName,
|
|
13980
|
-
chunkCount: 0,
|
|
13981
|
-
indexedAt: this.lastIndexedAt,
|
|
13982
|
-
diagnostics: [this.note(LIVE_NOTE, "datasource-empty", "info")]
|
|
13983
|
-
};
|
|
13984
|
-
} catch (error) {
|
|
13985
|
-
const message = boundDiagnosticText(error instanceof Error ? error.message : "spotlight probe failed");
|
|
13986
|
-
this.lastError = message;
|
|
13987
|
-
return this.fail("datasource-unavailable", message);
|
|
13988
|
-
}
|
|
13989
|
-
}
|
|
13990
|
-
retrievalMethods() {
|
|
13991
|
-
return [new SpotlightLiveMethod(this)];
|
|
13992
|
-
}
|
|
13993
|
-
describeSources() {
|
|
13994
|
-
return this.instances.map((instanceId) => ({
|
|
13995
|
-
source: datasourceSourcePath(this.skillName, instanceId),
|
|
13996
|
-
datasourceId: this.skillName,
|
|
13997
|
-
skill: this.skillName,
|
|
13998
|
-
instanceId,
|
|
13999
|
-
contentType: "document",
|
|
14000
|
-
metadata: { datasourceId: this.skillName, instanceId, tags: this.tags }
|
|
14001
|
-
}));
|
|
14002
|
-
}
|
|
14003
|
-
skillManifest() {
|
|
14004
|
-
const scopes = this.instances.map((instanceId) => `- \`${datasourceSourcePath(this.skillName, instanceId)}\``).join(`
|
|
14005
|
-
`);
|
|
14006
|
-
return {
|
|
14007
|
-
name: `datasource-${this.skillName}`,
|
|
14008
|
-
description: "Search the current macOS Spotlight index directly. AutoRAG keeps no file snapshot.",
|
|
14009
|
-
content: [
|
|
14010
|
-
"# macOS Spotlight (live mdfind search)",
|
|
14011
|
-
"",
|
|
14012
|
-
"Search files indexed by macOS Spotlight via the built-in `mdfind` CLI. macOS owns and refreshes the index; AutoRAG does not copy files into a local snapshot.",
|
|
14013
|
-
"",
|
|
14014
|
-
"## Platform requirements",
|
|
14015
|
-
"- macOS only: this datasource is unavailable on any other platform.",
|
|
14016
|
-
"- Spotlight indexing must be enabled (`sudo mdutil -i on /`; check with `mdutil -s /`).",
|
|
14017
|
-
"- Full Disk Access may be required to read protected hits. Grant it under System Settings -> Privacy & Security -> Full Disk Access, then restart the host app.",
|
|
14018
|
-
"",
|
|
14019
|
-
"## How to search",
|
|
14020
|
-
`Call the dedicated \`${datasourceSearchToolName(this.skillName)}\` tool with a query and optional narrowing scope. Available scopes:`,
|
|
14021
|
-
scopes,
|
|
14022
|
-
"Each configured Spotlight filter is AND-ed with the user query. Only the requested hits are read; refresh performs a readiness probe without running `mdfind`.",
|
|
14023
|
-
"Result metadata carries the real absolute file path (`path`) so hits are traceable back to disk. If paths must not leave this machine, run AutoRAG with a local LLM.",
|
|
14024
|
-
"",
|
|
14025
|
-
"## Native CLI",
|
|
14026
|
-
"- `mdfind <query>` searches Spotlight; `mdfind -onlyin <dir> <query>` scopes to a directory; `mdfind -name <text>` matches file names.",
|
|
14027
|
-
"- `mdls <file>` shows Spotlight metadata for one file.",
|
|
14028
|
-
"- `mdutil -s /` checks index status; `sudo mdutil -i on /` enables indexing."
|
|
14029
|
-
].join(`
|
|
14030
|
-
`)
|
|
14031
|
-
};
|
|
14032
|
-
}
|
|
14033
|
-
async search(query, options) {
|
|
14034
|
-
const trimmed = query.trim();
|
|
14035
|
-
if (trimmed.length === 0)
|
|
14036
|
-
return [];
|
|
14037
|
-
const scope = options.scope?.trim() ? normalizeVirtualPath2(options.scope) : undefined;
|
|
14038
|
-
if (scope !== undefined) {
|
|
14039
|
-
const chunksRoot = normalizeVirtualPath2(`/${this.skillName}/${this.instanceId}/chunks`);
|
|
14040
|
-
const scopeProbe = datasourceSourcePath(this.skillName, this.instanceId, "__scope_probe__");
|
|
14041
|
-
if (!matchesDatasourceScope(chunksRoot, scope) && !matchesDatasourceScope(scopeProbe, scope) && !scope.startsWith(`${chunksRoot}/`)) {
|
|
14042
|
-
return [];
|
|
14043
|
-
}
|
|
14044
|
-
}
|
|
14045
|
-
const topK = options.topK ?? DEFAULT_TOP_K9;
|
|
14046
|
-
const result = await this.connector.search(trimmed, topK, (docId) => matchesDatasourceScope(datasourceSourcePath(this.skillName, this.instanceId, docId), scope));
|
|
14047
|
-
if (!result.ok) {
|
|
14048
|
-
throw datasourceCliError(this.skillName, "search", {
|
|
14049
|
-
reason: result.reason,
|
|
14050
|
-
stderr: result.stderr ?? result.message,
|
|
14051
|
-
stdout: result.stdout,
|
|
14052
|
-
code: result.code
|
|
14053
|
-
});
|
|
14054
|
-
}
|
|
14055
|
-
const hits = [];
|
|
14056
|
-
for (const [index, document] of result.documents.entries()) {
|
|
14057
|
-
const source = datasourceSourcePath(this.skillName, this.instanceId, document.docId);
|
|
14058
|
-
if (!matchesDatasourceScope(source, options.scope))
|
|
14059
|
-
continue;
|
|
14060
|
-
hits.push({
|
|
14061
|
-
id: `${this.skillName}:${this.instanceId}:${document.docId}`,
|
|
14062
|
-
content: document.content,
|
|
14063
|
-
source,
|
|
14064
|
-
score: 1 / (index + 1),
|
|
14065
|
-
metadata: {
|
|
14066
|
-
...document.metadata,
|
|
14067
|
-
method: `${this.skillName}-live`,
|
|
14068
|
-
datasourceId: this.skillName,
|
|
14069
|
-
instanceId: this.instanceId,
|
|
14070
|
-
chunkId: document.docId,
|
|
14071
|
-
...document.title !== undefined ? { title: document.title } : {},
|
|
14072
|
-
...document.hierarchy !== undefined && document.hierarchy.length > 0 ? { hierarchy: document.hierarchy.join("/") } : {},
|
|
14073
|
-
...document.publishedAt !== undefined ? { publishedAt: document.publishedAt } : {},
|
|
14074
|
-
...result.warnings !== undefined && result.warnings.length > 0 ? { warnings: result.warnings } : {}
|
|
14075
|
-
}
|
|
14076
|
-
});
|
|
14077
|
-
if (hits.length >= topK)
|
|
14078
|
-
break;
|
|
14079
|
-
}
|
|
14080
|
-
return hits;
|
|
14081
|
-
}
|
|
14082
|
-
fail(code, message) {
|
|
14083
|
-
return {
|
|
14084
|
-
ok: false,
|
|
14085
|
-
instanceId: this.instanceId,
|
|
14086
|
-
skill: this.skillName,
|
|
14087
|
-
indexedAt: Date.now(),
|
|
14088
|
-
diagnostics: [
|
|
14089
|
-
this.note(message, code, code === "datasource-unavailable" || code === "datasource-empty" ? "warning" : "error")
|
|
14090
|
-
],
|
|
14091
|
-
error: message,
|
|
14092
|
-
code,
|
|
14093
|
-
message
|
|
14094
|
-
};
|
|
14095
|
-
}
|
|
14096
|
-
note(message, code, severity) {
|
|
14097
|
-
return { code, severity, message, instanceId: this.instanceId, source: this.skillName };
|
|
14098
|
-
}
|
|
14099
|
-
}
|
|
14100
|
-
|
|
14101
|
-
class SpotlightLiveMethod {
|
|
14102
|
-
skill;
|
|
14103
|
-
constructor(skill) {
|
|
14104
|
-
this.skill = skill;
|
|
14105
|
-
}
|
|
14106
|
-
describe() {
|
|
14107
|
-
const descriptor = this.skill.describe();
|
|
14108
|
-
return {
|
|
14109
|
-
name: `${descriptor.datasourceId ?? "spotlight"}-live`,
|
|
14110
|
-
type: "remote",
|
|
14111
|
-
description: "Live macOS Spotlight search via mdfind",
|
|
14112
|
-
status: "active",
|
|
14113
|
-
capabilities: ["live-search", "scoped", "external-cli", "path-opaque-sources"],
|
|
14114
|
-
datasourceId: descriptor.datasourceId,
|
|
14115
|
-
tags: descriptor.tags
|
|
14116
|
-
};
|
|
14117
|
-
}
|
|
14118
|
-
retrieve(query, options) {
|
|
14119
|
-
return this.skill.search(query, options);
|
|
14120
|
-
}
|
|
14121
|
-
}
|
|
14122
|
-
var DEFAULT_TAGS4, DEFAULT_TOP_K9 = 20, LIVE_NOTE = "live-search datasource: macOS owns the Spotlight index; AutoRAG stores no snapshot";
|
|
14123
|
-
var init_skill12 = __esm(() => {
|
|
14124
|
-
init_connector();
|
|
14125
|
-
init_errors2();
|
|
14126
|
-
init_scope2();
|
|
14127
|
-
init_connector5();
|
|
14128
|
-
DEFAULT_TAGS4 = ["spotlight", "files"];
|
|
14129
|
-
});
|
|
14130
|
-
|
|
14131
|
-
// src/datasource/skills/spotlight/index.ts
|
|
14132
|
-
var init_spotlight = __esm(() => {
|
|
14133
|
-
init_connector5();
|
|
14134
|
-
init_skill12();
|
|
14135
|
-
});
|
|
14136
|
-
|
|
14137
13777
|
// src/datasource/skills/telecrawl/index.ts
|
|
14138
13778
|
function globalArgs3(options) {
|
|
14139
13779
|
return [
|
|
@@ -14538,7 +14178,6 @@ var init_factory = __esm(() => {
|
|
|
14538
14178
|
init_obsidian();
|
|
14539
14179
|
init_rss();
|
|
14540
14180
|
init_slack();
|
|
14541
|
-
init_spotlight();
|
|
14542
14181
|
init_telecrawl();
|
|
14543
14182
|
init_wacrawl();
|
|
14544
14183
|
BUILDERS = {
|
|
@@ -14675,12 +14314,6 @@ var init_factory = __esm(() => {
|
|
|
14675
14314
|
...common(config, workspaceRoot),
|
|
14676
14315
|
skillName: registrationName,
|
|
14677
14316
|
connectorOptions: config.connector
|
|
14678
|
-
}),
|
|
14679
|
-
spotlight: (config, _workspaceRoot, registrationName) => new SpotlightSkill({
|
|
14680
|
-
...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
|
|
14681
|
-
...config.tags !== undefined ? { tags: config.tags } : {},
|
|
14682
|
-
skillName: registrationName,
|
|
14683
|
-
connectorOptions: config.connector
|
|
14684
14317
|
})
|
|
14685
14318
|
};
|
|
14686
14319
|
PLATFORM_ONLY_SKILLS = { kakao: "darwin" };
|
|
@@ -14734,7 +14367,7 @@ function createHttpEmbedder(options) {
|
|
|
14734
14367
|
method: "POST",
|
|
14735
14368
|
headers: headers(options),
|
|
14736
14369
|
body: requestBody(options, batch),
|
|
14737
|
-
signal: AbortSignal.timeout(options.timeoutMs ??
|
|
14370
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS6)
|
|
14738
14371
|
});
|
|
14739
14372
|
if (!response.ok) {
|
|
14740
14373
|
const detail = await response.text().catch(() => "");
|
|
@@ -14771,7 +14404,7 @@ function createHttpEmbedder(options) {
|
|
|
14771
14404
|
}
|
|
14772
14405
|
};
|
|
14773
14406
|
}
|
|
14774
|
-
var DEFAULT_EMBED_BATCH_SIZE = 32,
|
|
14407
|
+
var DEFAULT_EMBED_BATCH_SIZE = 32, DEFAULT_TIMEOUT_MS6 = 120000;
|
|
14775
14408
|
|
|
14776
14409
|
// src/embedding/in-process.ts
|
|
14777
14410
|
async function loadNativeEmbedder(options = {}) {
|
|
@@ -15301,7 +14934,7 @@ var init_types3 = __esm(() => {
|
|
|
15301
14934
|
// src/cli/config.ts
|
|
15302
14935
|
import { randomUUID as randomUUID4 } from "node:crypto";
|
|
15303
14936
|
import { existsSync as existsSync14, mkdirSync as mkdirSync13, readFileSync as readFileSync17, unlinkSync as unlinkSync2, writeFileSync as writeFileSync10 } from "node:fs";
|
|
15304
|
-
import { basename as
|
|
14937
|
+
import { basename as basename5, dirname as dirname13, isAbsolute as isAbsolute5, join as join25, resolve as resolve6 } from "node:path";
|
|
15305
14938
|
import { findEnvKeys, getEnvApiKey } from "@earendil-works/pi-ai/compat";
|
|
15306
14939
|
import { getAgentDir as getAgentDir2, ModelRuntime, SettingsManager } from "@earendil-works/pi-coding-agent";
|
|
15307
14940
|
function resolveConfigPath(input) {
|
|
@@ -15375,7 +15008,7 @@ function acquireConfigWriteLock(configPath) {
|
|
|
15375
15008
|
});
|
|
15376
15009
|
}
|
|
15377
15010
|
function replaceFileAtomically(path, contents, assertCommitAllowed) {
|
|
15378
|
-
const temporaryPath = join25(dirname13(path), `.${
|
|
15011
|
+
const temporaryPath = join25(dirname13(path), `.${basename5(path)}.${process.pid}.${randomUUID4()}.tmp`);
|
|
15379
15012
|
try {
|
|
15380
15013
|
writeFileSync10(temporaryPath, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
|
|
15381
15014
|
assertCommitAllowed?.();
|
|
@@ -16945,7 +16578,7 @@ function loopGuardPrompt(rule) {
|
|
|
16945
16578
|
}
|
|
16946
16579
|
|
|
16947
16580
|
// src/convert/cli.ts
|
|
16948
|
-
import { spawn as
|
|
16581
|
+
import { spawn as spawn12 } from "node:child_process";
|
|
16949
16582
|
import { accessSync as accessSync2, constants as constants2, existsSync as existsSync15 } from "node:fs";
|
|
16950
16583
|
import { delimiter as delimiter4, dirname as dirname14, join as join27, resolve as resolve7 } from "node:path";
|
|
16951
16584
|
async function convertWithFilee(request, options = {}) {
|
|
@@ -17023,7 +16656,7 @@ function spawnFilee(executable, args, options) {
|
|
|
17023
16656
|
}
|
|
17024
16657
|
const portable = portableSpawnCommand(executable, args);
|
|
17025
16658
|
const env = options.dataDir === undefined ? process.env : { ...process.env, FILEE_DATA_DIR: resolve7(options.dataDir) };
|
|
17026
|
-
const child =
|
|
16659
|
+
const child = spawn12(portable.command, [...portable.args], {
|
|
17027
16660
|
cwd: options.cwd,
|
|
17028
16661
|
env,
|
|
17029
16662
|
stdio: ["ignore", "pipe", "pipe"]
|
|
@@ -17095,7 +16728,7 @@ var init_cli = __esm(() => {
|
|
|
17095
16728
|
});
|
|
17096
16729
|
|
|
17097
16730
|
// src/convert/installer.ts
|
|
17098
|
-
import { createHash as
|
|
16731
|
+
import { createHash as createHash10 } from "node:crypto";
|
|
17099
16732
|
import { createReadStream as createReadStream2, createWriteStream as createWriteStream2, existsSync as existsSync16, mkdirSync as mkdirSync14, readdirSync as readdirSync4, renameSync as renameSync3, rmSync as rmSync6 } from "node:fs";
|
|
17100
16733
|
import { mkdtemp as mkdtemp2 } from "node:fs/promises";
|
|
17101
16734
|
import { join as join28 } from "node:path";
|
|
@@ -17168,7 +16801,7 @@ async function downloadFile2(url, destination) {
|
|
|
17168
16801
|
await pipeline2(Readable.fromWeb(response.body), createWriteStream2(destination));
|
|
17169
16802
|
}
|
|
17170
16803
|
async function verifySha2562(path, expected, name) {
|
|
17171
|
-
const hash =
|
|
16804
|
+
const hash = createHash10("sha256");
|
|
17172
16805
|
await pipeline2(createReadStream2(path), hash);
|
|
17173
16806
|
const actual = hash.digest("hex");
|
|
17174
16807
|
if (actual !== expected) {
|
|
@@ -17712,7 +17345,7 @@ var init_matrix = __esm(() => {
|
|
|
17712
17345
|
});
|
|
17713
17346
|
|
|
17714
17347
|
// src/dupey/cli.ts
|
|
17715
|
-
import { spawn as
|
|
17348
|
+
import { spawn as spawn13 } from "node:child_process";
|
|
17716
17349
|
import { resolve as resolve8 } from "node:path";
|
|
17717
17350
|
function runDupey(args, options) {
|
|
17718
17351
|
return options.run ? options.run(args) : spawnDupey(options.executable ?? "dupey", args, options);
|
|
@@ -17784,7 +17417,7 @@ function isScanResult(value) {
|
|
|
17784
17417
|
function spawnDupey(executable, args, options) {
|
|
17785
17418
|
return new Promise((resolveOutput, reject) => {
|
|
17786
17419
|
const portable = portableSpawnCommand(executable, args);
|
|
17787
|
-
const child =
|
|
17420
|
+
const child = spawn13(portable.command, [...portable.args], {
|
|
17788
17421
|
cwd: options.cwd,
|
|
17789
17422
|
stdio: ["ignore", "pipe", "pipe"]
|
|
17790
17423
|
});
|
|
@@ -17832,7 +17465,7 @@ var init_cli2 = __esm(() => {
|
|
|
17832
17465
|
});
|
|
17833
17466
|
|
|
17834
17467
|
// src/dupey/filter.ts
|
|
17835
|
-
import { stat as
|
|
17468
|
+
import { stat as stat2 } from "node:fs/promises";
|
|
17836
17469
|
import { resolve as resolve9 } from "node:path";
|
|
17837
17470
|
async function selectExactDuplicateExclusions(root, scan, isUnavailable) {
|
|
17838
17471
|
const byHash = new Map;
|
|
@@ -17887,7 +17520,7 @@ async function rankExactCandidates(paths, dupeyRanking, errors) {
|
|
|
17887
17520
|
}
|
|
17888
17521
|
const ranked = await Promise.all(paths.map(async (path) => {
|
|
17889
17522
|
try {
|
|
17890
|
-
const info = await
|
|
17523
|
+
const info = await stat2(path);
|
|
17891
17524
|
return { path, mtimeMs: info.mtimeMs };
|
|
17892
17525
|
} catch {
|
|
17893
17526
|
errors.push("dupey exact duplicate path disappeared before indexing.");
|
|
@@ -18056,7 +17689,7 @@ var init_retry = __esm(() => {
|
|
|
18056
17689
|
});
|
|
18057
17690
|
|
|
18058
17691
|
// src/everything/bundle.ts
|
|
18059
|
-
import { createHash as
|
|
17692
|
+
import { createHash as createHash11 } from "node:crypto";
|
|
18060
17693
|
import { existsSync as existsSync17, mkdirSync as mkdirSync15, readFileSync as readFileSync18, rmSync as rmSync7, writeFileSync as writeFileSync11 } from "node:fs";
|
|
18061
17694
|
import { dirname as dirname15, join as join29 } from "node:path";
|
|
18062
17695
|
import { fileURLToPath } from "node:url";
|
|
@@ -18174,7 +17807,7 @@ function writeAtomic(path, bytes) {
|
|
|
18174
17807
|
}
|
|
18175
17808
|
}
|
|
18176
17809
|
function sha256(bytes) {
|
|
18177
|
-
return
|
|
17810
|
+
return createHash11("sha256").update(bytes).digest("hex");
|
|
18178
17811
|
}
|
|
18179
17812
|
function errorMessage2(error) {
|
|
18180
17813
|
return error instanceof Error ? error.message : String(error);
|
|
@@ -18204,13 +17837,13 @@ function createPathExclusionMatcher(excludedPaths, platform = process.platform)
|
|
|
18204
17837
|
var init_path_exclusions = () => {};
|
|
18205
17838
|
|
|
18206
17839
|
// src/everything/client.ts
|
|
18207
|
-
import { spawn as
|
|
18208
|
-
import { createHash as
|
|
17840
|
+
import { spawn as spawn14 } from "node:child_process";
|
|
17841
|
+
import { createHash as createHash12 } from "node:crypto";
|
|
18209
17842
|
import { mkdirSync as mkdirSync16, writeFileSync as writeFileSync12 } from "node:fs";
|
|
18210
17843
|
import { join as join30, resolve as resolve10 } from "node:path";
|
|
18211
17844
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
18212
17845
|
function everythingInstanceName(root) {
|
|
18213
|
-
return `autorag-${
|
|
17846
|
+
return `autorag-${createHash12("sha256").update(resolve10(root).toLowerCase()).digest("hex").slice(0, 12)}`;
|
|
18214
17847
|
}
|
|
18215
17848
|
function buildEverythingIni(input) {
|
|
18216
17849
|
const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
|
|
@@ -18349,7 +17982,7 @@ class EverythingClient {
|
|
|
18349
17982
|
};
|
|
18350
17983
|
}
|
|
18351
17984
|
const es = { ok: true, esPath: binaries.esPath };
|
|
18352
|
-
const timeoutMs = this.options.timeoutMs ??
|
|
17985
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS7;
|
|
18353
17986
|
if ((this.options.excludeFolders?.length ?? 0) > 0) {
|
|
18354
17987
|
return this.searchWithExclusions(es.esPath, request, timeoutMs);
|
|
18355
17988
|
}
|
|
@@ -18434,7 +18067,7 @@ class EverythingClient {
|
|
|
18434
18067
|
const binaries = await this.resolveBinaries("cached");
|
|
18435
18068
|
if (!binaries.ok)
|
|
18436
18069
|
return;
|
|
18437
|
-
await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ??
|
|
18070
|
+
await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS7);
|
|
18438
18071
|
});
|
|
18439
18072
|
}
|
|
18440
18073
|
serialize(task) {
|
|
@@ -18464,7 +18097,7 @@ class EverythingClient {
|
|
|
18464
18097
|
const binaries = await this.resolveBinaries("install");
|
|
18465
18098
|
if (!binaries.ok)
|
|
18466
18099
|
return binaries;
|
|
18467
|
-
const timeoutMs = this.options.timeoutMs ??
|
|
18100
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS7;
|
|
18468
18101
|
let ping = await this.ping(binaries.esPath);
|
|
18469
18102
|
if (ping.code === 0 && !reconfigure)
|
|
18470
18103
|
return { ok: true, esPath: binaries.esPath };
|
|
@@ -18521,7 +18154,7 @@ class EverythingClient {
|
|
|
18521
18154
|
};
|
|
18522
18155
|
}
|
|
18523
18156
|
async ping(esPath) {
|
|
18524
|
-
const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ??
|
|
18157
|
+
const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS7);
|
|
18525
18158
|
return result.code === 0 && !VERSION_PATTERN.test(result.stdout.trim()) ? { ...result, code: -1 } : result;
|
|
18526
18159
|
}
|
|
18527
18160
|
failure(reason, esPath, args, result) {
|
|
@@ -18537,7 +18170,7 @@ function runProcess(command, args, timeoutMs) {
|
|
|
18537
18170
|
const promise = new Promise((resolve) => {
|
|
18538
18171
|
resolvePromise = resolve;
|
|
18539
18172
|
});
|
|
18540
|
-
const child =
|
|
18173
|
+
const child = spawn14(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
|
|
18541
18174
|
const stdout = [];
|
|
18542
18175
|
const stderr = [];
|
|
18543
18176
|
let bytes = 0;
|
|
@@ -18572,11 +18205,11 @@ killed by ${signal} after ${timeoutMs}ms` : err
|
|
|
18572
18205
|
return promise;
|
|
18573
18206
|
}
|
|
18574
18207
|
function launchDetached(command, args) {
|
|
18575
|
-
const child =
|
|
18208
|
+
const child = spawn14(command, args, { detached: true, stdio: "ignore", windowsHide: true });
|
|
18576
18209
|
child.on("error", () => {});
|
|
18577
18210
|
child.unref();
|
|
18578
18211
|
}
|
|
18579
|
-
var
|
|
18212
|
+
var DEFAULT_TIMEOUT_MS7 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS2, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
|
|
18580
18213
|
var init_client8 = __esm(() => {
|
|
18581
18214
|
init_path_exclusions();
|
|
18582
18215
|
init_bundle();
|
|
@@ -18592,8 +18225,8 @@ var init_everything = __esm(() => {
|
|
|
18592
18225
|
});
|
|
18593
18226
|
|
|
18594
18227
|
// src/fsearch/walk.ts
|
|
18595
|
-
import { readdir, stat as
|
|
18596
|
-
import { basename as
|
|
18228
|
+
import { readdir, stat as stat3 } from "node:fs/promises";
|
|
18229
|
+
import { basename as basename6, join as join31, sep as sep5 } from "node:path";
|
|
18597
18230
|
function compare(a, b) {
|
|
18598
18231
|
return a < b ? -1 : a > b ? 1 : 0;
|
|
18599
18232
|
}
|
|
@@ -18604,11 +18237,11 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
18604
18237
|
const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_DEADLINE_MS);
|
|
18605
18238
|
const matches = request.regex ? (() => {
|
|
18606
18239
|
const pattern = new RegExp(request.query, request.matchCase ? "" : "i");
|
|
18607
|
-
return (path) => pattern.test(request.matchPath ? path :
|
|
18240
|
+
return (path) => pattern.test(request.matchPath ? path : basename6(path));
|
|
18608
18241
|
})() : (() => {
|
|
18609
18242
|
const needle = request.matchCase ? request.query : request.query.toLowerCase();
|
|
18610
18243
|
return (path) => {
|
|
18611
|
-
const haystack = request.matchPath ? path :
|
|
18244
|
+
const haystack = request.matchPath ? path : basename6(path);
|
|
18612
18245
|
return (request.matchCase ? haystack : haystack.toLowerCase()).includes(needle);
|
|
18613
18246
|
};
|
|
18614
18247
|
})();
|
|
@@ -18656,14 +18289,14 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
18656
18289
|
if (!matches(path))
|
|
18657
18290
|
continue;
|
|
18658
18291
|
try {
|
|
18659
|
-
const info = await
|
|
18292
|
+
const info = await stat3(path);
|
|
18660
18293
|
if (request.kind === "folders" && !info.isDirectory())
|
|
18661
18294
|
continue;
|
|
18662
18295
|
if (request.kind === "files" && info.isDirectory())
|
|
18663
18296
|
continue;
|
|
18664
18297
|
collected.push({
|
|
18665
18298
|
path,
|
|
18666
|
-
name:
|
|
18299
|
+
name: basename6(path),
|
|
18667
18300
|
type: info.isDirectory() ? "folder" : "file",
|
|
18668
18301
|
size: info.isDirectory() ? undefined : info.size,
|
|
18669
18302
|
dateModified: info.mtime.toISOString()
|
|
@@ -18683,12 +18316,12 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
18683
18316
|
if (request.kind === "files")
|
|
18684
18317
|
continue;
|
|
18685
18318
|
try {
|
|
18686
|
-
const info = await
|
|
18319
|
+
const info = await stat3(folder);
|
|
18687
18320
|
if (!info.isDirectory())
|
|
18688
18321
|
continue;
|
|
18689
18322
|
collected.push({
|
|
18690
18323
|
path: folder,
|
|
18691
|
-
name:
|
|
18324
|
+
name: basename6(folder),
|
|
18692
18325
|
type: "folder",
|
|
18693
18326
|
size: undefined,
|
|
18694
18327
|
dateModified: info.mtime.toISOString()
|
|
@@ -18718,10 +18351,10 @@ var init_walk = __esm(() => {
|
|
|
18718
18351
|
});
|
|
18719
18352
|
|
|
18720
18353
|
// src/fsearch/client.ts
|
|
18721
|
-
import { spawn as
|
|
18722
|
-
import { createHash as
|
|
18354
|
+
import { spawn as spawn15 } from "node:child_process";
|
|
18355
|
+
import { createHash as createHash13 } from "node:crypto";
|
|
18723
18356
|
import { existsSync as existsSync18, mkdirSync as mkdirSync17, readFileSync as readFileSync19, realpathSync as realpathSync2, rmSync as rmSync8, writeFileSync as writeFileSync13 } from "node:fs";
|
|
18724
|
-
import { basename as
|
|
18357
|
+
import { basename as basename7, join as join32, resolve as resolve11, sep as sep6 } from "node:path";
|
|
18725
18358
|
import { setTimeout as sleep2 } from "node:timers/promises";
|
|
18726
18359
|
function fsearchSocketPath(dbPath) {
|
|
18727
18360
|
let canonical;
|
|
@@ -18730,7 +18363,7 @@ function fsearchSocketPath(dbPath) {
|
|
|
18730
18363
|
} catch {
|
|
18731
18364
|
canonical = resolve11(dbPath);
|
|
18732
18365
|
}
|
|
18733
|
-
const hash =
|
|
18366
|
+
const hash = createHash13("sha256").update(canonical).digest("hex").slice(0, 12);
|
|
18734
18367
|
return `/tmp/autorag-fsearch-${process.getuid?.() ?? 0}-${hash}.sock`;
|
|
18735
18368
|
}
|
|
18736
18369
|
function buildFSearchSearchArgs(dbPath, socketPath, request) {
|
|
@@ -18785,7 +18418,7 @@ function parseFSearchJsonLines(stdout) {
|
|
|
18785
18418
|
const path = String(record.path ?? "");
|
|
18786
18419
|
entries.push({
|
|
18787
18420
|
path,
|
|
18788
|
-
name:
|
|
18421
|
+
name: basename7(path),
|
|
18789
18422
|
type: record.type === "folder" ? "folder" : "file",
|
|
18790
18423
|
size: typeof record.size === "number" ? record.size : undefined,
|
|
18791
18424
|
dateModified: typeof record.mtime === "number" ? new Date(record.mtime * 1000).toISOString() : undefined
|
|
@@ -18863,7 +18496,7 @@ class FSearchClient {
|
|
|
18863
18496
|
const hasExclusions = (this.options.excludeFolders?.length ?? 0) > 0;
|
|
18864
18497
|
const nativeRequest = hasExclusions ? { ...request, offset: 0, maxResults: MAX_RESULTS_CAP } : request;
|
|
18865
18498
|
const args = buildFSearchSearchArgs(this.dbPath, this.socketPath(), nativeRequest);
|
|
18866
|
-
const result = await this.run(binary.path, args, this.options.timeoutMs ??
|
|
18499
|
+
const result = await this.run(binary.path, args, this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
18867
18500
|
if (result.code !== 0)
|
|
18868
18501
|
return this.failure("search-failed", binary.path, args, result);
|
|
18869
18502
|
let parsed;
|
|
@@ -18930,7 +18563,7 @@ class FSearchClient {
|
|
|
18930
18563
|
if (indexed.code !== 0)
|
|
18931
18564
|
return this.failure("index-failed", binaryPath, indexArgs, indexed);
|
|
18932
18565
|
const statsArgs = ["stats", "--db", this.dbPath, "--socket", this.socketPath()];
|
|
18933
|
-
const stats = await this.run(binaryPath, statsArgs, this.options.timeoutMs ??
|
|
18566
|
+
const stats = await this.run(binaryPath, statsArgs, this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
18934
18567
|
if (stats.code !== 0)
|
|
18935
18568
|
return this.failure("index-failed", binaryPath, statsArgs, stats);
|
|
18936
18569
|
let counts;
|
|
@@ -18978,7 +18611,7 @@ class FSearchClient {
|
|
|
18978
18611
|
}
|
|
18979
18612
|
}
|
|
18980
18613
|
async serving(binaryPath) {
|
|
18981
|
-
const stats = await this.run(binaryPath, ["stats", "--db", this.dbPath, "--socket", this.socketPath()], this.options.timeoutMs ??
|
|
18614
|
+
const stats = await this.run(binaryPath, ["stats", "--db", this.dbPath, "--socket", this.socketPath()], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
18982
18615
|
if (stats.code !== 0)
|
|
18983
18616
|
return false;
|
|
18984
18617
|
try {
|
|
@@ -19011,7 +18644,7 @@ class FSearchClient {
|
|
|
19011
18644
|
if (this.binary !== undefined)
|
|
19012
18645
|
return this.binary;
|
|
19013
18646
|
const path = this.options.binaryPath ?? "fsearch-cli";
|
|
19014
|
-
const probe = await this.run(path, ["--version"], this.options.timeoutMs ??
|
|
18647
|
+
const probe = await this.run(path, ["--version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
19015
18648
|
this.binary = probe.code === 0 ? { ok: true, path } : {
|
|
19016
18649
|
ok: false,
|
|
19017
18650
|
reason: "binary-missing",
|
|
@@ -19039,7 +18672,7 @@ function runProcess2(command, args, timeoutMs) {
|
|
|
19039
18672
|
const promise = new Promise((resolve) => {
|
|
19040
18673
|
resolvePromise = resolve;
|
|
19041
18674
|
});
|
|
19042
|
-
const child =
|
|
18675
|
+
const child = spawn15(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
|
|
19043
18676
|
const stdout = [];
|
|
19044
18677
|
const stderr = [];
|
|
19045
18678
|
let bytes = 0;
|
|
@@ -19074,7 +18707,7 @@ killed by ${signal} after ${timeoutMs}ms` : err
|
|
|
19074
18707
|
return promise;
|
|
19075
18708
|
}
|
|
19076
18709
|
function launchDetached2(command, args) {
|
|
19077
|
-
const child =
|
|
18710
|
+
const child = spawn15(command, args, { detached: true, stdio: "ignore", windowsHide: true });
|
|
19078
18711
|
child.on("error", () => {});
|
|
19079
18712
|
child.unref();
|
|
19080
18713
|
return child.pid;
|
|
@@ -19087,7 +18720,7 @@ function defaultIsProcessAlive(pid) {
|
|
|
19087
18720
|
return false;
|
|
19088
18721
|
}
|
|
19089
18722
|
}
|
|
19090
|
-
var
|
|
18723
|
+
var DEFAULT_TIMEOUT_MS8 = 30000, DEFAULT_STARTUP_TIMEOUT_MS2 = 1e4, STOP_TIMEOUT_MS2 = 2000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS2 = 250, DEFAULT_MAX_RESULTS2 = 100, MAX_RESULTS_CAP = 1000, MAX_OUTPUT_BYTES2, SORT_KEYS;
|
|
19091
18724
|
var init_client9 = __esm(() => {
|
|
19092
18725
|
init_path_exclusions();
|
|
19093
18726
|
init_walk();
|
|
@@ -19317,7 +18950,7 @@ var init_answer_pack = __esm(() => {
|
|
|
19317
18950
|
});
|
|
19318
18951
|
|
|
19319
18952
|
// src/jikji/installer.ts
|
|
19320
|
-
import { spawn as
|
|
18953
|
+
import { spawn as spawn16 } from "node:child_process";
|
|
19321
18954
|
import { existsSync as existsSync19 } from "node:fs";
|
|
19322
18955
|
import { delimiter as delimiter5, join as join34 } from "node:path";
|
|
19323
18956
|
function jikjiExecutableName(platform = process.platform) {
|
|
@@ -19344,7 +18977,7 @@ function lookupExecutableInPath(executable, env) {
|
|
|
19344
18977
|
}
|
|
19345
18978
|
function defaultRunner(cargo, env) {
|
|
19346
18979
|
return (args, timeoutMs) => new Promise((resolvePromise) => {
|
|
19347
|
-
const child =
|
|
18980
|
+
const child = spawn16(cargo, args, { env, stdio: ["ignore", "ignore", "pipe"] });
|
|
19348
18981
|
let stderr = "";
|
|
19349
18982
|
const timeout = setTimeout(() => {
|
|
19350
18983
|
if (!child.killed)
|
|
@@ -19458,7 +19091,7 @@ class SerialQueue {
|
|
|
19458
19091
|
}
|
|
19459
19092
|
|
|
19460
19093
|
// src/jikji/client.ts
|
|
19461
|
-
import { spawn as
|
|
19094
|
+
import { spawn as spawn17 } from "node:child_process";
|
|
19462
19095
|
import { existsSync as existsSync20, realpathSync as realpathSync4 } from "node:fs";
|
|
19463
19096
|
|
|
19464
19097
|
class JikjiClient {
|
|
@@ -19572,7 +19205,7 @@ class JikjiClient {
|
|
|
19572
19205
|
if (perFile <= 0)
|
|
19573
19206
|
return;
|
|
19574
19207
|
const files = await countFiles(root, options.includeHidden === true, signal);
|
|
19575
|
-
return scaledPrepareTimeoutMs(options.timeoutMs ??
|
|
19208
|
+
return scaledPrepareTimeoutMs(options.timeoutMs ?? DEFAULT_TIMEOUT_MS9, perFile, files);
|
|
19576
19209
|
}
|
|
19577
19210
|
async spawn(request) {
|
|
19578
19211
|
return spawnJikji({
|
|
@@ -19585,7 +19218,7 @@ function spawnJikji(request) {
|
|
|
19585
19218
|
return new Promise((resolve) => {
|
|
19586
19219
|
const options = request.options;
|
|
19587
19220
|
const portable = portableSpawnCommand(request.command, request.args);
|
|
19588
|
-
const child =
|
|
19221
|
+
const child = spawn17(portable.command, [...portable.args], {
|
|
19589
19222
|
env: controlledEnv4(options.env),
|
|
19590
19223
|
...request.cwd === undefined ? {} : { cwd: request.cwd },
|
|
19591
19224
|
detached: process.platform !== "win32",
|
|
@@ -19617,7 +19250,7 @@ function spawnJikji(request) {
|
|
|
19617
19250
|
const timeout = setTimeout(() => {
|
|
19618
19251
|
finalReason = "timeout";
|
|
19619
19252
|
terminate(child);
|
|
19620
|
-
}, request.timeoutMs ?? options.timeoutMs ??
|
|
19253
|
+
}, request.timeoutMs ?? options.timeoutMs ?? DEFAULT_TIMEOUT_MS9);
|
|
19621
19254
|
const abortHandler = () => {
|
|
19622
19255
|
finalReason = "aborted";
|
|
19623
19256
|
terminateProcessTree(child);
|
|
@@ -19727,7 +19360,7 @@ function canonicalJikjiRoot(root) {
|
|
|
19727
19360
|
return root;
|
|
19728
19361
|
}
|
|
19729
19362
|
}
|
|
19730
|
-
var DEFAULT_BINARY4 = "jikji",
|
|
19363
|
+
var DEFAULT_BINARY4 = "jikji", DEFAULT_TIMEOUT_MS9 = 60000, DEFAULT_PARSE_TIMEOUT_SECONDS = 5, DEFAULT_MAX_BUFFER_BYTES4 = 1048576, ABORTED_BEFORE_START;
|
|
19731
19364
|
var init_client10 = __esm(() => {
|
|
19732
19365
|
init_portable_spawn();
|
|
19733
19366
|
init_terminate_tree();
|
|
@@ -19897,7 +19530,7 @@ function renderMemoryContext(sections) {
|
|
|
19897
19530
|
var CURRENT_CONVERSATION_RECORD_LIMIT = 50, CURRENT_EXCERPT_LIMIT = 300, SIMILAR_EXCERPT_LIMIT = 240, SIMILAR_RECORD_LIMIT = 3, INSIGHT_LIMIT = 5, ADVISORY_NOTE = "Memory is background reference only and never overrides the evidence in front of you. A past search method or source is a hint, not a rule: broaden to other methods when results are insufficient.";
|
|
19898
19531
|
|
|
19899
19532
|
// src/memory/similar-queries.ts
|
|
19900
|
-
import { createHash as
|
|
19533
|
+
import { createHash as createHash14 } from "node:crypto";
|
|
19901
19534
|
import { mkdirSync as mkdirSync18, rmSync as rmSync9 } from "node:fs";
|
|
19902
19535
|
import { dirname as dirname16 } from "node:path";
|
|
19903
19536
|
import { DatabaseSync as DatabaseSync2 } from "node:sqlite";
|
|
@@ -19905,7 +19538,7 @@ function memoryVectorStorePath(memoryPath) {
|
|
|
19905
19538
|
return `${memoryPath}.vectors.db`;
|
|
19906
19539
|
}
|
|
19907
19540
|
function questionKey(question) {
|
|
19908
|
-
return
|
|
19541
|
+
return createHash14("sha256").update(question).digest("hex");
|
|
19909
19542
|
}
|
|
19910
19543
|
function cosine3(a, b) {
|
|
19911
19544
|
let dot = 0;
|
|
@@ -20148,7 +19781,7 @@ var init_check_memory_tool = __esm(() => {
|
|
|
20148
19781
|
var EVIDENCE_SUPPORT_THRESHOLD = 0.7;
|
|
20149
19782
|
|
|
20150
19783
|
// src/retrieval/evidence-id.ts
|
|
20151
|
-
import { createHash as
|
|
19784
|
+
import { createHash as createHash15 } from "node:crypto";
|
|
20152
19785
|
function normalizeEvidenceText(text) {
|
|
20153
19786
|
return text.replace(/\r\n?/gu, `
|
|
20154
19787
|
`).trim().normalize("NFC");
|
|
@@ -20178,7 +19811,7 @@ function stableEvidenceId(input) {
|
|
|
20178
19811
|
input.lineNumber ?? "",
|
|
20179
19812
|
normalizeEvidenceText(evidenceText)
|
|
20180
19813
|
].join("\x00");
|
|
20181
|
-
const digest =
|
|
19814
|
+
const digest = createHash15("sha256").update(payload).digest("hex").slice(0, 24);
|
|
20182
19815
|
return `${method}:${digest}`;
|
|
20183
19816
|
}
|
|
20184
19817
|
function normalizeEvidenceRef(input) {
|
|
@@ -20190,7 +19823,7 @@ function normalizeEvidenceRef(input) {
|
|
|
20190
19823
|
var init_evidence_id = () => {};
|
|
20191
19824
|
|
|
20192
19825
|
// src/memory/memory.ts
|
|
20193
|
-
import { createHash as
|
|
19826
|
+
import { createHash as createHash16, randomUUID as randomUUID5 } from "node:crypto";
|
|
20194
19827
|
import { existsSync as existsSync21, mkdirSync as mkdirSync19, readFileSync as readFileSync20, unlinkSync as unlinkSync3, writeFileSync as writeFileSync14 } from "node:fs";
|
|
20195
19828
|
import { dirname as dirname17 } from "node:path";
|
|
20196
19829
|
import { z as z2 } from "zod";
|
|
@@ -20284,7 +19917,7 @@ function normalizeEvidenceChunkRecord(record) {
|
|
|
20284
19917
|
return { ...base, ...normalizeEvidenceContext(record) };
|
|
20285
19918
|
}
|
|
20286
19919
|
function hashText(value) {
|
|
20287
|
-
return
|
|
19920
|
+
return createHash16("sha256").update(value).digest("hex");
|
|
20288
19921
|
}
|
|
20289
19922
|
function resultId(sessionId, number) {
|
|
20290
19923
|
return `${sessionId}:${number}`;
|
|
@@ -20818,7 +20451,7 @@ import {
|
|
|
20818
20451
|
rmSync as rmSync10,
|
|
20819
20452
|
writeFileSync as writeFileSync15
|
|
20820
20453
|
} from "node:fs";
|
|
20821
|
-
import { basename as
|
|
20454
|
+
import { basename as basename8, dirname as dirname18, isAbsolute as isAbsolute8, join as join37, relative as relative6, sep as sep8 } from "node:path";
|
|
20822
20455
|
function syncMinSyncWorkspace(root, options = {}) {
|
|
20823
20456
|
const workspacePath = options.workspacePath ?? minSyncWorkspaceRoot(root);
|
|
20824
20457
|
const filesRoot = join37(workspacePath, "files");
|
|
@@ -20859,7 +20492,7 @@ function syncMinSyncWorkspace(root, options = {}) {
|
|
|
20859
20492
|
return { workspacePath, entries, excluded };
|
|
20860
20493
|
}
|
|
20861
20494
|
function stagingStatePath(workspacePath) {
|
|
20862
|
-
return join37(dirname18(workspacePath), `.${
|
|
20495
|
+
return join37(dirname18(workspacePath), `.${basename8(workspacePath)}-autorag-staging.json`);
|
|
20863
20496
|
}
|
|
20864
20497
|
function loadStagingState2(workspacePath) {
|
|
20865
20498
|
const path = stagingStatePath(workspacePath);
|
|
@@ -20910,7 +20543,7 @@ function ensureManagedDirectory(filesRoot, directory) {
|
|
|
20910
20543
|
}
|
|
20911
20544
|
}
|
|
20912
20545
|
function copyFileAtomically(source, destination) {
|
|
20913
|
-
const temporaryPath = join37(dirname18(destination), `.${
|
|
20546
|
+
const temporaryPath = join37(dirname18(destination), `.${basename8(destination)}.${randomUUID6()}.tmp`);
|
|
20914
20547
|
try {
|
|
20915
20548
|
copyFileSync4(source, temporaryPath);
|
|
20916
20549
|
renameWithRetry(temporaryPath, destination);
|
|
@@ -20964,7 +20597,7 @@ var init_workspace = __esm(() => {
|
|
|
20964
20597
|
|
|
20965
20598
|
// src/minsync/method.ts
|
|
20966
20599
|
import { existsSync as existsSync23, readFileSync as readFileSync22, realpathSync as realpathSync5, statSync as statSync6 } from "node:fs";
|
|
20967
|
-
import { basename as
|
|
20600
|
+
import { basename as basename9, join as join38, normalize as normalize2 } from "node:path";
|
|
20968
20601
|
|
|
20969
20602
|
class MinSyncVectorMethod {
|
|
20970
20603
|
root;
|
|
@@ -21065,7 +20698,7 @@ class MinSyncVectorMethod {
|
|
|
21065
20698
|
const entry = byPath.get(hit.path);
|
|
21066
20699
|
if (!entry || !matchesVirtualPathScope(entry.virtualPath, options.scope))
|
|
21067
20700
|
continue;
|
|
21068
|
-
const chunkDiscriminator = hit.docId ??
|
|
20701
|
+
const chunkDiscriminator = hit.docId ?? basename9(hit.path);
|
|
21069
20702
|
const chunkId = `minsync:${entry.virtualPath}:${chunkDiscriminator}`;
|
|
21070
20703
|
const location = locateChunkInMirror(mirrorTexts, entry, hit.text);
|
|
21071
20704
|
results.push({
|
|
@@ -21306,9 +20939,9 @@ var init_isolated = __esm(() => {
|
|
|
21306
20939
|
});
|
|
21307
20940
|
|
|
21308
20941
|
// src/mirror/sync.ts
|
|
21309
|
-
import { createHash as
|
|
21310
|
-
import { opendir as opendir2, readFile as readFile2, stat as
|
|
21311
|
-
import { basename as
|
|
20942
|
+
import { createHash as createHash17 } from "node:crypto";
|
|
20943
|
+
import { opendir as opendir2, readFile as readFile2, stat as stat4 } from "node:fs/promises";
|
|
20944
|
+
import { basename as basename10, extname as extname6, join as join40, resolve as resolve14, sep as sep9 } from "node:path";
|
|
21312
20945
|
import { setTimeout as sleep3 } from "node:timers/promises";
|
|
21313
20946
|
function errorCode(error) {
|
|
21314
20947
|
if (!(error instanceof Error))
|
|
@@ -21358,7 +20991,7 @@ function parseFingerprint(options) {
|
|
|
21358
20991
|
}
|
|
21359
20992
|
return value;
|
|
21360
20993
|
};
|
|
21361
|
-
return
|
|
20994
|
+
return createHash17("sha256").update(JSON.stringify(stable({ format: 1, parser: outputParserOptions(options.parserOptions) }))).digest("hex");
|
|
21362
20995
|
}
|
|
21363
20996
|
function outputParserOptions(options = {}) {
|
|
21364
20997
|
const { fileTimeoutMs: _timeout, maxHeapMb: _heap, ocr, ...parser } = options;
|
|
@@ -21366,7 +20999,7 @@ function outputParserOptions(options = {}) {
|
|
|
21366
20999
|
}
|
|
21367
21000
|
function retryFingerprint(options, fingerprint) {
|
|
21368
21001
|
const parser = options.parserOptions;
|
|
21369
|
-
return
|
|
21002
|
+
return createHash17("sha256").update(JSON.stringify({
|
|
21370
21003
|
fingerprint,
|
|
21371
21004
|
fileTimeoutMs: parser?.fileTimeoutMs,
|
|
21372
21005
|
maxHeapMb: parser?.maxHeapMb,
|
|
@@ -21585,7 +21218,7 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
21585
21218
|
}
|
|
21586
21219
|
let info;
|
|
21587
21220
|
try {
|
|
21588
|
-
info = await
|
|
21221
|
+
info = await stat4(entry.sourcePath, { bigint: true });
|
|
21589
21222
|
} catch (error) {
|
|
21590
21223
|
await skipFile(entry, sourceReadFailure(error));
|
|
21591
21224
|
continue;
|
|
@@ -21617,7 +21250,7 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
21617
21250
|
continue;
|
|
21618
21251
|
}
|
|
21619
21252
|
await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
|
|
21620
|
-
path:
|
|
21253
|
+
path: basename10(outputPath),
|
|
21621
21254
|
size: Buffer.byteLength(markdown)
|
|
21622
21255
|
});
|
|
21623
21256
|
for (const diagnostic of parsed.diagnostics ?? [])
|
|
@@ -21693,7 +21326,7 @@ async function detectMirrorStaleness(options) {
|
|
|
21693
21326
|
}
|
|
21694
21327
|
}
|
|
21695
21328
|
function standaloneImageTooSmallMessage(virtualPath, bytes) {
|
|
21696
|
-
if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(
|
|
21329
|
+
if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname6(virtualPath))))
|
|
21697
21330
|
return;
|
|
21698
21331
|
return ocrImageTooSmallMessage(bytes);
|
|
21699
21332
|
}
|
|
@@ -21749,7 +21382,7 @@ function errnoCode2(error) {
|
|
|
21749
21382
|
async function probeSearchRoot(rootPath) {
|
|
21750
21383
|
let isDirectory;
|
|
21751
21384
|
try {
|
|
21752
|
-
isDirectory = (await
|
|
21385
|
+
isDirectory = (await stat4(rootPath)).isDirectory();
|
|
21753
21386
|
} catch (error) {
|
|
21754
21387
|
const code = errnoCode2(error);
|
|
21755
21388
|
return code === "ENOENT" || code === "ENOTDIR" ? "gone" : "unreadable";
|
|
@@ -21861,12 +21494,12 @@ async function collectFiles(state, directory, depth) {
|
|
|
21861
21494
|
}
|
|
21862
21495
|
if (!entry.isFile())
|
|
21863
21496
|
continue;
|
|
21864
|
-
const extension = normalizeExtension2(
|
|
21497
|
+
const extension = normalizeExtension2(extname6(entry.name));
|
|
21865
21498
|
if (!state.supportedExtensions.has(extension))
|
|
21866
21499
|
continue;
|
|
21867
21500
|
let fileStat;
|
|
21868
21501
|
try {
|
|
21869
|
-
fileStat = await
|
|
21502
|
+
fileStat = await stat4(sourcePath, { bigint: true });
|
|
21870
21503
|
} catch (error) {
|
|
21871
21504
|
const code = errnoCode2(error);
|
|
21872
21505
|
if (code === undefined)
|
|
@@ -26042,7 +25675,7 @@ var init_citations = __esm(() => {
|
|
|
26042
25675
|
});
|
|
26043
25676
|
|
|
26044
25677
|
// src/agent/evidence-ledger.ts
|
|
26045
|
-
import { createHash as
|
|
25678
|
+
import { createHash as createHash18 } from "node:crypto";
|
|
26046
25679
|
import { closeSync as closeSync2, openSync as openSync2, readSync, statSync as statSync7 } from "node:fs";
|
|
26047
25680
|
import { isAbsolute as isAbsolute9 } from "node:path";
|
|
26048
25681
|
import { Type as Type3 } from "typebox";
|
|
@@ -26077,10 +25710,10 @@ function readFileExcerpt(path) {
|
|
|
26077
25710
|
let descriptor;
|
|
26078
25711
|
try {
|
|
26079
25712
|
descriptor = openSync2(path, "r");
|
|
26080
|
-
const buffer = Buffer.alloc(
|
|
25713
|
+
const buffer = Buffer.alloc(MAX_CONTENT_CHARS3 * 4);
|
|
26081
25714
|
const bytes = readSync(descriptor, buffer, 0, buffer.length, 0);
|
|
26082
25715
|
const head = buffer.subarray(0, bytes);
|
|
26083
|
-
return head.includes(0) ? path : head.toString("utf8").slice(0,
|
|
25716
|
+
return head.includes(0) ? path : head.toString("utf8").slice(0, MAX_CONTENT_CHARS3);
|
|
26084
25717
|
} catch {
|
|
26085
25718
|
return path;
|
|
26086
25719
|
} finally {
|
|
@@ -26113,8 +25746,8 @@ class EvidenceLedger {
|
|
|
26113
25746
|
unsaved = new Set;
|
|
26114
25747
|
lastOrdinal = 0;
|
|
26115
25748
|
register(input) {
|
|
26116
|
-
const content = input.content.slice(0,
|
|
26117
|
-
const digest =
|
|
25749
|
+
const content = input.content.slice(0, MAX_CONTENT_CHARS3);
|
|
25750
|
+
const digest = createHash18("sha256").update(input.content.replace(/\s+/gu, " ").trim()).digest("hex");
|
|
26118
25751
|
const key = `${input.source}\x00${digest}`;
|
|
26119
25752
|
const existing = this.byKey.get(key);
|
|
26120
25753
|
if (existing !== undefined) {
|
|
@@ -26200,7 +25833,7 @@ class EvidenceLedger {
|
|
|
26200
25833
|
this.lastOrdinal = Math.max(this.lastOrdinal, ordinalFloor, highestOrdinal(this.entries));
|
|
26201
25834
|
}
|
|
26202
25835
|
}
|
|
26203
|
-
var
|
|
25836
|
+
var MAX_CONTENT_CHARS3 = 2000, EVIDENCE_ENTRY_TYPE = "autorag.evidence", ledgerEntrySchema, persistedEvidenceSchema;
|
|
26204
25837
|
var init_evidence_ledger = __esm(() => {
|
|
26205
25838
|
ledgerEntrySchema = Type3.Object({
|
|
26206
25839
|
id: Type3.String({ pattern: "^e[1-9][0-9]*$" }),
|
|
@@ -26432,7 +26065,7 @@ var init_ask_user = () => {};
|
|
|
26432
26065
|
|
|
26433
26066
|
// src/automation/config.ts
|
|
26434
26067
|
import { existsSync as existsSync26, readFileSync as readFileSync23, realpathSync as realpathSync6 } from "node:fs";
|
|
26435
|
-
import { basename as
|
|
26068
|
+
import { basename as basename11, dirname as dirname22, isAbsolute as isAbsolute11, join as join42, matchesGlob, relative as relative7, resolve as resolve15 } from "node:path";
|
|
26436
26069
|
function automationsConfigPath(env = process.env) {
|
|
26437
26070
|
return join42(resolveAutoRAGHome(env), "automations.json");
|
|
26438
26071
|
}
|
|
@@ -26565,7 +26198,7 @@ function parseAutomations(raw, env = process.env) {
|
|
|
26565
26198
|
...agent !== undefined ? { agent } : {},
|
|
26566
26199
|
appendFiles: obj.appendFiles === true,
|
|
26567
26200
|
debounceMs: parsePositiveInt(obj.debounceMs, `${obj.id}.debounceMs`, DEFAULT_DEBOUNCE_MS),
|
|
26568
|
-
timeoutMs: parsePositiveInt(obj.timeoutMs, `${obj.id}.timeoutMs`,
|
|
26201
|
+
timeoutMs: parsePositiveInt(obj.timeoutMs, `${obj.id}.timeoutMs`, DEFAULT_TIMEOUT_MS10)
|
|
26569
26202
|
};
|
|
26570
26203
|
});
|
|
26571
26204
|
}
|
|
@@ -26622,7 +26255,7 @@ function canonicalizePath(target) {
|
|
|
26622
26255
|
const parent = dirname22(resolved);
|
|
26623
26256
|
if (parent === resolved)
|
|
26624
26257
|
return resolved;
|
|
26625
|
-
return join42(canonicalizePath(parent),
|
|
26258
|
+
return join42(canonicalizePath(parent), basename11(resolved));
|
|
26626
26259
|
}
|
|
26627
26260
|
}
|
|
26628
26261
|
function matchesAutomationPath(automation, watchedRoot, filePath) {
|
|
@@ -26645,7 +26278,7 @@ function matchesAutomation(automation, watchedRoot, event) {
|
|
|
26645
26278
|
return false;
|
|
26646
26279
|
return matchesAutomationPath(automation, watchedRoot, event.path);
|
|
26647
26280
|
}
|
|
26648
|
-
var AUTOMATION_EVENT_TYPES, DEFAULT_DEBOUNCE_MS = 1000,
|
|
26281
|
+
var AUTOMATION_EVENT_TYPES, DEFAULT_DEBOUNCE_MS = 1000, DEFAULT_TIMEOUT_MS10 = 600000, DEFAULT_EXCLUDES, ID_PATTERN, KNOWN_KEYS, AutomationConfigError, AGENT_KEYS, RETRIEVAL_KEYS;
|
|
26649
26282
|
var init_config5 = __esm(() => {
|
|
26650
26283
|
init_home();
|
|
26651
26284
|
init_atomic_write();
|
|
@@ -26685,7 +26318,7 @@ var init_config5 = __esm(() => {
|
|
|
26685
26318
|
});
|
|
26686
26319
|
|
|
26687
26320
|
// src/automation/run.ts
|
|
26688
|
-
import { spawn as
|
|
26321
|
+
import { spawn as spawn18 } from "node:child_process";
|
|
26689
26322
|
import { appendFileSync as appendFileSync2, existsSync as existsSync27, mkdirSync as mkdirSync22, readFileSync as readFileSync24, renameSync as renameSync4, statSync as statSync8 } from "node:fs";
|
|
26690
26323
|
import { dirname as dirname23, join as join43 } from "node:path";
|
|
26691
26324
|
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
@@ -26797,7 +26430,7 @@ async function runAutomation(automation, events, options = {}) {
|
|
|
26797
26430
|
appendAutomationRunLog(home, record);
|
|
26798
26431
|
resolve(record);
|
|
26799
26432
|
};
|
|
26800
|
-
const child =
|
|
26433
|
+
const child = spawn18(command, args, {
|
|
26801
26434
|
cwd,
|
|
26802
26435
|
env: childEnv,
|
|
26803
26436
|
stdio: ["pipe", "pipe", "pipe"],
|
|
@@ -27203,7 +26836,7 @@ var init_daemon2 = __esm(() => {
|
|
|
27203
26836
|
|
|
27204
26837
|
// src/automation/service.ts
|
|
27205
26838
|
import { execFile as execFile2 } from "node:child_process";
|
|
27206
|
-
import { createHash as
|
|
26839
|
+
import { createHash as createHash19 } from "node:crypto";
|
|
27207
26840
|
import { existsSync as existsSync30, mkdirSync as mkdirSync24, rmSync as rmSync11, writeFileSync as writeFileSync16 } from "node:fs";
|
|
27208
26841
|
import { dirname as dirname24, join as join45, resolve as resolve17 } from "node:path";
|
|
27209
26842
|
import { setTimeout as sleep5 } from "node:timers/promises";
|
|
@@ -27234,7 +26867,7 @@ function automationServiceDescriptor(options = {}) {
|
|
|
27234
26867
|
const userHome = resolveUserHome(env);
|
|
27235
26868
|
const defaultHome = resolve17(join45(userHome, ".autorag"));
|
|
27236
26869
|
const home = resolve17(options.home ?? resolveAutoRAGHome(env));
|
|
27237
|
-
const suffix = home === defaultHome ? "" : `-${
|
|
26870
|
+
const suffix = home === defaultHome ? "" : `-${createHash19("sha256").update(home).digest("hex").slice(0, 8)}`;
|
|
27238
26871
|
const automationsDir = join45(home, "automations");
|
|
27239
26872
|
const envFilePath = join45(automationsDir, "service.env");
|
|
27240
26873
|
const logPath = join45(automationsDir, "daemon.log");
|
|
@@ -27670,7 +27303,7 @@ var init_automation_tool = __esm(() => {
|
|
|
27670
27303
|
});
|
|
27671
27304
|
|
|
27672
27305
|
// src/agent/citation-links.ts
|
|
27673
|
-
import { basename as
|
|
27306
|
+
import { basename as basename12, isAbsolute as isAbsolute12 } from "node:path";
|
|
27674
27307
|
import { pathToFileURL as pathToFileURL3 } from "node:url";
|
|
27675
27308
|
function openableTarget(source) {
|
|
27676
27309
|
if (/^https?:\/\//iu.test(source)) {
|
|
@@ -27688,7 +27321,7 @@ function linkCitations(markdown, sourceOf, isFile) {
|
|
|
27688
27321
|
for (const marker of answerMarkers(markdown, isFile)) {
|
|
27689
27322
|
if (marker.kind === "number")
|
|
27690
27323
|
continue;
|
|
27691
|
-
const labels = marker.kind === "file" ? [{ label:
|
|
27324
|
+
const labels = marker.kind === "file" ? [{ label: basename12(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
|
|
27692
27325
|
let linked = false;
|
|
27693
27326
|
const parts = labels.map(({ label, source }) => {
|
|
27694
27327
|
const target = source === undefined ? undefined : openableTarget(source);
|
|
@@ -29036,7 +28669,7 @@ class DazziJevBackend {
|
|
|
29036
28669
|
this.apiKey = apiKey;
|
|
29037
28670
|
this.baseUrl = options.baseUrl ?? resolveBaseUrl(env);
|
|
29038
28671
|
this.defaultModel = options.defaultModel ?? DAZZI_DEFAULT_JEV_MODEL;
|
|
29039
|
-
this.timeoutMs = options.timeoutMs ??
|
|
28672
|
+
this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS11;
|
|
29040
28673
|
this.maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
|
|
29041
28674
|
this.fetchImpl = options.fetchImpl;
|
|
29042
28675
|
}
|
|
@@ -29223,13 +28856,49 @@ async function extractErrorMessage(response) {
|
|
|
29223
28856
|
function sleep6(ms) {
|
|
29224
28857
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
29225
28858
|
}
|
|
29226
|
-
var JEV_PATH, DAZZI_DEFAULT_JEV_MODEL = "jev-1.13",
|
|
28859
|
+
var JEV_PATH, DAZZI_DEFAULT_JEV_MODEL = "jev-1.13", DEFAULT_TIMEOUT_MS11 = 1e4, DEFAULT_MAX_RETRIES = 2, RETRY_BASE_MS = 500, RETRY_MAX_MS = 5000;
|
|
29227
28860
|
var init_jev_backend = __esm(() => {
|
|
29228
28861
|
init_config();
|
|
29229
28862
|
init_credential();
|
|
29230
28863
|
JEV_PATH = `${API_PATH}/jev`;
|
|
29231
28864
|
});
|
|
29232
28865
|
|
|
28866
|
+
// src/agent/token-count.ts
|
|
28867
|
+
import { createHash as createHash20 } from "node:crypto";
|
|
28868
|
+
import { countTokens } from "gpt-tokenizer/encoding/o200k_base";
|
|
28869
|
+
function estimateTextTokens(text) {
|
|
28870
|
+
return Math.max(Math.ceil(text.length / 4), countText(text));
|
|
28871
|
+
}
|
|
28872
|
+
function tokensBeyondCharGuess(parts) {
|
|
28873
|
+
let excess = 0;
|
|
28874
|
+
for (const part of parts)
|
|
28875
|
+
excess += Math.max(0, countText(part) - Math.ceil(part.length / 4));
|
|
28876
|
+
return excess;
|
|
28877
|
+
}
|
|
28878
|
+
function countText(text) {
|
|
28879
|
+
if (text.length === 0)
|
|
28880
|
+
return 0;
|
|
28881
|
+
if (text.length < CACHE_MIN_CHARS)
|
|
28882
|
+
return countTokens(text, PLAIN_TEXT);
|
|
28883
|
+
const key = `${text.length}:${createHash20("sha1").update(text).digest("base64")}`;
|
|
28884
|
+
const hit = cache.get(key);
|
|
28885
|
+
if (hit !== undefined) {
|
|
28886
|
+
cache.delete(key);
|
|
28887
|
+
cache.set(key, hit);
|
|
28888
|
+
return hit;
|
|
28889
|
+
}
|
|
28890
|
+
const count = countTokens(text, PLAIN_TEXT);
|
|
28891
|
+
cache.set(key, count);
|
|
28892
|
+
if (cache.size > CACHE_ENTRIES)
|
|
28893
|
+
cache.delete(cache.keys().next().value);
|
|
28894
|
+
return count;
|
|
28895
|
+
}
|
|
28896
|
+
var PLAIN_TEXT, CACHE_MIN_CHARS = 2000, CACHE_ENTRIES = 64, cache;
|
|
28897
|
+
var init_token_count = __esm(() => {
|
|
28898
|
+
PLAIN_TEXT = { disallowedSpecial: new Set };
|
|
28899
|
+
cache = new Map;
|
|
28900
|
+
});
|
|
28901
|
+
|
|
29233
28902
|
// src/agent/context-token-guard.ts
|
|
29234
28903
|
import { calculateContextTokens, estimateTokens } from "@earendil-works/pi-coding-agent";
|
|
29235
28904
|
function createContextTokenGuardedTransform(model, base, fixedOverheadTokens = () => 0) {
|
|
@@ -29245,12 +28914,20 @@ function withBoundedOutput(model) {
|
|
|
29245
28914
|
return { ...model, maxTokens: MAX_OUTPUT_TOKENS };
|
|
29246
28915
|
}
|
|
29247
28916
|
function guardAgentContextMessages(model, messages, useReportedUsage = true, fixedOverheadTokens = 0) {
|
|
28917
|
+
const guarded = guardWithinBudget(model, messages, useReportedUsage, fixedOverheadTokens);
|
|
28918
|
+
if (model !== undefined && guarded.length > 0) {
|
|
28919
|
+
recordRequestEstimate(model, estimateContextMessageTokens(guarded, false) + fixedOverheadTokens);
|
|
28920
|
+
}
|
|
28921
|
+
return guarded;
|
|
28922
|
+
}
|
|
28923
|
+
function guardWithinBudget(model, messages, useReportedUsage = true, fixedOverheadTokens = 0) {
|
|
29248
28924
|
if (model === undefined || !Number.isFinite(model.contextWindow) || model.contextWindow <= 0 || messages.length === 0) {
|
|
29249
28925
|
return messages;
|
|
29250
28926
|
}
|
|
29251
28927
|
const usableContext = Math.floor(model.contextWindow / CONTEXT_HEADROOM);
|
|
29252
28928
|
const outputReserve = Math.min(Math.floor(usableContext / 4), MAX_OUTPUT_TOKENS, Number.isFinite(model.maxTokens) ? Math.max(0, model.maxTokens) : 0);
|
|
29253
|
-
const
|
|
28929
|
+
const overflowCeiling = contextOverflowCeiling(model);
|
|
28930
|
+
const inputBudget = Math.min(Math.max(0, usableContext - outputReserve), overflowCeiling ?? Number.POSITIVE_INFINITY);
|
|
29254
28931
|
const budget = Math.max(0, inputBudget - fixedOverheadTokens);
|
|
29255
28932
|
const reported = useReportedUsage && findLastAssistantUsage(messages) !== undefined;
|
|
29256
28933
|
if (estimateContextMessageTokens(messages, useReportedUsage) <= (reported ? inputBudget : budget)) {
|
|
@@ -29272,50 +28949,40 @@ function guardAgentContextMessages(model, messages, useReportedUsage = true, fix
|
|
|
29272
28949
|
}
|
|
29273
28950
|
let guarded = messages;
|
|
29274
28951
|
if (candidates.length > 0) {
|
|
29275
|
-
const
|
|
29276
|
-
const maximumOmitted = candidates.reduce((total, section) => total + section.blocks.length, 0);
|
|
28952
|
+
const entries = candidates.flatMap((section, sectionIndex) => section.blocks.map((block, blockIndex) => ({ block, sectionIndex, blockIndex })));
|
|
29277
28953
|
const omissionMarker = (count) => `
|
|
29278
28954
|
|
|
29279
28955
|
${GUARD_MARKER} Retrieval candidates omitted: ${count}.
|
|
29280
28956
|
`;
|
|
29281
|
-
|
|
29282
|
-
|
|
29283
|
-
|
|
29284
|
-
|
|
29285
|
-
for (const
|
|
29286
|
-
|
|
29287
|
-
|
|
29288
|
-
if (tokens <= remaining) {
|
|
29289
|
-
kept[sectionIndex]?.push(block);
|
|
29290
|
-
remaining -= tokens;
|
|
29291
|
-
continue;
|
|
29292
|
-
}
|
|
29293
|
-
omitted += section.blocks.length - (kept[sectionIndex]?.length ?? 0);
|
|
29294
|
-
for (const later of candidates.slice(sectionIndex + 1))
|
|
29295
|
-
omitted += later.blocks.length;
|
|
29296
|
-
firstOmitted = sectionIndex;
|
|
29297
|
-
break;
|
|
29298
|
-
}
|
|
29299
|
-
if (firstOmitted !== -1)
|
|
28957
|
+
const markerTokens = candidates.length * Math.ceil(omissionMarker(entries.length).length / 4);
|
|
28958
|
+
let remaining = Math.max(0, budget - estimateContextMessageTokens(base, false) - markerTokens);
|
|
28959
|
+
entries.sort((a, b) => a.block.rank - b.block.rank || a.sectionIndex - b.sectionIndex || a.blockIndex - b.blockIndex);
|
|
28960
|
+
const kept = new Set;
|
|
28961
|
+
for (const { block } of entries) {
|
|
28962
|
+
const tokens = Math.max(1, estimateTextTokens(block.text));
|
|
28963
|
+
if (tokens > remaining)
|
|
29300
28964
|
break;
|
|
28965
|
+
remaining -= tokens;
|
|
28966
|
+
kept.add(block);
|
|
29301
28967
|
}
|
|
29302
|
-
if (
|
|
29303
|
-
const
|
|
29304
|
-
for (const [sectionIndex, messageIndex] of candidateIndexes.entries())
|
|
29305
|
-
candidateByMessage.set(messageIndex, sectionIndex);
|
|
28968
|
+
if (kept.size < entries.length) {
|
|
28969
|
+
const sectionByMessage = new Map(candidateIndexes.map((messageIndex, i) => [messageIndex, candidates[i]]));
|
|
29306
28970
|
guarded = base.map((message, index) => {
|
|
29307
|
-
const
|
|
29308
|
-
if (
|
|
28971
|
+
const section = sectionByMessage.get(index);
|
|
28972
|
+
if (section === undefined)
|
|
29309
28973
|
return message;
|
|
29310
|
-
const
|
|
29311
|
-
const
|
|
29312
|
-
|
|
28974
|
+
const keptBlocks = section.blocks.filter((block) => kept.has(block));
|
|
28975
|
+
const omitted = section.blocks.length - keptBlocks.length;
|
|
28976
|
+
if (omitted === 0)
|
|
28977
|
+
return messages[index] ?? message;
|
|
28978
|
+
const marker = omissionMarker(omitted);
|
|
28979
|
+
return replaceTextContent(message, `${section.prefix}${keptBlocks.map((block) => block.text).join("")}${marker}${section.suffix}`);
|
|
29313
28980
|
});
|
|
29314
28981
|
}
|
|
29315
28982
|
}
|
|
29316
|
-
if (estimateContextMessageTokens(guarded, false) <= budget
|
|
28983
|
+
if (guarded !== messages && estimateContextMessageTokens(guarded, false) <= budget)
|
|
29317
28984
|
return guarded;
|
|
29318
|
-
return
|
|
28985
|
+
return fitNewestToolResult(guarded, budget);
|
|
29319
28986
|
}
|
|
29320
28987
|
function estimateContextMessageTokens(messages, useReportedUsage = false) {
|
|
29321
28988
|
if (useReportedUsage) {
|
|
@@ -29326,82 +28993,37 @@ function estimateContextMessageTokens(messages, useReportedUsage = false) {
|
|
|
29326
28993
|
}
|
|
29327
28994
|
return messages.reduce((total, message) => total + estimateMessageTokens(message), 0);
|
|
29328
28995
|
}
|
|
29329
|
-
function
|
|
29330
|
-
const
|
|
29331
|
-
const
|
|
29332
|
-
|
|
29333
|
-
|
|
29334
|
-
const
|
|
29335
|
-
|
|
29336
|
-
|
|
29337
|
-
|
|
29338
|
-
|
|
29339
|
-
trimmed[index] = fitMessageToTokens(message, Math.max(floorTokens, inputBudget - others));
|
|
29340
|
-
};
|
|
29341
|
-
for (const [index, message] of trimmed.entries()) {
|
|
29342
|
-
if (index === lastIndex || index === queryIndex || message.role === "system")
|
|
29343
|
-
continue;
|
|
29344
|
-
const replacement = replaceMessageWithMarker(message);
|
|
29345
|
-
if (replacement === message || estimateMessageTokens(replacement) >= estimateMessageTokens(message))
|
|
29346
|
-
continue;
|
|
29347
|
-
trimmed[index] = replacement;
|
|
29348
|
-
if (fits())
|
|
29349
|
-
return trimmed;
|
|
29350
|
-
}
|
|
29351
|
-
if (lastIndex !== queryIndex && trimmed[lastIndex]?.role !== "system") {
|
|
29352
|
-
fitInto(lastIndex, 0);
|
|
29353
|
-
if (fits())
|
|
29354
|
-
return trimmed;
|
|
29355
|
-
}
|
|
29356
|
-
for (const [index, message] of trimmed.entries()) {
|
|
29357
|
-
if (message.role !== "system")
|
|
29358
|
-
continue;
|
|
29359
|
-
fitInto(index, 0);
|
|
29360
|
-
if (fits())
|
|
29361
|
-
return trimmed;
|
|
29362
|
-
}
|
|
29363
|
-
if (queryIndex >= 0)
|
|
29364
|
-
fitInto(queryIndex, MIN_QUERY_TOKENS);
|
|
29365
|
-
return trimmed;
|
|
28996
|
+
function fitNewestToolResult(messages, inputBudget) {
|
|
28997
|
+
const lastIndex = messages.length - 1;
|
|
28998
|
+
const last = messages[lastIndex];
|
|
28999
|
+
if (last?.role !== "toolResult")
|
|
29000
|
+
return messages;
|
|
29001
|
+
const others = messages.reduce((total, message, index) => index === lastIndex ? total : total + estimateMessageTokens(message), 0);
|
|
29002
|
+
if (others >= inputBudget)
|
|
29003
|
+
return messages;
|
|
29004
|
+
const fitted = fitMessageToTokens(last, inputBudget - others);
|
|
29005
|
+
return fitted === last ? messages : [...messages.slice(0, lastIndex), fitted];
|
|
29366
29006
|
}
|
|
29367
29007
|
function fitMessageToTokens(message, maxTokens) {
|
|
29368
29008
|
const original = textContent(message);
|
|
29369
29009
|
if (original === undefined)
|
|
29370
|
-
return
|
|
29010
|
+
return message;
|
|
29371
29011
|
const marker = `
|
|
29372
29012
|
|
|
29373
29013
|
${GUARD_MARKER}
|
|
29374
29014
|
`;
|
|
29375
|
-
|
|
29376
|
-
|
|
29377
|
-
let
|
|
29378
|
-
|
|
29379
|
-
const
|
|
29380
|
-
const
|
|
29381
|
-
if (
|
|
29382
|
-
|
|
29383
|
-
|
|
29384
|
-
|
|
29385
|
-
high = middle - 1;
|
|
29386
|
-
}
|
|
29015
|
+
const overhead = estimateMessageTokens(replaceTextContent(message, ""));
|
|
29016
|
+
const density = Math.max(0, estimateMessageTokens(message) - overhead) / Math.max(1, original.length);
|
|
29017
|
+
let chars = density > 0 ? Math.floor((maxTokens - overhead) / density) : original.length;
|
|
29018
|
+
for (let attempt = 0;attempt < MAX_FIT_ATTEMPTS && chars > 0; attempt++) {
|
|
29019
|
+
const candidate = replaceTextContent(message, excerpt(original, chars, marker));
|
|
29020
|
+
const tokens = estimateMessageTokens(candidate);
|
|
29021
|
+
if (tokens <= maxTokens)
|
|
29022
|
+
return candidate;
|
|
29023
|
+
const scaled = Math.floor(chars * Math.max(0, maxTokens - overhead) / Math.max(1, tokens - overhead) * 0.98);
|
|
29024
|
+
chars = Math.min(chars - 1, scaled);
|
|
29387
29025
|
}
|
|
29388
|
-
return
|
|
29389
|
-
}
|
|
29390
|
-
function stripNonTextContent(message) {
|
|
29391
|
-
if (message.role !== "user" || typeof message.content === "string")
|
|
29392
|
-
return replaceMessageWithMarker(message);
|
|
29393
|
-
const marker = `
|
|
29394
|
-
|
|
29395
|
-
${GUARD_MARKER}
|
|
29396
|
-
`;
|
|
29397
|
-
let replacedImage = false;
|
|
29398
|
-
const content = message.content.map((block) => {
|
|
29399
|
-
if (block.type === "text")
|
|
29400
|
-
return block;
|
|
29401
|
-
replacedImage = true;
|
|
29402
|
-
return { type: "text", text: marker };
|
|
29403
|
-
});
|
|
29404
|
-
return replacedImage ? { ...message, content } : message;
|
|
29026
|
+
return replaceTextContent(message, marker);
|
|
29405
29027
|
}
|
|
29406
29028
|
function excerpt(text, maxChars, marker) {
|
|
29407
29029
|
if (text.length <= maxChars)
|
|
@@ -29411,32 +29033,54 @@ function excerpt(text, maxChars, marker) {
|
|
|
29411
29033
|
const tail = Math.max(0, available - head);
|
|
29412
29034
|
return `${text.slice(0, head)}${marker}${tail > 0 ? text.slice(-tail) : ""}`;
|
|
29413
29035
|
}
|
|
29414
|
-
function replaceMessageWithMarker(message) {
|
|
29415
|
-
return replaceTextContent(message, `
|
|
29416
|
-
|
|
29417
|
-
${GUARD_MARKER}
|
|
29418
|
-
`);
|
|
29419
|
-
}
|
|
29420
29036
|
function estimateMessageTokens(message) {
|
|
29421
29037
|
if (message.role === "system")
|
|
29422
29038
|
return estimateSystemMessageTokens(message);
|
|
29423
|
-
const estimate = estimateTokens(message);
|
|
29039
|
+
const estimate = estimateTokens(message) + tokensBeyondCharGuess(messageTextParts(message));
|
|
29424
29040
|
return message.role === "toolResult" ? estimate + Math.ceil(message.toolName.length / 4) : estimate;
|
|
29425
29041
|
}
|
|
29426
29042
|
function estimateSystemMessageTokens(message) {
|
|
29427
|
-
|
|
29428
|
-
|
|
29043
|
+
const parts = [];
|
|
29044
|
+
let chars = 0;
|
|
29045
|
+
if (typeof message.content === "string")
|
|
29046
|
+
parts.push(message.content);
|
|
29047
|
+
else {
|
|
29429
29048
|
for (const block of message.content) {
|
|
29430
29049
|
if (block.type === "text")
|
|
29431
|
-
|
|
29050
|
+
parts.push(block.text);
|
|
29432
29051
|
else if (block.type === "image")
|
|
29433
29052
|
chars += 4800;
|
|
29434
29053
|
}
|
|
29435
29054
|
}
|
|
29436
|
-
|
|
29437
|
-
|
|
29438
|
-
|
|
29439
|
-
return Math.ceil(chars / 4);
|
|
29055
|
+
parts.push(JSON.stringify(message.sections ?? {}), JSON.stringify(message.toolsAdded ?? []), JSON.stringify(message.toolsRemoved ?? []));
|
|
29056
|
+
for (const part of parts)
|
|
29057
|
+
chars += part.length;
|
|
29058
|
+
return Math.ceil(chars / 4) + tokensBeyondCharGuess(parts);
|
|
29059
|
+
}
|
|
29060
|
+
function messageTextParts(message) {
|
|
29061
|
+
switch (message.role) {
|
|
29062
|
+
case "user":
|
|
29063
|
+
case "custom":
|
|
29064
|
+
case "toolResult":
|
|
29065
|
+
return typeof message.content === "string" ? [message.content] : message.content.flatMap((block) => block.type === "text" ? [block.text] : []);
|
|
29066
|
+
case "assistant":
|
|
29067
|
+
return message.content.flatMap((block) => {
|
|
29068
|
+
if (block.type === "text")
|
|
29069
|
+
return [block.text];
|
|
29070
|
+
if (block.type === "thinking")
|
|
29071
|
+
return [block.thinking];
|
|
29072
|
+
if (block.type === "toolCall")
|
|
29073
|
+
return [`${block.name}${JSON.stringify(block.arguments)}`];
|
|
29074
|
+
return [];
|
|
29075
|
+
});
|
|
29076
|
+
case "bashExecution":
|
|
29077
|
+
return [message.command, message.output];
|
|
29078
|
+
case "branchSummary":
|
|
29079
|
+
case "compactionSummary":
|
|
29080
|
+
return [message.summary];
|
|
29081
|
+
default:
|
|
29082
|
+
return [];
|
|
29083
|
+
}
|
|
29440
29084
|
}
|
|
29441
29085
|
function findLastAssistantUsage(messages) {
|
|
29442
29086
|
for (let index = messages.length - 1;index >= 0; index--) {
|
|
@@ -29510,7 +29154,7 @@ function parseCandidateSection(message, text) {
|
|
|
29510
29154
|
const sectionEnd = findBaselineSectionEnd(text, sectionStart);
|
|
29511
29155
|
if (!/^\[e?\d+\] /mu.test(text.slice(sectionStart, sectionEnd)))
|
|
29512
29156
|
return;
|
|
29513
|
-
return splitCandidates(text, sectionStart, sectionEnd);
|
|
29157
|
+
return splitCandidates(text, sectionStart, sectionEnd, { lists: true });
|
|
29514
29158
|
}
|
|
29515
29159
|
if (message.role !== "toolResult")
|
|
29516
29160
|
return;
|
|
@@ -29521,7 +29165,9 @@ function parseCandidateSection(message, text) {
|
|
|
29521
29165
|
const directiveStart = text.indexOf(`
|
|
29522
29166
|
|
|
29523
29167
|
directive:`);
|
|
29524
|
-
return splitCandidates(text, 0, directiveStart >= 0 ? directiveStart : text.length,
|
|
29168
|
+
return splitCandidates(text, 0, directiveStart >= 0 ? directiveStart : text.length, {
|
|
29169
|
+
pattern: /^- [^\n]*(?:\n|$)/gmu
|
|
29170
|
+
});
|
|
29525
29171
|
}
|
|
29526
29172
|
const diagnosticsStart = text.indexOf(`
|
|
29527
29173
|
|
|
@@ -29543,17 +29189,23 @@ Formatting and content rules`
|
|
|
29543
29189
|
const positions = endMarkers.map((marker) => text.indexOf(marker, startIndex)).filter((index) => index >= 0);
|
|
29544
29190
|
return positions.length > 0 ? Math.min(...positions) : text.length;
|
|
29545
29191
|
}
|
|
29546
|
-
function splitCandidates(text, sectionStart, sectionEnd,
|
|
29192
|
+
function splitCandidates(text, sectionStart, sectionEnd, options = {}) {
|
|
29547
29193
|
const region = text.slice(sectionStart, sectionEnd);
|
|
29548
|
-
const matches = [...region.matchAll(pattern)];
|
|
29194
|
+
const matches = [...region.matchAll(options.pattern ?? /^\[e?\d+\] [^\n]*(?:\n|$)/gmu)];
|
|
29549
29195
|
const firstStart = matches[0]?.index;
|
|
29550
29196
|
if (firstStart === undefined)
|
|
29551
29197
|
return;
|
|
29552
29198
|
const blocks = [];
|
|
29199
|
+
let rank = 0;
|
|
29200
|
+
let heading = "";
|
|
29553
29201
|
for (const [index, match] of matches.entries()) {
|
|
29554
29202
|
const start = match.index ?? 0;
|
|
29555
29203
|
const end = index + 1 < matches.length ? matches[index + 1]?.index ?? region.length : region.length;
|
|
29556
|
-
|
|
29204
|
+
const raw = region.slice(start, end);
|
|
29205
|
+
const tail = options.lists === true ? LIST_HEADING_TAIL.exec(raw) : null;
|
|
29206
|
+
blocks.push({ text: heading + (tail === null ? raw : raw.slice(0, tail.index)), rank });
|
|
29207
|
+
rank = tail === null ? rank + 1 : 0;
|
|
29208
|
+
heading = tail === null ? "" : tail[0];
|
|
29557
29209
|
}
|
|
29558
29210
|
return {
|
|
29559
29211
|
prefix: text.slice(0, sectionStart) + region.slice(0, firstStart),
|
|
@@ -29561,8 +29213,10 @@ function splitCandidates(text, sectionStart, sectionEnd, pattern = /^\[e?\d+\] [
|
|
|
29561
29213
|
blocks
|
|
29562
29214
|
};
|
|
29563
29215
|
}
|
|
29564
|
-
var CONTEXT_HEADROOM = 1.2, MAX_OUTPUT_TOKENS = 32000,
|
|
29216
|
+
var CONTEXT_HEADROOM = 1.2, MAX_OUTPUT_TOKENS = 32000, GUARD_MARKER = "[AutoRAG context guard: omitted context to stay within the model context budget.]", RETRIEVAL_TOOL_PREFIX = "search_datasource_", RETRIEVAL_TOOL_NAMES, MAX_FIT_ATTEMPTS = 6, LIST_HEADING_TAIL;
|
|
29565
29217
|
var init_context_token_guard = __esm(() => {
|
|
29218
|
+
init_context_overflow();
|
|
29219
|
+
init_token_count();
|
|
29566
29220
|
RETRIEVAL_TOOL_NAMES = {
|
|
29567
29221
|
jikji_find: true,
|
|
29568
29222
|
search_all_documents: true,
|
|
@@ -29571,6 +29225,7 @@ var init_context_token_guard = __esm(() => {
|
|
|
29571
29225
|
everything_search: true,
|
|
29572
29226
|
fsearch_search: true
|
|
29573
29227
|
};
|
|
29228
|
+
LIST_HEADING_TAIL = /\n[^\n]*initial candidates[^\n]*:\n$/u;
|
|
29574
29229
|
});
|
|
29575
29230
|
|
|
29576
29231
|
// src/agent/current-date.ts
|
|
@@ -29907,7 +29562,7 @@ function guardContext(pi, contextTransform) {
|
|
|
29907
29562
|
pi.on("context", async (event, ctx) => {
|
|
29908
29563
|
const activeTools = new Set(pi.getActiveTools());
|
|
29909
29564
|
const toolDeclarations = pi.getAllTools().filter((tool) => activeTools.has(tool.name)).map(({ name, description, parameters }) => ({ name, description, parameters }));
|
|
29910
|
-
const fixedOverheadTokens =
|
|
29565
|
+
const fixedOverheadTokens = estimateTextTokens(ctx.getSystemPrompt()) + estimateTextTokens(JSON.stringify(toolDeclarations));
|
|
29911
29566
|
return {
|
|
29912
29567
|
messages: await createContextTokenGuardedTransform(ctx.model, contextTransform, () => fixedOverheadTokens)(event.messages)
|
|
29913
29568
|
};
|
|
@@ -30141,6 +29796,7 @@ var init_pi_session = __esm(() => {
|
|
|
30141
29796
|
init_intent_tracing();
|
|
30142
29797
|
init_mcp_client();
|
|
30143
29798
|
init_request_limits();
|
|
29799
|
+
init_token_count();
|
|
30144
29800
|
init_welcome_header();
|
|
30145
29801
|
PI_BUILTIN_TOOL_NAMES = ["read", "bash", "edit", "write", "grep", "find", "ls"];
|
|
30146
29802
|
});
|
|
@@ -31912,7 +31568,7 @@ var init_page_loader = __esm(() => {
|
|
|
31912
31568
|
});
|
|
31913
31569
|
|
|
31914
31570
|
// src/web/fetch/url-target.ts
|
|
31915
|
-
import { extname as
|
|
31571
|
+
import { extname as extname7 } from "node:path";
|
|
31916
31572
|
function repairCollapsedScheme(value) {
|
|
31917
31573
|
const m = value.match(/^(https?):\/(?!\/)/i);
|
|
31918
31574
|
return m ? `${m[1]}://${value.slice(m[0].length)}` : value;
|
|
@@ -31931,7 +31587,7 @@ function getFilenameExtensionHint(filename) {
|
|
|
31931
31587
|
const lower = filename.toLowerCase();
|
|
31932
31588
|
if (lower.endsWith(".tar.gz"))
|
|
31933
31589
|
return ".tar.gz";
|
|
31934
|
-
return
|
|
31590
|
+
return extname7(filename).toLowerCase();
|
|
31935
31591
|
}
|
|
31936
31592
|
function getExtensionHint(url, contentDisposition) {
|
|
31937
31593
|
if (contentDisposition) {
|
|
@@ -32108,7 +31764,7 @@ var init_convert2 = __esm(() => {
|
|
|
32108
31764
|
});
|
|
32109
31765
|
|
|
32110
31766
|
// src/web/fetch/feeds.ts
|
|
32111
|
-
import { extname as
|
|
31767
|
+
import { extname as extname8 } from "node:path";
|
|
32112
31768
|
import { XMLParser as XMLParser3 } from "fast-xml-parser";
|
|
32113
31769
|
function getHtmlAttribute(tag, attribute) {
|
|
32114
31770
|
const pattern = new RegExp(`\\b${attribute}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'=<>]+))`, "i");
|
|
@@ -32171,7 +31827,7 @@ function extractDocumentLinks(html, baseUrl) {
|
|
|
32171
31827
|
const href = getHtmlAttribute(tag, "href");
|
|
32172
31828
|
if (!href)
|
|
32173
31829
|
continue;
|
|
32174
|
-
const ext =
|
|
31830
|
+
const ext = extname8(href).toLowerCase();
|
|
32175
31831
|
if (!CONVERTIBLE_EXTENSIONS2.has(ext))
|
|
32176
31832
|
continue;
|
|
32177
31833
|
const resolved = href.startsWith("http") ? href : new URL(href, baseUrl).href;
|
|
@@ -33839,7 +33495,7 @@ var init_web_search_tool = __esm(() => {
|
|
|
33839
33495
|
// src/agent/agent.ts
|
|
33840
33496
|
import { randomUUID as randomUUID7 } from "node:crypto";
|
|
33841
33497
|
import { existsSync as existsSync34, watch as fsWatch, mkdirSync as mkdirSync25, realpathSync as realpathSync7, statSync as statSync9, writeFileSync as writeFileSync18 } from "node:fs";
|
|
33842
|
-
import { basename as
|
|
33498
|
+
import { basename as basename13, dirname as dirname26, join as join51, resolve as resolve18 } from "node:path";
|
|
33843
33499
|
import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
|
|
33844
33500
|
import { getAgentDir as getAgentDir5, keyText } from "@earendil-works/pi-coding-agent";
|
|
33845
33501
|
function positiveLimit(value, fallback, path) {
|
|
@@ -36762,7 +36418,7 @@ function toMinSyncReasonDiagnostic(reason) {
|
|
|
36762
36418
|
function pinExcludedPath(path) {
|
|
36763
36419
|
const resolvedPath = resolve18(path);
|
|
36764
36420
|
try {
|
|
36765
|
-
return join51(realpathSync7(dirname26(resolvedPath)),
|
|
36421
|
+
return join51(realpathSync7(dirname26(resolvedPath)), basename13(resolvedPath));
|
|
36766
36422
|
} catch {
|
|
36767
36423
|
return resolvedPath;
|
|
36768
36424
|
}
|