@autorag/librarian 2.13.3 → 2.13.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +3 -0
- package/README.md +0 -1
- package/dist/agent/context-token-guard.d.ts +11 -6
- package/dist/agent/token-count.d.ts +4 -0
- package/dist/cli/index.js +230 -631
- package/dist/core.js +146 -546
- package/dist/datasource/connector-skill.d.ts +1 -1
- package/dist/datasource/connector.d.ts +1 -1
- package/dist/datasource/index.d.ts +0 -1
- package/dist/index.js +148 -551
- package/dist/mcp/index.js +148 -548
- package/dist/profiles.js +231 -631
- package/package.json +2 -1
- package/skills/autorag-doctor/SKILL.md +1 -2
- package/skills/autorag-lite-setup/SKILL.md +1 -1
- package/skills/autorag-setup/SKILL.md +3 -4
- package/dist/datasource/skills/spotlight/connector.d.ts +0 -53
- package/dist/datasource/skills/spotlight/index.d.ts +0 -2
- package/dist/datasource/skills/spotlight/skill.d.ts +0 -33
package/dist/profiles.js
CHANGED
|
@@ -13774,413 +13774,6 @@ var init_slack = __esm(() => {
|
|
|
13774
13774
|
};
|
|
13775
13775
|
});
|
|
13776
13776
|
|
|
13777
|
-
// src/datasource/skills/spotlight/connector.ts
|
|
13778
|
-
import { spawn as spawn12 } from "node:child_process";
|
|
13779
|
-
import { createHash as createHash10 } from "node:crypto";
|
|
13780
|
-
import { readFile as fsReadFile, stat as stat2 } from "node:fs/promises";
|
|
13781
|
-
import { basename as basename5, extname as extname6 } from "node:path";
|
|
13782
|
-
|
|
13783
|
-
class SpotlightConnector {
|
|
13784
|
-
options;
|
|
13785
|
-
run;
|
|
13786
|
-
readFile;
|
|
13787
|
-
platform;
|
|
13788
|
-
constructor(options = {}) {
|
|
13789
|
-
this.options = options;
|
|
13790
|
-
this.run = options.run ?? spawnRunner;
|
|
13791
|
-
this.readFile = options.readFile ?? ((path) => fsReadFile(path, "utf8"));
|
|
13792
|
-
this.platform = options.platform ?? process.platform;
|
|
13793
|
-
}
|
|
13794
|
-
async probe() {
|
|
13795
|
-
const queries = (this.options.queries ?? []).map((query) => query.trim()).filter((query) => query.length > 0);
|
|
13796
|
-
const failure = await this.readinessFailure(queries);
|
|
13797
|
-
return failure ?? { ok: true, documents: [] };
|
|
13798
|
-
}
|
|
13799
|
-
async search(query, limit = DEFAULT_MAX_DOCUMENTS4, acceptDocument) {
|
|
13800
|
-
const trimmed = query.trim();
|
|
13801
|
-
if (trimmed.length === 0 || limit <= 0)
|
|
13802
|
-
return { ok: true, documents: [] };
|
|
13803
|
-
const queries = (this.options.queries ?? []).map((query) => query.trim()).filter((query) => query.length > 0);
|
|
13804
|
-
const failure = await this.readinessFailure(queries);
|
|
13805
|
-
if (failure !== undefined)
|
|
13806
|
-
return failure;
|
|
13807
|
-
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS6;
|
|
13808
|
-
const maxPerQuery = this.options.maxResultsPerQuery ?? DEFAULT_MAX_RESULTS_PER_QUERY;
|
|
13809
|
-
const maxDocuments = Math.min(this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS4, limit);
|
|
13810
|
-
const seen = new Set;
|
|
13811
|
-
const paths = [];
|
|
13812
|
-
const textQuery = trimmed.split(/\s+/u).map((term) => `"${term.replace(/\\/gu, "\\\\").replace(/"/gu, "\\\"")}"`).join(" AND ");
|
|
13813
|
-
let mdfindMissing = false;
|
|
13814
|
-
let failedQueries = 0;
|
|
13815
|
-
const failures = [];
|
|
13816
|
-
for (const filter of queries) {
|
|
13817
|
-
const query = `(${filter}) AND (${textQuery})`;
|
|
13818
|
-
const args = this.options.onlyIn !== undefined ? ["-onlyin", this.options.onlyIn, query] : [query];
|
|
13819
|
-
const result = await this.run("mdfind", args, timeoutMs);
|
|
13820
|
-
if (!result.ok) {
|
|
13821
|
-
if (result.error === "ENOENT") {
|
|
13822
|
-
mdfindMissing = true;
|
|
13823
|
-
break;
|
|
13824
|
-
}
|
|
13825
|
-
failedQueries += 1;
|
|
13826
|
-
failures.push(result);
|
|
13827
|
-
continue;
|
|
13828
|
-
}
|
|
13829
|
-
let kept = 0;
|
|
13830
|
-
for (const line of result.stdout.split(`
|
|
13831
|
-
`)) {
|
|
13832
|
-
const path = line.trim();
|
|
13833
|
-
if (path.length === 0 || seen.has(path))
|
|
13834
|
-
continue;
|
|
13835
|
-
if (kept >= maxPerQuery)
|
|
13836
|
-
break;
|
|
13837
|
-
seen.add(path);
|
|
13838
|
-
paths.push(path);
|
|
13839
|
-
kept += 1;
|
|
13840
|
-
}
|
|
13841
|
-
}
|
|
13842
|
-
if (mdfindMissing) {
|
|
13843
|
-
return { ok: false, reason: "unavailable", message: "mdfind binary not found; Spotlight requires macOS" };
|
|
13844
|
-
}
|
|
13845
|
-
if (failedQueries === queries.length) {
|
|
13846
|
-
return {
|
|
13847
|
-
ok: false,
|
|
13848
|
-
reason: "unavailable",
|
|
13849
|
-
message: "every configured mdfind query failed",
|
|
13850
|
-
stderr: failures[0]?.stderr,
|
|
13851
|
-
stdout: failures[0]?.stdout,
|
|
13852
|
-
code: failures[0]?.code
|
|
13853
|
-
};
|
|
13854
|
-
}
|
|
13855
|
-
const documents = [];
|
|
13856
|
-
let permissionDenied = 0;
|
|
13857
|
-
let readFailures = 0;
|
|
13858
|
-
const maxBytes = this.options.maxBytesPerFile ?? DEFAULT_MAX_BYTES_PER_FILE2;
|
|
13859
|
-
const maxContentChars = this.options.maxContentChars ?? MAX_CONTENT_CHARS3;
|
|
13860
|
-
for (const path of paths) {
|
|
13861
|
-
if (documents.length >= maxDocuments)
|
|
13862
|
-
break;
|
|
13863
|
-
const docId = createHash10("sha256").update(path).digest("hex").slice(0, 16);
|
|
13864
|
-
if (acceptDocument !== undefined && !acceptDocument(docId))
|
|
13865
|
-
continue;
|
|
13866
|
-
try {
|
|
13867
|
-
const info = await stat2(path);
|
|
13868
|
-
if (!info.isFile() || info.size > maxBytes)
|
|
13869
|
-
continue;
|
|
13870
|
-
const text = await this.readFile(path);
|
|
13871
|
-
const content = text.trim().slice(0, maxContentChars);
|
|
13872
|
-
if (content.length === 0)
|
|
13873
|
-
continue;
|
|
13874
|
-
documents.push({
|
|
13875
|
-
docId,
|
|
13876
|
-
hierarchy: ["files"],
|
|
13877
|
-
title: basename5(path),
|
|
13878
|
-
content,
|
|
13879
|
-
publishedAt: info.mtimeMs,
|
|
13880
|
-
metadata: { path, extension: extname6(path).toLowerCase() }
|
|
13881
|
-
});
|
|
13882
|
-
} catch (error) {
|
|
13883
|
-
const code = error.code;
|
|
13884
|
-
if (code === "EPERM" || code === "EACCES")
|
|
13885
|
-
permissionDenied += 1;
|
|
13886
|
-
else
|
|
13887
|
-
readFailures += 1;
|
|
13888
|
-
}
|
|
13889
|
-
}
|
|
13890
|
-
if (documents.length === 0 && permissionDenied > 0) {
|
|
13891
|
-
return {
|
|
13892
|
-
ok: false,
|
|
13893
|
-
reason: "permission",
|
|
13894
|
-
message: `macOS denied access to all Spotlight hits; ${FDA_GUIDANCE}`
|
|
13895
|
-
};
|
|
13896
|
-
}
|
|
13897
|
-
if (documents.length === 0 && failedQueries > 0) {
|
|
13898
|
-
return {
|
|
13899
|
-
ok: false,
|
|
13900
|
-
reason: "unavailable",
|
|
13901
|
-
message: "some mdfind queries failed and no readable hits remained",
|
|
13902
|
-
stderr: failures[0]?.stderr,
|
|
13903
|
-
stdout: failures[0]?.stdout,
|
|
13904
|
-
code: failures[0]?.code
|
|
13905
|
-
};
|
|
13906
|
-
}
|
|
13907
|
-
const warnings = [];
|
|
13908
|
-
if (failedQueries > 0)
|
|
13909
|
-
warnings.push(`${failedQueries} mdfind quer(y/ies) failed`);
|
|
13910
|
-
if (permissionDenied > 0) {
|
|
13911
|
-
warnings.push(`${permissionDenied} Spotlight hit(s) were denied by macOS privacy protections; ${FDA_GUIDANCE}`);
|
|
13912
|
-
}
|
|
13913
|
-
if (readFailures > 0)
|
|
13914
|
-
warnings.push(`${readFailures} Spotlight hit(s) failed to read`);
|
|
13915
|
-
return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
|
|
13916
|
-
}
|
|
13917
|
-
async readinessFailure(queries) {
|
|
13918
|
-
if (this.platform !== "darwin") {
|
|
13919
|
-
return { ok: false, reason: "unavailable", message: "spotlight datasource is supported on macOS only" };
|
|
13920
|
-
}
|
|
13921
|
-
if (queries.length === 0) {
|
|
13922
|
-
return { ok: false, reason: "not-configured", message: "no spotlight queries configured" };
|
|
13923
|
-
}
|
|
13924
|
-
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS6;
|
|
13925
|
-
const status = await this.run("mdutil", ["-s", "/"], timeoutMs);
|
|
13926
|
-
if (status.ok && /indexing disabled/iu.test(status.stdout)) {
|
|
13927
|
-
return {
|
|
13928
|
-
ok: false,
|
|
13929
|
-
reason: "unavailable",
|
|
13930
|
-
message: "Spotlight indexing is disabled; enable it with `sudo mdutil -i on /`"
|
|
13931
|
-
};
|
|
13932
|
-
}
|
|
13933
|
-
return;
|
|
13934
|
-
}
|
|
13935
|
-
}
|
|
13936
|
-
var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS4 = 300, DEFAULT_MAX_BYTES_PER_FILE2, DEFAULT_TIMEOUT_MS6 = 15000, MAX_CONTENT_CHARS3 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve) => {
|
|
13937
|
-
const child = spawn12(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
|
|
13938
|
-
let stdout = "";
|
|
13939
|
-
let stderr = "";
|
|
13940
|
-
let settled = false;
|
|
13941
|
-
const finish = (result) => {
|
|
13942
|
-
if (settled)
|
|
13943
|
-
return;
|
|
13944
|
-
settled = true;
|
|
13945
|
-
clearTimeout(timer);
|
|
13946
|
-
resolve(result);
|
|
13947
|
-
};
|
|
13948
|
-
const timer = setTimeout(() => {
|
|
13949
|
-
child.kill("SIGKILL");
|
|
13950
|
-
finish({ ok: false, stdout, stderr: "process timed out", code: null });
|
|
13951
|
-
}, timeoutMs);
|
|
13952
|
-
child.stdout.setEncoding("utf8");
|
|
13953
|
-
child.stderr.setEncoding("utf8");
|
|
13954
|
-
child.stdout.on("data", (chunk) => {
|
|
13955
|
-
if (stdout.length < MAX_BUFFER_BYTES)
|
|
13956
|
-
stdout += chunk;
|
|
13957
|
-
});
|
|
13958
|
-
child.stderr.on("data", (chunk) => {
|
|
13959
|
-
if (stderr.length < MAX_BUFFER_BYTES)
|
|
13960
|
-
stderr += chunk;
|
|
13961
|
-
});
|
|
13962
|
-
child.on("error", (error) => {
|
|
13963
|
-
finish({ ok: false, stdout, stderr: error.message, code: null, error: error.code });
|
|
13964
|
-
});
|
|
13965
|
-
child.on("close", (code) => {
|
|
13966
|
-
finish({ ok: code === 0, stdout, stderr, code });
|
|
13967
|
-
});
|
|
13968
|
-
});
|
|
13969
|
-
var init_connector5 = __esm(() => {
|
|
13970
|
-
DEFAULT_MAX_BYTES_PER_FILE2 = 256 * 1024;
|
|
13971
|
-
MAX_BUFFER_BYTES = 4 * 1024 * 1024;
|
|
13972
|
-
});
|
|
13973
|
-
|
|
13974
|
-
// src/datasource/skills/spotlight/skill.ts
|
|
13975
|
-
class SpotlightSkill {
|
|
13976
|
-
connector;
|
|
13977
|
-
skillName;
|
|
13978
|
-
instanceId;
|
|
13979
|
-
instances;
|
|
13980
|
-
tags;
|
|
13981
|
-
lastIndexedAt;
|
|
13982
|
-
lastError;
|
|
13983
|
-
constructor(options = {}) {
|
|
13984
|
-
this.skillName = options.skillName ?? "spotlight";
|
|
13985
|
-
this.instanceId = options.instanceId ?? "default";
|
|
13986
|
-
this.instances = options.instances !== undefined && options.instances.length > 0 ? options.instances : [this.instanceId];
|
|
13987
|
-
this.tags = options.tags ?? DEFAULT_TAGS4;
|
|
13988
|
-
this.connector = options.connector ?? new SpotlightConnector(options.connectorOptions ?? {});
|
|
13989
|
-
}
|
|
13990
|
-
describe() {
|
|
13991
|
-
return {
|
|
13992
|
-
name: this.skillName,
|
|
13993
|
-
id: this.skillName,
|
|
13994
|
-
type: "spotlight-search",
|
|
13995
|
-
description: "Live macOS Spotlight search via the built-in mdfind CLI",
|
|
13996
|
-
capabilities: ["files", "macos-only", "local-search", "live-search", "scoped", "external-cli"],
|
|
13997
|
-
tags: this.tags,
|
|
13998
|
-
status: "active",
|
|
13999
|
-
requiresExternalCli: true,
|
|
14000
|
-
datasourceId: this.skillName,
|
|
14001
|
-
instanceId: this.instanceId,
|
|
14002
|
-
instances: this.instances
|
|
14003
|
-
};
|
|
14004
|
-
}
|
|
14005
|
-
polling() {
|
|
14006
|
-
return {
|
|
14007
|
-
mode: "none",
|
|
14008
|
-
...this.lastIndexedAt !== undefined ? { lastIndexedAt: this.lastIndexedAt } : {},
|
|
14009
|
-
...this.lastError !== undefined ? { lastError: this.lastError } : {}
|
|
14010
|
-
};
|
|
14011
|
-
}
|
|
14012
|
-
async index() {
|
|
14013
|
-
try {
|
|
14014
|
-
const result = await this.connector.probe();
|
|
14015
|
-
if (!result.ok) {
|
|
14016
|
-
const code = connectorFailureToDiagnosticCode(result.reason);
|
|
14017
|
-
const message = boundDiagnosticText(result.message ?? result.reason);
|
|
14018
|
-
this.lastError = message;
|
|
14019
|
-
return this.fail(code, message);
|
|
14020
|
-
}
|
|
14021
|
-
this.lastError = undefined;
|
|
14022
|
-
this.lastIndexedAt = Date.now();
|
|
14023
|
-
return {
|
|
14024
|
-
ok: true,
|
|
14025
|
-
instanceId: this.instanceId,
|
|
14026
|
-
skill: this.skillName,
|
|
14027
|
-
chunkCount: 0,
|
|
14028
|
-
indexedAt: this.lastIndexedAt,
|
|
14029
|
-
diagnostics: [this.note(LIVE_NOTE, "datasource-empty", "info")]
|
|
14030
|
-
};
|
|
14031
|
-
} catch (error) {
|
|
14032
|
-
const message = boundDiagnosticText(error instanceof Error ? error.message : "spotlight probe failed");
|
|
14033
|
-
this.lastError = message;
|
|
14034
|
-
return this.fail("datasource-unavailable", message);
|
|
14035
|
-
}
|
|
14036
|
-
}
|
|
14037
|
-
retrievalMethods() {
|
|
14038
|
-
return [new SpotlightLiveMethod(this)];
|
|
14039
|
-
}
|
|
14040
|
-
describeSources() {
|
|
14041
|
-
return this.instances.map((instanceId) => ({
|
|
14042
|
-
source: datasourceSourcePath(this.skillName, instanceId),
|
|
14043
|
-
datasourceId: this.skillName,
|
|
14044
|
-
skill: this.skillName,
|
|
14045
|
-
instanceId,
|
|
14046
|
-
contentType: "document",
|
|
14047
|
-
metadata: { datasourceId: this.skillName, instanceId, tags: this.tags }
|
|
14048
|
-
}));
|
|
14049
|
-
}
|
|
14050
|
-
skillManifest() {
|
|
14051
|
-
const scopes = this.instances.map((instanceId) => `- \`${datasourceSourcePath(this.skillName, instanceId)}\``).join(`
|
|
14052
|
-
`);
|
|
14053
|
-
return {
|
|
14054
|
-
name: `datasource-${this.skillName}`,
|
|
14055
|
-
description: "Search the current macOS Spotlight index directly. AutoRAG keeps no file snapshot.",
|
|
14056
|
-
content: [
|
|
14057
|
-
"# macOS Spotlight (live mdfind search)",
|
|
14058
|
-
"",
|
|
14059
|
-
"Search files indexed by macOS Spotlight via the built-in `mdfind` CLI. macOS owns and refreshes the index; AutoRAG does not copy files into a local snapshot.",
|
|
14060
|
-
"",
|
|
14061
|
-
"## Platform requirements",
|
|
14062
|
-
"- macOS only: this datasource is unavailable on any other platform.",
|
|
14063
|
-
"- Spotlight indexing must be enabled (`sudo mdutil -i on /`; check with `mdutil -s /`).",
|
|
14064
|
-
"- Full Disk Access may be required to read protected hits. Grant it under System Settings -> Privacy & Security -> Full Disk Access, then restart the host app.",
|
|
14065
|
-
"",
|
|
14066
|
-
"## How to search",
|
|
14067
|
-
`Call the dedicated \`${datasourceSearchToolName(this.skillName)}\` tool with a query and optional narrowing scope. Available scopes:`,
|
|
14068
|
-
scopes,
|
|
14069
|
-
"Each configured Spotlight filter is AND-ed with the user query. Only the requested hits are read; refresh performs a readiness probe without running `mdfind`.",
|
|
14070
|
-
"Result metadata carries the real absolute file path (`path`) so hits are traceable back to disk. If paths must not leave this machine, run AutoRAG with a local LLM.",
|
|
14071
|
-
"",
|
|
14072
|
-
"## Native CLI",
|
|
14073
|
-
"- `mdfind <query>` searches Spotlight; `mdfind -onlyin <dir> <query>` scopes to a directory; `mdfind -name <text>` matches file names.",
|
|
14074
|
-
"- `mdls <file>` shows Spotlight metadata for one file.",
|
|
14075
|
-
"- `mdutil -s /` checks index status; `sudo mdutil -i on /` enables indexing."
|
|
14076
|
-
].join(`
|
|
14077
|
-
`)
|
|
14078
|
-
};
|
|
14079
|
-
}
|
|
14080
|
-
async search(query, options) {
|
|
14081
|
-
const trimmed = query.trim();
|
|
14082
|
-
if (trimmed.length === 0)
|
|
14083
|
-
return [];
|
|
14084
|
-
const scope = options.scope?.trim() ? normalizeVirtualPath2(options.scope) : undefined;
|
|
14085
|
-
if (scope !== undefined) {
|
|
14086
|
-
const chunksRoot = normalizeVirtualPath2(`/${this.skillName}/${this.instanceId}/chunks`);
|
|
14087
|
-
const scopeProbe = datasourceSourcePath(this.skillName, this.instanceId, "__scope_probe__");
|
|
14088
|
-
if (!matchesDatasourceScope(chunksRoot, scope) && !matchesDatasourceScope(scopeProbe, scope) && !scope.startsWith(`${chunksRoot}/`)) {
|
|
14089
|
-
return [];
|
|
14090
|
-
}
|
|
14091
|
-
}
|
|
14092
|
-
const topK = options.topK ?? DEFAULT_TOP_K9;
|
|
14093
|
-
const result = await this.connector.search(trimmed, topK, (docId) => matchesDatasourceScope(datasourceSourcePath(this.skillName, this.instanceId, docId), scope));
|
|
14094
|
-
if (!result.ok) {
|
|
14095
|
-
throw datasourceCliError(this.skillName, "search", {
|
|
14096
|
-
reason: result.reason,
|
|
14097
|
-
stderr: result.stderr ?? result.message,
|
|
14098
|
-
stdout: result.stdout,
|
|
14099
|
-
code: result.code
|
|
14100
|
-
});
|
|
14101
|
-
}
|
|
14102
|
-
const hits = [];
|
|
14103
|
-
for (const [index, document] of result.documents.entries()) {
|
|
14104
|
-
const source = datasourceSourcePath(this.skillName, this.instanceId, document.docId);
|
|
14105
|
-
if (!matchesDatasourceScope(source, options.scope))
|
|
14106
|
-
continue;
|
|
14107
|
-
hits.push({
|
|
14108
|
-
id: `${this.skillName}:${this.instanceId}:${document.docId}`,
|
|
14109
|
-
content: document.content,
|
|
14110
|
-
source,
|
|
14111
|
-
score: 1 / (index + 1),
|
|
14112
|
-
metadata: {
|
|
14113
|
-
...document.metadata,
|
|
14114
|
-
method: `${this.skillName}-live`,
|
|
14115
|
-
datasourceId: this.skillName,
|
|
14116
|
-
instanceId: this.instanceId,
|
|
14117
|
-
chunkId: document.docId,
|
|
14118
|
-
...document.title !== undefined ? { title: document.title } : {},
|
|
14119
|
-
...document.hierarchy !== undefined && document.hierarchy.length > 0 ? { hierarchy: document.hierarchy.join("/") } : {},
|
|
14120
|
-
...document.publishedAt !== undefined ? { publishedAt: document.publishedAt } : {},
|
|
14121
|
-
...result.warnings !== undefined && result.warnings.length > 0 ? { warnings: result.warnings } : {}
|
|
14122
|
-
}
|
|
14123
|
-
});
|
|
14124
|
-
if (hits.length >= topK)
|
|
14125
|
-
break;
|
|
14126
|
-
}
|
|
14127
|
-
return hits;
|
|
14128
|
-
}
|
|
14129
|
-
fail(code, message) {
|
|
14130
|
-
return {
|
|
14131
|
-
ok: false,
|
|
14132
|
-
instanceId: this.instanceId,
|
|
14133
|
-
skill: this.skillName,
|
|
14134
|
-
indexedAt: Date.now(),
|
|
14135
|
-
diagnostics: [
|
|
14136
|
-
this.note(message, code, code === "datasource-unavailable" || code === "datasource-empty" ? "warning" : "error")
|
|
14137
|
-
],
|
|
14138
|
-
error: message,
|
|
14139
|
-
code,
|
|
14140
|
-
message
|
|
14141
|
-
};
|
|
14142
|
-
}
|
|
14143
|
-
note(message, code, severity) {
|
|
14144
|
-
return { code, severity, message, instanceId: this.instanceId, source: this.skillName };
|
|
14145
|
-
}
|
|
14146
|
-
}
|
|
14147
|
-
|
|
14148
|
-
class SpotlightLiveMethod {
|
|
14149
|
-
skill;
|
|
14150
|
-
constructor(skill) {
|
|
14151
|
-
this.skill = skill;
|
|
14152
|
-
}
|
|
14153
|
-
describe() {
|
|
14154
|
-
const descriptor = this.skill.describe();
|
|
14155
|
-
return {
|
|
14156
|
-
name: `${descriptor.datasourceId ?? "spotlight"}-live`,
|
|
14157
|
-
type: "remote",
|
|
14158
|
-
description: "Live macOS Spotlight search via mdfind",
|
|
14159
|
-
status: "active",
|
|
14160
|
-
capabilities: ["live-search", "scoped", "external-cli", "path-opaque-sources"],
|
|
14161
|
-
datasourceId: descriptor.datasourceId,
|
|
14162
|
-
tags: descriptor.tags
|
|
14163
|
-
};
|
|
14164
|
-
}
|
|
14165
|
-
retrieve(query, options) {
|
|
14166
|
-
return this.skill.search(query, options);
|
|
14167
|
-
}
|
|
14168
|
-
}
|
|
14169
|
-
var DEFAULT_TAGS4, DEFAULT_TOP_K9 = 20, LIVE_NOTE = "live-search datasource: macOS owns the Spotlight index; AutoRAG stores no snapshot";
|
|
14170
|
-
var init_skill12 = __esm(() => {
|
|
14171
|
-
init_connector();
|
|
14172
|
-
init_errors2();
|
|
14173
|
-
init_scope2();
|
|
14174
|
-
init_connector5();
|
|
14175
|
-
DEFAULT_TAGS4 = ["spotlight", "files"];
|
|
14176
|
-
});
|
|
14177
|
-
|
|
14178
|
-
// src/datasource/skills/spotlight/index.ts
|
|
14179
|
-
var init_spotlight = __esm(() => {
|
|
14180
|
-
init_connector5();
|
|
14181
|
-
init_skill12();
|
|
14182
|
-
});
|
|
14183
|
-
|
|
14184
13777
|
// src/datasource/skills/telecrawl/index.ts
|
|
14185
13778
|
function globalArgs3(options) {
|
|
14186
13779
|
return [
|
|
@@ -14585,7 +14178,6 @@ var init_factory = __esm(() => {
|
|
|
14585
14178
|
init_obsidian();
|
|
14586
14179
|
init_rss();
|
|
14587
14180
|
init_slack();
|
|
14588
|
-
init_spotlight();
|
|
14589
14181
|
init_telecrawl();
|
|
14590
14182
|
init_wacrawl();
|
|
14591
14183
|
BUILDERS = {
|
|
@@ -14722,12 +14314,6 @@ var init_factory = __esm(() => {
|
|
|
14722
14314
|
...common(config, workspaceRoot),
|
|
14723
14315
|
skillName: registrationName,
|
|
14724
14316
|
connectorOptions: config.connector
|
|
14725
|
-
}),
|
|
14726
|
-
spotlight: (config, _workspaceRoot, registrationName) => new SpotlightSkill({
|
|
14727
|
-
...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
|
|
14728
|
-
...config.tags !== undefined ? { tags: config.tags } : {},
|
|
14729
|
-
skillName: registrationName,
|
|
14730
|
-
connectorOptions: config.connector
|
|
14731
14317
|
})
|
|
14732
14318
|
};
|
|
14733
14319
|
PLATFORM_ONLY_SKILLS = { kakao: "darwin" };
|
|
@@ -14781,7 +14367,7 @@ function createHttpEmbedder(options) {
|
|
|
14781
14367
|
method: "POST",
|
|
14782
14368
|
headers: headers(options),
|
|
14783
14369
|
body: requestBody(options, batch),
|
|
14784
|
-
signal: AbortSignal.timeout(options.timeoutMs ??
|
|
14370
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS6)
|
|
14785
14371
|
});
|
|
14786
14372
|
if (!response.ok) {
|
|
14787
14373
|
const detail = await response.text().catch(() => "");
|
|
@@ -14818,7 +14404,7 @@ function createHttpEmbedder(options) {
|
|
|
14818
14404
|
}
|
|
14819
14405
|
};
|
|
14820
14406
|
}
|
|
14821
|
-
var DEFAULT_EMBED_BATCH_SIZE = 32,
|
|
14407
|
+
var DEFAULT_EMBED_BATCH_SIZE = 32, DEFAULT_TIMEOUT_MS6 = 120000;
|
|
14822
14408
|
|
|
14823
14409
|
// src/embedding/in-process.ts
|
|
14824
14410
|
async function loadNativeEmbedder(options = {}) {
|
|
@@ -15348,7 +14934,7 @@ var init_types3 = __esm(() => {
|
|
|
15348
14934
|
// src/cli/config.ts
|
|
15349
14935
|
import { randomUUID as randomUUID4 } from "node:crypto";
|
|
15350
14936
|
import { existsSync as existsSync14, mkdirSync as mkdirSync13, readFileSync as readFileSync17, unlinkSync as unlinkSync2, writeFileSync as writeFileSync10 } from "node:fs";
|
|
15351
|
-
import { basename as
|
|
14937
|
+
import { basename as basename5, dirname as dirname13, isAbsolute as isAbsolute5, join as join25, resolve as resolve6 } from "node:path";
|
|
15352
14938
|
import { findEnvKeys, getEnvApiKey } from "@earendil-works/pi-ai/compat";
|
|
15353
14939
|
import { getAgentDir as getAgentDir2, ModelRuntime, SettingsManager } from "@earendil-works/pi-coding-agent";
|
|
15354
14940
|
function resolveConfigPath(input) {
|
|
@@ -15422,7 +15008,7 @@ function acquireConfigWriteLock(configPath) {
|
|
|
15422
15008
|
});
|
|
15423
15009
|
}
|
|
15424
15010
|
function replaceFileAtomically(path, contents, assertCommitAllowed) {
|
|
15425
|
-
const temporaryPath = join25(dirname13(path), `.${
|
|
15011
|
+
const temporaryPath = join25(dirname13(path), `.${basename5(path)}.${process.pid}.${randomUUID4()}.tmp`);
|
|
15426
15012
|
try {
|
|
15427
15013
|
writeFileSync10(temporaryPath, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
|
|
15428
15014
|
assertCommitAllowed?.();
|
|
@@ -16992,7 +16578,7 @@ function loopGuardPrompt(rule) {
|
|
|
16992
16578
|
}
|
|
16993
16579
|
|
|
16994
16580
|
// src/convert/cli.ts
|
|
16995
|
-
import { spawn as
|
|
16581
|
+
import { spawn as spawn12 } from "node:child_process";
|
|
16996
16582
|
import { accessSync as accessSync2, constants as constants2, existsSync as existsSync15 } from "node:fs";
|
|
16997
16583
|
import { delimiter as delimiter4, dirname as dirname14, join as join27, resolve as resolve7 } from "node:path";
|
|
16998
16584
|
async function convertWithFilee(request, options = {}) {
|
|
@@ -17070,7 +16656,7 @@ function spawnFilee(executable, args, options) {
|
|
|
17070
16656
|
}
|
|
17071
16657
|
const portable = portableSpawnCommand(executable, args);
|
|
17072
16658
|
const env = options.dataDir === undefined ? process.env : { ...process.env, FILEE_DATA_DIR: resolve7(options.dataDir) };
|
|
17073
|
-
const child =
|
|
16659
|
+
const child = spawn12(portable.command, [...portable.args], {
|
|
17074
16660
|
cwd: options.cwd,
|
|
17075
16661
|
env,
|
|
17076
16662
|
stdio: ["ignore", "pipe", "pipe"]
|
|
@@ -17142,7 +16728,7 @@ var init_cli = __esm(() => {
|
|
|
17142
16728
|
});
|
|
17143
16729
|
|
|
17144
16730
|
// src/convert/installer.ts
|
|
17145
|
-
import { createHash as
|
|
16731
|
+
import { createHash as createHash10 } from "node:crypto";
|
|
17146
16732
|
import { createReadStream as createReadStream2, createWriteStream as createWriteStream2, existsSync as existsSync16, mkdirSync as mkdirSync14, readdirSync as readdirSync4, renameSync as renameSync3, rmSync as rmSync6 } from "node:fs";
|
|
17147
16733
|
import { mkdtemp as mkdtemp2 } from "node:fs/promises";
|
|
17148
16734
|
import { join as join28 } from "node:path";
|
|
@@ -17215,7 +16801,7 @@ async function downloadFile2(url, destination) {
|
|
|
17215
16801
|
await pipeline2(Readable.fromWeb(response.body), createWriteStream2(destination));
|
|
17216
16802
|
}
|
|
17217
16803
|
async function verifySha2562(path, expected, name) {
|
|
17218
|
-
const hash =
|
|
16804
|
+
const hash = createHash10("sha256");
|
|
17219
16805
|
await pipeline2(createReadStream2(path), hash);
|
|
17220
16806
|
const actual = hash.digest("hex");
|
|
17221
16807
|
if (actual !== expected) {
|
|
@@ -17759,7 +17345,7 @@ var init_matrix = __esm(() => {
|
|
|
17759
17345
|
});
|
|
17760
17346
|
|
|
17761
17347
|
// src/dupey/cli.ts
|
|
17762
|
-
import { spawn as
|
|
17348
|
+
import { spawn as spawn13 } from "node:child_process";
|
|
17763
17349
|
import { resolve as resolve8 } from "node:path";
|
|
17764
17350
|
function runDupey(args, options) {
|
|
17765
17351
|
return options.run ? options.run(args) : spawnDupey(options.executable ?? "dupey", args, options);
|
|
@@ -17831,7 +17417,7 @@ function isScanResult(value) {
|
|
|
17831
17417
|
function spawnDupey(executable, args, options) {
|
|
17832
17418
|
return new Promise((resolveOutput, reject) => {
|
|
17833
17419
|
const portable = portableSpawnCommand(executable, args);
|
|
17834
|
-
const child =
|
|
17420
|
+
const child = spawn13(portable.command, [...portable.args], {
|
|
17835
17421
|
cwd: options.cwd,
|
|
17836
17422
|
stdio: ["ignore", "pipe", "pipe"]
|
|
17837
17423
|
});
|
|
@@ -17879,7 +17465,7 @@ var init_cli2 = __esm(() => {
|
|
|
17879
17465
|
});
|
|
17880
17466
|
|
|
17881
17467
|
// src/dupey/filter.ts
|
|
17882
|
-
import { stat as
|
|
17468
|
+
import { stat as stat2 } from "node:fs/promises";
|
|
17883
17469
|
import { resolve as resolve9 } from "node:path";
|
|
17884
17470
|
async function selectExactDuplicateExclusions(root, scan, isUnavailable) {
|
|
17885
17471
|
const byHash = new Map;
|
|
@@ -17934,7 +17520,7 @@ async function rankExactCandidates(paths, dupeyRanking, errors) {
|
|
|
17934
17520
|
}
|
|
17935
17521
|
const ranked = await Promise.all(paths.map(async (path) => {
|
|
17936
17522
|
try {
|
|
17937
|
-
const info = await
|
|
17523
|
+
const info = await stat2(path);
|
|
17938
17524
|
return { path, mtimeMs: info.mtimeMs };
|
|
17939
17525
|
} catch {
|
|
17940
17526
|
errors.push("dupey exact duplicate path disappeared before indexing.");
|
|
@@ -18103,7 +17689,7 @@ var init_retry = __esm(() => {
|
|
|
18103
17689
|
});
|
|
18104
17690
|
|
|
18105
17691
|
// src/everything/bundle.ts
|
|
18106
|
-
import { createHash as
|
|
17692
|
+
import { createHash as createHash11 } from "node:crypto";
|
|
18107
17693
|
import { existsSync as existsSync17, mkdirSync as mkdirSync15, readFileSync as readFileSync18, rmSync as rmSync7, writeFileSync as writeFileSync11 } from "node:fs";
|
|
18108
17694
|
import { dirname as dirname15, join as join29 } from "node:path";
|
|
18109
17695
|
import { fileURLToPath } from "node:url";
|
|
@@ -18221,7 +17807,7 @@ function writeAtomic(path, bytes) {
|
|
|
18221
17807
|
}
|
|
18222
17808
|
}
|
|
18223
17809
|
function sha256(bytes) {
|
|
18224
|
-
return
|
|
17810
|
+
return createHash11("sha256").update(bytes).digest("hex");
|
|
18225
17811
|
}
|
|
18226
17812
|
function errorMessage2(error) {
|
|
18227
17813
|
return error instanceof Error ? error.message : String(error);
|
|
@@ -18251,13 +17837,13 @@ function createPathExclusionMatcher(excludedPaths, platform = process.platform)
|
|
|
18251
17837
|
var init_path_exclusions = () => {};
|
|
18252
17838
|
|
|
18253
17839
|
// src/everything/client.ts
|
|
18254
|
-
import { spawn as
|
|
18255
|
-
import { createHash as
|
|
17840
|
+
import { spawn as spawn14 } from "node:child_process";
|
|
17841
|
+
import { createHash as createHash12 } from "node:crypto";
|
|
18256
17842
|
import { mkdirSync as mkdirSync16, writeFileSync as writeFileSync12 } from "node:fs";
|
|
18257
17843
|
import { join as join30, resolve as resolve10 } from "node:path";
|
|
18258
17844
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
18259
17845
|
function everythingInstanceName(root) {
|
|
18260
|
-
return `autorag-${
|
|
17846
|
+
return `autorag-${createHash12("sha256").update(resolve10(root).toLowerCase()).digest("hex").slice(0, 12)}`;
|
|
18261
17847
|
}
|
|
18262
17848
|
function buildEverythingIni(input) {
|
|
18263
17849
|
const list = (values) => values.map((value) => `"${value.replace(/\\/g, "\\\\")}"`).join(",");
|
|
@@ -18396,7 +17982,7 @@ class EverythingClient {
|
|
|
18396
17982
|
};
|
|
18397
17983
|
}
|
|
18398
17984
|
const es = { ok: true, esPath: binaries.esPath };
|
|
18399
|
-
const timeoutMs = this.options.timeoutMs ??
|
|
17985
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS7;
|
|
18400
17986
|
if ((this.options.excludeFolders?.length ?? 0) > 0) {
|
|
18401
17987
|
return this.searchWithExclusions(es.esPath, request, timeoutMs);
|
|
18402
17988
|
}
|
|
@@ -18481,7 +18067,7 @@ class EverythingClient {
|
|
|
18481
18067
|
const binaries = await this.resolveBinaries("cached");
|
|
18482
18068
|
if (!binaries.ok)
|
|
18483
18069
|
return;
|
|
18484
|
-
await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ??
|
|
18070
|
+
await this.run(binaries.esPath, ["-instance", this.instanceName, "-exit"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS7);
|
|
18485
18071
|
});
|
|
18486
18072
|
}
|
|
18487
18073
|
serialize(task) {
|
|
@@ -18511,7 +18097,7 @@ class EverythingClient {
|
|
|
18511
18097
|
const binaries = await this.resolveBinaries("install");
|
|
18512
18098
|
if (!binaries.ok)
|
|
18513
18099
|
return binaries;
|
|
18514
|
-
const timeoutMs = this.options.timeoutMs ??
|
|
18100
|
+
const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS7;
|
|
18515
18101
|
let ping = await this.ping(binaries.esPath);
|
|
18516
18102
|
if (ping.code === 0 && !reconfigure)
|
|
18517
18103
|
return { ok: true, esPath: binaries.esPath };
|
|
@@ -18568,7 +18154,7 @@ class EverythingClient {
|
|
|
18568
18154
|
};
|
|
18569
18155
|
}
|
|
18570
18156
|
async ping(esPath) {
|
|
18571
|
-
const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ??
|
|
18157
|
+
const result = await this.run(esPath, ["-instance", this.instanceName, "-get-everything-version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS7);
|
|
18572
18158
|
return result.code === 0 && !VERSION_PATTERN.test(result.stdout.trim()) ? { ...result, code: -1 } : result;
|
|
18573
18159
|
}
|
|
18574
18160
|
failure(reason, esPath, args, result) {
|
|
@@ -18584,7 +18170,7 @@ function runProcess(command, args, timeoutMs) {
|
|
|
18584
18170
|
const promise = new Promise((resolve) => {
|
|
18585
18171
|
resolvePromise = resolve;
|
|
18586
18172
|
});
|
|
18587
|
-
const child =
|
|
18173
|
+
const child = spawn14(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
|
|
18588
18174
|
const stdout = [];
|
|
18589
18175
|
const stderr = [];
|
|
18590
18176
|
let bytes = 0;
|
|
@@ -18619,11 +18205,11 @@ killed by ${signal} after ${timeoutMs}ms` : err
|
|
|
18619
18205
|
return promise;
|
|
18620
18206
|
}
|
|
18621
18207
|
function launchDetached(command, args) {
|
|
18622
|
-
const child =
|
|
18208
|
+
const child = spawn14(command, args, { detached: true, stdio: "ignore", windowsHide: true });
|
|
18623
18209
|
child.on("error", () => {});
|
|
18624
18210
|
child.unref();
|
|
18625
18211
|
}
|
|
18626
|
-
var
|
|
18212
|
+
var DEFAULT_TIMEOUT_MS7 = 30000, DEFAULT_STARTUP_TIMEOUT_MS = 20000, DEFAULT_INDEX_TIMEOUT_MS2, DEFAULT_POLL_INTERVAL_MS = 250, DEFAULT_MAX_RESULTS = 100, MAX_FILTER_CANDIDATES = 1000, MAX_OUTPUT_BYTES, VERSION_PATTERN;
|
|
18627
18213
|
var init_client8 = __esm(() => {
|
|
18628
18214
|
init_path_exclusions();
|
|
18629
18215
|
init_bundle();
|
|
@@ -18639,8 +18225,8 @@ var init_everything = __esm(() => {
|
|
|
18639
18225
|
});
|
|
18640
18226
|
|
|
18641
18227
|
// src/fsearch/walk.ts
|
|
18642
|
-
import { readdir, stat as
|
|
18643
|
-
import { basename as
|
|
18228
|
+
import { readdir, stat as stat3 } from "node:fs/promises";
|
|
18229
|
+
import { basename as basename6, join as join31, sep as sep5 } from "node:path";
|
|
18644
18230
|
function compare(a, b) {
|
|
18645
18231
|
return a < b ? -1 : a > b ? 1 : 0;
|
|
18646
18232
|
}
|
|
@@ -18651,11 +18237,11 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
18651
18237
|
const deadline = Date.now() + (options.deadlineMs ?? DEFAULT_DEADLINE_MS);
|
|
18652
18238
|
const matches = request.regex ? (() => {
|
|
18653
18239
|
const pattern = new RegExp(request.query, request.matchCase ? "" : "i");
|
|
18654
|
-
return (path) => pattern.test(request.matchPath ? path :
|
|
18240
|
+
return (path) => pattern.test(request.matchPath ? path : basename6(path));
|
|
18655
18241
|
})() : (() => {
|
|
18656
18242
|
const needle = request.matchCase ? request.query : request.query.toLowerCase();
|
|
18657
18243
|
return (path) => {
|
|
18658
|
-
const haystack = request.matchPath ? path :
|
|
18244
|
+
const haystack = request.matchPath ? path : basename6(path);
|
|
18659
18245
|
return (request.matchCase ? haystack : haystack.toLowerCase()).includes(needle);
|
|
18660
18246
|
};
|
|
18661
18247
|
})();
|
|
@@ -18703,14 +18289,14 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
18703
18289
|
if (!matches(path))
|
|
18704
18290
|
continue;
|
|
18705
18291
|
try {
|
|
18706
|
-
const info = await
|
|
18292
|
+
const info = await stat3(path);
|
|
18707
18293
|
if (request.kind === "folders" && !info.isDirectory())
|
|
18708
18294
|
continue;
|
|
18709
18295
|
if (request.kind === "files" && info.isDirectory())
|
|
18710
18296
|
continue;
|
|
18711
18297
|
collected.push({
|
|
18712
18298
|
path,
|
|
18713
|
-
name:
|
|
18299
|
+
name: basename6(path),
|
|
18714
18300
|
type: info.isDirectory() ? "folder" : "file",
|
|
18715
18301
|
size: info.isDirectory() ? undefined : info.size,
|
|
18716
18302
|
dateModified: info.mtime.toISOString()
|
|
@@ -18730,12 +18316,12 @@ async function walkFileSearch(folders, request, options = {}) {
|
|
|
18730
18316
|
if (request.kind === "files")
|
|
18731
18317
|
continue;
|
|
18732
18318
|
try {
|
|
18733
|
-
const info = await
|
|
18319
|
+
const info = await stat3(folder);
|
|
18734
18320
|
if (!info.isDirectory())
|
|
18735
18321
|
continue;
|
|
18736
18322
|
collected.push({
|
|
18737
18323
|
path: folder,
|
|
18738
|
-
name:
|
|
18324
|
+
name: basename6(folder),
|
|
18739
18325
|
type: "folder",
|
|
18740
18326
|
size: undefined,
|
|
18741
18327
|
dateModified: info.mtime.toISOString()
|
|
@@ -18765,10 +18351,10 @@ var init_walk = __esm(() => {
|
|
|
18765
18351
|
});
|
|
18766
18352
|
|
|
18767
18353
|
// src/fsearch/client.ts
|
|
18768
|
-
import { spawn as
|
|
18769
|
-
import { createHash as
|
|
18354
|
+
import { spawn as spawn15 } from "node:child_process";
|
|
18355
|
+
import { createHash as createHash13 } from "node:crypto";
|
|
18770
18356
|
import { existsSync as existsSync18, mkdirSync as mkdirSync17, readFileSync as readFileSync19, realpathSync as realpathSync2, rmSync as rmSync8, writeFileSync as writeFileSync13 } from "node:fs";
|
|
18771
|
-
import { basename as
|
|
18357
|
+
import { basename as basename7, join as join32, resolve as resolve11, sep as sep6 } from "node:path";
|
|
18772
18358
|
import { setTimeout as sleep2 } from "node:timers/promises";
|
|
18773
18359
|
function fsearchSocketPath(dbPath) {
|
|
18774
18360
|
let canonical;
|
|
@@ -18777,7 +18363,7 @@ function fsearchSocketPath(dbPath) {
|
|
|
18777
18363
|
} catch {
|
|
18778
18364
|
canonical = resolve11(dbPath);
|
|
18779
18365
|
}
|
|
18780
|
-
const hash =
|
|
18366
|
+
const hash = createHash13("sha256").update(canonical).digest("hex").slice(0, 12);
|
|
18781
18367
|
return `/tmp/autorag-fsearch-${process.getuid?.() ?? 0}-${hash}.sock`;
|
|
18782
18368
|
}
|
|
18783
18369
|
function buildFSearchSearchArgs(dbPath, socketPath, request) {
|
|
@@ -18832,7 +18418,7 @@ function parseFSearchJsonLines(stdout) {
|
|
|
18832
18418
|
const path = String(record.path ?? "");
|
|
18833
18419
|
entries.push({
|
|
18834
18420
|
path,
|
|
18835
|
-
name:
|
|
18421
|
+
name: basename7(path),
|
|
18836
18422
|
type: record.type === "folder" ? "folder" : "file",
|
|
18837
18423
|
size: typeof record.size === "number" ? record.size : undefined,
|
|
18838
18424
|
dateModified: typeof record.mtime === "number" ? new Date(record.mtime * 1000).toISOString() : undefined
|
|
@@ -18910,7 +18496,7 @@ class FSearchClient {
|
|
|
18910
18496
|
const hasExclusions = (this.options.excludeFolders?.length ?? 0) > 0;
|
|
18911
18497
|
const nativeRequest = hasExclusions ? { ...request, offset: 0, maxResults: MAX_RESULTS_CAP } : request;
|
|
18912
18498
|
const args = buildFSearchSearchArgs(this.dbPath, this.socketPath(), nativeRequest);
|
|
18913
|
-
const result = await this.run(binary.path, args, this.options.timeoutMs ??
|
|
18499
|
+
const result = await this.run(binary.path, args, this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
18914
18500
|
if (result.code !== 0)
|
|
18915
18501
|
return this.failure("search-failed", binary.path, args, result);
|
|
18916
18502
|
let parsed;
|
|
@@ -18977,7 +18563,7 @@ class FSearchClient {
|
|
|
18977
18563
|
if (indexed.code !== 0)
|
|
18978
18564
|
return this.failure("index-failed", binaryPath, indexArgs, indexed);
|
|
18979
18565
|
const statsArgs = ["stats", "--db", this.dbPath, "--socket", this.socketPath()];
|
|
18980
|
-
const stats = await this.run(binaryPath, statsArgs, this.options.timeoutMs ??
|
|
18566
|
+
const stats = await this.run(binaryPath, statsArgs, this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
18981
18567
|
if (stats.code !== 0)
|
|
18982
18568
|
return this.failure("index-failed", binaryPath, statsArgs, stats);
|
|
18983
18569
|
let counts;
|
|
@@ -19025,7 +18611,7 @@ class FSearchClient {
|
|
|
19025
18611
|
}
|
|
19026
18612
|
}
|
|
19027
18613
|
async serving(binaryPath) {
|
|
19028
|
-
const stats = await this.run(binaryPath, ["stats", "--db", this.dbPath, "--socket", this.socketPath()], this.options.timeoutMs ??
|
|
18614
|
+
const stats = await this.run(binaryPath, ["stats", "--db", this.dbPath, "--socket", this.socketPath()], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
19029
18615
|
if (stats.code !== 0)
|
|
19030
18616
|
return false;
|
|
19031
18617
|
try {
|
|
@@ -19058,7 +18644,7 @@ class FSearchClient {
|
|
|
19058
18644
|
if (this.binary !== undefined)
|
|
19059
18645
|
return this.binary;
|
|
19060
18646
|
const path = this.options.binaryPath ?? "fsearch-cli";
|
|
19061
|
-
const probe = await this.run(path, ["--version"], this.options.timeoutMs ??
|
|
18647
|
+
const probe = await this.run(path, ["--version"], this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS8);
|
|
19062
18648
|
this.binary = probe.code === 0 ? { ok: true, path } : {
|
|
19063
18649
|
ok: false,
|
|
19064
18650
|
reason: "binary-missing",
|
|
@@ -19086,7 +18672,7 @@ function runProcess2(command, args, timeoutMs) {
|
|
|
19086
18672
|
const promise = new Promise((resolve) => {
|
|
19087
18673
|
resolvePromise = resolve;
|
|
19088
18674
|
});
|
|
19089
|
-
const child =
|
|
18675
|
+
const child = spawn15(command, args, { windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
|
|
19090
18676
|
const stdout = [];
|
|
19091
18677
|
const stderr = [];
|
|
19092
18678
|
let bytes = 0;
|
|
@@ -19121,7 +18707,7 @@ killed by ${signal} after ${timeoutMs}ms` : err
|
|
|
19121
18707
|
return promise;
|
|
19122
18708
|
}
|
|
19123
18709
|
function launchDetached2(command, args) {
|
|
19124
|
-
const child =
|
|
18710
|
+
const child = spawn15(command, args, { detached: true, stdio: "ignore", windowsHide: true });
|
|
19125
18711
|
child.on("error", () => {});
|
|
19126
18712
|
child.unref();
|
|
19127
18713
|
return child.pid;
|
|
@@ -19134,7 +18720,7 @@ function defaultIsProcessAlive(pid) {
|
|
|
19134
18720
|
return false;
|
|
19135
18721
|
}
|
|
19136
18722
|
}
|
|
19137
|
-
var
|
|
18723
|
+
var DEFAULT_TIMEOUT_MS8 = 30000, DEFAULT_STARTUP_TIMEOUT_MS2 = 1e4, STOP_TIMEOUT_MS2 = 2000, DEFAULT_INDEX_TIMEOUT_MS3, DEFAULT_POLL_INTERVAL_MS2 = 250, DEFAULT_MAX_RESULTS2 = 100, MAX_RESULTS_CAP = 1000, MAX_OUTPUT_BYTES2, SORT_KEYS;
|
|
19138
18724
|
var init_client9 = __esm(() => {
|
|
19139
18725
|
init_path_exclusions();
|
|
19140
18726
|
init_walk();
|
|
@@ -19364,7 +18950,7 @@ var init_answer_pack = __esm(() => {
|
|
|
19364
18950
|
});
|
|
19365
18951
|
|
|
19366
18952
|
// src/jikji/installer.ts
|
|
19367
|
-
import { spawn as
|
|
18953
|
+
import { spawn as spawn16 } from "node:child_process";
|
|
19368
18954
|
import { existsSync as existsSync19 } from "node:fs";
|
|
19369
18955
|
import { delimiter as delimiter5, join as join34 } from "node:path";
|
|
19370
18956
|
function jikjiExecutableName(platform = process.platform) {
|
|
@@ -19391,7 +18977,7 @@ function lookupExecutableInPath(executable, env) {
|
|
|
19391
18977
|
}
|
|
19392
18978
|
function defaultRunner(cargo, env) {
|
|
19393
18979
|
return (args, timeoutMs) => new Promise((resolvePromise) => {
|
|
19394
|
-
const child =
|
|
18980
|
+
const child = spawn16(cargo, args, { env, stdio: ["ignore", "ignore", "pipe"] });
|
|
19395
18981
|
let stderr = "";
|
|
19396
18982
|
const timeout = setTimeout(() => {
|
|
19397
18983
|
if (!child.killed)
|
|
@@ -19505,7 +19091,7 @@ class SerialQueue {
|
|
|
19505
19091
|
}
|
|
19506
19092
|
|
|
19507
19093
|
// src/jikji/client.ts
|
|
19508
|
-
import { spawn as
|
|
19094
|
+
import { spawn as spawn17 } from "node:child_process";
|
|
19509
19095
|
import { existsSync as existsSync20, realpathSync as realpathSync4 } from "node:fs";
|
|
19510
19096
|
|
|
19511
19097
|
class JikjiClient {
|
|
@@ -19619,7 +19205,7 @@ class JikjiClient {
|
|
|
19619
19205
|
if (perFile <= 0)
|
|
19620
19206
|
return;
|
|
19621
19207
|
const files = await countFiles(root, options.includeHidden === true, signal);
|
|
19622
|
-
return scaledPrepareTimeoutMs(options.timeoutMs ??
|
|
19208
|
+
return scaledPrepareTimeoutMs(options.timeoutMs ?? DEFAULT_TIMEOUT_MS9, perFile, files);
|
|
19623
19209
|
}
|
|
19624
19210
|
async spawn(request) {
|
|
19625
19211
|
return spawnJikji({
|
|
@@ -19632,7 +19218,7 @@ function spawnJikji(request) {
|
|
|
19632
19218
|
return new Promise((resolve) => {
|
|
19633
19219
|
const options = request.options;
|
|
19634
19220
|
const portable = portableSpawnCommand(request.command, request.args);
|
|
19635
|
-
const child =
|
|
19221
|
+
const child = spawn17(portable.command, [...portable.args], {
|
|
19636
19222
|
env: controlledEnv4(options.env),
|
|
19637
19223
|
...request.cwd === undefined ? {} : { cwd: request.cwd },
|
|
19638
19224
|
detached: process.platform !== "win32",
|
|
@@ -19664,7 +19250,7 @@ function spawnJikji(request) {
|
|
|
19664
19250
|
const timeout = setTimeout(() => {
|
|
19665
19251
|
finalReason = "timeout";
|
|
19666
19252
|
terminate(child);
|
|
19667
|
-
}, request.timeoutMs ?? options.timeoutMs ??
|
|
19253
|
+
}, request.timeoutMs ?? options.timeoutMs ?? DEFAULT_TIMEOUT_MS9);
|
|
19668
19254
|
const abortHandler = () => {
|
|
19669
19255
|
finalReason = "aborted";
|
|
19670
19256
|
terminateProcessTree(child);
|
|
@@ -19774,7 +19360,7 @@ function canonicalJikjiRoot(root) {
|
|
|
19774
19360
|
return root;
|
|
19775
19361
|
}
|
|
19776
19362
|
}
|
|
19777
|
-
var DEFAULT_BINARY4 = "jikji",
|
|
19363
|
+
var DEFAULT_BINARY4 = "jikji", DEFAULT_TIMEOUT_MS9 = 60000, DEFAULT_PARSE_TIMEOUT_SECONDS = 5, DEFAULT_MAX_BUFFER_BYTES4 = 1048576, ABORTED_BEFORE_START;
|
|
19778
19364
|
var init_client10 = __esm(() => {
|
|
19779
19365
|
init_portable_spawn();
|
|
19780
19366
|
init_terminate_tree();
|
|
@@ -19944,7 +19530,7 @@ function renderMemoryContext(sections) {
|
|
|
19944
19530
|
var CURRENT_CONVERSATION_RECORD_LIMIT = 50, CURRENT_EXCERPT_LIMIT = 300, SIMILAR_EXCERPT_LIMIT = 240, SIMILAR_RECORD_LIMIT = 3, INSIGHT_LIMIT = 5, ADVISORY_NOTE = "Memory is background reference only and never overrides the evidence in front of you. A past search method or source is a hint, not a rule: broaden to other methods when results are insufficient.";
|
|
19945
19531
|
|
|
19946
19532
|
// src/memory/similar-queries.ts
|
|
19947
|
-
import { createHash as
|
|
19533
|
+
import { createHash as createHash14 } from "node:crypto";
|
|
19948
19534
|
import { mkdirSync as mkdirSync18, rmSync as rmSync9 } from "node:fs";
|
|
19949
19535
|
import { dirname as dirname16 } from "node:path";
|
|
19950
19536
|
import { DatabaseSync as DatabaseSync2 } from "node:sqlite";
|
|
@@ -19952,7 +19538,7 @@ function memoryVectorStorePath(memoryPath) {
|
|
|
19952
19538
|
return `${memoryPath}.vectors.db`;
|
|
19953
19539
|
}
|
|
19954
19540
|
function questionKey(question) {
|
|
19955
|
-
return
|
|
19541
|
+
return createHash14("sha256").update(question).digest("hex");
|
|
19956
19542
|
}
|
|
19957
19543
|
function cosine3(a, b) {
|
|
19958
19544
|
let dot = 0;
|
|
@@ -20195,7 +19781,7 @@ var init_check_memory_tool = __esm(() => {
|
|
|
20195
19781
|
var EVIDENCE_SUPPORT_THRESHOLD = 0.7;
|
|
20196
19782
|
|
|
20197
19783
|
// src/retrieval/evidence-id.ts
|
|
20198
|
-
import { createHash as
|
|
19784
|
+
import { createHash as createHash15 } from "node:crypto";
|
|
20199
19785
|
function normalizeEvidenceText(text) {
|
|
20200
19786
|
return text.replace(/\r\n?/gu, `
|
|
20201
19787
|
`).trim().normalize("NFC");
|
|
@@ -20225,7 +19811,7 @@ function stableEvidenceId(input) {
|
|
|
20225
19811
|
input.lineNumber ?? "",
|
|
20226
19812
|
normalizeEvidenceText(evidenceText)
|
|
20227
19813
|
].join("\x00");
|
|
20228
|
-
const digest =
|
|
19814
|
+
const digest = createHash15("sha256").update(payload).digest("hex").slice(0, 24);
|
|
20229
19815
|
return `${method}:${digest}`;
|
|
20230
19816
|
}
|
|
20231
19817
|
function normalizeEvidenceRef(input) {
|
|
@@ -20237,7 +19823,7 @@ function normalizeEvidenceRef(input) {
|
|
|
20237
19823
|
var init_evidence_id = () => {};
|
|
20238
19824
|
|
|
20239
19825
|
// src/memory/memory.ts
|
|
20240
|
-
import { createHash as
|
|
19826
|
+
import { createHash as createHash16, randomUUID as randomUUID5 } from "node:crypto";
|
|
20241
19827
|
import { existsSync as existsSync21, mkdirSync as mkdirSync19, readFileSync as readFileSync20, unlinkSync as unlinkSync3, writeFileSync as writeFileSync14 } from "node:fs";
|
|
20242
19828
|
import { dirname as dirname17 } from "node:path";
|
|
20243
19829
|
import { z as z2 } from "zod";
|
|
@@ -20331,7 +19917,7 @@ function normalizeEvidenceChunkRecord(record) {
|
|
|
20331
19917
|
return { ...base, ...normalizeEvidenceContext(record) };
|
|
20332
19918
|
}
|
|
20333
19919
|
function hashText(value) {
|
|
20334
|
-
return
|
|
19920
|
+
return createHash16("sha256").update(value).digest("hex");
|
|
20335
19921
|
}
|
|
20336
19922
|
function resultId(sessionId, number) {
|
|
20337
19923
|
return `${sessionId}:${number}`;
|
|
@@ -20865,7 +20451,7 @@ import {
|
|
|
20865
20451
|
rmSync as rmSync10,
|
|
20866
20452
|
writeFileSync as writeFileSync15
|
|
20867
20453
|
} from "node:fs";
|
|
20868
|
-
import { basename as
|
|
20454
|
+
import { basename as basename8, dirname as dirname18, isAbsolute as isAbsolute8, join as join37, relative as relative6, sep as sep8 } from "node:path";
|
|
20869
20455
|
function syncMinSyncWorkspace(root, options = {}) {
|
|
20870
20456
|
const workspacePath = options.workspacePath ?? minSyncWorkspaceRoot(root);
|
|
20871
20457
|
const filesRoot = join37(workspacePath, "files");
|
|
@@ -20906,7 +20492,7 @@ function syncMinSyncWorkspace(root, options = {}) {
|
|
|
20906
20492
|
return { workspacePath, entries, excluded };
|
|
20907
20493
|
}
|
|
20908
20494
|
function stagingStatePath(workspacePath) {
|
|
20909
|
-
return join37(dirname18(workspacePath), `.${
|
|
20495
|
+
return join37(dirname18(workspacePath), `.${basename8(workspacePath)}-autorag-staging.json`);
|
|
20910
20496
|
}
|
|
20911
20497
|
function loadStagingState2(workspacePath) {
|
|
20912
20498
|
const path = stagingStatePath(workspacePath);
|
|
@@ -20957,7 +20543,7 @@ function ensureManagedDirectory(filesRoot, directory) {
|
|
|
20957
20543
|
}
|
|
20958
20544
|
}
|
|
20959
20545
|
function copyFileAtomically(source, destination) {
|
|
20960
|
-
const temporaryPath = join37(dirname18(destination), `.${
|
|
20546
|
+
const temporaryPath = join37(dirname18(destination), `.${basename8(destination)}.${randomUUID6()}.tmp`);
|
|
20961
20547
|
try {
|
|
20962
20548
|
copyFileSync4(source, temporaryPath);
|
|
20963
20549
|
renameWithRetry(temporaryPath, destination);
|
|
@@ -21011,7 +20597,7 @@ var init_workspace = __esm(() => {
|
|
|
21011
20597
|
|
|
21012
20598
|
// src/minsync/method.ts
|
|
21013
20599
|
import { existsSync as existsSync23, readFileSync as readFileSync22, realpathSync as realpathSync5, statSync as statSync6 } from "node:fs";
|
|
21014
|
-
import { basename as
|
|
20600
|
+
import { basename as basename9, join as join38, normalize as normalize2 } from "node:path";
|
|
21015
20601
|
|
|
21016
20602
|
class MinSyncVectorMethod {
|
|
21017
20603
|
root;
|
|
@@ -21112,7 +20698,7 @@ class MinSyncVectorMethod {
|
|
|
21112
20698
|
const entry = byPath.get(hit.path);
|
|
21113
20699
|
if (!entry || !matchesVirtualPathScope(entry.virtualPath, options.scope))
|
|
21114
20700
|
continue;
|
|
21115
|
-
const chunkDiscriminator = hit.docId ??
|
|
20701
|
+
const chunkDiscriminator = hit.docId ?? basename9(hit.path);
|
|
21116
20702
|
const chunkId = `minsync:${entry.virtualPath}:${chunkDiscriminator}`;
|
|
21117
20703
|
const location = locateChunkInMirror(mirrorTexts, entry, hit.text);
|
|
21118
20704
|
results.push({
|
|
@@ -21353,9 +20939,9 @@ var init_isolated = __esm(() => {
|
|
|
21353
20939
|
});
|
|
21354
20940
|
|
|
21355
20941
|
// src/mirror/sync.ts
|
|
21356
|
-
import { createHash as
|
|
21357
|
-
import { opendir as opendir2, readFile as readFile2, stat as
|
|
21358
|
-
import { basename as
|
|
20942
|
+
import { createHash as createHash17 } from "node:crypto";
|
|
20943
|
+
import { opendir as opendir2, readFile as readFile2, stat as stat4 } from "node:fs/promises";
|
|
20944
|
+
import { basename as basename10, extname as extname6, join as join40, resolve as resolve14, sep as sep9 } from "node:path";
|
|
21359
20945
|
import { setTimeout as sleep3 } from "node:timers/promises";
|
|
21360
20946
|
function errorCode(error) {
|
|
21361
20947
|
if (!(error instanceof Error))
|
|
@@ -21405,7 +20991,7 @@ function parseFingerprint(options) {
|
|
|
21405
20991
|
}
|
|
21406
20992
|
return value;
|
|
21407
20993
|
};
|
|
21408
|
-
return
|
|
20994
|
+
return createHash17("sha256").update(JSON.stringify(stable({ format: 1, parser: outputParserOptions(options.parserOptions) }))).digest("hex");
|
|
21409
20995
|
}
|
|
21410
20996
|
function outputParserOptions(options = {}) {
|
|
21411
20997
|
const { fileTimeoutMs: _timeout, maxHeapMb: _heap, ocr, ...parser } = options;
|
|
@@ -21413,7 +20999,7 @@ function outputParserOptions(options = {}) {
|
|
|
21413
20999
|
}
|
|
21414
21000
|
function retryFingerprint(options, fingerprint) {
|
|
21415
21001
|
const parser = options.parserOptions;
|
|
21416
|
-
return
|
|
21002
|
+
return createHash17("sha256").update(JSON.stringify({
|
|
21417
21003
|
fingerprint,
|
|
21418
21004
|
fileTimeoutMs: parser?.fileTimeoutMs,
|
|
21419
21005
|
maxHeapMb: parser?.maxHeapMb,
|
|
@@ -21632,7 +21218,7 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
21632
21218
|
}
|
|
21633
21219
|
let info;
|
|
21634
21220
|
try {
|
|
21635
|
-
info = await
|
|
21221
|
+
info = await stat4(entry.sourcePath, { bigint: true });
|
|
21636
21222
|
} catch (error) {
|
|
21637
21223
|
await skipFile(entry, sourceReadFailure(error));
|
|
21638
21224
|
continue;
|
|
@@ -21664,7 +21250,7 @@ async function syncWithRegistry(options, registry, isolated) {
|
|
|
21664
21250
|
continue;
|
|
21665
21251
|
}
|
|
21666
21252
|
await session.commit(mirrorStageKey(entry.virtualPath), versions.get(entry.virtualPath), {
|
|
21667
|
-
path:
|
|
21253
|
+
path: basename10(outputPath),
|
|
21668
21254
|
size: Buffer.byteLength(markdown)
|
|
21669
21255
|
});
|
|
21670
21256
|
for (const diagnostic of parsed.diagnostics ?? [])
|
|
@@ -21740,7 +21326,7 @@ async function detectMirrorStaleness(options) {
|
|
|
21740
21326
|
}
|
|
21741
21327
|
}
|
|
21742
21328
|
function standaloneImageTooSmallMessage(virtualPath, bytes) {
|
|
21743
|
-
if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(
|
|
21329
|
+
if (!STANDALONE_OCR_IMAGE_EXTENSIONS.has(normalizeExtension2(extname6(virtualPath))))
|
|
21744
21330
|
return;
|
|
21745
21331
|
return ocrImageTooSmallMessage(bytes);
|
|
21746
21332
|
}
|
|
@@ -21796,7 +21382,7 @@ function errnoCode2(error) {
|
|
|
21796
21382
|
async function probeSearchRoot(rootPath) {
|
|
21797
21383
|
let isDirectory;
|
|
21798
21384
|
try {
|
|
21799
|
-
isDirectory = (await
|
|
21385
|
+
isDirectory = (await stat4(rootPath)).isDirectory();
|
|
21800
21386
|
} catch (error) {
|
|
21801
21387
|
const code = errnoCode2(error);
|
|
21802
21388
|
return code === "ENOENT" || code === "ENOTDIR" ? "gone" : "unreadable";
|
|
@@ -21908,12 +21494,12 @@ async function collectFiles(state, directory, depth) {
|
|
|
21908
21494
|
}
|
|
21909
21495
|
if (!entry.isFile())
|
|
21910
21496
|
continue;
|
|
21911
|
-
const extension = normalizeExtension2(
|
|
21497
|
+
const extension = normalizeExtension2(extname6(entry.name));
|
|
21912
21498
|
if (!state.supportedExtensions.has(extension))
|
|
21913
21499
|
continue;
|
|
21914
21500
|
let fileStat;
|
|
21915
21501
|
try {
|
|
21916
|
-
fileStat = await
|
|
21502
|
+
fileStat = await stat4(sourcePath, { bigint: true });
|
|
21917
21503
|
} catch (error) {
|
|
21918
21504
|
const code = errnoCode2(error);
|
|
21919
21505
|
if (code === undefined)
|
|
@@ -26089,7 +25675,7 @@ var init_citations = __esm(() => {
|
|
|
26089
25675
|
});
|
|
26090
25676
|
|
|
26091
25677
|
// src/agent/evidence-ledger.ts
|
|
26092
|
-
import { createHash as
|
|
25678
|
+
import { createHash as createHash18 } from "node:crypto";
|
|
26093
25679
|
import { closeSync as closeSync2, openSync as openSync2, readSync, statSync as statSync7 } from "node:fs";
|
|
26094
25680
|
import { isAbsolute as isAbsolute9 } from "node:path";
|
|
26095
25681
|
import { Type as Type3 } from "typebox";
|
|
@@ -26124,10 +25710,10 @@ function readFileExcerpt(path) {
|
|
|
26124
25710
|
let descriptor;
|
|
26125
25711
|
try {
|
|
26126
25712
|
descriptor = openSync2(path, "r");
|
|
26127
|
-
const buffer = Buffer.alloc(
|
|
25713
|
+
const buffer = Buffer.alloc(MAX_CONTENT_CHARS3 * 4);
|
|
26128
25714
|
const bytes = readSync(descriptor, buffer, 0, buffer.length, 0);
|
|
26129
25715
|
const head = buffer.subarray(0, bytes);
|
|
26130
|
-
return head.includes(0) ? path : head.toString("utf8").slice(0,
|
|
25716
|
+
return head.includes(0) ? path : head.toString("utf8").slice(0, MAX_CONTENT_CHARS3);
|
|
26131
25717
|
} catch {
|
|
26132
25718
|
return path;
|
|
26133
25719
|
} finally {
|
|
@@ -26160,8 +25746,8 @@ class EvidenceLedger {
|
|
|
26160
25746
|
unsaved = new Set;
|
|
26161
25747
|
lastOrdinal = 0;
|
|
26162
25748
|
register(input) {
|
|
26163
|
-
const content = input.content.slice(0,
|
|
26164
|
-
const digest =
|
|
25749
|
+
const content = input.content.slice(0, MAX_CONTENT_CHARS3);
|
|
25750
|
+
const digest = createHash18("sha256").update(input.content.replace(/\s+/gu, " ").trim()).digest("hex");
|
|
26165
25751
|
const key = `${input.source}\x00${digest}`;
|
|
26166
25752
|
const existing = this.byKey.get(key);
|
|
26167
25753
|
if (existing !== undefined) {
|
|
@@ -26247,7 +25833,7 @@ class EvidenceLedger {
|
|
|
26247
25833
|
this.lastOrdinal = Math.max(this.lastOrdinal, ordinalFloor, highestOrdinal(this.entries));
|
|
26248
25834
|
}
|
|
26249
25835
|
}
|
|
26250
|
-
var
|
|
25836
|
+
var MAX_CONTENT_CHARS3 = 2000, EVIDENCE_ENTRY_TYPE = "autorag.evidence", ledgerEntrySchema, persistedEvidenceSchema;
|
|
26251
25837
|
var init_evidence_ledger = __esm(() => {
|
|
26252
25838
|
ledgerEntrySchema = Type3.Object({
|
|
26253
25839
|
id: Type3.String({ pattern: "^e[1-9][0-9]*$" }),
|
|
@@ -26479,7 +26065,7 @@ var init_ask_user = () => {};
|
|
|
26479
26065
|
|
|
26480
26066
|
// src/automation/config.ts
|
|
26481
26067
|
import { existsSync as existsSync26, readFileSync as readFileSync23, realpathSync as realpathSync6 } from "node:fs";
|
|
26482
|
-
import { basename as
|
|
26068
|
+
import { basename as basename11, dirname as dirname22, isAbsolute as isAbsolute11, join as join42, matchesGlob, relative as relative7, resolve as resolve15 } from "node:path";
|
|
26483
26069
|
function automationsConfigPath(env = process.env) {
|
|
26484
26070
|
return join42(resolveAutoRAGHome(env), "automations.json");
|
|
26485
26071
|
}
|
|
@@ -26612,7 +26198,7 @@ function parseAutomations(raw, env = process.env) {
|
|
|
26612
26198
|
...agent !== undefined ? { agent } : {},
|
|
26613
26199
|
appendFiles: obj.appendFiles === true,
|
|
26614
26200
|
debounceMs: parsePositiveInt(obj.debounceMs, `${obj.id}.debounceMs`, DEFAULT_DEBOUNCE_MS),
|
|
26615
|
-
timeoutMs: parsePositiveInt(obj.timeoutMs, `${obj.id}.timeoutMs`,
|
|
26201
|
+
timeoutMs: parsePositiveInt(obj.timeoutMs, `${obj.id}.timeoutMs`, DEFAULT_TIMEOUT_MS10)
|
|
26616
26202
|
};
|
|
26617
26203
|
});
|
|
26618
26204
|
}
|
|
@@ -26669,7 +26255,7 @@ function canonicalizePath(target) {
|
|
|
26669
26255
|
const parent = dirname22(resolved);
|
|
26670
26256
|
if (parent === resolved)
|
|
26671
26257
|
return resolved;
|
|
26672
|
-
return join42(canonicalizePath(parent),
|
|
26258
|
+
return join42(canonicalizePath(parent), basename11(resolved));
|
|
26673
26259
|
}
|
|
26674
26260
|
}
|
|
26675
26261
|
function matchesAutomationPath(automation, watchedRoot, filePath) {
|
|
@@ -26692,7 +26278,7 @@ function matchesAutomation(automation, watchedRoot, event) {
|
|
|
26692
26278
|
return false;
|
|
26693
26279
|
return matchesAutomationPath(automation, watchedRoot, event.path);
|
|
26694
26280
|
}
|
|
26695
|
-
var AUTOMATION_EVENT_TYPES, DEFAULT_DEBOUNCE_MS = 1000,
|
|
26281
|
+
var AUTOMATION_EVENT_TYPES, DEFAULT_DEBOUNCE_MS = 1000, DEFAULT_TIMEOUT_MS10 = 600000, DEFAULT_EXCLUDES, ID_PATTERN, KNOWN_KEYS, AutomationConfigError, AGENT_KEYS, RETRIEVAL_KEYS;
|
|
26696
26282
|
var init_config5 = __esm(() => {
|
|
26697
26283
|
init_home();
|
|
26698
26284
|
init_atomic_write();
|
|
@@ -26732,7 +26318,7 @@ var init_config5 = __esm(() => {
|
|
|
26732
26318
|
});
|
|
26733
26319
|
|
|
26734
26320
|
// src/automation/run.ts
|
|
26735
|
-
import { spawn as
|
|
26321
|
+
import { spawn as spawn18 } from "node:child_process";
|
|
26736
26322
|
import { appendFileSync as appendFileSync2, existsSync as existsSync27, mkdirSync as mkdirSync22, readFileSync as readFileSync24, renameSync as renameSync4, statSync as statSync8 } from "node:fs";
|
|
26737
26323
|
import { dirname as dirname23, join as join43 } from "node:path";
|
|
26738
26324
|
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
@@ -26844,7 +26430,7 @@ async function runAutomation(automation, events, options = {}) {
|
|
|
26844
26430
|
appendAutomationRunLog(home, record);
|
|
26845
26431
|
resolve(record);
|
|
26846
26432
|
};
|
|
26847
|
-
const child =
|
|
26433
|
+
const child = spawn18(command, args, {
|
|
26848
26434
|
cwd,
|
|
26849
26435
|
env: childEnv,
|
|
26850
26436
|
stdio: ["pipe", "pipe", "pipe"],
|
|
@@ -27250,7 +26836,7 @@ var init_daemon2 = __esm(() => {
|
|
|
27250
26836
|
|
|
27251
26837
|
// src/automation/service.ts
|
|
27252
26838
|
import { execFile as execFile2 } from "node:child_process";
|
|
27253
|
-
import { createHash as
|
|
26839
|
+
import { createHash as createHash19 } from "node:crypto";
|
|
27254
26840
|
import { existsSync as existsSync30, mkdirSync as mkdirSync24, rmSync as rmSync11, writeFileSync as writeFileSync16 } from "node:fs";
|
|
27255
26841
|
import { dirname as dirname24, join as join45, resolve as resolve17 } from "node:path";
|
|
27256
26842
|
import { setTimeout as sleep5 } from "node:timers/promises";
|
|
@@ -27281,7 +26867,7 @@ function automationServiceDescriptor(options = {}) {
|
|
|
27281
26867
|
const userHome = resolveUserHome(env);
|
|
27282
26868
|
const defaultHome = resolve17(join45(userHome, ".autorag"));
|
|
27283
26869
|
const home = resolve17(options.home ?? resolveAutoRAGHome(env));
|
|
27284
|
-
const suffix = home === defaultHome ? "" : `-${
|
|
26870
|
+
const suffix = home === defaultHome ? "" : `-${createHash19("sha256").update(home).digest("hex").slice(0, 8)}`;
|
|
27285
26871
|
const automationsDir = join45(home, "automations");
|
|
27286
26872
|
const envFilePath = join45(automationsDir, "service.env");
|
|
27287
26873
|
const logPath = join45(automationsDir, "daemon.log");
|
|
@@ -27717,7 +27303,7 @@ var init_automation_tool = __esm(() => {
|
|
|
27717
27303
|
});
|
|
27718
27304
|
|
|
27719
27305
|
// src/agent/citation-links.ts
|
|
27720
|
-
import { basename as
|
|
27306
|
+
import { basename as basename12, isAbsolute as isAbsolute12 } from "node:path";
|
|
27721
27307
|
import { pathToFileURL as pathToFileURL3 } from "node:url";
|
|
27722
27308
|
function openableTarget(source) {
|
|
27723
27309
|
if (/^https?:\/\//iu.test(source)) {
|
|
@@ -27735,7 +27321,7 @@ function linkCitations(markdown, sourceOf, isFile) {
|
|
|
27735
27321
|
for (const marker of answerMarkers(markdown, isFile)) {
|
|
27736
27322
|
if (marker.kind === "number")
|
|
27737
27323
|
continue;
|
|
27738
|
-
const labels = marker.kind === "file" ? [{ label:
|
|
27324
|
+
const labels = marker.kind === "file" ? [{ label: basename12(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
|
|
27739
27325
|
let linked = false;
|
|
27740
27326
|
const parts = labels.map(({ label, source }) => {
|
|
27741
27327
|
const target = source === undefined ? undefined : openableTarget(source);
|
|
@@ -29083,7 +28669,7 @@ class DazziJevBackend {
|
|
|
29083
28669
|
this.apiKey = apiKey;
|
|
29084
28670
|
this.baseUrl = options.baseUrl ?? resolveBaseUrl(env);
|
|
29085
28671
|
this.defaultModel = options.defaultModel ?? DAZZI_DEFAULT_JEV_MODEL;
|
|
29086
|
-
this.timeoutMs = options.timeoutMs ??
|
|
28672
|
+
this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS11;
|
|
29087
28673
|
this.maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
|
|
29088
28674
|
this.fetchImpl = options.fetchImpl;
|
|
29089
28675
|
}
|
|
@@ -29270,13 +28856,49 @@ async function extractErrorMessage(response) {
|
|
|
29270
28856
|
function sleep6(ms) {
|
|
29271
28857
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
29272
28858
|
}
|
|
29273
|
-
var JEV_PATH, DAZZI_DEFAULT_JEV_MODEL = "jev-1.13",
|
|
28859
|
+
var JEV_PATH, DAZZI_DEFAULT_JEV_MODEL = "jev-1.13", DEFAULT_TIMEOUT_MS11 = 1e4, DEFAULT_MAX_RETRIES = 2, RETRY_BASE_MS = 500, RETRY_MAX_MS = 5000;
|
|
29274
28860
|
var init_jev_backend = __esm(() => {
|
|
29275
28861
|
init_config();
|
|
29276
28862
|
init_credential();
|
|
29277
28863
|
JEV_PATH = `${API_PATH}/jev`;
|
|
29278
28864
|
});
|
|
29279
28865
|
|
|
28866
|
+
// src/agent/token-count.ts
|
|
28867
|
+
import { createHash as createHash20 } from "node:crypto";
|
|
28868
|
+
import { countTokens } from "gpt-tokenizer/encoding/o200k_base";
|
|
28869
|
+
function estimateTextTokens(text) {
|
|
28870
|
+
return Math.max(Math.ceil(text.length / 4), countText(text));
|
|
28871
|
+
}
|
|
28872
|
+
function tokensBeyondCharGuess(parts) {
|
|
28873
|
+
let excess = 0;
|
|
28874
|
+
for (const part of parts)
|
|
28875
|
+
excess += Math.max(0, countText(part) - Math.ceil(part.length / 4));
|
|
28876
|
+
return excess;
|
|
28877
|
+
}
|
|
28878
|
+
function countText(text) {
|
|
28879
|
+
if (text.length === 0)
|
|
28880
|
+
return 0;
|
|
28881
|
+
if (text.length < CACHE_MIN_CHARS)
|
|
28882
|
+
return countTokens(text, PLAIN_TEXT);
|
|
28883
|
+
const key = `${text.length}:${createHash20("sha1").update(text).digest("base64")}`;
|
|
28884
|
+
const hit = cache.get(key);
|
|
28885
|
+
if (hit !== undefined) {
|
|
28886
|
+
cache.delete(key);
|
|
28887
|
+
cache.set(key, hit);
|
|
28888
|
+
return hit;
|
|
28889
|
+
}
|
|
28890
|
+
const count = countTokens(text, PLAIN_TEXT);
|
|
28891
|
+
cache.set(key, count);
|
|
28892
|
+
if (cache.size > CACHE_ENTRIES)
|
|
28893
|
+
cache.delete(cache.keys().next().value);
|
|
28894
|
+
return count;
|
|
28895
|
+
}
|
|
28896
|
+
var PLAIN_TEXT, CACHE_MIN_CHARS = 2000, CACHE_ENTRIES = 64, cache;
|
|
28897
|
+
var init_token_count = __esm(() => {
|
|
28898
|
+
PLAIN_TEXT = { disallowedSpecial: new Set };
|
|
28899
|
+
cache = new Map;
|
|
28900
|
+
});
|
|
28901
|
+
|
|
29280
28902
|
// src/agent/context-token-guard.ts
|
|
29281
28903
|
import { calculateContextTokens, estimateTokens } from "@earendil-works/pi-coding-agent";
|
|
29282
28904
|
function createContextTokenGuardedTransform(model, base, fixedOverheadTokens = () => 0) {
|
|
@@ -29327,50 +28949,40 @@ function guardWithinBudget(model, messages, useReportedUsage = true, fixedOverhe
|
|
|
29327
28949
|
}
|
|
29328
28950
|
let guarded = messages;
|
|
29329
28951
|
if (candidates.length > 0) {
|
|
29330
|
-
const
|
|
29331
|
-
const maximumOmitted = candidates.reduce((total, section) => total + section.blocks.length, 0);
|
|
28952
|
+
const entries = candidates.flatMap((section, sectionIndex) => section.blocks.map((block, blockIndex) => ({ block, sectionIndex, blockIndex })));
|
|
29332
28953
|
const omissionMarker = (count) => `
|
|
29333
28954
|
|
|
29334
28955
|
${GUARD_MARKER} Retrieval candidates omitted: ${count}.
|
|
29335
28956
|
`;
|
|
29336
|
-
|
|
29337
|
-
|
|
29338
|
-
|
|
29339
|
-
|
|
29340
|
-
for (const
|
|
29341
|
-
|
|
29342
|
-
|
|
29343
|
-
if (tokens <= remaining) {
|
|
29344
|
-
kept[sectionIndex]?.push(block);
|
|
29345
|
-
remaining -= tokens;
|
|
29346
|
-
continue;
|
|
29347
|
-
}
|
|
29348
|
-
omitted += section.blocks.length - (kept[sectionIndex]?.length ?? 0);
|
|
29349
|
-
for (const later of candidates.slice(sectionIndex + 1))
|
|
29350
|
-
omitted += later.blocks.length;
|
|
29351
|
-
firstOmitted = sectionIndex;
|
|
29352
|
-
break;
|
|
29353
|
-
}
|
|
29354
|
-
if (firstOmitted !== -1)
|
|
28957
|
+
const markerTokens = candidates.length * Math.ceil(omissionMarker(entries.length).length / 4);
|
|
28958
|
+
let remaining = Math.max(0, budget - estimateContextMessageTokens(base, false) - markerTokens);
|
|
28959
|
+
entries.sort((a, b) => a.block.rank - b.block.rank || a.sectionIndex - b.sectionIndex || a.blockIndex - b.blockIndex);
|
|
28960
|
+
const kept = new Set;
|
|
28961
|
+
for (const { block } of entries) {
|
|
28962
|
+
const tokens = Math.max(1, estimateTextTokens(block.text));
|
|
28963
|
+
if (tokens > remaining)
|
|
29355
28964
|
break;
|
|
28965
|
+
remaining -= tokens;
|
|
28966
|
+
kept.add(block);
|
|
29356
28967
|
}
|
|
29357
|
-
if (
|
|
29358
|
-
const
|
|
29359
|
-
for (const [sectionIndex, messageIndex] of candidateIndexes.entries())
|
|
29360
|
-
candidateByMessage.set(messageIndex, sectionIndex);
|
|
28968
|
+
if (kept.size < entries.length) {
|
|
28969
|
+
const sectionByMessage = new Map(candidateIndexes.map((messageIndex, i) => [messageIndex, candidates[i]]));
|
|
29361
28970
|
guarded = base.map((message, index) => {
|
|
29362
|
-
const
|
|
29363
|
-
if (
|
|
28971
|
+
const section = sectionByMessage.get(index);
|
|
28972
|
+
if (section === undefined)
|
|
29364
28973
|
return message;
|
|
29365
|
-
const
|
|
29366
|
-
const
|
|
29367
|
-
|
|
28974
|
+
const keptBlocks = section.blocks.filter((block) => kept.has(block));
|
|
28975
|
+
const omitted = section.blocks.length - keptBlocks.length;
|
|
28976
|
+
if (omitted === 0)
|
|
28977
|
+
return messages[index] ?? message;
|
|
28978
|
+
const marker = omissionMarker(omitted);
|
|
28979
|
+
return replaceTextContent(message, `${section.prefix}${keptBlocks.map((block) => block.text).join("")}${marker}${section.suffix}`);
|
|
29368
28980
|
});
|
|
29369
28981
|
}
|
|
29370
28982
|
}
|
|
29371
|
-
if (estimateContextMessageTokens(guarded, false) <= budget
|
|
28983
|
+
if (guarded !== messages && estimateContextMessageTokens(guarded, false) <= budget)
|
|
29372
28984
|
return guarded;
|
|
29373
|
-
return
|
|
28985
|
+
return fitNewestToolResult(guarded, budget);
|
|
29374
28986
|
}
|
|
29375
28987
|
function estimateContextMessageTokens(messages, useReportedUsage = false) {
|
|
29376
28988
|
if (useReportedUsage) {
|
|
@@ -29381,82 +28993,37 @@ function estimateContextMessageTokens(messages, useReportedUsage = false) {
|
|
|
29381
28993
|
}
|
|
29382
28994
|
return messages.reduce((total, message) => total + estimateMessageTokens(message), 0);
|
|
29383
28995
|
}
|
|
29384
|
-
function
|
|
29385
|
-
const
|
|
29386
|
-
const
|
|
29387
|
-
|
|
29388
|
-
|
|
29389
|
-
const
|
|
29390
|
-
|
|
29391
|
-
|
|
29392
|
-
|
|
29393
|
-
|
|
29394
|
-
trimmed[index] = fitMessageToTokens(message, Math.max(floorTokens, inputBudget - others));
|
|
29395
|
-
};
|
|
29396
|
-
for (const [index, message] of trimmed.entries()) {
|
|
29397
|
-
if (index === lastIndex || index === queryIndex || message.role === "system")
|
|
29398
|
-
continue;
|
|
29399
|
-
const replacement = replaceMessageWithMarker(message);
|
|
29400
|
-
if (replacement === message || estimateMessageTokens(replacement) >= estimateMessageTokens(message))
|
|
29401
|
-
continue;
|
|
29402
|
-
trimmed[index] = replacement;
|
|
29403
|
-
if (fits())
|
|
29404
|
-
return trimmed;
|
|
29405
|
-
}
|
|
29406
|
-
if (lastIndex !== queryIndex && trimmed[lastIndex]?.role !== "system") {
|
|
29407
|
-
fitInto(lastIndex, 0);
|
|
29408
|
-
if (fits())
|
|
29409
|
-
return trimmed;
|
|
29410
|
-
}
|
|
29411
|
-
for (const [index, message] of trimmed.entries()) {
|
|
29412
|
-
if (message.role !== "system")
|
|
29413
|
-
continue;
|
|
29414
|
-
fitInto(index, 0);
|
|
29415
|
-
if (fits())
|
|
29416
|
-
return trimmed;
|
|
29417
|
-
}
|
|
29418
|
-
if (queryIndex >= 0)
|
|
29419
|
-
fitInto(queryIndex, MIN_QUERY_TOKENS);
|
|
29420
|
-
return trimmed;
|
|
28996
|
+
function fitNewestToolResult(messages, inputBudget) {
|
|
28997
|
+
const lastIndex = messages.length - 1;
|
|
28998
|
+
const last = messages[lastIndex];
|
|
28999
|
+
if (last?.role !== "toolResult")
|
|
29000
|
+
return messages;
|
|
29001
|
+
const others = messages.reduce((total, message, index) => index === lastIndex ? total : total + estimateMessageTokens(message), 0);
|
|
29002
|
+
if (others >= inputBudget)
|
|
29003
|
+
return messages;
|
|
29004
|
+
const fitted = fitMessageToTokens(last, inputBudget - others);
|
|
29005
|
+
return fitted === last ? messages : [...messages.slice(0, lastIndex), fitted];
|
|
29421
29006
|
}
|
|
29422
29007
|
function fitMessageToTokens(message, maxTokens) {
|
|
29423
29008
|
const original = textContent(message);
|
|
29424
29009
|
if (original === undefined)
|
|
29425
|
-
return
|
|
29010
|
+
return message;
|
|
29426
29011
|
const marker = `
|
|
29427
29012
|
|
|
29428
29013
|
${GUARD_MARKER}
|
|
29429
29014
|
`;
|
|
29430
|
-
|
|
29431
|
-
|
|
29432
|
-
let
|
|
29433
|
-
|
|
29434
|
-
const
|
|
29435
|
-
const
|
|
29436
|
-
if (
|
|
29437
|
-
|
|
29438
|
-
|
|
29439
|
-
|
|
29440
|
-
high = middle - 1;
|
|
29441
|
-
}
|
|
29015
|
+
const overhead = estimateMessageTokens(replaceTextContent(message, ""));
|
|
29016
|
+
const density = Math.max(0, estimateMessageTokens(message) - overhead) / Math.max(1, original.length);
|
|
29017
|
+
let chars = density > 0 ? Math.floor((maxTokens - overhead) / density) : original.length;
|
|
29018
|
+
for (let attempt = 0;attempt < MAX_FIT_ATTEMPTS && chars > 0; attempt++) {
|
|
29019
|
+
const candidate = replaceTextContent(message, excerpt(original, chars, marker));
|
|
29020
|
+
const tokens = estimateMessageTokens(candidate);
|
|
29021
|
+
if (tokens <= maxTokens)
|
|
29022
|
+
return candidate;
|
|
29023
|
+
const scaled = Math.floor(chars * Math.max(0, maxTokens - overhead) / Math.max(1, tokens - overhead) * 0.98);
|
|
29024
|
+
chars = Math.min(chars - 1, scaled);
|
|
29442
29025
|
}
|
|
29443
|
-
return
|
|
29444
|
-
}
|
|
29445
|
-
function stripNonTextContent(message) {
|
|
29446
|
-
if (message.role !== "user" || typeof message.content === "string")
|
|
29447
|
-
return replaceMessageWithMarker(message);
|
|
29448
|
-
const marker = `
|
|
29449
|
-
|
|
29450
|
-
${GUARD_MARKER}
|
|
29451
|
-
`;
|
|
29452
|
-
let replacedImage = false;
|
|
29453
|
-
const content = message.content.map((block) => {
|
|
29454
|
-
if (block.type === "text")
|
|
29455
|
-
return block;
|
|
29456
|
-
replacedImage = true;
|
|
29457
|
-
return { type: "text", text: marker };
|
|
29458
|
-
});
|
|
29459
|
-
return replacedImage ? { ...message, content } : message;
|
|
29026
|
+
return replaceTextContent(message, marker);
|
|
29460
29027
|
}
|
|
29461
29028
|
function excerpt(text, maxChars, marker) {
|
|
29462
29029
|
if (text.length <= maxChars)
|
|
@@ -29466,32 +29033,54 @@ function excerpt(text, maxChars, marker) {
|
|
|
29466
29033
|
const tail = Math.max(0, available - head);
|
|
29467
29034
|
return `${text.slice(0, head)}${marker}${tail > 0 ? text.slice(-tail) : ""}`;
|
|
29468
29035
|
}
|
|
29469
|
-
function replaceMessageWithMarker(message) {
|
|
29470
|
-
return replaceTextContent(message, `
|
|
29471
|
-
|
|
29472
|
-
${GUARD_MARKER}
|
|
29473
|
-
`);
|
|
29474
|
-
}
|
|
29475
29036
|
function estimateMessageTokens(message) {
|
|
29476
29037
|
if (message.role === "system")
|
|
29477
29038
|
return estimateSystemMessageTokens(message);
|
|
29478
|
-
const estimate = estimateTokens(message);
|
|
29039
|
+
const estimate = estimateTokens(message) + tokensBeyondCharGuess(messageTextParts(message));
|
|
29479
29040
|
return message.role === "toolResult" ? estimate + Math.ceil(message.toolName.length / 4) : estimate;
|
|
29480
29041
|
}
|
|
29481
29042
|
function estimateSystemMessageTokens(message) {
|
|
29482
|
-
|
|
29483
|
-
|
|
29043
|
+
const parts = [];
|
|
29044
|
+
let chars = 0;
|
|
29045
|
+
if (typeof message.content === "string")
|
|
29046
|
+
parts.push(message.content);
|
|
29047
|
+
else {
|
|
29484
29048
|
for (const block of message.content) {
|
|
29485
29049
|
if (block.type === "text")
|
|
29486
|
-
|
|
29050
|
+
parts.push(block.text);
|
|
29487
29051
|
else if (block.type === "image")
|
|
29488
29052
|
chars += 4800;
|
|
29489
29053
|
}
|
|
29490
29054
|
}
|
|
29491
|
-
|
|
29492
|
-
|
|
29493
|
-
|
|
29494
|
-
return Math.ceil(chars / 4);
|
|
29055
|
+
parts.push(JSON.stringify(message.sections ?? {}), JSON.stringify(message.toolsAdded ?? []), JSON.stringify(message.toolsRemoved ?? []));
|
|
29056
|
+
for (const part of parts)
|
|
29057
|
+
chars += part.length;
|
|
29058
|
+
return Math.ceil(chars / 4) + tokensBeyondCharGuess(parts);
|
|
29059
|
+
}
|
|
29060
|
+
function messageTextParts(message) {
|
|
29061
|
+
switch (message.role) {
|
|
29062
|
+
case "user":
|
|
29063
|
+
case "custom":
|
|
29064
|
+
case "toolResult":
|
|
29065
|
+
return typeof message.content === "string" ? [message.content] : message.content.flatMap((block) => block.type === "text" ? [block.text] : []);
|
|
29066
|
+
case "assistant":
|
|
29067
|
+
return message.content.flatMap((block) => {
|
|
29068
|
+
if (block.type === "text")
|
|
29069
|
+
return [block.text];
|
|
29070
|
+
if (block.type === "thinking")
|
|
29071
|
+
return [block.thinking];
|
|
29072
|
+
if (block.type === "toolCall")
|
|
29073
|
+
return [`${block.name}${JSON.stringify(block.arguments)}`];
|
|
29074
|
+
return [];
|
|
29075
|
+
});
|
|
29076
|
+
case "bashExecution":
|
|
29077
|
+
return [message.command, message.output];
|
|
29078
|
+
case "branchSummary":
|
|
29079
|
+
case "compactionSummary":
|
|
29080
|
+
return [message.summary];
|
|
29081
|
+
default:
|
|
29082
|
+
return [];
|
|
29083
|
+
}
|
|
29495
29084
|
}
|
|
29496
29085
|
function findLastAssistantUsage(messages) {
|
|
29497
29086
|
for (let index = messages.length - 1;index >= 0; index--) {
|
|
@@ -29565,7 +29154,7 @@ function parseCandidateSection(message, text) {
|
|
|
29565
29154
|
const sectionEnd = findBaselineSectionEnd(text, sectionStart);
|
|
29566
29155
|
if (!/^\[e?\d+\] /mu.test(text.slice(sectionStart, sectionEnd)))
|
|
29567
29156
|
return;
|
|
29568
|
-
return splitCandidates(text, sectionStart, sectionEnd);
|
|
29157
|
+
return splitCandidates(text, sectionStart, sectionEnd, { lists: true });
|
|
29569
29158
|
}
|
|
29570
29159
|
if (message.role !== "toolResult")
|
|
29571
29160
|
return;
|
|
@@ -29576,7 +29165,9 @@ function parseCandidateSection(message, text) {
|
|
|
29576
29165
|
const directiveStart = text.indexOf(`
|
|
29577
29166
|
|
|
29578
29167
|
directive:`);
|
|
29579
|
-
return splitCandidates(text, 0, directiveStart >= 0 ? directiveStart : text.length,
|
|
29168
|
+
return splitCandidates(text, 0, directiveStart >= 0 ? directiveStart : text.length, {
|
|
29169
|
+
pattern: /^- [^\n]*(?:\n|$)/gmu
|
|
29170
|
+
});
|
|
29580
29171
|
}
|
|
29581
29172
|
const diagnosticsStart = text.indexOf(`
|
|
29582
29173
|
|
|
@@ -29598,17 +29189,23 @@ Formatting and content rules`
|
|
|
29598
29189
|
const positions = endMarkers.map((marker) => text.indexOf(marker, startIndex)).filter((index) => index >= 0);
|
|
29599
29190
|
return positions.length > 0 ? Math.min(...positions) : text.length;
|
|
29600
29191
|
}
|
|
29601
|
-
function splitCandidates(text, sectionStart, sectionEnd,
|
|
29192
|
+
function splitCandidates(text, sectionStart, sectionEnd, options = {}) {
|
|
29602
29193
|
const region = text.slice(sectionStart, sectionEnd);
|
|
29603
|
-
const matches = [...region.matchAll(pattern)];
|
|
29194
|
+
const matches = [...region.matchAll(options.pattern ?? /^\[e?\d+\] [^\n]*(?:\n|$)/gmu)];
|
|
29604
29195
|
const firstStart = matches[0]?.index;
|
|
29605
29196
|
if (firstStart === undefined)
|
|
29606
29197
|
return;
|
|
29607
29198
|
const blocks = [];
|
|
29199
|
+
let rank = 0;
|
|
29200
|
+
let heading = "";
|
|
29608
29201
|
for (const [index, match] of matches.entries()) {
|
|
29609
29202
|
const start = match.index ?? 0;
|
|
29610
29203
|
const end = index + 1 < matches.length ? matches[index + 1]?.index ?? region.length : region.length;
|
|
29611
|
-
|
|
29204
|
+
const raw = region.slice(start, end);
|
|
29205
|
+
const tail = options.lists === true ? LIST_HEADING_TAIL.exec(raw) : null;
|
|
29206
|
+
blocks.push({ text: heading + (tail === null ? raw : raw.slice(0, tail.index)), rank });
|
|
29207
|
+
rank = tail === null ? rank + 1 : 0;
|
|
29208
|
+
heading = tail === null ? "" : tail[0];
|
|
29612
29209
|
}
|
|
29613
29210
|
return {
|
|
29614
29211
|
prefix: text.slice(0, sectionStart) + region.slice(0, firstStart),
|
|
@@ -29616,9 +29213,10 @@ function splitCandidates(text, sectionStart, sectionEnd, pattern = /^\[e?\d+\] [
|
|
|
29616
29213
|
blocks
|
|
29617
29214
|
};
|
|
29618
29215
|
}
|
|
29619
|
-
var CONTEXT_HEADROOM = 1.2, MAX_OUTPUT_TOKENS = 32000,
|
|
29216
|
+
var CONTEXT_HEADROOM = 1.2, MAX_OUTPUT_TOKENS = 32000, GUARD_MARKER = "[AutoRAG context guard: omitted context to stay within the model context budget.]", RETRIEVAL_TOOL_PREFIX = "search_datasource_", RETRIEVAL_TOOL_NAMES, MAX_FIT_ATTEMPTS = 6, LIST_HEADING_TAIL;
|
|
29620
29217
|
var init_context_token_guard = __esm(() => {
|
|
29621
29218
|
init_context_overflow();
|
|
29219
|
+
init_token_count();
|
|
29622
29220
|
RETRIEVAL_TOOL_NAMES = {
|
|
29623
29221
|
jikji_find: true,
|
|
29624
29222
|
search_all_documents: true,
|
|
@@ -29627,6 +29225,7 @@ var init_context_token_guard = __esm(() => {
|
|
|
29627
29225
|
everything_search: true,
|
|
29628
29226
|
fsearch_search: true
|
|
29629
29227
|
};
|
|
29228
|
+
LIST_HEADING_TAIL = /\n[^\n]*initial candidates[^\n]*:\n$/u;
|
|
29630
29229
|
});
|
|
29631
29230
|
|
|
29632
29231
|
// src/agent/current-date.ts
|
|
@@ -29963,7 +29562,7 @@ function guardContext(pi, contextTransform) {
|
|
|
29963
29562
|
pi.on("context", async (event, ctx) => {
|
|
29964
29563
|
const activeTools = new Set(pi.getActiveTools());
|
|
29965
29564
|
const toolDeclarations = pi.getAllTools().filter((tool) => activeTools.has(tool.name)).map(({ name, description, parameters }) => ({ name, description, parameters }));
|
|
29966
|
-
const fixedOverheadTokens =
|
|
29565
|
+
const fixedOverheadTokens = estimateTextTokens(ctx.getSystemPrompt()) + estimateTextTokens(JSON.stringify(toolDeclarations));
|
|
29967
29566
|
return {
|
|
29968
29567
|
messages: await createContextTokenGuardedTransform(ctx.model, contextTransform, () => fixedOverheadTokens)(event.messages)
|
|
29969
29568
|
};
|
|
@@ -30197,6 +29796,7 @@ var init_pi_session = __esm(() => {
|
|
|
30197
29796
|
init_intent_tracing();
|
|
30198
29797
|
init_mcp_client();
|
|
30199
29798
|
init_request_limits();
|
|
29799
|
+
init_token_count();
|
|
30200
29800
|
init_welcome_header();
|
|
30201
29801
|
PI_BUILTIN_TOOL_NAMES = ["read", "bash", "edit", "write", "grep", "find", "ls"];
|
|
30202
29802
|
});
|
|
@@ -31968,7 +31568,7 @@ var init_page_loader = __esm(() => {
|
|
|
31968
31568
|
});
|
|
31969
31569
|
|
|
31970
31570
|
// src/web/fetch/url-target.ts
|
|
31971
|
-
import { extname as
|
|
31571
|
+
import { extname as extname7 } from "node:path";
|
|
31972
31572
|
function repairCollapsedScheme(value) {
|
|
31973
31573
|
const m = value.match(/^(https?):\/(?!\/)/i);
|
|
31974
31574
|
return m ? `${m[1]}://${value.slice(m[0].length)}` : value;
|
|
@@ -31987,7 +31587,7 @@ function getFilenameExtensionHint(filename) {
|
|
|
31987
31587
|
const lower = filename.toLowerCase();
|
|
31988
31588
|
if (lower.endsWith(".tar.gz"))
|
|
31989
31589
|
return ".tar.gz";
|
|
31990
|
-
return
|
|
31590
|
+
return extname7(filename).toLowerCase();
|
|
31991
31591
|
}
|
|
31992
31592
|
function getExtensionHint(url, contentDisposition) {
|
|
31993
31593
|
if (contentDisposition) {
|
|
@@ -32164,7 +31764,7 @@ var init_convert2 = __esm(() => {
|
|
|
32164
31764
|
});
|
|
32165
31765
|
|
|
32166
31766
|
// src/web/fetch/feeds.ts
|
|
32167
|
-
import { extname as
|
|
31767
|
+
import { extname as extname8 } from "node:path";
|
|
32168
31768
|
import { XMLParser as XMLParser3 } from "fast-xml-parser";
|
|
32169
31769
|
function getHtmlAttribute(tag, attribute) {
|
|
32170
31770
|
const pattern = new RegExp(`\\b${attribute}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'=<>]+))`, "i");
|
|
@@ -32227,7 +31827,7 @@ function extractDocumentLinks(html, baseUrl) {
|
|
|
32227
31827
|
const href = getHtmlAttribute(tag, "href");
|
|
32228
31828
|
if (!href)
|
|
32229
31829
|
continue;
|
|
32230
|
-
const ext =
|
|
31830
|
+
const ext = extname8(href).toLowerCase();
|
|
32231
31831
|
if (!CONVERTIBLE_EXTENSIONS2.has(ext))
|
|
32232
31832
|
continue;
|
|
32233
31833
|
const resolved = href.startsWith("http") ? href : new URL(href, baseUrl).href;
|
|
@@ -33895,7 +33495,7 @@ var init_web_search_tool = __esm(() => {
|
|
|
33895
33495
|
// src/agent/agent.ts
|
|
33896
33496
|
import { randomUUID as randomUUID7 } from "node:crypto";
|
|
33897
33497
|
import { existsSync as existsSync34, watch as fsWatch, mkdirSync as mkdirSync25, realpathSync as realpathSync7, statSync as statSync9, writeFileSync as writeFileSync18 } from "node:fs";
|
|
33898
|
-
import { basename as
|
|
33498
|
+
import { basename as basename13, dirname as dirname26, join as join51, resolve as resolve18 } from "node:path";
|
|
33899
33499
|
import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
|
|
33900
33500
|
import { getAgentDir as getAgentDir5, keyText } from "@earendil-works/pi-coding-agent";
|
|
33901
33501
|
function positiveLimit(value, fallback, path) {
|
|
@@ -36818,7 +36418,7 @@ function toMinSyncReasonDiagnostic(reason) {
|
|
|
36818
36418
|
function pinExcludedPath(path) {
|
|
36819
36419
|
const resolvedPath = resolve18(path);
|
|
36820
36420
|
try {
|
|
36821
|
-
return join51(realpathSync7(dirname26(resolvedPath)),
|
|
36421
|
+
return join51(realpathSync7(dirname26(resolvedPath)), basename13(resolvedPath));
|
|
36822
36422
|
} catch {
|
|
36823
36423
|
return resolvedPath;
|
|
36824
36424
|
}
|