agentic-pi 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +140 -29
- package/dist/args.d.ts +27 -0
- package/dist/args.js +46 -0
- package/dist/args.js.map +1 -1
- package/dist/extensions/file-search/index.d.ts +66 -0
- package/dist/extensions/file-search/index.js +86 -0
- package/dist/extensions/file-search/index.js.map +1 -0
- package/dist/extensions/web-search/extract.d.ts +18 -0
- package/dist/extensions/web-search/extract.js +110 -0
- package/dist/extensions/web-search/extract.js.map +1 -0
- package/dist/extensions/web-search/index.d.ts +43 -0
- package/dist/extensions/web-search/index.js +86 -0
- package/dist/extensions/web-search/index.js.map +1 -0
- package/dist/extensions/web-search/providers/brave.d.ts +21 -0
- package/dist/extensions/web-search/providers/brave.js +73 -0
- package/dist/extensions/web-search/providers/brave.js.map +1 -0
- package/dist/extensions/web-search/providers/exa.d.ts +16 -0
- package/dist/extensions/web-search/providers/exa.js +85 -0
- package/dist/extensions/web-search/providers/exa.js.map +1 -0
- package/dist/extensions/web-search/providers/tavily.d.ts +18 -0
- package/dist/extensions/web-search/providers/tavily.js +85 -0
- package/dist/extensions/web-search/providers/tavily.js.map +1 -0
- package/dist/extensions/web-search/rate-limit.d.ts +14 -0
- package/dist/extensions/web-search/rate-limit.js +24 -0
- package/dist/extensions/web-search/rate-limit.js.map +1 -0
- package/dist/extensions/web-search/safe-fetch.d.ts +54 -0
- package/dist/extensions/web-search/safe-fetch.js +172 -0
- package/dist/extensions/web-search/safe-fetch.js.map +1 -0
- package/dist/extensions/web-search/selection.d.ts +42 -0
- package/dist/extensions/web-search/selection.js +64 -0
- package/dist/extensions/web-search/selection.js.map +1 -0
- package/dist/extensions/web-search/tools.d.ts +13 -0
- package/dist/extensions/web-search/tools.js +136 -0
- package/dist/extensions/web-search/tools.js.map +1 -0
- package/dist/extensions/web-search/types.d.ts +65 -0
- package/dist/extensions/web-search/types.js +10 -0
- package/dist/extensions/web-search/types.js.map +1 -0
- package/dist/run.d.ts +45 -0
- package/dist/run.js +24 -0
- package/dist/run.js.map +1 -1
- package/dist/runner.js +85 -2
- package/dist/runner.js.map +1 -1
- package/dist/sandbox/gondolin.d.ts +13 -4
- package/dist/sandbox/gondolin.js +10 -3
- package/dist/sandbox/gondolin.js.map +1 -1
- package/package.json +2 -1
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Safe-by-default HTTP GET wrapper for web_fetch.
|
|
3
|
+
*
|
|
4
|
+
* Rails:
|
|
5
|
+
* - scheme must be http or https
|
|
6
|
+
* - timeout via AbortController (default 15s)
|
|
7
|
+
* - max response bytes hard-capped (default 1 MiB); streaming is aborted
|
|
8
|
+
* once the cap is exceeded
|
|
9
|
+
* - redirects followed manually, at most 3; scheme re-checked at each hop
|
|
10
|
+
* - caller passes the content-type allowlist; non-matching responses
|
|
11
|
+
* raise so the tool layer can return a structured error
|
|
12
|
+
*
|
|
13
|
+
* No SSRF private-range blocking is performed (deliberate per user
|
|
14
|
+
* decision). Operators who care should run this behind their own egress
|
|
15
|
+
* firewall.
|
|
16
|
+
*/
|
|
17
|
+
import type { FetchImpl } from "./types.js";
|
|
18
|
+
export interface SafeFetchOptions {
|
|
19
|
+
/** Defaults to globalThis.fetch. Injected in tests. */
|
|
20
|
+
fetchImpl?: FetchImpl;
|
|
21
|
+
/** Per-request timeout in ms. Default 15_000. */
|
|
22
|
+
timeoutMs?: number;
|
|
23
|
+
/** Hard cap on response body bytes. Default 1 MiB. */
|
|
24
|
+
maxBytes?: number;
|
|
25
|
+
/** Max redirect hops. Default 3. */
|
|
26
|
+
maxRedirects?: number;
|
|
27
|
+
/**
|
|
28
|
+
* Allowed content-types as case-insensitive prefixes. A response is
|
|
29
|
+
* accepted if its Content-Type starts with any entry. Default: text/*,
|
|
30
|
+
* application/(xhtml+xml|xml|json).
|
|
31
|
+
*/
|
|
32
|
+
allowedContentTypePrefixes?: string[];
|
|
33
|
+
}
|
|
34
|
+
export interface SafeFetchResult {
|
|
35
|
+
status: number;
|
|
36
|
+
contentType?: string;
|
|
37
|
+
body: string;
|
|
38
|
+
finalUrl: string;
|
|
39
|
+
}
|
|
40
|
+
export declare const SAFE_FETCH_DEFAULTS: {
|
|
41
|
+
timeoutMs: number;
|
|
42
|
+
maxBytes: number;
|
|
43
|
+
maxRedirects: number;
|
|
44
|
+
};
|
|
45
|
+
export declare class SafeFetchError extends Error {
|
|
46
|
+
readonly code: string;
|
|
47
|
+
readonly status?: number | undefined;
|
|
48
|
+
constructor(message: string, code: string, status?: number | undefined);
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Issue a GET, follow redirects manually, cap byte count, enforce timeout
|
|
52
|
+
* and content-type. Returns the decoded body as a UTF-8 string.
|
|
53
|
+
*/
|
|
54
|
+
export declare function safeFetch(rawUrl: string, options?: SafeFetchOptions): Promise<SafeFetchResult>;
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Safe-by-default HTTP GET wrapper for web_fetch.
|
|
3
|
+
*
|
|
4
|
+
* Rails:
|
|
5
|
+
* - scheme must be http or https
|
|
6
|
+
* - timeout via AbortController (default 15s)
|
|
7
|
+
* - max response bytes hard-capped (default 1 MiB); streaming is aborted
|
|
8
|
+
* once the cap is exceeded
|
|
9
|
+
* - redirects followed manually, at most 3; scheme re-checked at each hop
|
|
10
|
+
* - caller passes the content-type allowlist; non-matching responses
|
|
11
|
+
* raise so the tool layer can return a structured error
|
|
12
|
+
*
|
|
13
|
+
* No SSRF private-range blocking is performed (deliberate per user
|
|
14
|
+
* decision). Operators who care should run this behind their own egress
|
|
15
|
+
* firewall.
|
|
16
|
+
*/
|
|
17
|
+
const DEFAULT_ALLOWED_PREFIXES = [
|
|
18
|
+
"text/",
|
|
19
|
+
"application/xhtml+xml",
|
|
20
|
+
"application/xml",
|
|
21
|
+
"application/json",
|
|
22
|
+
];
|
|
23
|
+
export const SAFE_FETCH_DEFAULTS = {
|
|
24
|
+
timeoutMs: 15_000,
|
|
25
|
+
maxBytes: 1024 * 1024,
|
|
26
|
+
maxRedirects: 3,
|
|
27
|
+
};
|
|
28
|
+
export class SafeFetchError extends Error {
|
|
29
|
+
code;
|
|
30
|
+
status;
|
|
31
|
+
constructor(message, code, status) {
|
|
32
|
+
super(message);
|
|
33
|
+
this.code = code;
|
|
34
|
+
this.status = status;
|
|
35
|
+
this.name = "SafeFetchError";
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
function assertHttpScheme(url) {
|
|
39
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
40
|
+
throw new SafeFetchError(`unsupported url scheme '${url.protocol}' (only http/https allowed)`, "bad-scheme");
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
function isAllowedContentType(ct, prefixes) {
|
|
44
|
+
if (!ct)
|
|
45
|
+
return false;
|
|
46
|
+
const lower = ct.toLowerCase();
|
|
47
|
+
return prefixes.some((p) => lower.startsWith(p));
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Issue a GET, follow redirects manually, cap byte count, enforce timeout
|
|
51
|
+
* and content-type. Returns the decoded body as a UTF-8 string.
|
|
52
|
+
*/
|
|
53
|
+
export async function safeFetch(rawUrl, options = {}) {
|
|
54
|
+
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
|
55
|
+
if (!fetchImpl) {
|
|
56
|
+
throw new SafeFetchError("no fetch implementation available", "no-fetch");
|
|
57
|
+
}
|
|
58
|
+
const timeoutMs = options.timeoutMs ?? SAFE_FETCH_DEFAULTS.timeoutMs;
|
|
59
|
+
const maxBytes = options.maxBytes ?? SAFE_FETCH_DEFAULTS.maxBytes;
|
|
60
|
+
const maxRedirects = options.maxRedirects ?? SAFE_FETCH_DEFAULTS.maxRedirects;
|
|
61
|
+
const allowedPrefixes = options.allowedContentTypePrefixes ?? DEFAULT_ALLOWED_PREFIXES;
|
|
62
|
+
let currentUrl;
|
|
63
|
+
try {
|
|
64
|
+
currentUrl = new URL(rawUrl);
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
throw new SafeFetchError(`invalid url '${rawUrl}'`, "bad-url");
|
|
68
|
+
}
|
|
69
|
+
assertHttpScheme(currentUrl);
|
|
70
|
+
const controller = new AbortController();
|
|
71
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
72
|
+
try {
|
|
73
|
+
for (let hop = 0; hop <= maxRedirects; hop++) {
|
|
74
|
+
const response = await fetchImpl(currentUrl.toString(), {
|
|
75
|
+
method: "GET",
|
|
76
|
+
redirect: "manual",
|
|
77
|
+
signal: controller.signal,
|
|
78
|
+
headers: { "user-agent": "agentic-pi/web-search" },
|
|
79
|
+
});
|
|
80
|
+
const status = response.status;
|
|
81
|
+
if (status >= 300 && status < 400) {
|
|
82
|
+
const loc = response.headers.get("location");
|
|
83
|
+
if (!loc) {
|
|
84
|
+
throw new SafeFetchError(`redirect (${status}) without Location header`, "bad-redirect", status);
|
|
85
|
+
}
|
|
86
|
+
if (hop === maxRedirects) {
|
|
87
|
+
throw new SafeFetchError(`too many redirects (>${maxRedirects})`, "too-many-redirects");
|
|
88
|
+
}
|
|
89
|
+
try {
|
|
90
|
+
currentUrl = new URL(loc, currentUrl);
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
throw new SafeFetchError(`invalid redirect target '${loc}'`, "bad-url");
|
|
94
|
+
}
|
|
95
|
+
assertHttpScheme(currentUrl);
|
|
96
|
+
// Drain/close any body the redirect carried, just in case.
|
|
97
|
+
try {
|
|
98
|
+
await response.body?.cancel();
|
|
99
|
+
}
|
|
100
|
+
catch { /* ignore */ }
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (status >= 400) {
|
|
104
|
+
try {
|
|
105
|
+
await response.body?.cancel();
|
|
106
|
+
}
|
|
107
|
+
catch { /* ignore */ }
|
|
108
|
+
throw new SafeFetchError(`http ${status}`, "http-error", status);
|
|
109
|
+
}
|
|
110
|
+
const contentType = response.headers.get("content-type") ?? undefined;
|
|
111
|
+
if (!isAllowedContentType(contentType, allowedPrefixes)) {
|
|
112
|
+
try {
|
|
113
|
+
await response.body?.cancel();
|
|
114
|
+
}
|
|
115
|
+
catch { /* ignore */ }
|
|
116
|
+
throw new SafeFetchError(`disallowed content-type '${contentType ?? "(none)"}'`, "bad-content-type", status);
|
|
117
|
+
}
|
|
118
|
+
const body = await readCapped(response, maxBytes);
|
|
119
|
+
return {
|
|
120
|
+
status,
|
|
121
|
+
contentType,
|
|
122
|
+
body,
|
|
123
|
+
finalUrl: currentUrl.toString(),
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
// Unreachable — the for-loop returns or throws every path.
|
|
127
|
+
throw new SafeFetchError("redirect loop fell through", "internal");
|
|
128
|
+
}
|
|
129
|
+
finally {
|
|
130
|
+
clearTimeout(timer);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
async function readCapped(response, maxBytes) {
|
|
134
|
+
// Prefer streaming so we can abort when the cap is hit without buffering
|
|
135
|
+
// the entire body first.
|
|
136
|
+
if (response.body && typeof response.body.getReader === "function") {
|
|
137
|
+
const reader = response.body.getReader();
|
|
138
|
+
const chunks = [];
|
|
139
|
+
let total = 0;
|
|
140
|
+
while (true) {
|
|
141
|
+
const { done, value } = await reader.read();
|
|
142
|
+
if (done)
|
|
143
|
+
break;
|
|
144
|
+
if (!value)
|
|
145
|
+
continue;
|
|
146
|
+
total += value.byteLength;
|
|
147
|
+
if (total > maxBytes) {
|
|
148
|
+
try {
|
|
149
|
+
await reader.cancel();
|
|
150
|
+
}
|
|
151
|
+
catch { /* ignore */ }
|
|
152
|
+
throw new SafeFetchError(`response exceeded ${maxBytes} bytes`, "too-large");
|
|
153
|
+
}
|
|
154
|
+
chunks.push(value);
|
|
155
|
+
}
|
|
156
|
+
const buf = new Uint8Array(total);
|
|
157
|
+
let offset = 0;
|
|
158
|
+
for (const c of chunks) {
|
|
159
|
+
buf.set(c, offset);
|
|
160
|
+
offset += c.byteLength;
|
|
161
|
+
}
|
|
162
|
+
return new TextDecoder("utf-8", { fatal: false }).decode(buf);
|
|
163
|
+
}
|
|
164
|
+
// Fallback: text() without streaming cap — only triggers when the
|
|
165
|
+
// injected fetch returns a stub Response.
|
|
166
|
+
const text = await response.text();
|
|
167
|
+
if (text.length > maxBytes) {
|
|
168
|
+
throw new SafeFetchError(`response exceeded ${maxBytes} bytes`, "too-large");
|
|
169
|
+
}
|
|
170
|
+
return text;
|
|
171
|
+
}
|
|
172
|
+
//# sourceMappingURL=safe-fetch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"safe-fetch.js","sourceRoot":"","sources":["../../../src/extensions/web-search/safe-fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AA4BH,MAAM,wBAAwB,GAAG;IAC/B,OAAO;IACP,uBAAuB;IACvB,iBAAiB;IACjB,kBAAkB;CACnB,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,SAAS,EAAE,MAAM;IACjB,QAAQ,EAAE,IAAI,GAAG,IAAI;IACrB,YAAY,EAAE,CAAC;CAChB,CAAC;AAEF,MAAM,OAAO,cAAe,SAAQ,KAAK;IACD;IAAuB;IAA7D,YAAY,OAAe,EAAW,IAAY,EAAW,MAAe;QAC1E,KAAK,CAAC,OAAO,CAAC,CAAC;QADqB,SAAI,GAAJ,IAAI,CAAQ;QAAW,WAAM,GAAN,MAAM,CAAS;QAE1E,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED,SAAS,gBAAgB,CAAC,GAAQ;IAChC,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,cAAc,CACtB,2BAA2B,GAAG,CAAC,QAAQ,6BAA6B,EACpE,YAAY,CACb,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAC3B,EAAsB,EACtB,QAAkB;IAElB,IAAI,CAAC,EAAE;QAAE,OAAO,KAAK,CAAC;IACtB,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/B,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,MAAc,EACd,UAA4B,EAAE;IAE9B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAK,UAAU,CAAC,KAAmB,CAAC;IACvE,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,cAAc,CAAC,mCAAmC,EAAE,UAAU,CAAC,CAAC;IAC5E,CAAC;IACD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,mBAAmB,CAAC,SAAS,CAAC;IACrE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,mBAAmB,CAAC,QAAQ,CAAC;IAClE,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,mBAAmB,CAAC,YAAY,CAAC;IAC9E,MAAM,eAAe,GACnB,OAAO,CAAC,0BAA0B,IAAI,wBAAwB,CAAC;IAEjE,IAAI,UAAe,CAAC;IACpB,IAAI,CAAC;QACH,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,cAAc,CAAC,gBAAgB,MAAM,GAAG,EAAE,SAAS,CAAC,CAAC;IACjE,CAAC;IACD,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAE7B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,YAAY,EAAE,GAAG,EAAE,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,UAAU,CAAC,QAAQ,EAAE,EAAE;gBACtD,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,OAAO,EAAE,EAAE,YAAY,EAAE,uBAAuB,EAAE;aACnD,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;YAC/B,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;gBAClC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAC7C,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,MAAM,IAAI,cAAc,CACtB,aAAa,MAAM,2BAA2B,EAC9C,cAAc,EACd,MAAM,CACP,CAAC;gBACJ,CAAC;gBACD,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;oBACzB,MAAM,IAAI,cAAc,CACtB,wBAAwB,YAAY,GAAG,EACvC,oBAAoB,CACrB,CAAC;gBACJ,CAAC;gBACD,IAAI,CAAC;oBACH,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;gBACxC,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,IAAI,cAAc,CAAC,4BAA4B,GAAG,GAAG,EAAE,SAAS,CAAC,CAAC;gBAC1E,CAAC;gBACD,gBAAgB,CAAC,UAAU,CAAC,CAAC;gBAC7B,2DAA2D;gBAC3D,IAAI,CAAC;oBAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;gBAC7D,SAAS;YACX,CAAC;YAED,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;gBAClB,IAAI,CAAC;oBAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;gBAC7D,MAAM,IAAI,cAAc,CAAC,QAAQ,MAAM,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;YACnE,CAAC;YAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,SAAS,CAAC;YACtE,IAAI,CAAC,oBAAoB,CAAC,WAAW,EAAE,eAAe,CAAC,EAAE,CAAC;gBACxD,IAAI,CAAC;oBAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;gBAC7D,MAAM,IAAI,cAAc,CACtB,4BAA4B,WAAW,IAAI,QAAQ,GAAG,EACtD,kBAAkB,EAClB,MAAM,CACP,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAElD,OAAO;gBACL,MAAM;gBACN,WAAW;gBACX,IAAI;gBACJ,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE;aAChC,CAAC;QACJ,CAAC;QACD,2DAA2D;QAC3D,MAAM,IAAI,cAAc,CAAC,4BAA4B,EAAE,UAAU,CAAC,CAAC;IACrE,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,QAAkB,EAAE,QAAgB;IAC5D,yEAAyE;IACzE,yBAAyB;IACzB,IAAI,QAAQ,CAAC,IAAI,IAAI,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;QACnE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QACzC,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI;gBAAE,MAAM;YAChB,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;YAC1B,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;gBACrB,IAAI,CAAC;oBAAC,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;gBACrD,MAAM,IAAI,cAAc,CACtB,qBAAqB,QAAQ,QAAQ,EACrC,WAAW,CACZ,CAAC;YACJ,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACnB,MAAM,IAAI,CAAC,CAAC,UAAU,CAAC;QACzB,CAAC;QACD,OAAO,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAChE,CAAC;IACD,kEAAkE;IAClE,0CAA0C;IAC1C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,cAAc,CACtB,qBAAqB,QAAQ,QAAQ,EACrC,WAAW,CACZ,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider selection logic — pure, no IO, easy to unit-test.
|
|
3
|
+
*
|
|
4
|
+
* Resolution order:
|
|
5
|
+
* 1. config.webSearch === false → skipped (disabled-by-flag).
|
|
6
|
+
* 2. Explicit provider (config.webSearchProvider OR WEB_SEARCH_PROVIDER env):
|
|
7
|
+
* - key present → selected.
|
|
8
|
+
* - key missing → skipped (no-credentials) naming the env var.
|
|
9
|
+
* 3. Auto: scan env keys in fixed priority Tavily → Exa → Brave.
|
|
10
|
+
* - first hit wins; if more than one is set, attach an advisory
|
|
11
|
+
* message so the user can override via WEB_SEARCH_PROVIDER.
|
|
12
|
+
* 4. None present → skipped (no-credentials), silent.
|
|
13
|
+
* 5. Unknown explicit provider name → throw (config error).
|
|
14
|
+
*/
|
|
15
|
+
import { type ProviderName, type WebSearchSkipReason } from "./types.js";
|
|
16
|
+
export interface SelectionInput {
|
|
17
|
+
/** When false, force-skip with reason `disabled-by-flag`. */
|
|
18
|
+
webSearch: boolean;
|
|
19
|
+
/** Explicit provider from --web-search-provider (overrides env). */
|
|
20
|
+
webSearchProvider?: string;
|
|
21
|
+
/** Process env or test override. */
|
|
22
|
+
env: Record<string, string | undefined>;
|
|
23
|
+
}
|
|
24
|
+
export interface SelectedProvider {
|
|
25
|
+
status: "configured";
|
|
26
|
+
provider: ProviderName;
|
|
27
|
+
apiKey: string;
|
|
28
|
+
/** Optional advisory note (e.g. multi-key collision). */
|
|
29
|
+
message?: string;
|
|
30
|
+
}
|
|
31
|
+
export interface SkippedProvider {
|
|
32
|
+
status: "skipped";
|
|
33
|
+
reason: WebSearchSkipReason;
|
|
34
|
+
message?: string;
|
|
35
|
+
/** Echoed back when the user explicitly asked for one. */
|
|
36
|
+
provider?: ProviderName;
|
|
37
|
+
}
|
|
38
|
+
export type SelectionResult = SelectedProvider | SkippedProvider;
|
|
39
|
+
/** env var name that holds each provider's key. */
|
|
40
|
+
export declare const PROVIDER_ENV_VAR: Record<ProviderName, string>;
|
|
41
|
+
export declare function isProviderName(s: string): s is ProviderName;
|
|
42
|
+
export declare function selectProvider(input: SelectionInput): SelectionResult;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider selection logic — pure, no IO, easy to unit-test.
|
|
3
|
+
*
|
|
4
|
+
* Resolution order:
|
|
5
|
+
* 1. config.webSearch === false → skipped (disabled-by-flag).
|
|
6
|
+
* 2. Explicit provider (config.webSearchProvider OR WEB_SEARCH_PROVIDER env):
|
|
7
|
+
* - key present → selected.
|
|
8
|
+
* - key missing → skipped (no-credentials) naming the env var.
|
|
9
|
+
* 3. Auto: scan env keys in fixed priority Tavily → Exa → Brave.
|
|
10
|
+
* - first hit wins; if more than one is set, attach an advisory
|
|
11
|
+
* message so the user can override via WEB_SEARCH_PROVIDER.
|
|
12
|
+
* 4. None present → skipped (no-credentials), silent.
|
|
13
|
+
* 5. Unknown explicit provider name → throw (config error).
|
|
14
|
+
*/
|
|
15
|
+
import { PROVIDER_NAMES, } from "./types.js";
|
|
16
|
+
/** env var name that holds each provider's key. */
|
|
17
|
+
export const PROVIDER_ENV_VAR = {
|
|
18
|
+
tavily: "TAVILY_API_KEY",
|
|
19
|
+
exa: "EXA_API_KEY",
|
|
20
|
+
brave: "BRAVE_SEARCH_API_KEY",
|
|
21
|
+
};
|
|
22
|
+
/** Priority order for auto-detection. */
|
|
23
|
+
const AUTO_PRIORITY = ["tavily", "exa", "brave"];
|
|
24
|
+
export function isProviderName(s) {
|
|
25
|
+
return PROVIDER_NAMES.includes(s);
|
|
26
|
+
}
|
|
27
|
+
export function selectProvider(input) {
|
|
28
|
+
if (input.webSearch === false) {
|
|
29
|
+
return { status: "skipped", reason: "disabled-by-flag" };
|
|
30
|
+
}
|
|
31
|
+
const envExplicit = input.env.WEB_SEARCH_PROVIDER?.trim();
|
|
32
|
+
const explicit = input.webSearchProvider ?? envExplicit;
|
|
33
|
+
if (explicit) {
|
|
34
|
+
if (!isProviderName(explicit)) {
|
|
35
|
+
throw new Error(`Unknown web-search provider '${explicit}'. Expected one of: ${PROVIDER_NAMES.join(", ")}`);
|
|
36
|
+
}
|
|
37
|
+
const envVar = PROVIDER_ENV_VAR[explicit];
|
|
38
|
+
const apiKey = input.env[envVar]?.trim();
|
|
39
|
+
if (!apiKey) {
|
|
40
|
+
return {
|
|
41
|
+
status: "skipped",
|
|
42
|
+
reason: "no-credentials",
|
|
43
|
+
provider: explicit,
|
|
44
|
+
message: `--web-search-provider=${explicit} set but ${envVar} is empty`,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
return { status: "configured", provider: explicit, apiKey };
|
|
48
|
+
}
|
|
49
|
+
const present = AUTO_PRIORITY.filter((p) => (input.env[PROVIDER_ENV_VAR[p]] ?? "").trim().length > 0);
|
|
50
|
+
if (present.length === 0) {
|
|
51
|
+
return { status: "skipped", reason: "no-credentials" };
|
|
52
|
+
}
|
|
53
|
+
const chosen = present[0];
|
|
54
|
+
const message = present.length > 1
|
|
55
|
+
? `multiple provider keys present (${present.join(", ")}); using ${chosen} — set WEB_SEARCH_PROVIDER to override`
|
|
56
|
+
: undefined;
|
|
57
|
+
return {
|
|
58
|
+
status: "configured",
|
|
59
|
+
provider: chosen,
|
|
60
|
+
apiKey: input.env[PROVIDER_ENV_VAR[chosen]].trim(),
|
|
61
|
+
message,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=selection.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"selection.js","sourceRoot":"","sources":["../../../src/extensions/web-search/selection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EACL,cAAc,GAGf,MAAM,YAAY,CAAC;AA6BpB,mDAAmD;AACnD,MAAM,CAAC,MAAM,gBAAgB,GAAiC;IAC5D,MAAM,EAAE,gBAAgB;IACxB,GAAG,EAAE,aAAa;IAClB,KAAK,EAAE,sBAAsB;CAC9B,CAAC;AAEF,yCAAyC;AACzC,MAAM,aAAa,GAA4B,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;AAE1E,MAAM,UAAU,cAAc,CAAC,CAAS;IACtC,OAAQ,cAAoC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAqB;IAClD,IAAI,KAAK,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;QAC9B,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;IAC3D,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,EAAE,CAAC;IAC1D,MAAM,QAAQ,GAAG,KAAK,CAAC,iBAAiB,IAAI,WAAW,CAAC;IAExD,IAAI,QAAQ,EAAE,CAAC;QACb,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,gCAAgC,QAAQ,uBAAuB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC3F,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC;QACzC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO;gBACL,MAAM,EAAE,SAAS;gBACjB,MAAM,EAAE,gBAAgB;gBACxB,QAAQ,EAAE,QAAQ;gBAClB,OAAO,EAAE,yBAAyB,QAAQ,YAAY,MAAM,WAAW;aACxE,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;IAC9D,CAAC;IAED,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAClC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAChE,CAAC;IAEF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;IACzD,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1B,MAAM,OAAO,GACX,OAAO,CAAC,MAAM,GAAG,CAAC;QAChB,CAAC,CAAC,mCAAmC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,MAAM,wCAAwC;QACjH,CAAC,CAAC,SAAS,CAAC;IAEhB,OAAO;QACL,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE,MAAM;QAChB,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAE,CAAC,IAAI,EAAE;QACnD,OAAO;KACR,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Native Pi tools `web_search` and `web_fetch`.
|
|
3
|
+
*
|
|
4
|
+
* Both tools follow the GitHub-extension convention: errors return a
|
|
5
|
+
* structured JSON payload (instead of throwing) and the JSON payload is
|
|
6
|
+
* stuffed into a single text content block via `jsonContent`. The agent
|
|
7
|
+
* sees the active provider name in every result so it can attribute
|
|
8
|
+
* findings.
|
|
9
|
+
*/
|
|
10
|
+
import { type ToolDefinition } from "@earendil-works/pi-coding-agent";
|
|
11
|
+
import type { RateLimiter } from "./rate-limit.js";
|
|
12
|
+
import type { Provider } from "./types.js";
|
|
13
|
+
export declare function buildWebSearchTools(provider: Provider, limiter: RateLimiter): ToolDefinition<any>[];
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Native Pi tools `web_search` and `web_fetch`.
|
|
3
|
+
*
|
|
4
|
+
* Both tools follow the GitHub-extension convention: errors return a
|
|
5
|
+
* structured JSON payload (instead of throwing) and the JSON payload is
|
|
6
|
+
* stuffed into a single text content block via `jsonContent`. The agent
|
|
7
|
+
* sees the active provider name in every result so it can attribute
|
|
8
|
+
* findings.
|
|
9
|
+
*/
|
|
10
|
+
import { Type } from "@sinclair/typebox";
|
|
11
|
+
import { defineTool } from "@earendil-works/pi-coding-agent";
|
|
12
|
+
import { extractTitle, htmlToText } from "./extract.js";
|
|
13
|
+
import { safeFetch, SafeFetchError } from "./safe-fetch.js";
|
|
14
|
+
const MAX_RESULTS = 10;
|
|
15
|
+
const DEFAULT_RESULTS = 5;
|
|
16
|
+
function jsonContent(data) {
|
|
17
|
+
return {
|
|
18
|
+
content: [{ type: "text", text: JSON.stringify(data, null, 2) }],
|
|
19
|
+
details: {},
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
function errorPayload(provider, err, hint) {
|
|
23
|
+
const e = err;
|
|
24
|
+
return {
|
|
25
|
+
error: e?.message ?? String(err),
|
|
26
|
+
provider,
|
|
27
|
+
code: e?.code,
|
|
28
|
+
status: e?.status,
|
|
29
|
+
hint: hint ?? null,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
function rateLimitPayload(provider) {
|
|
33
|
+
return {
|
|
34
|
+
error: "web-search rate limit reached for this run",
|
|
35
|
+
provider,
|
|
36
|
+
code: "rate-limited",
|
|
37
|
+
hint: "Increase the budget with --web-search-max-calls (CLI) or webSearchMaxCalls (run options), " +
|
|
38
|
+
"or summarize earlier findings instead of searching again.",
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
42
|
+
export function buildWebSearchTools(provider, limiter) {
|
|
43
|
+
const searchSchema = Type.Object({
|
|
44
|
+
query: Type.String({ description: "Search query", minLength: 1 }),
|
|
45
|
+
max_results: Type.Optional(Type.Integer({
|
|
46
|
+
description: `Max results to return (1-${MAX_RESULTS}, default ${DEFAULT_RESULTS}).`,
|
|
47
|
+
minimum: 1,
|
|
48
|
+
maximum: MAX_RESULTS,
|
|
49
|
+
})),
|
|
50
|
+
include_domains: Type.Optional(Type.Array(Type.String(), {
|
|
51
|
+
description: "Only return results whose URL host matches one of these domains.",
|
|
52
|
+
})),
|
|
53
|
+
exclude_domains: Type.Optional(Type.Array(Type.String(), {
|
|
54
|
+
description: "Drop results whose URL host matches any of these domains.",
|
|
55
|
+
})),
|
|
56
|
+
search_depth: Type.Optional(Type.Union([Type.Literal("basic"), Type.Literal("advanced")], {
|
|
57
|
+
description: "Advisory: providers that support a depth knob honor it; Brave ignores.",
|
|
58
|
+
})),
|
|
59
|
+
include_content: Type.Optional(Type.Boolean({
|
|
60
|
+
description: "Ask the provider to return extracted page content inline when supported (Tavily, Exa). Brave ignores.",
|
|
61
|
+
})),
|
|
62
|
+
});
|
|
63
|
+
const fetchSchema = Type.Object({
|
|
64
|
+
url: Type.String({
|
|
65
|
+
description: "Absolute http:// or https:// URL to download and extract readable text from.",
|
|
66
|
+
}),
|
|
67
|
+
});
|
|
68
|
+
const searchTool = defineTool({
|
|
69
|
+
name: "web_search",
|
|
70
|
+
label: "web_search",
|
|
71
|
+
description: `Search the web via ${provider.name}. Returns a ranked list of {title, url, snippet, ...} ` +
|
|
72
|
+
`entries plus (for some providers) an optional 'answer' summary. Use this when you need ` +
|
|
73
|
+
`external context not available in the local workspace.`,
|
|
74
|
+
parameters: searchSchema,
|
|
75
|
+
async execute(_id, params) {
|
|
76
|
+
if (!limiter.consume()) {
|
|
77
|
+
return jsonContent(rateLimitPayload(provider.name));
|
|
78
|
+
}
|
|
79
|
+
const max = Math.min(MAX_RESULTS, params.max_results ?? DEFAULT_RESULTS);
|
|
80
|
+
try {
|
|
81
|
+
const result = await provider.search({
|
|
82
|
+
query: params.query,
|
|
83
|
+
maxResults: max,
|
|
84
|
+
includeDomains: params.include_domains,
|
|
85
|
+
excludeDomains: params.exclude_domains,
|
|
86
|
+
searchDepth: params.search_depth,
|
|
87
|
+
includeContent: params.include_content,
|
|
88
|
+
});
|
|
89
|
+
return jsonContent(result);
|
|
90
|
+
}
|
|
91
|
+
catch (err) {
|
|
92
|
+
return jsonContent(errorPayload(provider.name, err));
|
|
93
|
+
}
|
|
94
|
+
},
|
|
95
|
+
});
|
|
96
|
+
const fetchTool = defineTool({
|
|
97
|
+
name: "web_fetch",
|
|
98
|
+
label: "web_fetch",
|
|
99
|
+
description: `Download a single web page (http/https only) and return its extracted readable text. ` +
|
|
100
|
+
`Uses ${provider.fetch ? `${provider.name}'s content endpoint` : "a generic safe-fetch with HTML stripping"}. ` +
|
|
101
|
+
`Body is capped at ~1 MiB and extracted text at ~200 KiB.`,
|
|
102
|
+
parameters: fetchSchema,
|
|
103
|
+
async execute(_id, params) {
|
|
104
|
+
if (!limiter.consume()) {
|
|
105
|
+
return jsonContent(rateLimitPayload(provider.name));
|
|
106
|
+
}
|
|
107
|
+
try {
|
|
108
|
+
if (provider.fetch) {
|
|
109
|
+
const result = await provider.fetch({ url: params.url });
|
|
110
|
+
return jsonContent(result);
|
|
111
|
+
}
|
|
112
|
+
const r = await safeFetch(params.url);
|
|
113
|
+
const text = htmlToText(r.body);
|
|
114
|
+
const title = extractTitle(r.body);
|
|
115
|
+
return jsonContent({
|
|
116
|
+
provider: "safe-fetch",
|
|
117
|
+
url: params.url,
|
|
118
|
+
resolvedUrl: r.finalUrl,
|
|
119
|
+
status: r.status,
|
|
120
|
+
contentType: r.contentType,
|
|
121
|
+
title,
|
|
122
|
+
text,
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
catch (err) {
|
|
126
|
+
const code = err instanceof SafeFetchError ? err.code : undefined;
|
|
127
|
+
return jsonContent({
|
|
128
|
+
...errorPayload(provider.name, err),
|
|
129
|
+
code: code ?? err?.code,
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
},
|
|
133
|
+
});
|
|
134
|
+
return [searchTool, fetchTool];
|
|
135
|
+
}
|
|
136
|
+
//# sourceMappingURL=tools.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../../../src/extensions/web-search/tools.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,IAAI,EAAe,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAuB,MAAM,iCAAiC,CAAC;AAElF,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAExD,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAG5D,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,SAAS,WAAW,CAAC,IAAa;IAChC,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;QACzE,OAAO,EAAE,EAAE;KACZ,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,QAAgB,EAAE,GAAY,EAAE,IAAa;IACjE,MAAM,CAAC,GAAG,GAAiD,CAAC;IAC5D,OAAO;QACL,KAAK,EAAE,CAAC,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC;QAChC,QAAQ;QACR,IAAI,EAAE,CAAC,EAAE,IAAI;QACb,MAAM,EAAE,CAAC,EAAE,MAAM;QACjB,IAAI,EAAE,IAAI,IAAI,IAAI;KACnB,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAgB;IACxC,OAAO;QACL,KAAK,EAAE,4CAA4C;QACnD,QAAQ;QACR,IAAI,EAAE,cAAc;QACpB,IAAI,EACF,4FAA4F;YAC5F,2DAA2D;KAC9D,CAAC;AACJ,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,mBAAmB,CACjC,QAAkB,EAClB,OAAoB;IAGpB,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC;QAC/B,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,cAAc,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;QACjE,WAAW,EAAE,IAAI,CAAC,QAAQ,CACxB,IAAI,CAAC,OAAO,CAAC;YACX,WAAW,EAAE,4BAA4B,WAAW,aAAa,eAAe,IAAI;YACpF,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,WAAW;SACrB,CAAC,CACH;QACD,eAAe,EAAE,IAAI,CAAC,QAAQ,CAC5B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE;YACxB,WAAW,EAAE,kEAAkE;SAChF,CAAC,CACH;QACD,eAAe,EAAE,IAAI,CAAC,QAAQ,CAC5B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE;YACxB,WAAW,EAAE,2DAA2D;SACzE,CAAC,CACH;QACD,YAAY,EAAE,IAAI,CAAC,QAAQ,CACzB,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE;YAC5D,WAAW,EAAE,wEAAwE;SACtF,CAAC,CACH;QACD,eAAe,EAAE,IAAI,CAAC,QAAQ,CAC5B,IAAI,CAAC,OAAO,CAAC;YACX,WAAW,EACT,uGAAuG;SAC1G,CAAC,CACH;KACF,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC;QAC9B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC;YACf,WAAW,EAAE,8EAA8E;SAC5F,CAAC;KACH,CAAC,CAAC;IAKH,MAAM,UAAU,GAAG,UAAU,CAAC;QAC5B,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,YAAY;QACnB,WAAW,EACT,sBAAsB,QAAQ,CAAC,IAAI,wDAAwD;YAC3F,yFAAyF;YACzF,wDAAwD;QAC1D,UAAU,EAAE,YAAY;QACxB,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,MAAmB;YACpC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;gBACvB,OAAO,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;YACtD,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,eAAe,CAAC,CAAC;YACzE,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;oBACnC,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,UAAU,EAAE,GAAG;oBACf,cAAc,EAAE,MAAM,CAAC,eAAe;oBACtC,cAAc,EAAE,MAAM,CAAC,eAAe;oBACtC,WAAW,EAAE,MAAM,CAAC,YAAY;oBAChC,cAAc,EAAE,MAAM,CAAC,eAAe;iBACvC,CAAC,CAAC;gBACH,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;YAC7B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,WAAW,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,UAAU,CAAC;QAC3B,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,WAAW;QAClB,WAAW,EACT,uFAAuF;YACvF,QAAQ,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,qBAAqB,CAAC,CAAC,CAAC,0CAA0C,IAAI;YAC/G,0DAA0D;QAC5D,UAAU,EAAE,WAAW;QACvB,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,MAAkB;YACnC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;gBACvB,OAAO,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;YACtD,CAAC;YACD,IAAI,CAAC;gBACH,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;oBACnB,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;oBACzD,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;gBAC7B,CAAC;gBACD,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACtC,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAChC,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACnC,OAAO,WAAW,CAAC;oBACjB,QAAQ,EAAE,YAAY;oBACtB,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,WAAW,EAAE,CAAC,CAAC,QAAQ;oBACvB,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,WAAW,EAAE,CAAC,CAAC,WAAW;oBAC1B,KAAK;oBACL,IAAI;iBACL,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,GAAG,GAAG,YAAY,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;gBAClE,OAAO,WAAW,CAAC;oBACjB,GAAG,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC;oBACnC,IAAI,EAAE,IAAI,IAAK,GAAyB,EAAE,IAAI;iBAC/C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;AACjC,CAAC"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared types for the web-search extension.
|
|
3
|
+
*
|
|
4
|
+
* The same `Provider` interface is implemented by tavily / brave / exa.
|
|
5
|
+
* Callers of the extension treat all three uniformly: the tool layer never
|
|
6
|
+
* branches on provider name except to surface `provider` in the result
|
|
7
|
+
* payload.
|
|
8
|
+
*/
|
|
9
|
+
export type ProviderName = "tavily" | "brave" | "exa";
|
|
10
|
+
export declare const PROVIDER_NAMES: readonly ProviderName[];
|
|
11
|
+
export interface SearchParams {
|
|
12
|
+
query: string;
|
|
13
|
+
/** Hard-capped at 10 by the tool layer. */
|
|
14
|
+
maxResults: number;
|
|
15
|
+
includeDomains?: string[];
|
|
16
|
+
excludeDomains?: string[];
|
|
17
|
+
/** Advisory: providers that have a "depth" knob map this; Brave ignores. */
|
|
18
|
+
searchDepth?: "basic" | "advanced";
|
|
19
|
+
/** Advisory: providers that can return extracted content do; Brave ignores. */
|
|
20
|
+
includeContent?: boolean;
|
|
21
|
+
}
|
|
22
|
+
export interface SearchResultItem {
|
|
23
|
+
title: string;
|
|
24
|
+
url: string;
|
|
25
|
+
snippet?: string;
|
|
26
|
+
content?: string;
|
|
27
|
+
score?: number;
|
|
28
|
+
publishedDate?: string;
|
|
29
|
+
}
|
|
30
|
+
export interface NormalizedSearchResult {
|
|
31
|
+
provider: ProviderName;
|
|
32
|
+
query: string;
|
|
33
|
+
results: SearchResultItem[];
|
|
34
|
+
/** Optional provider-supplied summary answer (Tavily can return this). */
|
|
35
|
+
answer?: string;
|
|
36
|
+
}
|
|
37
|
+
export interface FetchParams {
|
|
38
|
+
url: string;
|
|
39
|
+
}
|
|
40
|
+
export interface NormalizedFetchResult {
|
|
41
|
+
provider: ProviderName | "safe-fetch";
|
|
42
|
+
url: string;
|
|
43
|
+
/** Final URL after redirects (when known). */
|
|
44
|
+
resolvedUrl?: string;
|
|
45
|
+
status?: number;
|
|
46
|
+
contentType?: string;
|
|
47
|
+
/** Extracted readable text. Capped at ~200 KiB upstream. */
|
|
48
|
+
text: string;
|
|
49
|
+
title?: string;
|
|
50
|
+
}
|
|
51
|
+
/** Minimal fetch signature used by providers and safe-fetch for DI in tests. */
|
|
52
|
+
export type FetchImpl = (input: string | URL, init?: RequestInit) => Promise<Response>;
|
|
53
|
+
export interface Provider {
|
|
54
|
+
readonly name: ProviderName;
|
|
55
|
+
/** True if the provider's search endpoint can return extracted page content inline. */
|
|
56
|
+
readonly supportsExtractedContent: boolean;
|
|
57
|
+
search(params: SearchParams): Promise<NormalizedSearchResult>;
|
|
58
|
+
/**
|
|
59
|
+
* Optional native fetch endpoint. Tavily has /extract; Exa has /contents;
|
|
60
|
+
* Brave has nothing here. When omitted, the tool layer falls back to
|
|
61
|
+
* `safeFetch` + the HTML extractor.
|
|
62
|
+
*/
|
|
63
|
+
fetch?(params: FetchParams): Promise<NormalizedFetchResult>;
|
|
64
|
+
}
|
|
65
|
+
export type WebSearchSkipReason = "disabled-by-flag" | "no-credentials" | "invalid-config";
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared types for the web-search extension.
|
|
3
|
+
*
|
|
4
|
+
* The same `Provider` interface is implemented by tavily / brave / exa.
|
|
5
|
+
* Callers of the extension treat all three uniformly: the tool layer never
|
|
6
|
+
* branches on provider name except to surface `provider` in the result
|
|
7
|
+
* payload.
|
|
8
|
+
*/
|
|
9
|
+
export const PROVIDER_NAMES = ["tavily", "brave", "exa"];
|
|
10
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/extensions/web-search/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,MAAM,CAAC,MAAM,cAAc,GAA4B,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC"}
|
package/dist/run.d.ts
CHANGED
|
@@ -71,6 +71,36 @@ export interface RunOptions {
|
|
|
71
71
|
* Ignored when `sandbox: "none"`.
|
|
72
72
|
*/
|
|
73
73
|
allowedHttpHosts?: string[] | null;
|
|
74
|
+
/**
|
|
75
|
+
* Web-search extension toggle. Default: `true` (auto-enables when a
|
|
76
|
+
* provider API key env var is present). Set to `false` to suppress the
|
|
77
|
+
* `web_search` / `web_fetch` tools entirely.
|
|
78
|
+
*/
|
|
79
|
+
webSearch?: boolean;
|
|
80
|
+
/**
|
|
81
|
+
* Force a specific web-search provider: `"tavily" | "brave" | "exa"`.
|
|
82
|
+
* Overrides env-var-based auto-detection. The matching API key env var
|
|
83
|
+
* (`TAVILY_API_KEY`, `BRAVE_SEARCH_API_KEY`, `EXA_API_KEY`) must still
|
|
84
|
+
* be present; otherwise the extension skips with a warning.
|
|
85
|
+
*/
|
|
86
|
+
webSearchProvider?: "tavily" | "brave" | "exa";
|
|
87
|
+
/**
|
|
88
|
+
* Per-run cap on combined `web_search` + `web_fetch` calls. Default: 30.
|
|
89
|
+
* Once exceeded, further calls return a structured rate-limit error
|
|
90
|
+
* payload (the agent can recover).
|
|
91
|
+
*/
|
|
92
|
+
webSearchMaxCalls?: number;
|
|
93
|
+
/**
|
|
94
|
+
* File-search extension (FFF) toggle. Default: `true` — bundled and
|
|
95
|
+
* enabled for every run. Set to `false` to fall back to Pi's built-in
|
|
96
|
+
* `find`/`grep`.
|
|
97
|
+
*/
|
|
98
|
+
fileSearch?: boolean;
|
|
99
|
+
/**
|
|
100
|
+
* FFF mode. Default: `"override"` (FFF replaces built-in `find`/`grep`
|
|
101
|
+
* under the same names). The `PI_FFF_MODE` env var, if set, wins.
|
|
102
|
+
*/
|
|
103
|
+
fileSearchMode?: "override" | "tools-only" | "tools-and-ui";
|
|
74
104
|
/**
|
|
75
105
|
* Called for every emitted JSONL record in order. Same shape that the
|
|
76
106
|
* CLI writes to stdout, with `sessionId` and `timestamp` already injected.
|
|
@@ -143,6 +173,21 @@ export interface RunResult {
|
|
|
143
173
|
profile?: string;
|
|
144
174
|
toolCount: number;
|
|
145
175
|
};
|
|
176
|
+
webSearch?: {
|
|
177
|
+
status: "configured" | "skipped";
|
|
178
|
+
reason?: string;
|
|
179
|
+
message?: string;
|
|
180
|
+
provider?: string;
|
|
181
|
+
toolCount: number;
|
|
182
|
+
maxCalls?: number;
|
|
183
|
+
};
|
|
184
|
+
fileSearch?: {
|
|
185
|
+
status: "configured" | "skipped";
|
|
186
|
+
reason?: string;
|
|
187
|
+
message?: string;
|
|
188
|
+
mode?: string;
|
|
189
|
+
toolCount: number;
|
|
190
|
+
};
|
|
146
191
|
/** Every JSONL record the run emitted, in order. */
|
|
147
192
|
records: EmitterRecord[];
|
|
148
193
|
/** Warnings that would have gone to stderr in CLI mode. */
|