@cruxy/cli 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/session-factory.js +11 -0
- package/dist/config/schema.d.ts +134 -8
- package/dist/config/schema.js +45 -1
- package/dist/errors/constructors.d.ts +29 -0
- package/dist/errors/constructors.js +77 -0
- package/dist/errors/types.d.ts +17 -0
- package/dist/errors/types.js +26 -0
- package/dist/web/demarcate.d.ts +13 -0
- package/dist/web/demarcate.js +78 -0
- package/dist/web/fetch.d.ts +11 -0
- package/dist/web/fetch.js +174 -0
- package/dist/web/index.d.ts +7 -0
- package/dist/web/index.js +7 -0
- package/dist/web/provider.d.ts +29 -0
- package/dist/web/provider.js +77 -0
- package/dist/web/search.d.ts +17 -0
- package/dist/web/search.js +42 -0
- package/dist/web/ssrf.d.ts +55 -0
- package/dist/web/ssrf.js +223 -0
- package/dist/web/tools.d.ts +20 -0
- package/dist/web/tools.js +81 -0
- package/dist/web/types.d.ts +62 -0
- package/dist/web/types.js +1 -0
- package/package.json +2 -1
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { webBlockedHost, webFetchFailed } from "../errors/index.js";
|
|
2
|
+
import { demarcatePage } from "./demarcate.js";
|
|
3
|
+
import { BlockedHostError, HostUnresolvedError, assertFetchable, createPinnedDispatcher, defaultResolveHost, } from "./ssrf.js";
|
|
4
|
+
import { fetch as undiciFetch } from "undici";
|
|
5
|
+
/** Content types `web_fetch` will read as text; anything else is refused. */
|
|
6
|
+
function isTextualType(contentType) {
|
|
7
|
+
const t = contentType.toLowerCase();
|
|
8
|
+
return (t.startsWith("text/") ||
|
|
9
|
+
t === "application/json" ||
|
|
10
|
+
t === "application/xml" ||
|
|
11
|
+
t === "application/xhtml+xml" ||
|
|
12
|
+
t === "application/javascript" ||
|
|
13
|
+
t.endsWith("+json") ||
|
|
14
|
+
t.endsWith("+xml"));
|
|
15
|
+
}
|
|
16
|
+
/** Read a response body up to `maxBytes`, stopping early; reports truncation. */
|
|
17
|
+
async function readCapped(res, maxBytes) {
|
|
18
|
+
const reader = res.body?.getReader?.();
|
|
19
|
+
if (!reader) {
|
|
20
|
+
// No stream (e.g. some test doubles): fall back to a full read, then cap.
|
|
21
|
+
const buf = new Uint8Array(await res.arrayBuffer());
|
|
22
|
+
if (buf.byteLength <= maxBytes)
|
|
23
|
+
return { bytes: buf, truncated: false };
|
|
24
|
+
return { bytes: buf.subarray(0, maxBytes), truncated: true };
|
|
25
|
+
}
|
|
26
|
+
const chunks = [];
|
|
27
|
+
let total = 0;
|
|
28
|
+
let truncated = false;
|
|
29
|
+
for (;;) {
|
|
30
|
+
const { done, value } = await reader.read();
|
|
31
|
+
if (done)
|
|
32
|
+
break;
|
|
33
|
+
if (value) {
|
|
34
|
+
chunks.push(value);
|
|
35
|
+
total += value.byteLength;
|
|
36
|
+
if (total >= maxBytes) {
|
|
37
|
+
truncated = true;
|
|
38
|
+
await reader.cancel().catch(() => { });
|
|
39
|
+
break;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
const joined = new Uint8Array(total);
|
|
44
|
+
let off = 0;
|
|
45
|
+
for (const c of chunks) {
|
|
46
|
+
joined.set(c, off);
|
|
47
|
+
off += c.byteLength;
|
|
48
|
+
}
|
|
49
|
+
const bytes = joined.byteLength > maxBytes ? joined.subarray(0, maxBytes) : joined;
|
|
50
|
+
return { bytes, truncated };
|
|
51
|
+
}
|
|
52
|
+
/** Collapse an HTML document to readable text: drop script/style, strip tags. */
|
|
53
|
+
function htmlToText(html) {
|
|
54
|
+
return html
|
|
55
|
+
.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, " ")
|
|
56
|
+
.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, " ")
|
|
57
|
+
.replace(/<!--[\s\S]*?-->/g, " ")
|
|
58
|
+
.replace(/<[^>]+>/g, " ")
|
|
59
|
+
.replace(/ /gi, " ")
|
|
60
|
+
.replace(/&/gi, "&")
|
|
61
|
+
.replace(/</gi, "<")
|
|
62
|
+
.replace(/>/gi, ">")
|
|
63
|
+
.replace(/[ \t\r\f\v]+/g, " ")
|
|
64
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
65
|
+
.trim();
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Fetch one URL as text, enforcing every bound. Returns a {@link FetchResult}.
|
|
69
|
+
* Throws {@link webBlockedHost} for an SSRF-refused URL (never dispatched),
|
|
70
|
+
* {@link webFetchFailed} for a network error / timeout / non-text or over-redirect
|
|
71
|
+
* response. A page fetched successfully but empty of text is a valid result with
|
|
72
|
+
* empty `text` (the tool surfaces it as `ok:true`, not an error).
|
|
73
|
+
*/
|
|
74
|
+
export async function fetchUrl(rawUrl, config, deps = {}) {
|
|
75
|
+
// Default to undici's OWN fetch, not the global one. The SSRF guard pins the
|
|
76
|
+
// connection with an undici `Agent` dispatcher (createPinnedDispatcher); that
|
|
77
|
+
// dispatcher must be driven by the SAME undici that produced it. The global
|
|
78
|
+
// `fetch` is backed by the undici BUNDLED IN NODE, whose version drifts by Node
|
|
79
|
+
// release (v6 on Node 20, v8 on Node 24+), and pairing our dep's v6 Agent with a
|
|
80
|
+
// bundled-v8 fetch handler throws `InvalidArgumentError: invalid onError method`.
|
|
81
|
+
// Using our dep's fetch keeps dispatcher + handler on one version on every Node.
|
|
82
|
+
// (Cast: we only touch the shared WHATWG Response surface — status/headers/body.)
|
|
83
|
+
const fetchImpl = deps.fetchImpl ?? undiciFetch;
|
|
84
|
+
const resolveHost = deps.resolveHost ?? defaultResolveHost;
|
|
85
|
+
let url;
|
|
86
|
+
try {
|
|
87
|
+
url = new URL(rawUrl);
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
throw webFetchFailed(rawUrl, new Error("not a valid absolute URL"));
|
|
91
|
+
}
|
|
92
|
+
const controller = new AbortController();
|
|
93
|
+
const timer = setTimeout(() => controller.abort(), config.timeoutMs);
|
|
94
|
+
// Dispatchers pin each hop's connection to its validated address; closed at the
|
|
95
|
+
// end (not per-hop) because the response body is streamed after the loop.
|
|
96
|
+
const dispatchers = [];
|
|
97
|
+
try {
|
|
98
|
+
let hops = 0;
|
|
99
|
+
for (;;) {
|
|
100
|
+
// Re-run the SSRF guard on EVERY hop so a redirect can't reach a private IP.
|
|
101
|
+
let validated;
|
|
102
|
+
try {
|
|
103
|
+
validated = await assertFetchable(url, resolveHost, config.allowPrivateHosts);
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
if (err instanceof BlockedHostError)
|
|
107
|
+
throw webBlockedHost(url.toString(), err.message);
|
|
108
|
+
if (err instanceof HostUnresolvedError)
|
|
109
|
+
throw webFetchFailed(url.toString(), err);
|
|
110
|
+
throw err;
|
|
111
|
+
}
|
|
112
|
+
// Pin the connection to the address(es) the guard just validated so a
|
|
113
|
+
// rebind can't flip the hostname to an internal IP between check and connect.
|
|
114
|
+
let dispatcher;
|
|
115
|
+
if (validated.length > 0) {
|
|
116
|
+
dispatcher = createPinnedDispatcher(validated);
|
|
117
|
+
dispatchers.push(dispatcher);
|
|
118
|
+
}
|
|
119
|
+
let res;
|
|
120
|
+
try {
|
|
121
|
+
res = await fetchImpl(url.toString(), {
|
|
122
|
+
method: "GET",
|
|
123
|
+
redirect: "manual",
|
|
124
|
+
signal: controller.signal,
|
|
125
|
+
headers: {
|
|
126
|
+
accept: "text/html,text/plain,application/json;q=0.9,*/*;q=0.1",
|
|
127
|
+
},
|
|
128
|
+
...(dispatcher ? { dispatcher } : {}),
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
catch (err) {
|
|
132
|
+
throw webFetchFailed(url.toString(), err);
|
|
133
|
+
}
|
|
134
|
+
// Manual redirect handling — re-validate the next hop through the guard.
|
|
135
|
+
if (res.status >= 300 && res.status < 400) {
|
|
136
|
+
const location = res.headers.get("location");
|
|
137
|
+
if (!location)
|
|
138
|
+
throw webFetchFailed(url.toString(), new Error(`redirect ${res.status} with no Location header`));
|
|
139
|
+
if (hops >= config.maxRedirects)
|
|
140
|
+
throw webFetchFailed(rawUrl, new Error(`exceeded ${config.maxRedirects} redirects`));
|
|
141
|
+
hops += 1;
|
|
142
|
+
url = new URL(location, url); // resolve relative redirects
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
if (!res.ok)
|
|
146
|
+
throw webFetchFailed(url.toString(), new Error(`server responded ${res.status} ${res.statusText}`));
|
|
147
|
+
const contentType = (res.headers.get("content-type") ?? "")
|
|
148
|
+
.split(";")[0]
|
|
149
|
+
.trim()
|
|
150
|
+
.toLowerCase();
|
|
151
|
+
if (contentType !== "" && !isTextualType(contentType))
|
|
152
|
+
throw webFetchFailed(url.toString(), new Error(`unsupported content type "${contentType}" (text only)`));
|
|
153
|
+
const { bytes, truncated } = await readCapped(res, config.fetchMaxBytes);
|
|
154
|
+
const decoded = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
|
|
155
|
+
const text = contentType === "text/html" || contentType === "application/xhtml+xml"
|
|
156
|
+
? htmlToText(decoded)
|
|
157
|
+
: decoded;
|
|
158
|
+
return { url: url.toString(), contentType, text, truncated };
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
finally {
|
|
162
|
+
clearTimeout(timer);
|
|
163
|
+
for (const d of dispatchers)
|
|
164
|
+
void d.close().catch(() => { });
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
/** Fetch a URL and render it as a demarcated, scrubbed, untrusted-data block. */
|
|
168
|
+
export async function runWebFetch(rawUrl, config, deps = {}) {
|
|
169
|
+
const result = await fetchUrl(rawUrl, config, deps);
|
|
170
|
+
const note = result.truncated
|
|
171
|
+
? `content truncated at ${config.fetchMaxBytes} bytes`
|
|
172
|
+
: undefined;
|
|
173
|
+
return demarcatePage(result.url, result.text, note);
|
|
174
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { SearchProvider, SearchResult, WebConfig, WebDeps } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Search-provider seam (C.20). `createSearchProvider` resolves the configured
|
|
4
|
+
* backend and its API key. There is no gateway search endpoint today (the SDK
|
|
5
|
+
* Provider only streams chat), so the direct Tavily provider sits behind the seam;
|
|
6
|
+
* a gateway-backed provider would slot in here with no change to the tools.
|
|
7
|
+
*
|
|
8
|
+
* A missing key or unknown provider throws {@link webUnavailable} — a coded,
|
|
9
|
+
* actionable failure, NEVER a silent empty result (the honesty rule: "no provider"
|
|
10
|
+
* must not read as "no results").
|
|
11
|
+
*/
|
|
12
|
+
export declare function createSearchProvider(config: WebConfig, deps?: WebDeps): SearchProvider;
|
|
13
|
+
/**
|
|
14
|
+
* Tavily direct provider. The API key travels ONLY in the `Authorization` header
|
|
15
|
+
* (never in the query text or URL, never logged), so the outbound query carries no
|
|
16
|
+
* secret. Any non-2xx / network / timeout outcome becomes a thrown
|
|
17
|
+
* {@link webSearchFailed}; an empty `results` array is a legitimate zero-result
|
|
18
|
+
* outcome and is returned as `[]` (the tool renders "no results" as `ok:true`).
|
|
19
|
+
*/
|
|
20
|
+
export declare class TavilyProvider implements SearchProvider {
|
|
21
|
+
private readonly apiKey;
|
|
22
|
+
private readonly fetchImpl;
|
|
23
|
+
readonly name = "tavily";
|
|
24
|
+
constructor(apiKey: string, fetchImpl: typeof fetch);
|
|
25
|
+
search(query: string, opts: {
|
|
26
|
+
maxResults: number;
|
|
27
|
+
signal: AbortSignal;
|
|
28
|
+
}): Promise<SearchResult[]>;
|
|
29
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { webSearchFailed, webUnavailable } from "../errors/index.js";
|
|
2
|
+
/**
|
|
3
|
+
* Search-provider seam (C.20). `createSearchProvider` resolves the configured
|
|
4
|
+
* backend and its API key. There is no gateway search endpoint today (the SDK
|
|
5
|
+
* Provider only streams chat), so the direct Tavily provider sits behind the seam;
|
|
6
|
+
* a gateway-backed provider would slot in here with no change to the tools.
|
|
7
|
+
*
|
|
8
|
+
* A missing key or unknown provider throws {@link webUnavailable} — a coded,
|
|
9
|
+
* actionable failure, NEVER a silent empty result (the honesty rule: "no provider"
|
|
10
|
+
* must not read as "no results").
|
|
11
|
+
*/
|
|
12
|
+
export function createSearchProvider(config, deps = {}) {
|
|
13
|
+
const env = deps.env ?? process.env;
|
|
14
|
+
const apiKey = env[config.apiKeyEnv];
|
|
15
|
+
if (!apiKey || apiKey.trim() === "") {
|
|
16
|
+
throw webUnavailable(config.provider, config.apiKeyEnv);
|
|
17
|
+
}
|
|
18
|
+
const fetchImpl = deps.fetchImpl ?? fetch;
|
|
19
|
+
switch (config.provider) {
|
|
20
|
+
case "tavily":
|
|
21
|
+
return new TavilyProvider(apiKey, fetchImpl);
|
|
22
|
+
default:
|
|
23
|
+
// Exhaustive today (the enum has one member); keeps the seam honest if a
|
|
24
|
+
// provider is added to the schema but not wired here.
|
|
25
|
+
throw webUnavailable(config.provider, config.apiKeyEnv);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
const TAVILY_ENDPOINT = "https://api.tavily.com/search";
|
|
29
|
+
/**
|
|
30
|
+
* Tavily direct provider. The API key travels ONLY in the `Authorization` header
|
|
31
|
+
* (never in the query text or URL, never logged), so the outbound query carries no
|
|
32
|
+
* secret. Any non-2xx / network / timeout outcome becomes a thrown
|
|
33
|
+
* {@link webSearchFailed}; an empty `results` array is a legitimate zero-result
|
|
34
|
+
* outcome and is returned as `[]` (the tool renders "no results" as `ok:true`).
|
|
35
|
+
*/
|
|
36
|
+
export class TavilyProvider {
|
|
37
|
+
apiKey;
|
|
38
|
+
fetchImpl;
|
|
39
|
+
name = "tavily";
|
|
40
|
+
constructor(apiKey, fetchImpl) {
|
|
41
|
+
this.apiKey = apiKey;
|
|
42
|
+
this.fetchImpl = fetchImpl;
|
|
43
|
+
}
|
|
44
|
+
async search(query, opts) {
|
|
45
|
+
let res;
|
|
46
|
+
try {
|
|
47
|
+
res = await this.fetchImpl(TAVILY_ENDPOINT, {
|
|
48
|
+
method: "POST",
|
|
49
|
+
headers: {
|
|
50
|
+
"content-type": "application/json",
|
|
51
|
+
authorization: `Bearer ${this.apiKey}`,
|
|
52
|
+
},
|
|
53
|
+
body: JSON.stringify({ query, max_results: opts.maxResults }),
|
|
54
|
+
signal: opts.signal,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
catch (err) {
|
|
58
|
+
throw webSearchFailed(err);
|
|
59
|
+
}
|
|
60
|
+
if (!res.ok) {
|
|
61
|
+
throw webSearchFailed(new Error(`provider responded ${res.status} ${res.statusText}`));
|
|
62
|
+
}
|
|
63
|
+
let data;
|
|
64
|
+
try {
|
|
65
|
+
data = (await res.json());
|
|
66
|
+
}
|
|
67
|
+
catch (err) {
|
|
68
|
+
throw webSearchFailed(err);
|
|
69
|
+
}
|
|
70
|
+
const results = Array.isArray(data.results) ? data.results : [];
|
|
71
|
+
return results.map((r) => ({
|
|
72
|
+
title: typeof r.title === "string" ? r.title : "",
|
|
73
|
+
url: typeof r.url === "string" ? r.url : "",
|
|
74
|
+
snippet: typeof r.content === "string" ? r.content : "",
|
|
75
|
+
}));
|
|
76
|
+
}
|
|
77
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { SearchResult, WebConfig, WebDeps } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Apply the top-N and per-snippet caps. The provider is asked for `maxResults`,
|
|
4
|
+
* but we re-cap defensively (a provider may over-return) and truncate each snippet
|
|
5
|
+
* with a visible marker so a long body can't blow the context budget.
|
|
6
|
+
*/
|
|
7
|
+
export declare function boundResults(results: SearchResult[], config: WebConfig): SearchResult[];
|
|
8
|
+
/**
|
|
9
|
+
* Run one web search and return the demarcated, bounded, scrubbed result block.
|
|
10
|
+
*
|
|
11
|
+
* Throws coded errors for the two failure modes the caller must NOT collapse into
|
|
12
|
+
* an empty result: {@link webUnavailable} (no provider/key — via
|
|
13
|
+
* `createSearchProvider`) and {@link webSearchFailed} (the search errored — via the
|
|
14
|
+
* provider). A search that runs and finds nothing returns a demarcated "(no
|
|
15
|
+
* results)" block — an ordinary success the tool surfaces as `ok:true`.
|
|
16
|
+
*/
|
|
17
|
+
export declare function runWebSearch(query: string, config: WebConfig, deps?: WebDeps): Promise<string>;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { createSearchProvider } from "./provider.js";
|
|
2
|
+
import { demarcateSearchResults } from "./demarcate.js";
|
|
3
|
+
/**
|
|
4
|
+
* Apply the top-N and per-snippet caps. The provider is asked for `maxResults`,
|
|
5
|
+
* but we re-cap defensively (a provider may over-return) and truncate each snippet
|
|
6
|
+
* with a visible marker so a long body can't blow the context budget.
|
|
7
|
+
*/
|
|
8
|
+
export function boundResults(results, config) {
|
|
9
|
+
return results.slice(0, config.maxResults).map((r) => {
|
|
10
|
+
if (r.snippet.length <= config.snippetMaxChars)
|
|
11
|
+
return r;
|
|
12
|
+
return {
|
|
13
|
+
...r,
|
|
14
|
+
snippet: r.snippet.slice(0, config.snippetMaxChars) + " …[snippet truncated]",
|
|
15
|
+
};
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Run one web search and return the demarcated, bounded, scrubbed result block.
|
|
20
|
+
*
|
|
21
|
+
* Throws coded errors for the two failure modes the caller must NOT collapse into
|
|
22
|
+
* an empty result: {@link webUnavailable} (no provider/key — via
|
|
23
|
+
* `createSearchProvider`) and {@link webSearchFailed} (the search errored — via the
|
|
24
|
+
* provider). A search that runs and finds nothing returns a demarcated "(no
|
|
25
|
+
* results)" block — an ordinary success the tool surfaces as `ok:true`.
|
|
26
|
+
*/
|
|
27
|
+
export async function runWebSearch(query, config, deps = {}) {
|
|
28
|
+
const provider = createSearchProvider(config, deps);
|
|
29
|
+
const controller = new AbortController();
|
|
30
|
+
const timer = setTimeout(() => controller.abort(), config.timeoutMs);
|
|
31
|
+
try {
|
|
32
|
+
const raw = await provider.search(query, {
|
|
33
|
+
maxResults: config.maxResults,
|
|
34
|
+
signal: controller.signal,
|
|
35
|
+
});
|
|
36
|
+
const bounded = boundResults(raw, config);
|
|
37
|
+
return demarcateSearchResults(query, bounded);
|
|
38
|
+
}
|
|
39
|
+
finally {
|
|
40
|
+
clearTimeout(timer);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { type Dispatcher } from "undici";
|
|
2
|
+
import type { HostResolver } from "./types.js";
|
|
3
|
+
/**
|
|
4
|
+
* SSRF guard (C.20). A URL the MODEL chose must not be able to reach the user's
|
|
5
|
+
* internal network, cloud metadata service, or loopback interface. Three layers:
|
|
6
|
+
*
|
|
7
|
+
* 1. Scheme allowlist — only `http`/`https` (blocks `file:`, `data:`, `gopher:`…).
|
|
8
|
+
* 2. Address check — the hostname is RESOLVED and every returned address is
|
|
9
|
+
* checked against private/loopback/link-local/reserved ranges (v4 and v6,
|
|
10
|
+
* including IPv4-mapped and alternate IP encodings).
|
|
11
|
+
* 3. Connection pinning — the connection is pinned to the exact address the check
|
|
12
|
+
* validated (see {@link createPinnedDispatcher}). Without this, resolving-then-
|
|
13
|
+
* fetching re-resolves the hostname at connect time, so a DNS-rebind attacker
|
|
14
|
+
* can pass the check with a public IP and have the socket land on 127.0.0.1
|
|
15
|
+
* (a TOCTOU hole). Pinning closes it: the connection can only reach a validated
|
|
16
|
+
* address, and the host header / TLS SNI still carry the original hostname.
|
|
17
|
+
*
|
|
18
|
+
* The check runs BEFORE any request is dispatched, and again on every redirect hop
|
|
19
|
+
* (see fetch.ts). A block is a security refusal, distinct from a network failure.
|
|
20
|
+
*/
|
|
21
|
+
/** Default resolver: node's `dns.lookup` returning ALL addresses. */
|
|
22
|
+
export declare const defaultResolveHost: HostResolver;
|
|
23
|
+
/** Thrown when a URL is refused pre-dispatch; carries a human reason. */
|
|
24
|
+
export declare class BlockedHostError extends Error {
|
|
25
|
+
}
|
|
26
|
+
/** Thrown when the host could not be resolved (a network failure, not a block). */
|
|
27
|
+
export declare class HostUnresolvedError extends Error {
|
|
28
|
+
}
|
|
29
|
+
/** True if an address (v4 or v6) is in a range `web_fetch` must never reach. */
|
|
30
|
+
export declare function isBlockedAddress(addr: string): boolean;
|
|
31
|
+
/**
|
|
32
|
+
* A `dns.lookup`-compatible function that ignores the hostname and always hands
|
|
33
|
+
* back one of the pre-validated `addresses`. This is what pins a connection to the
|
|
34
|
+
* address the SSRF check already approved, defeating DNS rebinding: the socket can
|
|
35
|
+
* only reach a validated IP, never a value re-resolved at connect time.
|
|
36
|
+
*/
|
|
37
|
+
export declare function pinnedLookup(addresses: string[]): (_hostname: string, options: unknown, callback: (err: NodeJS.ErrnoException | null, address: string | {
|
|
38
|
+
address: string;
|
|
39
|
+
family: number;
|
|
40
|
+
}[], family?: number) => void) => void;
|
|
41
|
+
/** An undici dispatcher whose connections are pinned to `addresses`. */
|
|
42
|
+
export declare function createPinnedDispatcher(addresses: string[]): Dispatcher;
|
|
43
|
+
/**
|
|
44
|
+
* Assert that `url` may be fetched and return the validated addresses to pin the
|
|
45
|
+
* connection to. Throws {@link BlockedHostError} for a bad scheme or a host
|
|
46
|
+
* resolving into a blocked range, or {@link HostUnresolvedError} if the host cannot
|
|
47
|
+
* be resolved.
|
|
48
|
+
*
|
|
49
|
+
* The returned list is the exact set of addresses the caller must restrict the
|
|
50
|
+
* connection to (via {@link createPinnedDispatcher}). An empty list means "do not
|
|
51
|
+
* pin" — only returned under `allowPrivate`, the deliberate internal-network escape
|
|
52
|
+
* hatch (from `web.allowPrivateHosts`), which bypasses the address check and lets
|
|
53
|
+
* the transport resolve normally. The scheme check always applies.
|
|
54
|
+
*/
|
|
55
|
+
export declare function assertFetchable(url: URL, resolveHost: HostResolver, allowPrivate: boolean): Promise<string[]>;
|
package/dist/web/ssrf.js
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import { lookup } from "node:dns";
|
|
2
|
+
import { Agent } from "undici";
|
|
3
|
+
/**
|
|
4
|
+
* SSRF guard (C.20). A URL the MODEL chose must not be able to reach the user's
|
|
5
|
+
* internal network, cloud metadata service, or loopback interface. Three layers:
|
|
6
|
+
*
|
|
7
|
+
* 1. Scheme allowlist — only `http`/`https` (blocks `file:`, `data:`, `gopher:`…).
|
|
8
|
+
* 2. Address check — the hostname is RESOLVED and every returned address is
|
|
9
|
+
* checked against private/loopback/link-local/reserved ranges (v4 and v6,
|
|
10
|
+
* including IPv4-mapped and alternate IP encodings).
|
|
11
|
+
* 3. Connection pinning — the connection is pinned to the exact address the check
|
|
12
|
+
* validated (see {@link createPinnedDispatcher}). Without this, resolving-then-
|
|
13
|
+
* fetching re-resolves the hostname at connect time, so a DNS-rebind attacker
|
|
14
|
+
* can pass the check with a public IP and have the socket land on 127.0.0.1
|
|
15
|
+
* (a TOCTOU hole). Pinning closes it: the connection can only reach a validated
|
|
16
|
+
* address, and the host header / TLS SNI still carry the original hostname.
|
|
17
|
+
*
|
|
18
|
+
* The check runs BEFORE any request is dispatched, and again on every redirect hop
|
|
19
|
+
* (see fetch.ts). A block is a security refusal, distinct from a network failure.
|
|
20
|
+
*/
|
|
21
|
+
/** Default resolver: node's `dns.lookup` returning ALL addresses. */
|
|
22
|
+
export const defaultResolveHost = (host) => new Promise((resolve, reject) => {
|
|
23
|
+
lookup(host, { all: true }, (err, addresses) => {
|
|
24
|
+
if (err)
|
|
25
|
+
reject(err);
|
|
26
|
+
else
|
|
27
|
+
resolve(addresses.map((a) => a.address));
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
/** Thrown when a URL is refused pre-dispatch; carries a human reason. */
|
|
31
|
+
export class BlockedHostError extends Error {
|
|
32
|
+
}
|
|
33
|
+
/** Thrown when the host could not be resolved (a network failure, not a block). */
|
|
34
|
+
export class HostUnresolvedError extends Error {
|
|
35
|
+
}
|
|
36
|
+
/** Parse a dotted-quad IPv4 string into its four octets, or null. */
|
|
37
|
+
function parseIpv4(ip) {
|
|
38
|
+
const m = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(ip);
|
|
39
|
+
if (!m)
|
|
40
|
+
return null;
|
|
41
|
+
const octets = m.slice(1, 5).map((s) => Number(s));
|
|
42
|
+
if (octets.some((o) => o > 255))
|
|
43
|
+
return null;
|
|
44
|
+
return octets;
|
|
45
|
+
}
|
|
46
|
+
/** True if an IPv4 address falls in a private/loopback/link-local/reserved range. */
|
|
47
|
+
function isBlockedIpv4(ip) {
|
|
48
|
+
const octets = parseIpv4(ip);
|
|
49
|
+
if (!octets)
|
|
50
|
+
return false;
|
|
51
|
+
const [a, b] = octets;
|
|
52
|
+
if (a === 0)
|
|
53
|
+
return true; // 0.0.0.0/8 "this network" / unspecified
|
|
54
|
+
if (a === 10)
|
|
55
|
+
return true; // 10.0.0.0/8 private
|
|
56
|
+
if (a === 127)
|
|
57
|
+
return true; // 127.0.0.0/8 loopback
|
|
58
|
+
if (a === 169 && b === 254)
|
|
59
|
+
return true; // 169.254.0.0/16 link-local (incl. 169.254.169.254 metadata)
|
|
60
|
+
if (a === 172 && b >= 16 && b <= 31)
|
|
61
|
+
return true; // 172.16.0.0/12 private
|
|
62
|
+
if (a === 192 && b === 168)
|
|
63
|
+
return true; // 192.168.0.0/16 private
|
|
64
|
+
if (a === 100 && b >= 64 && b <= 127)
|
|
65
|
+
return true; // 100.64.0.0/10 CGNAT
|
|
66
|
+
if (a === 198 && (b === 18 || b === 19))
|
|
67
|
+
return true; // 198.18.0.0/15 benchmarking
|
|
68
|
+
if (a === 255 && b === 255)
|
|
69
|
+
return true; // broadcast-ish
|
|
70
|
+
return false;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Expand an IPv6 literal into its 8 sixteen-bit groups, or null if unparseable.
|
|
74
|
+
* Handles `::` compression and an embedded IPv4 tail (`::ffff:127.0.0.1`).
|
|
75
|
+
*/
|
|
76
|
+
function parseIpv6(input) {
|
|
77
|
+
let s = input;
|
|
78
|
+
const tail = [];
|
|
79
|
+
// Peel off a trailing dotted-quad (IPv4-mapped/-compatible forms).
|
|
80
|
+
const v4 = /(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/.exec(s);
|
|
81
|
+
if (v4) {
|
|
82
|
+
const o = parseIpv4(v4[1]);
|
|
83
|
+
if (!o)
|
|
84
|
+
return null;
|
|
85
|
+
tail.push((o[0] << 8) | o[1], (o[2] << 8) | o[3]);
|
|
86
|
+
s = s.slice(0, v4.index); // leaves a trailing ':' before the compression split
|
|
87
|
+
}
|
|
88
|
+
const halves = s.split("::");
|
|
89
|
+
if (halves.length > 2)
|
|
90
|
+
return null; // more than one "::" is illegal
|
|
91
|
+
const head = halves[0] ? halves[0].split(":").filter(Boolean) : [];
|
|
92
|
+
const rest = halves[1] ? halves[1].split(":").filter(Boolean) : [];
|
|
93
|
+
const toNums = (groups) => {
|
|
94
|
+
const out = [];
|
|
95
|
+
for (const g of groups) {
|
|
96
|
+
if (!/^[0-9a-f]{1,4}$/.test(g))
|
|
97
|
+
return null;
|
|
98
|
+
out.push(parseInt(g, 16));
|
|
99
|
+
}
|
|
100
|
+
return out;
|
|
101
|
+
};
|
|
102
|
+
const headNums = toNums(head);
|
|
103
|
+
const restNums = toNums(rest);
|
|
104
|
+
if (!headNums || !restNums)
|
|
105
|
+
return null;
|
|
106
|
+
let groups;
|
|
107
|
+
if (halves.length === 2) {
|
|
108
|
+
const fill = 8 - (headNums.length + restNums.length + tail.length);
|
|
109
|
+
if (fill < 0)
|
|
110
|
+
return null;
|
|
111
|
+
groups = [
|
|
112
|
+
...headNums,
|
|
113
|
+
...Array(fill).fill(0),
|
|
114
|
+
...restNums,
|
|
115
|
+
...tail,
|
|
116
|
+
];
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
groups = [...headNums, ...tail];
|
|
120
|
+
}
|
|
121
|
+
return groups.length === 8 ? groups : null;
|
|
122
|
+
}
|
|
123
|
+
/** True if an expanded IPv6 address is in a range `web_fetch` must never reach. */
|
|
124
|
+
function isBlockedIpv6(g) {
|
|
125
|
+
// IPv4-mapped (::ffff:a.b.c.d) and IPv4-compatible (::a.b.c.d): check the v4 part.
|
|
126
|
+
const firstFiveZero = g.slice(0, 5).every((x) => x === 0);
|
|
127
|
+
const firstSixZero = firstFiveZero && g[5] === 0;
|
|
128
|
+
const embedded = `${g[6] >> 8}.${g[6] & 0xff}.${g[7] >> 8}.${g[7] & 0xff}`;
|
|
129
|
+
if (firstFiveZero && g[5] === 0xffff)
|
|
130
|
+
return isBlockedIpv4(embedded); // ::ffff:x
|
|
131
|
+
if (firstSixZero &&
|
|
132
|
+
!(g[6] === 0 && g[7] === 0) &&
|
|
133
|
+
!(g[6] === 0 && g[7] === 1))
|
|
134
|
+
return isBlockedIpv4(embedded); // ::x.y.z.w (IPv4-compatible, deprecated)
|
|
135
|
+
if (g.every((x) => x === 0))
|
|
136
|
+
return true; // :: unspecified
|
|
137
|
+
if (firstSixZero && g[6] === 0 && g[7] === 1)
|
|
138
|
+
return true; // ::1 loopback
|
|
139
|
+
if ((g[0] & 0xffc0) === 0xfe80)
|
|
140
|
+
return true; // fe80::/10 link-local (fe80–febf)
|
|
141
|
+
if ((g[0] & 0xfe00) === 0xfc00)
|
|
142
|
+
return true; // fc00::/7 unique-local (fc00–fdff)
|
|
143
|
+
if ((g[0] & 0xff00) === 0xff00)
|
|
144
|
+
return true; // ff00::/8 multicast
|
|
145
|
+
return false;
|
|
146
|
+
}
|
|
147
|
+
/** True if an address (v4 or v6) is in a range `web_fetch` must never reach. */
|
|
148
|
+
export function isBlockedAddress(addr) {
|
|
149
|
+
// Strip IPv6 brackets and a scope/zone id (e.g. fe80::1%eth0).
|
|
150
|
+
const ip = addr
|
|
151
|
+
.trim()
|
|
152
|
+
.toLowerCase()
|
|
153
|
+
.replace(/^\[|\]$/g, "")
|
|
154
|
+
.split("%")[0];
|
|
155
|
+
if (ip.includes(":")) {
|
|
156
|
+
const groups = parseIpv6(ip);
|
|
157
|
+
if (!groups)
|
|
158
|
+
return true; // fail closed: an unparseable colon-address is refused
|
|
159
|
+
return isBlockedIpv6(groups);
|
|
160
|
+
}
|
|
161
|
+
return isBlockedIpv4(ip);
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* A `dns.lookup`-compatible function that ignores the hostname and always hands
|
|
165
|
+
* back one of the pre-validated `addresses`. This is what pins a connection to the
|
|
166
|
+
* address the SSRF check already approved, defeating DNS rebinding: the socket can
|
|
167
|
+
* only reach a validated IP, never a value re-resolved at connect time.
|
|
168
|
+
*/
|
|
169
|
+
export function pinnedLookup(addresses) {
|
|
170
|
+
const resolved = addresses.map((address) => ({
|
|
171
|
+
address,
|
|
172
|
+
family: address.includes(":") ? 6 : 4,
|
|
173
|
+
}));
|
|
174
|
+
return (_hostname, options, callback) => {
|
|
175
|
+
const all = typeof options === "object" && options !== null && "all" in options
|
|
176
|
+
? options.all
|
|
177
|
+
: false;
|
|
178
|
+
if (all)
|
|
179
|
+
callback(null, resolved);
|
|
180
|
+
else
|
|
181
|
+
callback(null, resolved[0].address, resolved[0].family);
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
/** An undici dispatcher whose connections are pinned to `addresses`. */
|
|
185
|
+
export function createPinnedDispatcher(addresses) {
|
|
186
|
+
return new Agent({ connect: { lookup: pinnedLookup(addresses) } });
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Assert that `url` may be fetched and return the validated addresses to pin the
|
|
190
|
+
* connection to. Throws {@link BlockedHostError} for a bad scheme or a host
|
|
191
|
+
* resolving into a blocked range, or {@link HostUnresolvedError} if the host cannot
|
|
192
|
+
* be resolved.
|
|
193
|
+
*
|
|
194
|
+
* The returned list is the exact set of addresses the caller must restrict the
|
|
195
|
+
* connection to (via {@link createPinnedDispatcher}). An empty list means "do not
|
|
196
|
+
* pin" — only returned under `allowPrivate`, the deliberate internal-network escape
|
|
197
|
+
* hatch (from `web.allowPrivateHosts`), which bypasses the address check and lets
|
|
198
|
+
* the transport resolve normally. The scheme check always applies.
|
|
199
|
+
*/
|
|
200
|
+
export async function assertFetchable(url, resolveHost, allowPrivate) {
|
|
201
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
202
|
+
throw new BlockedHostError(`only http(s) URLs may be fetched (got "${url.protocol.replace(/:$/, "")}")`);
|
|
203
|
+
}
|
|
204
|
+
if (allowPrivate)
|
|
205
|
+
return [];
|
|
206
|
+
const host = url.hostname.replace(/^\[|\]$/g, ""); // strip IPv6 brackets
|
|
207
|
+
let addresses;
|
|
208
|
+
try {
|
|
209
|
+
addresses = await resolveHost(host);
|
|
210
|
+
}
|
|
211
|
+
catch (err) {
|
|
212
|
+
throw new HostUnresolvedError(`could not resolve host "${host}": ${err.message}`);
|
|
213
|
+
}
|
|
214
|
+
if (addresses.length === 0) {
|
|
215
|
+
throw new HostUnresolvedError(`host "${host}" resolved to no addresses`);
|
|
216
|
+
}
|
|
217
|
+
for (const addr of addresses) {
|
|
218
|
+
if (isBlockedAddress(addr)) {
|
|
219
|
+
throw new BlockedHostError(`host "${host}" resolves to ${addr}, a private/loopback/link-local address`);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
return addresses;
|
|
223
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { Tool } from "../tools/types.js";
|
|
3
|
+
import type { WebDeps } from "./types.js";
|
|
4
|
+
declare const searchParams: z.ZodObject<{
|
|
5
|
+
query: z.ZodString;
|
|
6
|
+
}, "strip", z.ZodTypeAny, {
|
|
7
|
+
query: string;
|
|
8
|
+
}, {
|
|
9
|
+
query: string;
|
|
10
|
+
}>;
|
|
11
|
+
export declare function createWebSearchTool(deps?: WebDeps): Tool<typeof searchParams>;
|
|
12
|
+
declare const fetchParams: z.ZodObject<{
|
|
13
|
+
url: z.ZodString;
|
|
14
|
+
}, "strip", z.ZodTypeAny, {
|
|
15
|
+
url: string;
|
|
16
|
+
}, {
|
|
17
|
+
url: string;
|
|
18
|
+
}>;
|
|
19
|
+
export declare function createWebFetchTool(deps?: WebDeps): Tool<typeof fetchParams>;
|
|
20
|
+
export {};
|