@easynet/agent-tool-buildin 0.0.68 → 0.0.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-tool-extension-compat.d.ts.map +1 -1
- package/dist/agent-tool-extension-compat.js +14 -4
- package/dist/agent-tool-extension-compat.js.map +1 -1
- package/dist/core-tools-manifest.json +6 -503
- package/dist/src/analyze/analyzeProjectRepo.constants.d.ts +6 -0
- package/dist/src/analyze/analyzeProjectRepo.constants.d.ts.map +1 -0
- package/dist/src/analyze/analyzeProjectRepo.constants.js +53 -0
- package/dist/src/analyze/analyzeProjectRepo.constants.js.map +1 -0
- package/dist/src/analyze/analyzeProjectRepo.d.ts +4 -9
- package/dist/src/analyze/analyzeProjectRepo.d.ts.map +1 -1
- package/dist/src/analyze/analyzeProjectRepo.js +100 -122
- package/dist/src/analyze/analyzeProjectRepo.js.map +1 -1
- package/dist/src/analyze/askProjectRepo.d.ts +0 -2
- package/dist/src/analyze/askProjectRepo.d.ts.map +1 -1
- package/dist/src/analyze/askProjectRepo.helpers.d.ts +57 -0
- package/dist/src/analyze/askProjectRepo.helpers.d.ts.map +1 -0
- package/dist/src/analyze/askProjectRepo.helpers.js +233 -0
- package/dist/src/analyze/askProjectRepo.helpers.js.map +1 -0
- package/dist/src/analyze/askProjectRepo.js +132 -354
- package/dist/src/analyze/askProjectRepo.js.map +1 -1
- package/dist/src/exec/runCommand.d.ts +1 -6
- package/dist/src/exec/runCommand.d.ts.map +1 -1
- package/dist/src/exec/runCommand.js +88 -69
- package/dist/src/exec/runCommand.js.map +1 -1
- package/dist/src/http/downloadFile.d.ts +1 -3
- package/dist/src/http/downloadFile.d.ts.map +1 -1
- package/dist/src/http/downloadFile.js +99 -56
- package/dist/src/http/downloadFile.js.map +1 -1
- package/dist/src/http/duckduckgoSearch.d.ts +0 -9
- package/dist/src/http/duckduckgoSearch.d.ts.map +1 -1
- package/dist/src/http/duckduckgoSearch.js +104 -94
- package/dist/src/http/duckduckgoSearch.js.map +1 -1
- package/dist/src/http/fetchJson.d.ts +1 -2
- package/dist/src/http/fetchJson.d.ts.map +1 -1
- package/dist/src/http/fetchJson.js +91 -53
- package/dist/src/http/fetchJson.js.map +1 -1
- package/dist/src/http/fetchPageMainContent.d.ts +1 -3
- package/dist/src/http/fetchPageMainContent.d.ts.map +1 -1
- package/dist/src/http/fetchPageMainContent.js +80 -66
- package/dist/src/http/fetchPageMainContent.js.map +1 -1
- package/dist/src/http/fetchText.d.ts +1 -2
- package/dist/src/http/fetchText.d.ts.map +1 -1
- package/dist/src/http/fetchText.js +100 -67
- package/dist/src/http/fetchText.js.map +1 -1
- package/dist/src/http/yahooFinance.d.ts +1 -4
- package/dist/src/http/yahooFinance.d.ts.map +1 -1
- package/dist/src/http/yahooFinance.js +122 -93
- package/dist/src/http/yahooFinance.js.map +1 -1
- package/dist/src/iterm/itermRunCommandInSession.d.ts +0 -18
- package/dist/src/iterm/itermRunCommandInSession.d.ts.map +1 -1
- package/dist/src/iterm/itermRunCommandInSession.js +57 -86
- package/dist/src/iterm/itermRunCommandInSession.js.map +1 -1
- package/dist/src/platform/securityCompat.d.ts.map +1 -1
- package/dist/src/platform/securityCompat.js +14 -6
- package/dist/src/platform/securityCompat.js.map +1 -1
- package/dist/src/reminders/remindersSyncFromCalendar.d.ts +0 -6
- package/dist/src/reminders/remindersSyncFromCalendar.d.ts.map +1 -1
- package/dist/src/reminders/remindersSyncFromCalendar.js +62 -57
- package/dist/src/reminders/remindersSyncFromCalendar.js.map +1 -1
- package/package.json +1 -1
|
@@ -6,15 +6,7 @@ import { validateUrl } from "../platform/securityCompat.js";
|
|
|
6
6
|
import { overrideWithConfig } from "../util/overrideWithConfig.js";
|
|
7
7
|
const DUCKDUCKGO_API = "https://api.duckduckgo.com/";
|
|
8
8
|
const DUCKDUCKGO_HTML = "https://html.duckduckgo.com/html/";
|
|
9
|
-
|
|
10
|
-
* Search the web via DuckDuckGo (no API key required), with HTML fallback when no instant answers.
|
|
11
|
-
* When the API returns no web results, falls back to HTML search (html.duckduckgo.com).
|
|
12
|
-
* Requires api.duckduckgo.com in allowedHosts; add html.duckduckgo.com for fallback results.
|
|
13
|
-
* @tool
|
|
14
|
-
* @effect none
|
|
15
|
-
*/
|
|
16
|
-
export async function duckduckgoSearch(args) {
|
|
17
|
-
// Step 1: Defaults, then override with extension config; validate query input.
|
|
9
|
+
function getDuckConfig() {
|
|
18
10
|
const defaults = {
|
|
19
11
|
timeoutMs: DEFAULT_TIMEOUT_MS,
|
|
20
12
|
maxBytes: DEFAULT_MAX_HTTP_BYTES,
|
|
@@ -23,118 +15,96 @@ export async function duckduckgoSearch(args) {
|
|
|
23
15
|
allowedHosts: [],
|
|
24
16
|
blockedHosts: [],
|
|
25
17
|
};
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
const timeoutMs = args.timeoutMs ?? cfg.timeoutMs ?? defaults.timeoutMs;
|
|
33
|
-
const maxResults = args.maxResults ?? 10;
|
|
34
|
-
const maxBytes = cfg.maxBytes ?? defaults.maxBytes;
|
|
35
|
-
const blockedCidrs = Array.isArray(cfg.blockedCidrs) ? cfg.blockedCidrs : defaults.blockedCidrs;
|
|
36
|
-
const userAgent = cfg.userAgent ?? defaults.userAgent;
|
|
37
|
-
// Step 2: Build API URL, validate it, and fetch with timeout.
|
|
38
|
-
const url = `${DUCKDUCKGO_API}?q=${encodeURIComponent(query)}&format=json`;
|
|
39
|
-
await validateUrl(url, {
|
|
18
|
+
const cfg = overrideWithConfig(defaults, getExtensionContext().config);
|
|
19
|
+
return {
|
|
20
|
+
timeoutMs: cfg.timeoutMs ?? defaults.timeoutMs,
|
|
21
|
+
maxBytes: cfg.maxBytes ?? defaults.maxBytes,
|
|
22
|
+
userAgent: cfg.userAgent ?? defaults.userAgent,
|
|
23
|
+
blockedCidrs: Array.isArray(cfg.blockedCidrs) ? cfg.blockedCidrs : defaults.blockedCidrs,
|
|
40
24
|
allowedHosts: Array.isArray(cfg.allowedHosts) ? cfg.allowedHosts : defaults.allowedHosts,
|
|
41
25
|
blockedHosts: Array.isArray(cfg.blockedHosts) ? cfg.blockedHosts : defaults.blockedHosts,
|
|
42
|
-
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function requireQuery(args) {
|
|
29
|
+
const query = (args.query ?? args.q)?.trim();
|
|
30
|
+
if (!query)
|
|
31
|
+
throw createTaggedError("DUCKDUCKGO_INVALID", "query is required (pass 'query' or 'q')", {});
|
|
32
|
+
return query;
|
|
33
|
+
}
|
|
34
|
+
async function fetchTextWithTimeout(args) {
|
|
35
|
+
await validateUrl(args.url, {
|
|
36
|
+
allowedHosts: args.allowedHosts,
|
|
37
|
+
blockedHosts: args.blockedHosts,
|
|
38
|
+
blockedCidrs: args.blockedCidrs,
|
|
43
39
|
});
|
|
44
40
|
const controller = new AbortController();
|
|
45
|
-
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
41
|
+
const timer = setTimeout(() => controller.abort(), args.timeoutMs);
|
|
46
42
|
let response;
|
|
47
43
|
try {
|
|
48
|
-
response = await fetch(url, {
|
|
44
|
+
response = await fetch(args.url, {
|
|
49
45
|
method: "GET",
|
|
50
|
-
headers: { "User-Agent": userAgent },
|
|
46
|
+
headers: { "User-Agent": args.userAgent },
|
|
51
47
|
signal: controller.signal,
|
|
52
48
|
});
|
|
53
49
|
}
|
|
54
50
|
catch (err) {
|
|
55
51
|
clearTimeout(timer);
|
|
56
52
|
if (err instanceof Error && err.name === "AbortError") {
|
|
57
|
-
throw createTaggedError("HTTP_TIMEOUT", `DuckDuckGo search timed out after ${timeoutMs}ms`, {
|
|
53
|
+
throw createTaggedError("HTTP_TIMEOUT", `DuckDuckGo search timed out after ${args.timeoutMs}ms`, {
|
|
54
|
+
timeoutMs: args.timeoutMs,
|
|
55
|
+
});
|
|
58
56
|
}
|
|
59
|
-
throw createTaggedError("UPSTREAM_ERROR", `DuckDuckGo search failed: ${err instanceof Error ? err.message : String(err)}`, {
|
|
57
|
+
throw createTaggedError("UPSTREAM_ERROR", `DuckDuckGo search failed: ${err instanceof Error ? err.message : String(err)}`, {});
|
|
60
58
|
}
|
|
61
59
|
clearTimeout(timer);
|
|
62
60
|
const text = await response.text();
|
|
63
61
|
const bytes = Buffer.byteLength(text, "utf-8");
|
|
64
|
-
if (bytes > maxBytes) {
|
|
65
|
-
throw createTaggedError("HTTP_TOO_LARGE", `DuckDuckGo response ${bytes} bytes exceeds limit of ${maxBytes} bytes`, {
|
|
62
|
+
if (bytes > args.maxBytes) {
|
|
63
|
+
throw createTaggedError("HTTP_TOO_LARGE", `DuckDuckGo response ${bytes} bytes exceeds limit of ${args.maxBytes} bytes`, {
|
|
64
|
+
bytes,
|
|
65
|
+
limit: args.maxBytes,
|
|
66
|
+
});
|
|
66
67
|
}
|
|
67
|
-
|
|
68
|
-
|
|
68
|
+
return text;
|
|
69
|
+
}
|
|
70
|
+
function parseDuckApi(text, query) {
|
|
69
71
|
try {
|
|
70
|
-
|
|
72
|
+
return JSON.parse(text);
|
|
71
73
|
}
|
|
72
74
|
catch {
|
|
73
|
-
throw createTaggedError("UPSTREAM_ERROR",
|
|
75
|
+
throw createTaggedError("UPSTREAM_ERROR", "DuckDuckGo returned invalid JSON", {
|
|
76
|
+
query,
|
|
77
|
+
textPreview: text.slice(0, 200),
|
|
78
|
+
});
|
|
74
79
|
}
|
|
80
|
+
}
|
|
81
|
+
function collectInstantApiResults(raw, maxResults) {
|
|
75
82
|
const results = [];
|
|
76
83
|
if (Array.isArray(raw.Results)) {
|
|
77
|
-
for (const
|
|
78
|
-
if (
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
}
|
|
84
|
+
for (const item of raw.Results.slice(0, maxResults)) {
|
|
85
|
+
if (!item.FirstURL)
|
|
86
|
+
continue;
|
|
87
|
+
results.push({
|
|
88
|
+
url: item.FirstURL,
|
|
89
|
+
title: item.Text ?? item.FirstURL,
|
|
90
|
+
snippet: item.Text,
|
|
91
|
+
});
|
|
85
92
|
}
|
|
86
93
|
}
|
|
87
94
|
const relatedTopics = [];
|
|
88
95
|
if (Array.isArray(raw.RelatedTopics)) {
|
|
89
|
-
for (const
|
|
90
|
-
const text =
|
|
91
|
-
if (text)
|
|
92
|
-
relatedTopics.push({ text, url:
|
|
93
|
-
}
|
|
96
|
+
for (const topic of raw.RelatedTopics.slice(0, maxResults)) {
|
|
97
|
+
const text = topic.Text ?? topic.Result;
|
|
98
|
+
if (text)
|
|
99
|
+
relatedTopics.push({ text, url: topic.FirstURL });
|
|
94
100
|
}
|
|
95
101
|
}
|
|
96
|
-
|
|
97
|
-
// Requires html.duckduckgo.com in allowedHosts.
|
|
98
|
-
if (results.length === 0 && relatedTopics.length === 0) {
|
|
99
|
-
const htmlResults = await fetchDuckDuckGoHtmlSearch(query, maxResults, timeoutMs, maxBytes, userAgent, cfg.allowedHosts ?? defaults.allowedHosts, cfg.blockedHosts ?? defaults.blockedHosts, blockedCidrs);
|
|
100
|
-
if (htmlResults.length > 0) {
|
|
101
|
-
results.push(...htmlResults);
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
const result = {
|
|
105
|
-
query,
|
|
106
|
-
abstract: raw.Abstract ?? raw.AbstractText ?? undefined,
|
|
107
|
-
abstractUrl: raw.AbstractURL ?? undefined,
|
|
108
|
-
abstractSource: raw.AbstractSource ?? undefined,
|
|
109
|
-
heading: raw.Heading ?? undefined,
|
|
110
|
-
results,
|
|
111
|
-
relatedTopics,
|
|
112
|
-
};
|
|
113
|
-
// Step 4: Return normalized result and evidence.
|
|
114
|
-
return {
|
|
115
|
-
result,
|
|
116
|
-
evidence: [
|
|
117
|
-
{
|
|
118
|
-
type: "url",
|
|
119
|
-
ref: url,
|
|
120
|
-
summary: `DuckDuckGo search: "${query}" → ${results.length} results`,
|
|
121
|
-
createdAt: new Date().toISOString(),
|
|
122
|
-
},
|
|
123
|
-
],
|
|
124
|
-
};
|
|
102
|
+
return { results, relatedTopics };
|
|
125
103
|
}
|
|
126
|
-
/**
|
|
127
|
-
* Fetch DuckDuckGo HTML search (html.duckduckgo.com) and parse result links/snippets.
|
|
128
|
-
* Used when the Instant Answer API returns no results.
|
|
129
|
-
*/
|
|
130
104
|
async function fetchDuckDuckGoHtmlSearch(query, maxResults, timeoutMs, maxBytes, userAgent, allowedHosts, blockedHosts, blockedCidrs) {
|
|
131
105
|
const htmlUrl = `${DUCKDUCKGO_HTML}?q=${encodeURIComponent(query)}`;
|
|
132
106
|
try {
|
|
133
|
-
await validateUrl(htmlUrl, {
|
|
134
|
-
allowedHosts,
|
|
135
|
-
blockedHosts,
|
|
136
|
-
blockedCidrs,
|
|
137
|
-
});
|
|
107
|
+
await validateUrl(htmlUrl, { allowedHosts, blockedHosts, blockedCidrs });
|
|
138
108
|
}
|
|
139
109
|
catch {
|
|
140
110
|
return [];
|
|
@@ -157,24 +127,64 @@ async function fetchDuckDuckGoHtmlSearch(query, maxResults, timeoutMs, maxBytes,
|
|
|
157
127
|
if (!response.ok)
|
|
158
128
|
return [];
|
|
159
129
|
const text = await response.text();
|
|
160
|
-
|
|
161
|
-
if (bytes > maxBytes)
|
|
130
|
+
if (Buffer.byteLength(text, "utf-8") > maxBytes)
|
|
162
131
|
return [];
|
|
163
132
|
const root = parse(text, { comment: false });
|
|
164
133
|
const out = [];
|
|
165
|
-
// DuckDuckGo HTML: result blocks have .result, link in .result__a, snippet in .result__snippet
|
|
166
134
|
const resultBlocks = root.querySelectorAll(".result");
|
|
167
|
-
for (let i = 0; i < Math.min(resultBlocks.length, maxResults); i
|
|
135
|
+
for (let i = 0; i < Math.min(resultBlocks.length, maxResults); i += 1) {
|
|
168
136
|
const block = resultBlocks[i];
|
|
169
137
|
const linkEl = block?.querySelector(".result__a");
|
|
170
138
|
const href = linkEl?.getAttribute("href");
|
|
171
139
|
const title = (linkEl?.textContent ?? "").trim().replace(/\s+/g, " ");
|
|
172
|
-
const
|
|
173
|
-
|
|
174
|
-
|
|
140
|
+
const snippet = (block?.querySelector(".result__snippet")?.textContent ?? "")
|
|
141
|
+
.trim()
|
|
142
|
+
.replace(/\s+/g, " ")
|
|
143
|
+
.slice(0, 500);
|
|
144
|
+
if (href && title)
|
|
175
145
|
out.push({ url: href, title, snippet: snippet || undefined });
|
|
176
|
-
}
|
|
177
146
|
}
|
|
178
147
|
return out;
|
|
179
148
|
}
|
|
149
|
+
export async function duckduckgoSearch(args) {
|
|
150
|
+
const cfg = getDuckConfig();
|
|
151
|
+
const query = requireQuery(args);
|
|
152
|
+
const maxResults = args.maxResults ?? 10;
|
|
153
|
+
const timeoutMs = args.timeoutMs ?? cfg.timeoutMs;
|
|
154
|
+
const maxBytes = args.maxBytes ?? cfg.maxBytes;
|
|
155
|
+
const url = `${DUCKDUCKGO_API}?q=${encodeURIComponent(query)}&format=json`;
|
|
156
|
+
const text = await fetchTextWithTimeout({
|
|
157
|
+
url,
|
|
158
|
+
timeoutMs,
|
|
159
|
+
userAgent: cfg.userAgent,
|
|
160
|
+
maxBytes,
|
|
161
|
+
allowedHosts: cfg.allowedHosts,
|
|
162
|
+
blockedHosts: cfg.blockedHosts,
|
|
163
|
+
blockedCidrs: cfg.blockedCidrs,
|
|
164
|
+
});
|
|
165
|
+
const raw = parseDuckApi(text, query);
|
|
166
|
+
const { results, relatedTopics } = collectInstantApiResults(raw, maxResults);
|
|
167
|
+
if (results.length === 0 && relatedTopics.length === 0) {
|
|
168
|
+
const htmlResults = await fetchDuckDuckGoHtmlSearch(query, maxResults, timeoutMs, maxBytes, cfg.userAgent, cfg.allowedHosts, cfg.blockedHosts, cfg.blockedCidrs);
|
|
169
|
+
if (htmlResults.length > 0)
|
|
170
|
+
results.push(...htmlResults);
|
|
171
|
+
}
|
|
172
|
+
return {
|
|
173
|
+
result: {
|
|
174
|
+
query,
|
|
175
|
+
abstract: raw.Abstract ?? raw.AbstractText ?? undefined,
|
|
176
|
+
abstractUrl: raw.AbstractURL ?? undefined,
|
|
177
|
+
abstractSource: raw.AbstractSource ?? undefined,
|
|
178
|
+
heading: raw.Heading ?? undefined,
|
|
179
|
+
results,
|
|
180
|
+
relatedTopics,
|
|
181
|
+
},
|
|
182
|
+
evidence: [{
|
|
183
|
+
type: "url",
|
|
184
|
+
ref: url,
|
|
185
|
+
summary: `DuckDuckGo search: "${query}" → ${results.length} results`,
|
|
186
|
+
createdAt: new Date().toISOString(),
|
|
187
|
+
}],
|
|
188
|
+
};
|
|
189
|
+
}
|
|
180
190
|
//# sourceMappingURL=duckduckgoSearch.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"duckduckgoSearch.js","sourceRoot":"","sources":["../../../src/http/duckduckgoSearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAEnE,MAAM,cAAc,GAAG,6BAA6B,CAAC;AACrD,MAAM,eAAe,GAAG,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"duckduckgoSearch.js","sourceRoot":"","sources":["../../../src/http/duckduckgoSearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAEnE,MAAM,cAAc,GAAG,6BAA6B,CAAC;AACrD,MAAM,eAAe,GAAG,mCAAmC,CAAC;AAgC5D,SAAS,aAAa;IACpB,MAAM,QAAQ,GAAG;QACf,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,sBAAsB;QAChC,SAAS,EAAE,uBAAuB;QAClC,YAAY,EAAE,qBAAqB;QACnC,YAAY,EAAE,EAAc;QAC5B,YAAY,EAAE,EAAc;KAC7B,CAAC;IACF,MAAM,GAAG,GAAG,kBAAkB,CAAC,QAAQ,EAAE,mBAAmB,EAAE,CAAC,MAAM,CAAC,CAAC;IACvE,OAAO;QACL,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS;QAC9C,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ;QAC3C,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS;QAC9C,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;QACxF,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;QACxF,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;KACzF,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,IAAmC;IACvD,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;IAC7C,IAAI,CAAC,KAAK;QAAE,MAAM,iBAAiB,CAAC,oBAAoB,EAAE,yCAAyC,EAAE,EAAE,CAAC,CAAC;IACzG,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,IAQnC;IACC,MAAM,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE;QAC1B,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,YAAY,EAAE,IAAI,CAAC,YAAY;KAChC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;YAC/B,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;YACzC,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACtD,MAAM,iBAAiB,CAAC,cAAc,EAAE,qCAAqC,IAAI,CAAC,SAAS,IAAI,EAAE;gBAC/F,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B,CAAC,CAAC;QACL,CAAC;QACD,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,6BAA6B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACjI,CAAC;IACD,YAAY,CAAC,KAAK,CAAC,CAAC;IAEpB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC1B,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,uBAAuB,KAAK,2BAA2B,IAAI,CAAC,QAAQ,QAAQ,EAAE;YACtH,KAAK;YACL,KAAK,EAAE,IAAI,CAAC,QAAQ;SACrB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,KAAa;IAC/C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAuB,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,kCAAkC,EAAE;YAC5E,KAAK;YACL,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;SAChC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,GAAuB,EAAE,UAAkB;IAC3E,MAAM,OAAO,GAA4D,EAAE,CAAC;IAC5E,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAC7B,OAAO,CAAC,IAAI,CAAC;gBACX,GAAG,EAAE,IAAI,CAAC,QAAQ;gBAClB,KAAK,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ;gBACjC,OAAO,EAAE,IAAI,CAAC,IAAI;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAA0C,EAAE,CAAC;IAChE,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;YAC3D,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;YACxC,IAAI,IAAI;gBAAE,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AACpC,CAAC;AAED,KAAK,UAAU,yBAAyB,CACtC,KAAa,EACb,UAAkB,EAClB,SAAiB,EACjB,QAAgB,EAChB,SAAiB,EACjB,YAAsB,EACtB,YAAsB,EACtB,YAAsB;IAEtB,MAAM,OAAO,GAAG,GAAG,eAAe,MAAM,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;IACpE,IAAI,CAAC;QACH,MAAM,WAAW,CAAC,OAAO,EAAE,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9D,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAC9B,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE;YACpC,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,YAAY,CAAC,KAAK,CAAC,CAAC;IAEpB,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,QAAQ;QAAE,OAAO,EAAE,CAAC;IAE3D,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7C,MAAM,GAAG,GAA4D,EAAE,CAAC;IACxE,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACtE,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,EAAE,aAAa,CAAC,YAAY,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,CAAC,KAAK,EAAE,aAAa,CAAC,kBAAkB,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC;aAC1E,IAAI,EAAE;aACN,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;aACpB,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACjB,IAAI,IAAI,IAAI,KAAK;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,IAAI,SAAS,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAMtC;IACC,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC;IAC/C,MAAM,GAAG,GAAG,GAAG,cAAc,MAAM,kBAAkB,CAAC,KAAK,CAAC,cAAc,CAAC;IAE3E,MAAM,IAAI,GAAG,MAAM,oBAAoB,CAAC;QACtC,GAAG;QACH,SAAS;QACT,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,QAAQ;QACR,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,YAAY,EAAE,GAAG,CAAC,YAAY;KAC/B,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,wBAAwB,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAC7E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvD,MAAM,WAAW,GAAG,MAAM,yBAAyB,CACjD,KAAK,EACL,UAAU,EACV,SAAS,EACT,QAAQ,EACR,GAAG,CAAC,SAAS,EACb,GAAG,CAAC,YAAY,EAChB,GAAG,CAAC,YAAY,EAChB,GAAG,CAAC,YAAY,CACjB,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO;QACL,MAAM,EAAE;YACN,KAAK;YACL,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,YAAY,IAAI,SAAS;YACvD,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS;YACzC,cAAc,EAAE,GAAG,CAAC,cAAc,IAAI,SAAS;YAC/C,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,SAAS;YACjC,OAAO;YACP,aAAa;SACd;QACD,QAAQ,EAAE,CAAC;gBACT,IAAI,EAAE,KAAK;gBACX,GAAG,EAAE,GAAG;gBACR,OAAO,EAAE,uBAAuB,KAAK,OAAO,OAAO,CAAC,MAAM,UAAU;gBACpE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC;KACH,CAAC;AACJ,CAAC"}
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Fetch
|
|
2
|
+
* Fetch JSON from HTTP endpoint with SSRF/safety checks and size limits.
|
|
3
3
|
* @tool
|
|
4
4
|
* @effect none
|
|
5
5
|
*/
|
|
6
6
|
export declare function fetchJson(args: {
|
|
7
7
|
url?: string;
|
|
8
|
-
/** Alias for url (accepted when agents send "uri" instead of "url"). */
|
|
9
8
|
uri?: string;
|
|
10
9
|
method?: "GET" | "POST";
|
|
11
10
|
headers?: Record<string, string>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetchJson.d.ts","sourceRoot":"","sources":["../../../src/http/fetchJson.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fetchJson.d.ts","sourceRoot":"","sources":["../../../src/http/fetchJson.ts"],"names":[],"mappings":"AA4HA;;;;GAIG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE;IACpC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;;;;;;;;;;;;;GAuCA"}
|
|
@@ -3,13 +3,7 @@ import { DEFAULT_TIMEOUT_MS, DEFAULT_MAX_HTTP_BYTES, DEFAULT_HTTP_USER_AGENT, DE
|
|
|
3
3
|
import { createTaggedError } from "@easynet/agent-tool/runtime";
|
|
4
4
|
import { validateUrl } from "../platform/securityCompat.js";
|
|
5
5
|
import { overrideWithConfig } from "../util/overrideWithConfig.js";
|
|
6
|
-
|
|
7
|
-
* Fetch a URL and return the response as parsed JSON.
|
|
8
|
-
* @tool
|
|
9
|
-
* @effect none
|
|
10
|
-
*/
|
|
11
|
-
export async function fetchJson(args) {
|
|
12
|
-
// Step 1: Defaults, then override with extension config; validate url input.
|
|
6
|
+
function getFetchJsonConfig() {
|
|
13
7
|
const defaults = {
|
|
14
8
|
timeoutMs: DEFAULT_TIMEOUT_MS,
|
|
15
9
|
maxBytes: DEFAULT_MAX_HTTP_BYTES,
|
|
@@ -18,84 +12,128 @@ export async function fetchJson(args) {
|
|
|
18
12
|
allowedHosts: [],
|
|
19
13
|
blockedHosts: [],
|
|
20
14
|
};
|
|
21
|
-
const
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
const method = args.method ?? "GET";
|
|
28
|
-
const headers = args.headers ?? {};
|
|
29
|
-
const body = args.body ?? undefined;
|
|
30
|
-
const timeoutMs = args.timeoutMs ?? cfg.timeoutMs ?? defaults.timeoutMs;
|
|
31
|
-
const maxBytes = args.maxBytes ?? cfg.maxBytes ?? defaults.maxBytes;
|
|
32
|
-
const blockedCidrs = Array.isArray(cfg.blockedCidrs) ? cfg.blockedCidrs : defaults.blockedCidrs;
|
|
33
|
-
const userAgent = cfg.userAgent ?? defaults.userAgent;
|
|
34
|
-
// Step 2: Validate URL and set Accept/User-Agent headers.
|
|
35
|
-
await validateUrl(url, {
|
|
15
|
+
const cfg = overrideWithConfig(defaults, getExtensionContext().config);
|
|
16
|
+
return {
|
|
17
|
+
timeoutMs: cfg.timeoutMs ?? defaults.timeoutMs,
|
|
18
|
+
maxBytes: cfg.maxBytes ?? defaults.maxBytes,
|
|
19
|
+
userAgent: cfg.userAgent ?? defaults.userAgent,
|
|
20
|
+
blockedCidrs: Array.isArray(cfg.blockedCidrs) ? cfg.blockedCidrs : defaults.blockedCidrs,
|
|
36
21
|
allowedHosts: Array.isArray(cfg.allowedHosts) ? cfg.allowedHosts : defaults.allowedHosts,
|
|
37
22
|
blockedHosts: Array.isArray(cfg.blockedHosts) ? cfg.blockedHosts : defaults.blockedHosts,
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
function parseFetchJsonInput(args) {
|
|
26
|
+
const url = (args.url ?? args.uri)?.trim();
|
|
27
|
+
if (!url)
|
|
28
|
+
throw createTaggedError("HTTP_INVALID", "url is required (pass 'url' or 'uri')", {});
|
|
29
|
+
return {
|
|
30
|
+
url,
|
|
31
|
+
method: args.method ?? "GET",
|
|
32
|
+
headers: { ...(args.headers ?? {}) },
|
|
33
|
+
body: args.body ?? undefined,
|
|
34
|
+
timeoutMs: args.timeoutMs,
|
|
35
|
+
maxBytes: args.maxBytes,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
async function fetchJsonResponse(args) {
|
|
47
39
|
const controller = new AbortController();
|
|
48
|
-
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
49
|
-
let response;
|
|
40
|
+
const timer = setTimeout(() => controller.abort(), args.timeoutMs);
|
|
50
41
|
try {
|
|
51
|
-
|
|
52
|
-
method,
|
|
53
|
-
headers,
|
|
54
|
-
body: body
|
|
42
|
+
return await fetch(args.url, {
|
|
43
|
+
method: args.method,
|
|
44
|
+
headers: args.headers,
|
|
45
|
+
body: args.body,
|
|
55
46
|
signal: controller.signal,
|
|
56
47
|
});
|
|
57
48
|
}
|
|
58
49
|
catch (err) {
|
|
59
50
|
if (err instanceof Error && err.name === "AbortError") {
|
|
60
|
-
throw createTaggedError("HTTP_TIMEOUT", `Request to ${url} timed out after ${timeoutMs}ms`, {
|
|
51
|
+
throw createTaggedError("HTTP_TIMEOUT", `Request to ${args.url} timed out after ${args.timeoutMs}ms`, {
|
|
52
|
+
url: args.url,
|
|
53
|
+
timeoutMs: args.timeoutMs,
|
|
54
|
+
});
|
|
61
55
|
}
|
|
62
|
-
throw createTaggedError("UPSTREAM_ERROR", `Fetch failed for ${url}: ${err instanceof Error ? err.message : String(err)}`, {
|
|
56
|
+
throw createTaggedError("UPSTREAM_ERROR", `Fetch failed for ${args.url}: ${err instanceof Error ? err.message : String(err)}`, {
|
|
57
|
+
url: args.url,
|
|
58
|
+
});
|
|
63
59
|
}
|
|
64
60
|
finally {
|
|
65
61
|
clearTimeout(timer);
|
|
66
62
|
}
|
|
63
|
+
}
|
|
64
|
+
function checkBodySizeLimit(url, response, text, maxBytes) {
|
|
67
65
|
const contentLength = response.headers.get("content-length");
|
|
68
66
|
if (contentLength && parseInt(contentLength, 10) > maxBytes) {
|
|
69
|
-
throw createTaggedError("HTTP_TOO_LARGE", `Response Content-Length ${contentLength} exceeds limit of ${maxBytes} bytes`, {
|
|
67
|
+
throw createTaggedError("HTTP_TOO_LARGE", `Response Content-Length ${contentLength} exceeds limit of ${maxBytes} bytes`, {
|
|
68
|
+
url,
|
|
69
|
+
contentLength: parseInt(contentLength, 10),
|
|
70
|
+
limit: maxBytes,
|
|
71
|
+
});
|
|
70
72
|
}
|
|
71
|
-
const text = await response.text();
|
|
72
73
|
const bytes = Buffer.byteLength(text, "utf-8");
|
|
73
74
|
if (bytes > maxBytes) {
|
|
74
|
-
throw createTaggedError("HTTP_TOO_LARGE", `Response body ${bytes} bytes exceeds limit of ${maxBytes} bytes`, {
|
|
75
|
+
throw createTaggedError("HTTP_TOO_LARGE", `Response body ${bytes} bytes exceeds limit of ${maxBytes} bytes`, {
|
|
76
|
+
url,
|
|
77
|
+
bytes,
|
|
78
|
+
limit: maxBytes,
|
|
79
|
+
});
|
|
75
80
|
}
|
|
76
|
-
|
|
81
|
+
return bytes;
|
|
82
|
+
}
|
|
83
|
+
function parseJsonOrThrow(url, response, text) {
|
|
77
84
|
try {
|
|
78
|
-
|
|
85
|
+
return JSON.parse(text);
|
|
79
86
|
}
|
|
80
87
|
catch {
|
|
81
|
-
throw createTaggedError("UPSTREAM_ERROR", `Failed to parse JSON response from ${url}: ${text.slice(0, 200)}`, {
|
|
88
|
+
throw createTaggedError("UPSTREAM_ERROR", `Failed to parse JSON response from ${url}: ${text.slice(0, 200)}`, {
|
|
89
|
+
url,
|
|
90
|
+
status: response.status,
|
|
91
|
+
textPreview: text.slice(0, 500),
|
|
92
|
+
});
|
|
82
93
|
}
|
|
83
|
-
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Fetch JSON from HTTP endpoint with SSRF/safety checks and size limits.
|
|
97
|
+
* @tool
|
|
98
|
+
* @effect none
|
|
99
|
+
*/
|
|
100
|
+
export async function fetchJson(args) {
|
|
101
|
+
const cfg = getFetchJsonConfig();
|
|
102
|
+
const input = parseFetchJsonInput(args);
|
|
103
|
+
const timeoutMs = input.timeoutMs ?? cfg.timeoutMs;
|
|
104
|
+
const maxBytes = input.maxBytes ?? cfg.maxBytes;
|
|
105
|
+
await validateUrl(input.url, {
|
|
106
|
+
allowedHosts: cfg.allowedHosts,
|
|
107
|
+
blockedHosts: cfg.blockedHosts,
|
|
108
|
+
blockedCidrs: cfg.blockedCidrs,
|
|
109
|
+
});
|
|
110
|
+
if (!input.headers.Accept && !input.headers.accept)
|
|
111
|
+
input.headers.Accept = "application/json";
|
|
112
|
+
if (!input.headers["User-Agent"] && !input.headers["user-agent"])
|
|
113
|
+
input.headers["User-Agent"] = cfg.userAgent;
|
|
114
|
+
const response = await fetchJsonResponse({
|
|
115
|
+
url: input.url,
|
|
116
|
+
method: input.method,
|
|
117
|
+
headers: input.headers,
|
|
118
|
+
body: input.body,
|
|
119
|
+
timeoutMs,
|
|
120
|
+
});
|
|
121
|
+
const text = await response.text();
|
|
122
|
+
const bytes = checkBodySizeLimit(input.url, response, text, maxBytes);
|
|
123
|
+
const json = parseJsonOrThrow(input.url, response, text);
|
|
84
124
|
return {
|
|
85
125
|
result: {
|
|
86
|
-
url,
|
|
126
|
+
url: input.url,
|
|
87
127
|
status: response.status,
|
|
88
128
|
json,
|
|
89
129
|
bytes,
|
|
90
130
|
},
|
|
91
|
-
evidence: [
|
|
92
|
-
{
|
|
131
|
+
evidence: [{
|
|
93
132
|
type: "url",
|
|
94
|
-
ref: url,
|
|
95
|
-
summary: `${method} ${url} → ${response.status} JSON (${bytes} bytes)`,
|
|
133
|
+
ref: input.url,
|
|
134
|
+
summary: `${input.method} ${input.url} → ${response.status} JSON (${bytes} bytes)`,
|
|
96
135
|
createdAt: new Date().toISOString(),
|
|
97
|
-
},
|
|
98
|
-
],
|
|
136
|
+
}],
|
|
99
137
|
};
|
|
100
138
|
}
|
|
101
139
|
//# sourceMappingURL=fetchJson.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetchJson.js","sourceRoot":"","sources":["../../../src/http/fetchJson.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;
|
|
1
|
+
{"version":3,"file":"fetchJson.js","sourceRoot":"","sources":["../../../src/http/fetchJson.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAWnE,SAAS,kBAAkB;IACzB,MAAM,QAAQ,GAAG;QACf,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,sBAAsB;QAChC,SAAS,EAAE,uBAAuB;QAClC,YAAY,EAAE,qBAAqB;QACnC,YAAY,EAAE,EAAc;QAC5B,YAAY,EAAE,EAAc;KAC7B,CAAC;IACF,MAAM,GAAG,GAAG,kBAAkB,CAAC,QAAQ,EAAE,mBAAmB,EAAE,CAAC,MAAM,CAAC,CAAC;IACvE,OAAO;QACL,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS;QAC9C,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ;QAC3C,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS;QAC9C,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;QACxF,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;QACxF,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;KACzF,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,IAQ5B;IACC,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;IAC3C,IAAI,CAAC,GAAG;QAAE,MAAM,iBAAiB,CAAC,cAAc,EAAE,uCAAuC,EAAE,EAAE,CAAC,CAAC;IAC/F,OAAO;QACL,GAAG;QACH,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,KAAK;QAC5B,OAAO,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE;QACpC,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,SAAS;QAC5B,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;KACxB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,IAMhC;IACC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,IAAI,CAAC;QACH,OAAO,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;YAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACtD,MAAM,iBAAiB,CAAC,cAAc,EAAE,cAAc,IAAI,CAAC,GAAG,oBAAoB,IAAI,CAAC,SAAS,IAAI,EAAE;gBACpG,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B,CAAC,CAAC;QACL,CAAC;QACD,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,oBAAoB,IAAI,CAAC,GAAG,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;YAC7H,GAAG,EAAE,IAAI,CAAC,GAAG;SACd,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW,EAAE,QAAkB,EAAE,IAAY,EAAE,QAAgB;IACzF,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC7D,IAAI,aAAa,IAAI,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC;QAC5D,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,2BAA2B,aAAa,qBAAqB,QAAQ,QAAQ,EAAE;YACvH,GAAG;YACH,aAAa,EAAE,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC;YAC1C,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;IACL,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;QACrB,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,iBAAiB,KAAK,2BAA2B,QAAQ,QAAQ,EAAE;YAC3G,GAAG;YACH,KAAK;YACL,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW,EAAE,QAAkB,EAAE,IAAY;IACrE,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,sCAAsC,GAAG,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE;YAC5G,GAAG;YACH,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;SAChC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAQ/B;IACC,MAAM,GAAG,GAAG,kBAAkB,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC;IACnD,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC;IAEhD,MAAM,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE;QAC3B,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,YAAY,EAAE,GAAG,CAAC,YAAY;KAC/B,CAAC,CAAC;IACH,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM;QAAE,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,kBAAkB,CAAC;IAC9F,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;QAAE,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC;IAE9G,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC;QACvC,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,SAAS;KACV,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IACtE,MAAM,IAAI,GAAG,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAEzD,OAAO;QACL,MAAM,EAAE;YACN,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,IAAI;YACJ,KAAK;SACN;QACD,QAAQ,EAAE,CAAC;gBACT,IAAI,EAAE,KAAK;gBACX,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,OAAO,EAAE,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,QAAQ,CAAC,MAAM,UAAU,KAAK,SAAS;gBAClF,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC;KACH,CAAC;AACJ,CAAC"}
|
|
@@ -1,12 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Fetch
|
|
3
|
-
* For HTML responses, extracts main content; for non-HTML, returns raw text.
|
|
2
|
+
* Fetch page and extract main text content heuristically.
|
|
4
3
|
* @tool
|
|
5
4
|
* @effect none
|
|
6
5
|
*/
|
|
7
6
|
export declare function fetchPageMainContent(args: {
|
|
8
7
|
url?: string;
|
|
9
|
-
/** Alias for url (accepted when agents send "uri" instead of "url"). */
|
|
10
8
|
uri?: string;
|
|
11
9
|
timeoutMs?: number;
|
|
12
10
|
maxBytes?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetchPageMainContent.d.ts","sourceRoot":"","sources":["../../../src/http/fetchPageMainContent.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fetchPageMainContent.d.ts","sourceRoot":"","sources":["../../../src/http/fetchPageMainContent.ts"],"names":[],"mappings":"AAoIA;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;;;;;;;;;;;;;;;GAgDA"}
|