chatccc 0.2.243 → 0.2.245
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/bin/cccagent.mjs +17 -17
- package/deepccc-agent/README.md +14 -8
- package/deepccc-agent/bin/deepccc.mjs +26 -26
- package/deepccc-agent/os-prompts/darwin.md +8 -0
- package/deepccc-agent/os-prompts/linux.md +8 -0
- package/deepccc-agent/os-prompts/win32.md +9 -0
- package/deepccc-agent/package-lock.json +2 -2
- package/deepccc-agent/package.json +63 -62
- package/deepccc-agent/src/__tests__/chat-session.test.ts +682 -578
- package/deepccc-agent/src/__tests__/cli-json.test.ts +49 -49
- package/deepccc-agent/src/__tests__/config.test.ts +26 -26
- package/deepccc-agent/src/__tests__/context.test.ts +319 -319
- package/deepccc-agent/src/__tests__/file-tools.test.ts +240 -240
- package/deepccc-agent/src/__tests__/permissions.test.ts +195 -195
- package/deepccc-agent/src/__tests__/privacy.test.ts +8 -5
- package/deepccc-agent/src/__tests__/progress-reducer.test.ts +121 -121
- package/deepccc-agent/src/__tests__/session-search.test.ts +262 -262
- package/deepccc-agent/src/__tests__/session-select.test.ts +116 -116
- package/deepccc-agent/src/__tests__/sigint.test.ts +56 -56
- package/deepccc-agent/src/__tests__/skills.test.ts +284 -284
- package/deepccc-agent/src/__tests__/terminal-renderer.test.ts +247 -247
- package/deepccc-agent/src/__tests__/web-tools.test.ts +220 -220
- package/deepccc-agent/src/cli.ts +7 -6
- package/deepccc-agent/src/config.ts +88 -84
- package/deepccc-agent/src/context.ts +465 -465
- package/deepccc-agent/src/file-log.ts +38 -38
- package/deepccc-agent/src/index.ts +103 -36
- package/deepccc-agent/src/proc-tree-kill.ts +61 -61
- package/deepccc-agent/src/progress/cards-helpers.ts +76 -76
- package/deepccc-agent/src/progress/reducer.ts +113 -113
- package/deepccc-agent/src/progress/terminal-renderer.ts +294 -294
- package/deepccc-agent/src/progress/view.ts +77 -77
- package/deepccc-agent/src/raw-stream-log.ts +124 -124
- package/deepccc-agent/src/session-search.ts +370 -370
- package/deepccc-agent/src/session-select.ts +48 -48
- package/deepccc-agent/src/sigint.ts +50 -50
- package/deepccc-agent/src/skills.ts +205 -205
- package/deepccc-agent/src/web-tools.ts +313 -313
- package/deepccc-agent/tsconfig.build.json +13 -13
- package/deepccc-agent/tsconfig.json +13 -13
- package/deepccc-agent/vitest.config.ts +7 -7
- package/package.json +1 -1
- package/src/__tests__/builtin-chat-session.test.ts +522 -522
- package/src/__tests__/builtin-config.test.ts +26 -26
- package/src/__tests__/builtin-context.test.ts +319 -319
- package/src/__tests__/builtin-file-tools.test.ts +240 -240
- package/src/__tests__/builtin-permissions.test.ts +211 -211
- package/src/__tests__/builtin-session-search.test.ts +262 -262
- package/src/__tests__/builtin-session-select.test.ts +116 -116
- package/src/__tests__/builtin-sigint.test.ts +56 -56
- package/src/__tests__/builtin-skills.test.ts +284 -284
- package/src/__tests__/builtin-web-tools.test.ts +220 -220
- package/src/__tests__/ccc-adapter.test.ts +15 -0
- package/src/__tests__/config.test.ts +17 -17
- package/src/__tests__/progress-reducer.test.ts +121 -121
- package/src/__tests__/session-ccc-config.test.ts +45 -45
- package/src/__tests__/session.test.ts +369 -306
- package/src/adapters/adapter-interface.ts +8 -2
- package/src/adapters/ccc-adapter.ts +149 -145
- package/src/config-utils.ts +13 -13
- package/src/config.ts +13 -13
- package/src/progress/reducer.ts +113 -113
- package/src/session-chat-binding.ts +83 -83
- package/src/session.ts +323 -320
|
@@ -1,313 +1,313 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* web-tools.ts — websearch / webfetch 内置工具(agent 端联网,业界主流做法)
|
|
3
|
-
*
|
|
4
|
-
* 与 Claude Code 的 WebSearch/WebFetch、Codex 的 web search 相同形态:
|
|
5
|
-
* 模型通过 function calling 调用,agent 进程执行 HTTP 请求,结果回填上下文。
|
|
6
|
-
* 两者都是只读外部网络操作,不触碰本地文件系统,无需权限询问。
|
|
7
|
-
*
|
|
8
|
-
* - websearch:DuckDuckGo HTML 端点(免 API key),返回标题 + URL + 摘要
|
|
9
|
-
* - webfetch:HTTP GET + HTML 转纯文本,控制大小与超时
|
|
10
|
-
*
|
|
11
|
-
* 零新依赖:使用 Node 20 内置 fetch / AbortSignal / Buffer。
|
|
12
|
-
* 解析逻辑拆成纯函数导出,便于单测;fetch 可通过 options 注入 mock。
|
|
13
|
-
*/
|
|
14
|
-
|
|
15
|
-
export interface WebSearchInput {
|
|
16
|
-
query: string;
|
|
17
|
-
/** 返回结果条数上限,默认 5,上限 10 */
|
|
18
|
-
maxResults?: number;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
export interface WebSearchResult {
|
|
22
|
-
title: string;
|
|
23
|
-
url: string;
|
|
24
|
-
snippet: string;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
export interface WebSearchOutput {
|
|
28
|
-
query: string;
|
|
29
|
-
results: WebSearchResult[];
|
|
30
|
-
truncated: boolean;
|
|
31
|
-
durationMs: number;
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
export interface WebFetchInput {
|
|
35
|
-
url: string;
|
|
36
|
-
/** 返回纯文本字符数上限,默认 10000,上限 100000 */
|
|
37
|
-
maxChars?: number;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
export interface WebFetchOutput {
|
|
41
|
-
url: string;
|
|
42
|
-
contentType: string;
|
|
43
|
-
title: string;
|
|
44
|
-
text: string;
|
|
45
|
-
chars: number;
|
|
46
|
-
truncated: boolean;
|
|
47
|
-
durationMs: number;
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
export const WEB_SEARCH_TIMEOUT_MS = 15_000;
|
|
51
|
-
export const WEB_FETCH_TIMEOUT_MS = 20_000;
|
|
52
|
-
export const WEB_SEARCH_DEFAULT_RESULTS = 5;
|
|
53
|
-
export const WEB_SEARCH_MAX_RESULTS = 10;
|
|
54
|
-
export const WEB_FETCH_DEFAULT_CHARS = 10_000;
|
|
55
|
-
export const WEB_FETCH_MAX_CHARS = 100_000;
|
|
56
|
-
/** webfetch 读取响应体的大小上限(超出即截断),防止把整个大文件拉进上下文 */
|
|
57
|
-
export const WEB_FETCH_MAX_BYTES = 512 * 1024;
|
|
58
|
-
|
|
59
|
-
const SEARCH_UA =
|
|
60
|
-
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36";
|
|
61
|
-
|
|
62
|
-
export type FetchLike = (
|
|
63
|
-
input: string | URL | Request,
|
|
64
|
-
init?: RequestInit,
|
|
65
|
-
) => Promise<Response>;
|
|
66
|
-
|
|
67
|
-
export interface WebToolOptions {
|
|
68
|
-
abortSignal?: AbortSignal;
|
|
69
|
-
/** 测试注入用,默认全局 fetch */
|
|
70
|
-
fetchImpl?: FetchLike;
|
|
71
|
-
timeoutMs?: number;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
function normalizeMaxResults(value: number | undefined): number {
|
|
75
|
-
if (value === undefined) return WEB_SEARCH_DEFAULT_RESULTS;
|
|
76
|
-
if (!Number.isInteger(value) || value <= 0) {
|
|
77
|
-
throw new Error("maxResults must be a positive integer when provided");
|
|
78
|
-
}
|
|
79
|
-
return Math.min(value, WEB_SEARCH_MAX_RESULTS);
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
function normalizeMaxChars(value: number | undefined): number {
|
|
83
|
-
if (value === undefined) return WEB_FETCH_DEFAULT_CHARS;
|
|
84
|
-
if (!Number.isInteger(value) || value <= 0) {
|
|
85
|
-
throw new Error("maxChars must be a positive integer when provided");
|
|
86
|
-
}
|
|
87
|
-
return Math.min(value, WEB_FETCH_MAX_CHARS);
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
function buildSignal(timeoutMs: number, abortSignal?: AbortSignal): AbortSignal {
|
|
91
|
-
const signals: AbortSignal[] = [AbortSignal.timeout(timeoutMs)];
|
|
92
|
-
if (abortSignal) signals.push(abortSignal);
|
|
93
|
-
return signals.length === 1 ? signals[0] : AbortSignal.any(signals);
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
function decodeEntities(s: string): string {
|
|
97
|
-
return s
|
|
98
|
-
.replace(/</gi, "<")
|
|
99
|
-
.replace(/>/gi, ">")
|
|
100
|
-
.replace(/"/gi, '"')
|
|
101
|
-
.replace(/�?39;/gi, "'")
|
|
102
|
-
.replace(/ /gi, " ")
|
|
103
|
-
.replace(/&#(\d+);/g, (_, n: string) => {
|
|
104
|
-
try {
|
|
105
|
-
return String.fromCodePoint(Number(n));
|
|
106
|
-
} catch {
|
|
107
|
-
return "";
|
|
108
|
-
}
|
|
109
|
-
})
|
|
110
|
-
.replace(/&#x([0-9a-f]+);/gi, (_, h: string) => {
|
|
111
|
-
try {
|
|
112
|
-
return String.fromCodePoint(parseInt(h, 16));
|
|
113
|
-
} catch {
|
|
114
|
-
return "";
|
|
115
|
-
}
|
|
116
|
-
})
|
|
117
|
-
.replace(/&/gi, "&");
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
function stripTags(s: string): string {
|
|
121
|
-
return decodeEntities(s.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim());
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
/** 提取 <title> 文本(用于 webfetch 结果),无则返回空字符串 */
|
|
125
|
-
export function extractHtmlTitle(html: string): string {
|
|
126
|
-
const m = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html);
|
|
127
|
-
return m ? stripTags(m[1]) : "";
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
/**
|
|
131
|
-
* HTML 转纯文本:
|
|
132
|
-
* 去掉 script/style/noscript/svg/注释 → 块级元素补换行 → 去标签 → 解实体 → 压缩空白。
|
|
133
|
-
*/
|
|
134
|
-
export function htmlToPlainText(html: string): string {
|
|
135
|
-
let text = html
|
|
136
|
-
.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, " ")
|
|
137
|
-
.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, " ")
|
|
138
|
-
.replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, " ")
|
|
139
|
-
.replace(/<svg\b[^>]*>[\s\S]*?<\/svg>/gi, " ")
|
|
140
|
-
.replace(/<!--[\s\S]*?-->/g, " ")
|
|
141
|
-
.replace(/<\/(p|div|li|tr|h[1-6]|pre|blockquote|section|article|table|ul|ol|header|footer|nav|dl|dd|dt)>/gi, "\n")
|
|
142
|
-
.replace(/<(br|hr)\s*\/?>/gi, "\n")
|
|
143
|
-
.replace(/<[^>]+>/g, "");
|
|
144
|
-
|
|
145
|
-
text = decodeEntities(text);
|
|
146
|
-
|
|
147
|
-
return text
|
|
148
|
-
.split("\n")
|
|
149
|
-
.map((line) => line.replace(/\s+/g, " ").trim())
|
|
150
|
-
.filter((line, i, arr) => line !== "" || (i > 0 && arr[i - 1] !== ""))
|
|
151
|
-
.join("\n")
|
|
152
|
-
.trim();
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
/** 解码 DuckDuckGo 结果链接:/l/?uddg=<encoded> 还原为目标 URL */
|
|
156
|
-
export function decodeDdgHref(href: string): string {
|
|
157
|
-
let h = href.trim();
|
|
158
|
-
if (h.startsWith("//")) h = "https:" + h;
|
|
159
|
-
try {
|
|
160
|
-
const u = new URL(h);
|
|
161
|
-
if (u.hostname === "duckduckgo.com" && u.pathname.startsWith("/l/")) {
|
|
162
|
-
const uddg = u.searchParams.get("uddg");
|
|
163
|
-
if (uddg) return decodeURIComponent(uddg);
|
|
164
|
-
}
|
|
165
|
-
return u.toString();
|
|
166
|
-
} catch {
|
|
167
|
-
return h;
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
/**
|
|
172
|
-
* 解析 DuckDuckGo HTML 搜索结果页(html.duckduckgo.com/html)。
|
|
173
|
-
* 提取 result__a(标题 + href)与 result__snippet(摘要)。
|
|
174
|
-
*/
|
|
175
|
-
export function parseDuckDuckGoHtml(html: string): WebSearchResult[] {
|
|
176
|
-
const results: WebSearchResult[] = [];
|
|
177
|
-
const titleRe = /<a[^>]*class="result__a"[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi;
|
|
178
|
-
const snippetRe = /<a[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/a>/gi;
|
|
179
|
-
|
|
180
|
-
const titleMatches = [...html.matchAll(titleRe)];
|
|
181
|
-
const snippetMatches = [...html.matchAll(snippetRe)];
|
|
182
|
-
|
|
183
|
-
titleMatches.forEach((m, i) => {
|
|
184
|
-
const title = stripTags(m[2]);
|
|
185
|
-
const url = decodeDdgHref(m[1]);
|
|
186
|
-
const snippet = snippetMatches[i] ? stripTags(snippetMatches[i][1]) : "";
|
|
187
|
-
if (!title && !url) return;
|
|
188
|
-
results.push({ title, url, snippet });
|
|
189
|
-
});
|
|
190
|
-
|
|
191
|
-
return results;
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
/** 读取响应体并限制字节数(超出截断),避免整页大文件进入上下文 */
|
|
195
|
-
async function readBodyWithLimit(res: Response, maxBytes: number): Promise<{ text: string; truncated: boolean }> {
|
|
196
|
-
if (!res.body) {
|
|
197
|
-
const text = await res.text();
|
|
198
|
-
return { text, truncated: Buffer.byteLength(text, "utf8") > maxBytes };
|
|
199
|
-
}
|
|
200
|
-
const reader = res.body.getReader();
|
|
201
|
-
const chunks: Uint8Array[] = [];
|
|
202
|
-
let total = 0;
|
|
203
|
-
let truncated = false;
|
|
204
|
-
for (;;) {
|
|
205
|
-
const { done, value } = await reader.read();
|
|
206
|
-
if (done) break;
|
|
207
|
-
if (!value) continue;
|
|
208
|
-
if (total + value.byteLength > maxBytes) {
|
|
209
|
-
const remaining = maxBytes - total;
|
|
210
|
-
if (remaining > 0) chunks.push(value.subarray(0, remaining));
|
|
211
|
-
truncated = true;
|
|
212
|
-
await reader.cancel().catch(() => {});
|
|
213
|
-
break;
|
|
214
|
-
}
|
|
215
|
-
chunks.push(value);
|
|
216
|
-
total += value.byteLength;
|
|
217
|
-
}
|
|
218
|
-
return { text: Buffer.concat(chunks.map((c) => Buffer.from(c))).toString("utf8"), truncated };
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
/**
|
|
222
|
-
* websearch:DuckDuckGo HTML 搜索(免 API key),返回标题 + URL + 摘要。
|
|
223
|
-
* 网络失败抛错(成为 tool-error),搜索无结果返回空数组(不是错误)。
|
|
224
|
-
*/
|
|
225
|
-
export async function webSearchForTool(
|
|
226
|
-
input: WebSearchInput,
|
|
227
|
-
options: WebToolOptions = {},
|
|
228
|
-
): Promise<WebSearchOutput> {
|
|
229
|
-
const query = input.query?.trim();
|
|
230
|
-
if (!query) throw new Error("query is required");
|
|
231
|
-
|
|
232
|
-
const maxResults = normalizeMaxResults(input.maxResults);
|
|
233
|
-
const startedAt = Date.now();
|
|
234
|
-
const fetchImpl = options.fetchImpl ?? fetch;
|
|
235
|
-
const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
|
236
|
-
|
|
237
|
-
const res = await fetchImpl(url, {
|
|
238
|
-
headers: {
|
|
239
|
-
"User-Agent": SEARCH_UA,
|
|
240
|
-
Accept: "text/html,application/xhtml+xml",
|
|
241
|
-
},
|
|
242
|
-
redirect: "follow",
|
|
243
|
-
signal: buildSignal(options.timeoutMs ?? WEB_SEARCH_TIMEOUT_MS, options.abortSignal),
|
|
244
|
-
});
|
|
245
|
-
if (!res.ok) {
|
|
246
|
-
throw new Error(`web search failed: HTTP ${res.status} ${res.statusText}`);
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
const { text } = await readBodyWithLimit(res, WEB_FETCH_MAX_BYTES);
|
|
250
|
-
const all = parseDuckDuckGoHtml(text);
|
|
251
|
-
const results = all.slice(0, maxResults);
|
|
252
|
-
|
|
253
|
-
return {
|
|
254
|
-
query,
|
|
255
|
-
results,
|
|
256
|
-
truncated: all.length > results.length,
|
|
257
|
-
durationMs: Date.now() - startedAt,
|
|
258
|
-
};
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
/**
|
|
262
|
-
* webfetch:HTTP GET + HTML 转纯文本。
|
|
263
|
-
* 只允许 http/https(防 file:// 等本地协议),非 2xx 抛错。
|
|
264
|
-
*/
|
|
265
|
-
export async function webFetchForTool(
|
|
266
|
-
input: WebFetchInput,
|
|
267
|
-
options: WebToolOptions = {},
|
|
268
|
-
): Promise<WebFetchOutput> {
|
|
269
|
-
const raw = input.url?.trim();
|
|
270
|
-
if (!raw) throw new Error("url is required");
|
|
271
|
-
|
|
272
|
-
let parsed: URL;
|
|
273
|
-
try {
|
|
274
|
-
parsed = new URL(raw);
|
|
275
|
-
} catch {
|
|
276
|
-
throw new Error(`invalid URL: ${raw}`);
|
|
277
|
-
}
|
|
278
|
-
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
279
|
-
throw new Error(`unsupported protocol: ${parsed.protocol} (only http/https allowed)`);
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
const maxChars = normalizeMaxChars(input.maxChars);
|
|
283
|
-
const startedAt = Date.now();
|
|
284
|
-
const fetchImpl = options.fetchImpl ?? fetch;
|
|
285
|
-
|
|
286
|
-
const res = await fetchImpl(parsed.toString(), {
|
|
287
|
-
headers: {
|
|
288
|
-
"User-Agent": SEARCH_UA,
|
|
289
|
-
Accept: "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.8",
|
|
290
|
-
},
|
|
291
|
-
redirect: "follow",
|
|
292
|
-
signal: buildSignal(options.timeoutMs ?? WEB_FETCH_TIMEOUT_MS, options.abortSignal),
|
|
293
|
-
});
|
|
294
|
-
if (!res.ok) {
|
|
295
|
-
throw new Error(`HTTP ${res.status} ${res.statusText} for ${parsed.toString()}`);
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
const contentType = res.headers.get("content-type") ?? "";
|
|
299
|
-
const { text: body, truncated: bodyTruncated } = await readBodyWithLimit(res, WEB_FETCH_MAX_BYTES);
|
|
300
|
-
const title = extractHtmlTitle(body);
|
|
301
|
-
const plain = htmlToPlainText(body);
|
|
302
|
-
const text = plain.length > maxChars ? plain.slice(0, maxChars) : plain;
|
|
303
|
-
|
|
304
|
-
return {
|
|
305
|
-
url: parsed.toString(),
|
|
306
|
-
contentType,
|
|
307
|
-
title,
|
|
308
|
-
text,
|
|
309
|
-
chars: text.length,
|
|
310
|
-
truncated: bodyTruncated || plain.length > maxChars,
|
|
311
|
-
durationMs: Date.now() - startedAt,
|
|
312
|
-
};
|
|
313
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* web-tools.ts — websearch / webfetch 内置工具(agent 端联网,业界主流做法)
|
|
3
|
+
*
|
|
4
|
+
* 与 Claude Code 的 WebSearch/WebFetch、Codex 的 web search 相同形态:
|
|
5
|
+
* 模型通过 function calling 调用,agent 进程执行 HTTP 请求,结果回填上下文。
|
|
6
|
+
* 两者都是只读外部网络操作,不触碰本地文件系统,无需权限询问。
|
|
7
|
+
*
|
|
8
|
+
* - websearch:DuckDuckGo HTML 端点(免 API key),返回标题 + URL + 摘要
|
|
9
|
+
* - webfetch:HTTP GET + HTML 转纯文本,控制大小与超时
|
|
10
|
+
*
|
|
11
|
+
* 零新依赖:使用 Node 20 内置 fetch / AbortSignal / Buffer。
|
|
12
|
+
* 解析逻辑拆成纯函数导出,便于单测;fetch 可通过 options 注入 mock。
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
export interface WebSearchInput {
|
|
16
|
+
query: string;
|
|
17
|
+
/** 返回结果条数上限,默认 5,上限 10 */
|
|
18
|
+
maxResults?: number;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface WebSearchResult {
|
|
22
|
+
title: string;
|
|
23
|
+
url: string;
|
|
24
|
+
snippet: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface WebSearchOutput {
|
|
28
|
+
query: string;
|
|
29
|
+
results: WebSearchResult[];
|
|
30
|
+
truncated: boolean;
|
|
31
|
+
durationMs: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface WebFetchInput {
|
|
35
|
+
url: string;
|
|
36
|
+
/** 返回纯文本字符数上限,默认 10000,上限 100000 */
|
|
37
|
+
maxChars?: number;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface WebFetchOutput {
|
|
41
|
+
url: string;
|
|
42
|
+
contentType: string;
|
|
43
|
+
title: string;
|
|
44
|
+
text: string;
|
|
45
|
+
chars: number;
|
|
46
|
+
truncated: boolean;
|
|
47
|
+
durationMs: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export const WEB_SEARCH_TIMEOUT_MS = 15_000;
|
|
51
|
+
export const WEB_FETCH_TIMEOUT_MS = 20_000;
|
|
52
|
+
export const WEB_SEARCH_DEFAULT_RESULTS = 5;
|
|
53
|
+
export const WEB_SEARCH_MAX_RESULTS = 10;
|
|
54
|
+
export const WEB_FETCH_DEFAULT_CHARS = 10_000;
|
|
55
|
+
export const WEB_FETCH_MAX_CHARS = 100_000;
|
|
56
|
+
/** webfetch 读取响应体的大小上限(超出即截断),防止把整个大文件拉进上下文 */
|
|
57
|
+
export const WEB_FETCH_MAX_BYTES = 512 * 1024;
|
|
58
|
+
|
|
59
|
+
const SEARCH_UA =
|
|
60
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36";
|
|
61
|
+
|
|
62
|
+
export type FetchLike = (
|
|
63
|
+
input: string | URL | Request,
|
|
64
|
+
init?: RequestInit,
|
|
65
|
+
) => Promise<Response>;
|
|
66
|
+
|
|
67
|
+
export interface WebToolOptions {
|
|
68
|
+
abortSignal?: AbortSignal;
|
|
69
|
+
/** 测试注入用,默认全局 fetch */
|
|
70
|
+
fetchImpl?: FetchLike;
|
|
71
|
+
timeoutMs?: number;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function normalizeMaxResults(value: number | undefined): number {
|
|
75
|
+
if (value === undefined) return WEB_SEARCH_DEFAULT_RESULTS;
|
|
76
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
77
|
+
throw new Error("maxResults must be a positive integer when provided");
|
|
78
|
+
}
|
|
79
|
+
return Math.min(value, WEB_SEARCH_MAX_RESULTS);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function normalizeMaxChars(value: number | undefined): number {
|
|
83
|
+
if (value === undefined) return WEB_FETCH_DEFAULT_CHARS;
|
|
84
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
85
|
+
throw new Error("maxChars must be a positive integer when provided");
|
|
86
|
+
}
|
|
87
|
+
return Math.min(value, WEB_FETCH_MAX_CHARS);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function buildSignal(timeoutMs: number, abortSignal?: AbortSignal): AbortSignal {
|
|
91
|
+
const signals: AbortSignal[] = [AbortSignal.timeout(timeoutMs)];
|
|
92
|
+
if (abortSignal) signals.push(abortSignal);
|
|
93
|
+
return signals.length === 1 ? signals[0] : AbortSignal.any(signals);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function decodeEntities(s: string): string {
|
|
97
|
+
return s
|
|
98
|
+
.replace(/</gi, "<")
|
|
99
|
+
.replace(/>/gi, ">")
|
|
100
|
+
.replace(/"/gi, '"')
|
|
101
|
+
.replace(/�?39;/gi, "'")
|
|
102
|
+
.replace(/ /gi, " ")
|
|
103
|
+
.replace(/&#(\d+);/g, (_, n: string) => {
|
|
104
|
+
try {
|
|
105
|
+
return String.fromCodePoint(Number(n));
|
|
106
|
+
} catch {
|
|
107
|
+
return "";
|
|
108
|
+
}
|
|
109
|
+
})
|
|
110
|
+
.replace(/&#x([0-9a-f]+);/gi, (_, h: string) => {
|
|
111
|
+
try {
|
|
112
|
+
return String.fromCodePoint(parseInt(h, 16));
|
|
113
|
+
} catch {
|
|
114
|
+
return "";
|
|
115
|
+
}
|
|
116
|
+
})
|
|
117
|
+
.replace(/&/gi, "&");
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function stripTags(s: string): string {
|
|
121
|
+
return decodeEntities(s.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim());
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** 提取 <title> 文本(用于 webfetch 结果),无则返回空字符串 */
|
|
125
|
+
export function extractHtmlTitle(html: string): string {
|
|
126
|
+
const m = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html);
|
|
127
|
+
return m ? stripTags(m[1]) : "";
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* HTML 转纯文本:
|
|
132
|
+
* 去掉 script/style/noscript/svg/注释 → 块级元素补换行 → 去标签 → 解实体 → 压缩空白。
|
|
133
|
+
*/
|
|
134
|
+
export function htmlToPlainText(html: string): string {
|
|
135
|
+
let text = html
|
|
136
|
+
.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, " ")
|
|
137
|
+
.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, " ")
|
|
138
|
+
.replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, " ")
|
|
139
|
+
.replace(/<svg\b[^>]*>[\s\S]*?<\/svg>/gi, " ")
|
|
140
|
+
.replace(/<!--[\s\S]*?-->/g, " ")
|
|
141
|
+
.replace(/<\/(p|div|li|tr|h[1-6]|pre|blockquote|section|article|table|ul|ol|header|footer|nav|dl|dd|dt)>/gi, "\n")
|
|
142
|
+
.replace(/<(br|hr)\s*\/?>/gi, "\n")
|
|
143
|
+
.replace(/<[^>]+>/g, "");
|
|
144
|
+
|
|
145
|
+
text = decodeEntities(text);
|
|
146
|
+
|
|
147
|
+
return text
|
|
148
|
+
.split("\n")
|
|
149
|
+
.map((line) => line.replace(/\s+/g, " ").trim())
|
|
150
|
+
.filter((line, i, arr) => line !== "" || (i > 0 && arr[i - 1] !== ""))
|
|
151
|
+
.join("\n")
|
|
152
|
+
.trim();
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** 解码 DuckDuckGo 结果链接:/l/?uddg=<encoded> 还原为目标 URL */
|
|
156
|
+
export function decodeDdgHref(href: string): string {
|
|
157
|
+
let h = href.trim();
|
|
158
|
+
if (h.startsWith("//")) h = "https:" + h;
|
|
159
|
+
try {
|
|
160
|
+
const u = new URL(h);
|
|
161
|
+
if (u.hostname === "duckduckgo.com" && u.pathname.startsWith("/l/")) {
|
|
162
|
+
const uddg = u.searchParams.get("uddg");
|
|
163
|
+
if (uddg) return decodeURIComponent(uddg);
|
|
164
|
+
}
|
|
165
|
+
return u.toString();
|
|
166
|
+
} catch {
|
|
167
|
+
return h;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* 解析 DuckDuckGo HTML 搜索结果页(html.duckduckgo.com/html)。
|
|
173
|
+
* 提取 result__a(标题 + href)与 result__snippet(摘要)。
|
|
174
|
+
*/
|
|
175
|
+
export function parseDuckDuckGoHtml(html: string): WebSearchResult[] {
|
|
176
|
+
const results: WebSearchResult[] = [];
|
|
177
|
+
const titleRe = /<a[^>]*class="result__a"[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi;
|
|
178
|
+
const snippetRe = /<a[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/a>/gi;
|
|
179
|
+
|
|
180
|
+
const titleMatches = [...html.matchAll(titleRe)];
|
|
181
|
+
const snippetMatches = [...html.matchAll(snippetRe)];
|
|
182
|
+
|
|
183
|
+
titleMatches.forEach((m, i) => {
|
|
184
|
+
const title = stripTags(m[2]);
|
|
185
|
+
const url = decodeDdgHref(m[1]);
|
|
186
|
+
const snippet = snippetMatches[i] ? stripTags(snippetMatches[i][1]) : "";
|
|
187
|
+
if (!title && !url) return;
|
|
188
|
+
results.push({ title, url, snippet });
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
return results;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** 读取响应体并限制字节数(超出截断),避免整页大文件进入上下文 */
|
|
195
|
+
async function readBodyWithLimit(res: Response, maxBytes: number): Promise<{ text: string; truncated: boolean }> {
|
|
196
|
+
if (!res.body) {
|
|
197
|
+
const text = await res.text();
|
|
198
|
+
return { text, truncated: Buffer.byteLength(text, "utf8") > maxBytes };
|
|
199
|
+
}
|
|
200
|
+
const reader = res.body.getReader();
|
|
201
|
+
const chunks: Uint8Array[] = [];
|
|
202
|
+
let total = 0;
|
|
203
|
+
let truncated = false;
|
|
204
|
+
for (;;) {
|
|
205
|
+
const { done, value } = await reader.read();
|
|
206
|
+
if (done) break;
|
|
207
|
+
if (!value) continue;
|
|
208
|
+
if (total + value.byteLength > maxBytes) {
|
|
209
|
+
const remaining = maxBytes - total;
|
|
210
|
+
if (remaining > 0) chunks.push(value.subarray(0, remaining));
|
|
211
|
+
truncated = true;
|
|
212
|
+
await reader.cancel().catch(() => {});
|
|
213
|
+
break;
|
|
214
|
+
}
|
|
215
|
+
chunks.push(value);
|
|
216
|
+
total += value.byteLength;
|
|
217
|
+
}
|
|
218
|
+
return { text: Buffer.concat(chunks.map((c) => Buffer.from(c))).toString("utf8"), truncated };
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* websearch:DuckDuckGo HTML 搜索(免 API key),返回标题 + URL + 摘要。
|
|
223
|
+
* 网络失败抛错(成为 tool-error),搜索无结果返回空数组(不是错误)。
|
|
224
|
+
*/
|
|
225
|
+
export async function webSearchForTool(
|
|
226
|
+
input: WebSearchInput,
|
|
227
|
+
options: WebToolOptions = {},
|
|
228
|
+
): Promise<WebSearchOutput> {
|
|
229
|
+
const query = input.query?.trim();
|
|
230
|
+
if (!query) throw new Error("query is required");
|
|
231
|
+
|
|
232
|
+
const maxResults = normalizeMaxResults(input.maxResults);
|
|
233
|
+
const startedAt = Date.now();
|
|
234
|
+
const fetchImpl = options.fetchImpl ?? fetch;
|
|
235
|
+
const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
|
236
|
+
|
|
237
|
+
const res = await fetchImpl(url, {
|
|
238
|
+
headers: {
|
|
239
|
+
"User-Agent": SEARCH_UA,
|
|
240
|
+
Accept: "text/html,application/xhtml+xml",
|
|
241
|
+
},
|
|
242
|
+
redirect: "follow",
|
|
243
|
+
signal: buildSignal(options.timeoutMs ?? WEB_SEARCH_TIMEOUT_MS, options.abortSignal),
|
|
244
|
+
});
|
|
245
|
+
if (!res.ok) {
|
|
246
|
+
throw new Error(`web search failed: HTTP ${res.status} ${res.statusText}`);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const { text } = await readBodyWithLimit(res, WEB_FETCH_MAX_BYTES);
|
|
250
|
+
const all = parseDuckDuckGoHtml(text);
|
|
251
|
+
const results = all.slice(0, maxResults);
|
|
252
|
+
|
|
253
|
+
return {
|
|
254
|
+
query,
|
|
255
|
+
results,
|
|
256
|
+
truncated: all.length > results.length,
|
|
257
|
+
durationMs: Date.now() - startedAt,
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* webfetch:HTTP GET + HTML 转纯文本。
|
|
263
|
+
* 只允许 http/https(防 file:// 等本地协议),非 2xx 抛错。
|
|
264
|
+
*/
|
|
265
|
+
export async function webFetchForTool(
|
|
266
|
+
input: WebFetchInput,
|
|
267
|
+
options: WebToolOptions = {},
|
|
268
|
+
): Promise<WebFetchOutput> {
|
|
269
|
+
const raw = input.url?.trim();
|
|
270
|
+
if (!raw) throw new Error("url is required");
|
|
271
|
+
|
|
272
|
+
let parsed: URL;
|
|
273
|
+
try {
|
|
274
|
+
parsed = new URL(raw);
|
|
275
|
+
} catch {
|
|
276
|
+
throw new Error(`invalid URL: ${raw}`);
|
|
277
|
+
}
|
|
278
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
279
|
+
throw new Error(`unsupported protocol: ${parsed.protocol} (only http/https allowed)`);
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
const maxChars = normalizeMaxChars(input.maxChars);
|
|
283
|
+
const startedAt = Date.now();
|
|
284
|
+
const fetchImpl = options.fetchImpl ?? fetch;
|
|
285
|
+
|
|
286
|
+
const res = await fetchImpl(parsed.toString(), {
|
|
287
|
+
headers: {
|
|
288
|
+
"User-Agent": SEARCH_UA,
|
|
289
|
+
Accept: "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.8",
|
|
290
|
+
},
|
|
291
|
+
redirect: "follow",
|
|
292
|
+
signal: buildSignal(options.timeoutMs ?? WEB_FETCH_TIMEOUT_MS, options.abortSignal),
|
|
293
|
+
});
|
|
294
|
+
if (!res.ok) {
|
|
295
|
+
throw new Error(`HTTP ${res.status} ${res.statusText} for ${parsed.toString()}`);
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const contentType = res.headers.get("content-type") ?? "";
|
|
299
|
+
const { text: body, truncated: bodyTruncated } = await readBodyWithLimit(res, WEB_FETCH_MAX_BYTES);
|
|
300
|
+
const title = extractHtmlTitle(body);
|
|
301
|
+
const plain = htmlToPlainText(body);
|
|
302
|
+
const text = plain.length > maxChars ? plain.slice(0, maxChars) : plain;
|
|
303
|
+
|
|
304
|
+
return {
|
|
305
|
+
url: parsed.toString(),
|
|
306
|
+
contentType,
|
|
307
|
+
title,
|
|
308
|
+
text,
|
|
309
|
+
chars: text.length,
|
|
310
|
+
truncated: bodyTruncated || plain.length > maxChars,
|
|
311
|
+
durationMs: Date.now() - startedAt,
|
|
312
|
+
};
|
|
313
|
+
}
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
{
|
|
2
|
-
"extends": "./tsconfig.json",
|
|
3
|
-
"compilerOptions": {
|
|
4
|
-
"noEmit": false,
|
|
5
|
-
"outDir": "./dist",
|
|
6
|
-
"rootDir": "./src",
|
|
7
|
-
"declaration": true,
|
|
8
|
-
"declarationMap": false,
|
|
9
|
-
"sourceMap": false
|
|
10
|
-
},
|
|
11
|
-
"include": ["src/*.ts"],
|
|
12
|
-
"exclude": ["src/__tests__/**"]
|
|
13
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"extends": "./tsconfig.json",
|
|
3
|
+
"compilerOptions": {
|
|
4
|
+
"noEmit": false,
|
|
5
|
+
"outDir": "./dist",
|
|
6
|
+
"rootDir": "./src",
|
|
7
|
+
"declaration": true,
|
|
8
|
+
"declarationMap": false,
|
|
9
|
+
"sourceMap": false
|
|
10
|
+
},
|
|
11
|
+
"include": ["src/*.ts"],
|
|
12
|
+
"exclude": ["src/__tests__/**"]
|
|
13
|
+
}
|