@echo-agent/coding 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -8
- package/package.json +6 -5
- package/src/agent.ts +34 -31
- package/src/cli.ts +21 -6
- package/src/extensions.ts +6 -65
- package/src/index.ts +4 -6
- package/src/permission.ts +27 -49
- package/src/prompt.ts +1 -41
- package/src/tools/bash.ts +0 -384
- package/src/tools/fs.ts +0 -424
- package/src/tools/search.ts +0 -273
- package/src/tools/web.ts +0 -434
package/src/tools/web.ts
DELETED
|
@@ -1,434 +0,0 @@
|
|
|
1
|
-
// web 一组(2026-09-03/04):`web_fetch` 取一个 URL 的正文;`web_search` 走 Brave Search API 拿结果列表。
|
|
2
|
-
// 不做出网确认(用户拍板:缺省全放行);但 web_fetch 不取内网地址(2026-11-03,见 `isPrivateAddress`)。
|
|
3
|
-
// HTML 在这层剥成可读文本,不加依赖。
|
|
4
|
-
// 两件都是延迟工具(`deferred`):多数编码任务用不上,经 tool_search 取过才上菜单。
|
|
5
|
-
//
|
|
6
|
-
// 搜索源选 Brave(2026-09-04 用户拍板):与模型无关,一把 key 五家 provider 通用;厂商自带搜索要改 core 的
|
|
7
|
-
// dialect 且 deepseek 没有,另议。key 的解析顺序与模型 key 一样:环境变量 `BRAVE_API_KEY` → 凭据 store 的
|
|
8
|
-
// `brave` 条目 → 没有(工具如实报「没配」,不猜)。key 的值不进任何错误信息。
|
|
9
|
-
|
|
10
|
-
import { lookup } from "node:dns/promises";
|
|
11
|
-
import { isIP } from "node:net";
|
|
12
|
-
import { toolError, toolOk, type CredentialStore, type ModelTool } from "@echo-agent/core";
|
|
13
|
-
|
|
14
|
-
const TIMEOUT_MS = 30_000;
|
|
15
|
-
/** 重定向最多跟这么多跳(与 opencode 的 webfetch 同数):每跳都重查地址,所以手动跟、不交给 fetch。 */
|
|
16
|
-
const MAX_REDIRECTS = 10;
|
|
17
|
-
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
18
|
-
/** 响应体最多读这么多字节:超过就截,避免一个大文件把内存与上下文一起吃掉。 */
|
|
19
|
-
const MAX_BYTES = 4_000_000;
|
|
20
|
-
const DEFAULT_MAX_CHARS = 20_000;
|
|
21
|
-
|
|
22
|
-
const BRAVE_ENDPOINT = "https://api.search.brave.com/res/v1/web/search";
|
|
23
|
-
const BRAVE_ENV = "BRAVE_API_KEY";
|
|
24
|
-
/** 凭据 store 里的条目名:`{ "brave": { "apiKey": "…" } }`。 */
|
|
25
|
-
const BRAVE_CREDENTIAL = "brave";
|
|
26
|
-
const SEARCH_TIMEOUT_MS = 20_000;
|
|
27
|
-
const SEARCH_COUNT_DEFAULT = 5;
|
|
28
|
-
const SEARCH_COUNT_MAX = 10;
|
|
29
|
-
|
|
30
|
-
export type WebDeps = {
|
|
31
|
-
/** 搜索 key 的第二来源(第一是环境变量)。不给 = 只认环境变量。 */
|
|
32
|
-
credentials?: Pick<CredentialStore, "read">;
|
|
33
|
-
/** 测试用:把 Brave 的接口指到假服务。 */
|
|
34
|
-
searchEndpoint?: string;
|
|
35
|
-
/**
|
|
36
|
-
* 测试用:web_fetch 判「是不是内网」之前那一步主机名解析(缺省:IP 字面量原样,其余走系统 DNS)。
|
|
37
|
-
* 本机起的假服务在 127.0.0.1,测试把它解析成公网地址才发得出去;它只换「查什么」,判据本身不放开。
|
|
38
|
-
*/
|
|
39
|
-
resolveHost?: (host: string) => Promise<readonly string[]>;
|
|
40
|
-
};
|
|
41
|
-
|
|
42
|
-
export function makeWebTools(deps: WebDeps = {}): ModelTool[] {
|
|
43
|
-
return [webFetchTool(deps), webSearchTool(deps)] as ModelTool[];
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
/* ─────────────────── 不取内网地址 ─────────────────── */
|
|
47
|
-
|
|
48
|
-
/**
|
|
49
|
-
* web_fetch 只取公网地址(2026-11-03 拍板,`docs/decisions/implemented/2026-11-03-web-fetch-refuses-private-addresses.md`):
|
|
50
|
-
* 抓回来的网页是不可信内容,一句注入就能让模型去取 `http://169.254.169.254/` 的云凭据或本机的管理口,
|
|
51
|
-
* 再把结果带进对话。所以每一跳发出之前都查:主机名解析出的**任一**地址落在下面这些段就拒。
|
|
52
|
-
* 已知的缺口:DNS rebinding(查的那次与 fetch 自己解析的那次之间换了答案)不在防线内——fetch 不让指定连哪个地址。
|
|
53
|
-
*/
|
|
54
|
-
export function isPrivateAddress(ip: string): boolean {
|
|
55
|
-
const bare = ip.replace(/%.*$/, ""); // IPv6 的 zone id(fe80::1%lo0)
|
|
56
|
-
const kind = isIP(bare);
|
|
57
|
-
if (kind === 4) return privateV4(bare.split(".").map(Number) as [number, number, number, number]);
|
|
58
|
-
if (kind === 6) return privateV6(bare);
|
|
59
|
-
return true; // 认不出的不放行
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
function privateV4([a, b]: [number, number, number, number]): boolean {
|
|
63
|
-
return (
|
|
64
|
-
a === 0 || // 0.0.0.0/8:未指定 / 本网络
|
|
65
|
-
a === 10 || // 10/8
|
|
66
|
-
a === 127 || // loopback
|
|
67
|
-
(a === 169 && b === 254) || // 链路本地,含云元数据 169.254.169.254
|
|
68
|
-
(a === 172 && b >= 16 && b <= 31) || // 172.16/12
|
|
69
|
-
(a === 192 && b === 168) || // 192.168/16
|
|
70
|
-
(a === 100 && b >= 64 && b <= 127) // CGNAT 100.64/10
|
|
71
|
-
);
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
function privateV6(ip: string): boolean {
|
|
75
|
-
const g = ipv6Groups(ip);
|
|
76
|
-
if (g === null) return true;
|
|
77
|
-
const zeroHead = g.slice(0, 5).every((x) => x === 0);
|
|
78
|
-
if (zeroHead && g[5] === 0xffff) return privateV4([g[6]! >> 8, g[6]! & 0xff, g[7]! >> 8, g[7]! & 0xff]); // ::ffff:a.b.c.d(IPv4 映射)
|
|
79
|
-
if (zeroHead && g[5] === 0) {
|
|
80
|
-
if (g[6] === 0 && (g[7] === 0 || g[7] === 1)) return true; // :: 未指定、::1 loopback
|
|
81
|
-
return privateV4([g[6]! >> 8, g[6]! & 0xff, g[7]! >> 8, g[7]! & 0xff]); // ::a.b.c.d(已废弃的 IPv4 兼容写法)
|
|
82
|
-
}
|
|
83
|
-
return (g[0]! & 0xfe00) === 0xfc00 || (g[0]! & 0xffc0) === 0xfe80; // fc00::/7 唯一本地、fe80::/10 链路本地
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
/** IPv6 文本 → 8 组 16 位。末尾可以是点分 IPv4。格式不对 = null。 */
|
|
87
|
-
function ipv6Groups(ip: string): number[] | null {
|
|
88
|
-
let text = ip.toLowerCase();
|
|
89
|
-
const v4 = /(\d+)\.(\d+)\.(\d+)\.(\d+)$/.exec(text);
|
|
90
|
-
if (v4 !== null) {
|
|
91
|
-
const [a, b, c, d] = v4.slice(1).map(Number) as [number, number, number, number];
|
|
92
|
-
text = `${text.slice(0, v4.index)}${((a << 8) | b).toString(16)}:${((c << 8) | d).toString(16)}`;
|
|
93
|
-
}
|
|
94
|
-
const halves = text.split("::");
|
|
95
|
-
if (halves.length > 2) return null;
|
|
96
|
-
const parse = (s: string): number[] => (s === "" ? [] : s.split(":").map((h) => Number.parseInt(h, 16)));
|
|
97
|
-
const head = parse(halves[0]!);
|
|
98
|
-
const tail = halves.length === 2 ? parse(halves[1]!) : [];
|
|
99
|
-
const fill = 8 - head.length - tail.length;
|
|
100
|
-
if (fill < 0 || (halves.length === 1 && fill !== 0)) return null;
|
|
101
|
-
const groups = [...head, ...Array<number>(halves.length === 2 ? fill : 0).fill(0), ...tail];
|
|
102
|
-
return groups.length === 8 && groups.every((x) => Number.isInteger(x) && x >= 0 && x <= 0xffff) ? groups : null;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
/** 缺省的主机名解析:IP 字面量原样,其余走系统 DNS,取全部地址。 */
|
|
106
|
-
async function systemResolve(host: string): Promise<readonly string[]> {
|
|
107
|
-
if (isIP(host) !== 0) return [host];
|
|
108
|
-
return (await lookup(host, { all: true, verbatim: true })).map((a) => a.address);
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
/** 这一跳能不能发:null = 能;否则是给模型看的拒绝理由。 */
|
|
112
|
-
async function refusedHost(url: URL, resolveHost: (host: string) => Promise<readonly string[]>, via: string | null): Promise<string | null> {
|
|
113
|
-
const host = url.hostname.replace(/^\[|\]$/g, "");
|
|
114
|
-
let addresses: readonly string[];
|
|
115
|
-
try {
|
|
116
|
-
addresses = await resolveHost(host);
|
|
117
|
-
} catch (e) {
|
|
118
|
-
return `Fetch failed for ${url.href}: could not resolve ${host} (${e instanceof Error ? e.message : String(e)})`;
|
|
119
|
-
}
|
|
120
|
-
const bad = addresses.find(isPrivateAddress);
|
|
121
|
-
if (bad === undefined) return null;
|
|
122
|
-
const how = via === null ? url.href : `${via}, which redirects to ${url.href}`;
|
|
123
|
-
return (
|
|
124
|
-
`Refused to fetch ${how}: ${host} resolves to ${bad}, a loopback, private, link-local or otherwise non-public address. ` +
|
|
125
|
-
"web_fetch only reaches public internet addresses; this is a safety restriction, not a network error."
|
|
126
|
-
);
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
/** 环境变量 → 凭据 store → 没有。值只在这里经手,不进返回给模型的任何文字。 */
|
|
130
|
-
async function searchKey(deps: WebDeps): Promise<string | undefined> {
|
|
131
|
-
const env = process.env[BRAVE_ENV];
|
|
132
|
-
if (env !== undefined && env !== "") return env;
|
|
133
|
-
const stored = await deps.credentials?.read(BRAVE_CREDENTIAL);
|
|
134
|
-
return stored?.type === "api_key" && stored.key !== "" ? stored.key : undefined;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
type BraveResult = { title?: unknown; url?: unknown; description?: unknown; age?: unknown; page_age?: unknown };
|
|
138
|
-
|
|
139
|
-
const FRESHNESS: Record<string, string> = { day: "pd", week: "pw", month: "pm", year: "py" };
|
|
140
|
-
|
|
141
|
-
function webSearchTool(deps: WebDeps): ModelTool<{ query: string; count?: number; freshness?: string }> {
|
|
142
|
-
return {
|
|
143
|
-
kind: "model",
|
|
144
|
-
name: "web_search",
|
|
145
|
-
label: "搜网页",
|
|
146
|
-
deferred: true,
|
|
147
|
-
description:
|
|
148
|
-
"Search the web (Brave Search) and return the top results as title, URL and snippet; read a result with web_fetch. " +
|
|
149
|
-
"count is 1-10 (default 5); freshness limits results to the last day, week, month or year. " +
|
|
150
|
-
`Needs a Brave Search API key: the ${BRAVE_ENV} environment variable or a '${BRAVE_CREDENTIAL}' entry in the credentials file.`,
|
|
151
|
-
parameters: {
|
|
152
|
-
type: "object",
|
|
153
|
-
properties: {
|
|
154
|
-
query: { type: "string" },
|
|
155
|
-
count: { type: "number", description: "How many results, 1-10 (default 5)" },
|
|
156
|
-
freshness: { type: "string", enum: ["day", "week", "month", "year"], description: "Only results from the last day/week/month/year" },
|
|
157
|
-
},
|
|
158
|
-
required: ["query"],
|
|
159
|
-
},
|
|
160
|
-
async execute({ query, count, freshness }, ctx) {
|
|
161
|
-
const q = query.trim();
|
|
162
|
-
if (q === "") return toolError("query is empty");
|
|
163
|
-
const key = await searchKey(deps);
|
|
164
|
-
if (key === undefined) {
|
|
165
|
-
return toolError(
|
|
166
|
-
`No search service configured: set ${BRAVE_ENV}, or add { "${BRAVE_CREDENTIAL}": { "apiKey": "…" } } to $ECHO_HOME/credentials.json ` +
|
|
167
|
-
"(a Brave Search API key, see https://brave.com/search/api/).",
|
|
168
|
-
);
|
|
169
|
-
}
|
|
170
|
-
const url = new URL(deps.searchEndpoint ?? BRAVE_ENDPOINT);
|
|
171
|
-
url.searchParams.set("q", q);
|
|
172
|
-
url.searchParams.set("count", String(Math.min(SEARCH_COUNT_MAX, Math.max(1, Math.floor(count ?? SEARCH_COUNT_DEFAULT)))));
|
|
173
|
-
if (freshness !== undefined) {
|
|
174
|
-
const code = FRESHNESS[freshness];
|
|
175
|
-
if (code === undefined) return toolError(`freshness must be one of day, week, month, year (got '${freshness}')`);
|
|
176
|
-
url.searchParams.set("freshness", code);
|
|
177
|
-
}
|
|
178
|
-
const timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
|
|
179
|
-
const signal = ctx.signal === undefined ? timeout : AbortSignal.any([ctx.signal, timeout]);
|
|
180
|
-
let res: Response;
|
|
181
|
-
let text: string;
|
|
182
|
-
try {
|
|
183
|
-
res = await fetch(url, { signal, headers: { accept: "application/json", "x-subscription-token": key } });
|
|
184
|
-
text = await res.text();
|
|
185
|
-
} catch (e) {
|
|
186
|
-
return toolError(`Search request failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
187
|
-
}
|
|
188
|
-
if (res.status === 401 || res.status === 403) return toolError(`Brave Search rejected the API key (HTTP ${res.status}); check ${BRAVE_ENV} or the '${BRAVE_CREDENTIAL}' credential`);
|
|
189
|
-
if (res.status === 429) return toolError("Brave Search rate limit or quota exceeded (HTTP 429); wait or check the plan");
|
|
190
|
-
if (!res.ok) return toolError(`Brave Search returned HTTP ${res.status} ${res.statusText}\n${text.slice(0, 300)}`);
|
|
191
|
-
let results: BraveResult[];
|
|
192
|
-
try {
|
|
193
|
-
const parsed = JSON.parse(text) as { web?: { results?: unknown } };
|
|
194
|
-
results = Array.isArray(parsed.web?.results) ? (parsed.web.results as BraveResult[]) : [];
|
|
195
|
-
} catch {
|
|
196
|
-
return toolError("Brave Search returned a response that is not JSON");
|
|
197
|
-
}
|
|
198
|
-
if (results.length === 0) return toolOk(`No results for "${q}"`, { query: q, count: 0 });
|
|
199
|
-
const lines = results.map((r, i) => {
|
|
200
|
-
const title = typeof r.title === "string" ? stripTags(r.title) : "(untitled)";
|
|
201
|
-
const link = typeof r.url === "string" ? r.url : "";
|
|
202
|
-
const snippet = typeof r.description === "string" ? stripTags(r.description) : "";
|
|
203
|
-
const age = typeof r.age === "string" ? r.age : typeof r.page_age === "string" ? r.page_age : "";
|
|
204
|
-
return `${i + 1}. ${title}${age === "" ? "" : ` (${age})`}\n ${link}${snippet === "" ? "" : `\n ${snippet}`}`;
|
|
205
|
-
});
|
|
206
|
-
return toolOk(`Results for "${q}":\n${lines.join("\n")}`, { query: q, count: results.length });
|
|
207
|
-
},
|
|
208
|
-
};
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
/** Brave 的标题与摘要里带 `<strong>` 之类的高亮标签:剥掉、解实体、压空白。 */
|
|
212
|
-
function stripTags(s: string): string {
|
|
213
|
-
return decodeEntities(s.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
function webFetchTool(deps: WebDeps): ModelTool<{ url: string; max_chars?: number }> {
|
|
217
|
-
const resolveHost = deps.resolveHost ?? systemResolve;
|
|
218
|
-
return {
|
|
219
|
-
kind: "model",
|
|
220
|
-
name: "web_fetch",
|
|
221
|
-
label: "取网页",
|
|
222
|
-
deferred: true,
|
|
223
|
-
description:
|
|
224
|
-
"Fetch a URL (http or https) and return its content as text: HTML is reduced to readable text with headings, " +
|
|
225
|
-
"links kept as [text](url) and <pre> blocks kept verbatim as code blocks; JSON and plain text come back as they are. " +
|
|
226
|
-
`Follows up to ${MAX_REDIRECTS} redirects, 30 s timeout, output capped at max_chars (default 20000). ` +
|
|
227
|
-
"Only public internet addresses: loopback, private-network and link-local hosts (localhost, 10.x, 192.168.x, 169.254.x, ...) are refused. " +
|
|
228
|
-
"Find pages with web_search.",
|
|
229
|
-
parameters: {
|
|
230
|
-
type: "object",
|
|
231
|
-
properties: {
|
|
232
|
-
url: { type: "string" },
|
|
233
|
-
max_chars: { type: "number", description: "Cap on returned characters, default 20000" },
|
|
234
|
-
},
|
|
235
|
-
required: ["url"],
|
|
236
|
-
},
|
|
237
|
-
async execute({ url, max_chars }, ctx) {
|
|
238
|
-
let target: URL;
|
|
239
|
-
try {
|
|
240
|
-
target = new URL(url);
|
|
241
|
-
} catch {
|
|
242
|
-
return toolError(`Not a valid URL: ${url}`);
|
|
243
|
-
}
|
|
244
|
-
if (target.protocol !== "http:" && target.protocol !== "https:") return toolError(`Only http and https are supported: ${url}`);
|
|
245
|
-
const timeout = AbortSignal.timeout(TIMEOUT_MS);
|
|
246
|
-
const signal = ctx.signal === undefined ? timeout : AbortSignal.any([ctx.signal, timeout]);
|
|
247
|
-
let res: Response;
|
|
248
|
-
let bytes: Uint8Array;
|
|
249
|
-
let clipped = false;
|
|
250
|
-
let current = target; // 跟到第几跳了:最终 URL 也是相对链接的基准
|
|
251
|
-
try {
|
|
252
|
-
// 重定向手动逐跳跟:fetch 自己跟的话中间那几跳不经检查,公网页面一个 302 就能把请求带进内网
|
|
253
|
-
for (let hop = 0; ; hop++) {
|
|
254
|
-
const refused = await refusedHost(current, resolveHost, hop === 0 ? null : target.href);
|
|
255
|
-
if (refused !== null) return toolError(refused);
|
|
256
|
-
res = await fetch(current, {
|
|
257
|
-
signal,
|
|
258
|
-
redirect: "manual",
|
|
259
|
-
headers: { "user-agent": "echo-coding", accept: "text/html, text/plain, application/json;q=0.9, */*;q=0.5" },
|
|
260
|
-
});
|
|
261
|
-
const location = res.headers.get("location");
|
|
262
|
-
if (!REDIRECT_STATUSES.has(res.status) || location === null) break;
|
|
263
|
-
await res.body?.cancel();
|
|
264
|
-
if (hop >= MAX_REDIRECTS) return toolError(`Too many redirects (more than ${MAX_REDIRECTS}) starting from ${target.href}`);
|
|
265
|
-
let next: URL;
|
|
266
|
-
try {
|
|
267
|
-
next = new URL(location, current);
|
|
268
|
-
} catch {
|
|
269
|
-
return toolError(`${current.href} redirects to an invalid location: ${location}`);
|
|
270
|
-
}
|
|
271
|
-
if (next.protocol !== "http:" && next.protocol !== "https:") return toolError(`${current.href} redirects to an unsupported scheme: ${next.href}`);
|
|
272
|
-
current = next;
|
|
273
|
-
}
|
|
274
|
-
// 流式读、到上限就取消:`arrayBuffer()` 会先把整个响应体收进内存再截,MAX_BYTES 那时限不住内存(review 2026-09-07)
|
|
275
|
-
const chunks: Uint8Array[] = [];
|
|
276
|
-
let total = 0;
|
|
277
|
-
const reader = res.body?.getReader();
|
|
278
|
-
if (reader !== undefined) {
|
|
279
|
-
for (;;) {
|
|
280
|
-
const { done, value } = await reader.read();
|
|
281
|
-
if (done) break;
|
|
282
|
-
if (value === undefined) continue;
|
|
283
|
-
if (total + value.byteLength > MAX_BYTES) {
|
|
284
|
-
chunks.push(value.subarray(0, MAX_BYTES - total));
|
|
285
|
-
total = MAX_BYTES;
|
|
286
|
-
clipped = true;
|
|
287
|
-
await reader.cancel();
|
|
288
|
-
break;
|
|
289
|
-
}
|
|
290
|
-
chunks.push(value);
|
|
291
|
-
total += value.byteLength;
|
|
292
|
-
}
|
|
293
|
-
}
|
|
294
|
-
bytes = new Uint8Array(total);
|
|
295
|
-
let offset = 0;
|
|
296
|
-
for (const c of chunks) {
|
|
297
|
-
bytes.set(c, offset);
|
|
298
|
-
offset += c.byteLength;
|
|
299
|
-
}
|
|
300
|
-
} catch (e) {
|
|
301
|
-
return toolError(`Fetch failed for ${url}: ${e instanceof Error ? e.message : String(e)}`);
|
|
302
|
-
}
|
|
303
|
-
const type = (res.headers.get("content-type") ?? "").toLowerCase();
|
|
304
|
-
const raw = decodeBody(bytes, type);
|
|
305
|
-
const finalUrl = current.href;
|
|
306
|
-
if (!res.ok) return toolError(`HTTP ${res.status} ${res.statusText} for ${finalUrl}\n${raw.slice(0, 500)}`);
|
|
307
|
-
let text: string;
|
|
308
|
-
if (type.includes("html")) text = htmlToText(raw, finalUrl);
|
|
309
|
-
else if (type === "" || type.startsWith("text/") || type.includes("json") || type.includes("xml")) text = raw;
|
|
310
|
-
else return toolError(`Unsupported content type '${type}' (${clipped ? `over ${MAX_BYTES}` : String(bytes.byteLength)} bytes) at ${url}`);
|
|
311
|
-
const cap = Math.max(1, max_chars ?? DEFAULT_MAX_CHARS);
|
|
312
|
-
const head = finalUrl !== target.href ? `(redirected to ${finalUrl})\n` : "";
|
|
313
|
-
const body =
|
|
314
|
-
text.length > cap ? `${text.slice(0, cap)}\n…[truncated: ${text.length} characters in total; raise max_chars to see more]` : text;
|
|
315
|
-
return toolOk(`${head}${body === "" ? "(empty)" : body}${clipped ? `\n…[response body over ${MAX_BYTES} bytes, cut]` : ""}`, {
|
|
316
|
-
url: finalUrl,
|
|
317
|
-
status: res.status,
|
|
318
|
-
contentType: type,
|
|
319
|
-
});
|
|
320
|
-
},
|
|
321
|
-
};
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
/**
|
|
325
|
-
* 响应体字节 → 文本:先认 `Content-Type` 的 charset;没写、又是 HTML(或没有类型)就看文档开头的 `<meta charset>` /
|
|
326
|
-
* `<meta http-equiv="Content-Type" content="…; charset=…">`;都没有或认不出这个编码名,按 UTF-8 解。
|
|
327
|
-
*/
|
|
328
|
-
function decodeBody(bytes: Uint8Array, contentType: string): string {
|
|
329
|
-
const declared =
|
|
330
|
-
/charset\s*=\s*["']?([^;"'\s]+)/i.exec(contentType)?.[1] ??
|
|
331
|
-
(contentType === "" || contentType.includes("html") ? sniffMetaCharset(bytes) : undefined);
|
|
332
|
-
if (declared !== undefined) {
|
|
333
|
-
try {
|
|
334
|
-
return new TextDecoder(declared).decode(bytes);
|
|
335
|
-
} catch {
|
|
336
|
-
// 编码名认不出:退 UTF-8
|
|
337
|
-
}
|
|
338
|
-
}
|
|
339
|
-
return new TextDecoder().decode(bytes);
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
/** 文档开头 4 KB 里的 `<meta … charset=…>`。按 latin1 读——编码名本身是 ASCII,读的时候还不知道编码。 */
|
|
343
|
-
function sniffMetaCharset(bytes: Uint8Array): string | undefined {
|
|
344
|
-
const head = new TextDecoder("latin1").decode(bytes.subarray(0, 4096));
|
|
345
|
-
return /<meta\b[^>]*?charset\s*=\s*["']?([^"'\s/>;]+)/i.exec(head)?.[1];
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
/** 链接目标 → 绝对 http(s) URL;本来就是绝对的原样返回。页内锚点、解析不了、不是 http(s) = null(只留文字)。 */
|
|
349
|
-
function absoluteLink(href: string, baseUrl: string | undefined): string | null {
|
|
350
|
-
if (/^https?:\/\//i.test(href)) return href;
|
|
351
|
-
if (href.startsWith("#") || baseUrl === undefined) return null;
|
|
352
|
-
try {
|
|
353
|
-
const u = new URL(href, baseUrl);
|
|
354
|
-
return u.protocol === "http:" || u.protocol === "https:" ? u.href : null;
|
|
355
|
-
} catch {
|
|
356
|
-
return null;
|
|
357
|
-
}
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
/**
|
|
361
|
-
* HTML → 可读文本:去 script/style/head,标题成 `#`、列表成 `-`、链接留成 `[text](url)`,块级标签换行,实体解码。
|
|
362
|
-
* `<pre>` 整块原样(缩进、空行都不压),成一段 ``` 代码块。给了页面 URL(`baseUrl`)就把相对链接解析成绝对链接;
|
|
363
|
-
* 没给只认本来就是绝对的 http(s) 链接。页内锚点(`#…`)与非 http(s) 链接只留文字。
|
|
364
|
-
*/
|
|
365
|
-
export function htmlToText(html: string, baseUrl?: string): string {
|
|
366
|
-
let s = html.replace(/<!--[\s\S]*?-->/g, "").replace(/<(script|style|noscript|svg|template)\b[\s\S]*?<\/\1\s*>/gi, "");
|
|
367
|
-
const title = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(s)?.[1];
|
|
368
|
-
s = s.replace(/<head\b[\s\S]*?<\/head\s*>/i, "");
|
|
369
|
-
// pre 先摘出来换成占位(私用区字符):后面逐行压空白、折叠空行都碰不到它,最后原样放回
|
|
370
|
-
const blocks: string[] = [];
|
|
371
|
-
s = s.replace(/<pre\b[^>]*>([\s\S]*?)<\/pre\s*>/gi, (_m, inner: string) => {
|
|
372
|
-
const code = decodeEntities(inner.replace(/<br\b[^>]*\/?>/gi, "\n").replace(/<[^>]+>/g, ""))
|
|
373
|
-
.replace(/^\r?\n/, "") // 紧跟 <pre> 的那个换行按 HTML 规则不算内容
|
|
374
|
-
.replace(/\s+$/, "");
|
|
375
|
-
blocks.push(code);
|
|
376
|
-
return `\n\n${blocks.length - 1}\n\n`; // 前后各空一行:成段
|
|
377
|
-
});
|
|
378
|
-
s = s.replace(/<a\b[^>]*?href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a\s*>/gi, (_m, href: string, inner: string) => {
|
|
379
|
-
const label = inner.replace(/<[^>]+>/g, "").trim();
|
|
380
|
-
const link = absoluteLink(href, baseUrl);
|
|
381
|
-
return link !== null && label !== "" && label !== href && label !== link ? `[${label}](${link})` : label;
|
|
382
|
-
});
|
|
383
|
-
s = s
|
|
384
|
-
.replace(/<(br|hr)\b[^>]*\/?>/gi, "\n")
|
|
385
|
-
.replace(/<h([1-6])\b[^>]*>/gi, (_m, n: string) => `\n${"#".repeat(Number(n))} `)
|
|
386
|
-
.replace(/<\/(p|div|section|article|header|footer|main|nav|aside|h[1-6]|li|tr|blockquote|table|ul|ol|dd|dt|figure)\s*>/gi, "\n")
|
|
387
|
-
.replace(/<li\b[^>]*>/gi, "- ")
|
|
388
|
-
.replace(/<(td|th)\b[^>]*>/gi, " ")
|
|
389
|
-
.replace(/<[^>]+>/g, "");
|
|
390
|
-
s = decodeEntities(s);
|
|
391
|
-
const text = s
|
|
392
|
-
.split("\n")
|
|
393
|
-
.map((l) => l.replace(/[ \t\r ]+/g, " ").trim())
|
|
394
|
-
.join("\n")
|
|
395
|
-
.replace(/\n{3,}/g, "\n\n")
|
|
396
|
-
.trim()
|
|
397
|
-
.replace(/(\d+)/g, (_m, i: string) => `\`\`\`\n${blocks[Number(i)]}\n\`\`\``);
|
|
398
|
-
const t = title === undefined ? "" : decodeEntities(title).trim();
|
|
399
|
-
return t === "" ? text : `${t}\n\n${text}`;
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
const NAMED_ENTITIES: Record<string, string> = {
|
|
403
|
-
amp: "&",
|
|
404
|
-
lt: "<",
|
|
405
|
-
gt: ">",
|
|
406
|
-
quot: '"',
|
|
407
|
-
apos: "'",
|
|
408
|
-
nbsp: " ",
|
|
409
|
-
copy: "©",
|
|
410
|
-
reg: "®",
|
|
411
|
-
mdash: "—",
|
|
412
|
-
ndash: "–",
|
|
413
|
-
hellip: "…",
|
|
414
|
-
laquo: "«",
|
|
415
|
-
raquo: "»",
|
|
416
|
-
lsquo: "‘",
|
|
417
|
-
rsquo: "’",
|
|
418
|
-
ldquo: "“",
|
|
419
|
-
rdquo: "”",
|
|
420
|
-
};
|
|
421
|
-
|
|
422
|
-
function decodeEntities(s: string): string {
|
|
423
|
-
return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e: string) => {
|
|
424
|
-
if (e.startsWith("#")) {
|
|
425
|
-
const code = e[1]?.toLowerCase() === "x" ? Number.parseInt(e.slice(2), 16) : Number.parseInt(e.slice(1), 10);
|
|
426
|
-
try {
|
|
427
|
-
return String.fromCodePoint(code);
|
|
428
|
-
} catch {
|
|
429
|
-
return m;
|
|
430
|
-
}
|
|
431
|
-
}
|
|
432
|
-
return NAMED_ENTITIES[e.toLowerCase()] ?? m;
|
|
433
|
-
});
|
|
434
|
-
}
|