dsh-read-url 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -1,1663 +1,1988 @@
1
- // dsh-read-url — DeepSeek Harness URL reader plugin
2
- // Zero external dependencies (Node 20+ built-ins only).
3
- // Aligned with DSH architecture:
4
- // - all network access goes through the ctx.web capability seam (official
5
- // docs/capability-seams.md), falling back to global fetch when absent
6
- // - cache is registered under ctx.effect, so unload fully reverts it
7
- // (temporal composability, docs/cordis-primer.md)
8
- // - cooperative tool-call timeout via ToolDefinition.timeoutMs + exec.signal
9
- // Focus: token economy + smarter extraction for DSH agents.
10
- // - charset auto-detect (BOM UTF-16 / GBK/GB2312/UTF-8/Big5/Shift-JIS)
11
- // - content dispatch: HTML / JSON APIs / RSS-Atom feeds natively
12
- // - clean main-content extraction (heuristic; optional @mozilla/readability upgrade path)
13
- // + text-density fallback on the body path
14
- // - auto-joined pagination, page metadata (published/author)
15
- // - HTML -> Markdown / plain text, paragraph-aligned smart truncation
16
- // - session-level cache, compact text render (lowest token cost)
17
- import { TextDecoder } from 'node:util'
18
- import { looksLikeSpa, renderPage, closeBrowser, looksLikeChallenge } from './spa.js'
19
- import { detectProxy, fetchViaCurlProxy, looksBinary } from './proxy-fallback.js'
20
- // Re-export for tests / programmatic (PTC) cleanup.
21
- export { closeBrowser, renderPage, looksLikeSpa, looksLikeChallenge } from './spa.js'
22
-
23
- export const name = 'dsh-read-url'
24
- export const inject = ['tools']
25
-
26
- // Plugin-level configuration (overridable via cordis.patch.yml `config:` row).
27
- const DEFAULTS = {
28
- timeoutMs: 15000,
29
- maxBytes: 3 * 1024 * 1024,
30
- maxChars: 6000,
31
- maxLinks: 20,
32
- cacheTtlMs: 300000,
33
- cacheMax: 32,
34
- spaRender: true,
35
- paginate: true,
36
- paginateMax: 3,
37
- userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36',
38
- }
39
-
40
- // Session-level cache: url -> { time, full }. Failures live in a separate
41
- // small map so 30 s error entries never evict hot success pages from the
42
- // main cache (they used to share the FIFO budget).
43
- const cache = new Map()
44
- const failCache = new Map()
45
- const FAIL_CACHE_MAX = 8
46
- function cacheFail(key, error) {
47
- if (failCache.size >= FAIL_CACHE_MAX) failCache.delete(failCache.keys().next().value)
48
- failCache.set(key, { time: Date.now(), error })
49
- }
50
- function cacheStore(key, full, cacheMax) {
51
- if (cache.size >= cacheMax) cache.delete(cache.keys().next().value)
52
- cache.set(key, { time: Date.now(), full })
53
- }
54
- const decoders = new Map()
55
-
56
- function getDecoder(enc) {
57
- let d = decoders.get(enc)
58
- if (!d) {
59
- try {
60
- d = new TextDecoder(enc, { fatal: false })
61
- } catch {
62
- d = new TextDecoder('utf-8')
63
- }
64
- decoders.set(enc, d)
65
- }
66
- return d
67
- }
68
-
69
- function sniffCharset(buffer, contentType) {
70
- // BOM first — byte-level evidence beats any declared charset (a UTF-16 body
71
- // with a latin1-read <meta> probe garbles the meta match anyway).
72
- if (buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) return 'utf-8'
73
- if (buffer[0] === 0xff && buffer[1] === 0xfe) return 'utf-16le'
74
- if (buffer[0] === 0xfe && buffer[1] === 0xff) return 'utf-16be'
75
- const m = /charset=["']?([\w-]+)/i.exec(contentType || '')
76
- if (m) return m[1]
77
- const head = buffer.subarray(0, 2048).toString('latin1').toLowerCase()
78
- const meta = /<meta[^>]+charset=["']?([\w-]+)/i.exec(head)
79
- if (meta) return meta[1]
80
- return null
81
- }
82
-
83
- export function decodeBuffer(buffer, contentType) {
84
- let enc = sniffCharset(buffer, contentType)
85
- if (!enc) enc = 'utf-8'
86
- enc = enc.toLowerCase().replace('gb2312', 'gbk').replace('gb_2312', 'gbk')
87
- let text
88
- try {
89
- text = getDecoder(enc).decode(buffer)
90
- } catch {
91
- text = new TextDecoder('utf-8').decode(buffer)
92
- }
93
- if (enc !== 'utf-8' && enc !== 'utf8' && /\uFFFD/.test(text.slice(0, 2000))) {
94
- const fixed = new TextDecoder('utf-8').decode(buffer)
95
- if (!/\uFFFD/.test(fixed.slice(0, 2000))) return { text: fixed, charset: 'utf-8' }
96
- }
97
- return { text, charset: enc }
98
- }
99
-
100
- // Content types this plugin can consume. Beyond HTML: JSON (data APIs) and
101
- // XML (RSS/Atom feeds) are read natively — see readUrl's type dispatch.
102
- // Note "application/rss+xml": the separator before xml is '+', not '/'.
103
- const FETCHABLE_CT = /text\/html|application\/xhtml|text\/plain|\/json|[+/]xml/i
104
-
105
- // Retry-After header (seconds form) → ms, capped; undefined when absent/invalid.
106
- function retryAfterMs(res) {
107
- const v = res.headers.get('retry-after')
108
- if (!v) return undefined
109
- const sec = Number(v)
110
- if (!Number.isFinite(sec) || sec < 0) return undefined
111
- return Math.min(sec * 1000, 5000)
112
- }
113
-
114
- // Single fetch attempt. Never throws; maps abort/timeout/network to { error }.
115
- async function directFetchOnce(url, signal, cfg) {
116
- try {
117
- const res = await fetch(url, {
118
- redirect: 'follow',
119
- signal,
120
- headers: { 'user-agent': cfg.userAgent, accept: 'text/html,application/xhtml+xml,*/*;q=0.8' },
121
- })
122
- // Early-return paths must cancel the body stream, or the connection is
123
- // held open undrained and cannot return to the keep-alive pool.
124
- const discardBody = () => { if (res.body) res.body.cancel().catch(() => {}) }
125
- if (!res.ok) {
126
- // Rate-limit / temporary-overload: honor Retry-After with one retry
127
- // (capped at 5s so the cooperative timeout still bounds the call).
128
- if (res.status === 429 || res.status === 503) {
129
- const ra = retryAfterMs(res)
130
- discardBody()
131
- if (ra !== undefined) return { retryAfterMs: ra }
132
- }
133
- discardBody()
134
- return { error: `HTTP ${res.status} ${res.statusText}` }
135
- }
136
- const contentType = res.headers.get('content-type') || ''
137
- if (contentType && !FETCHABLE_CT.test(contentType)) {
138
- discardBody()
139
- return { error: `Unsupported content-type: ${contentType.split(';')[0]}` }
140
- }
141
- if (!res.body) return { buffer: Buffer.alloc(0), contentType, finalUrl: res.url }
142
- const chunks = []
143
- let size = 0
144
- for await (const chunk of res.body) {
145
- size += chunk.length
146
- if (size > cfg.maxBytes) { discardBody(); return { error: `Page exceeds ${cfg.maxBytes} bytes` } }
147
- chunks.push(chunk)
148
- }
149
- const buffer = Buffer.concat(chunks)
150
- // Headerless responses: gate binary bodies out of the HTML pipeline.
151
- if (!contentType && looksBinary(buffer)) {
152
- return { error: 'Unsupported content-type (no header, binary body)' }
153
- }
154
- return { buffer, contentType, finalUrl: res.url }
155
- } catch (e) {
156
- if (e.name === 'AbortError' || e.name === 'TimeoutError') {
157
- return { error: `Timeout after ${cfg.timeoutMs}ms or cancelled` }
158
- }
159
- // undici wraps the real reason in e.cause (ENOTFOUND / ECONNREFUSED /
160
- // connect-timeout / TLS) — surface it so the model can tell "bad domain"
161
- // from "blocked network" and act accordingly instead of retrying blindly
162
- const cause = e.cause && e.cause.message ? String(e.cause.message).slice(0, 90) : ''
163
- return { error: cause || (e.message || '').slice(0, 120) }
164
- }
165
- }
166
-
167
- // Direct-connect fetch with a single Retry-After-aware retry on 429/503.
168
- async function directFetch(url, signal, cfg) {
169
- const first = await directFetchOnce(url, signal, cfg)
170
- if (first && first.retryAfterMs !== undefined && !(signal && signal.aborted)) {
171
- await new Promise((resolve) => {
172
- const t = setTimeout(resolve, first.retryAfterMs)
173
- if (signal) signal.addEventListener('abort', () => { clearTimeout(t); resolve() }, { once: true })
174
- })
175
- return directFetchOnce(url, signal, cfg)
176
- }
177
- return first
178
- }
179
-
180
- // Settle-all race that resolves with the FIRST SUCCESSFUL result (a value
181
- // with a buffer and no error). If every attempt fails it resolves with
182
- // { success:false, failures:[...] } once the slowest one has settled —
183
- // the proxy channel (curl --max-time ~20s) is the natural ceiling, and the
184
- // direct fetch aborts on the same total budget.
185
- export function raceFirstSuccess(promises) {
186
- return new Promise((resolve) => {
187
- if (promises.length === 0) {
188
- resolve({ success: false, failures: [] })
189
- return
190
- }
191
- const failures = []
192
- let settled = 0
193
- promises.forEach((p, i) => {
194
- p.then((v) => {
195
- if (v && !v.error && v.buffer) {
196
- resolve({ success: true, value: v })
197
- return
198
- }
199
- failures[i] = v
200
- settled += 1
201
- if (settled === promises.length) resolve({ success: false, failures })
202
- }).catch((e) => {
203
- failures[i] = { error: String((e && e.message) || e) }
204
- settled += 1
205
- if (settled === promises.length) resolve({ success: false, failures })
206
- })
207
- })
208
- })
209
- }
210
- async function raceFetch(url, externalSignal, cfg, proxy) {
211
- const directCtrl = new AbortController()
212
- const proxyCtrl = new AbortController()
213
- const withExt = (ctrl) =>
214
- externalSignal ? AbortSignal.any([externalSignal, ctrl.signal]) : ctrl.signal
215
- const directSignal = AbortSignal.any([withExt(directCtrl), AbortSignal.timeout(cfg.timeoutMs)])
216
- const direct = directFetch(url, directSignal, cfg)
217
- const proxied = fetchViaCurlProxy(url, cfg, withExt(proxyCtrl), proxy)
218
- const winner = await raceFirstSuccess([direct, proxied])
219
- directCtrl.abort()
220
- proxyCtrl.abort()
221
- if (!winner.success) {
222
- // Attach channel labels where the attempt order is defined, so callers
223
- // never have to guess which slot is the proxy error.
224
- const labels = ['direct', 'proxy']
225
- winner.failures = winner.failures.map((f, i) => ({
226
- label: labels[i],
227
- error: f && f.error ? String(f.error) : 'unknown error',
228
- }))
229
- }
230
- return winner
231
- }
232
-
233
- async function fetchPage(url, externalSignal, cfg) {
234
- const proxy = await detectProxy()
235
- if (proxy) {
236
- const winner = await raceFetch(url, externalSignal, cfg, proxy)
237
- if (winner.success) return winner.value
238
- const dErr = winner.failures.find((f) => f.label === 'direct')
239
- const pErr = winner.failures.find((f) => f.label === 'proxy')
240
- const px = pErr ? `,代理返回 ${pErr.error}` : ',代理未连通'
241
- return { error: `Fetch failed: ${dErr ? dErr.error : 'unknown error'}(直连失败;已尝试代理 ${proxy}${px})` }
242
- }
243
- // No proxy configured: plain direct connect (the original behaviour).
244
- const timeoutSignal = AbortSignal.timeout(cfg.timeoutMs)
245
- const signal = externalSignal ? AbortSignal.any([externalSignal, timeoutSignal]) : timeoutSignal
246
- const r = await directFetch(url, signal, cfg)
247
- if (r.error) return { error: `Fetch failed: ${r.error}(直连失败,未检测到代理配置)` }
248
- return r
249
- }
250
-
251
- // Prefer the ctx.web capability seam (official docs/capability-seams.md):
252
- // the web provider already decodes the document, so we skip our charset
253
- // layer and only re-extract. Falls back to global fetch when absent.
254
- // Seam calls get the same cooperative timeout as fetchPage (a hanging
255
- // provider should not block the tool call forever).
256
- async function fetchViaWebSeam(ctx, url, externalSignal, cfg) {
257
- try {
258
- // ctx.get for a non-injected service throws on strict cordis hosts (seen
259
- // with 'settings') — treat that as "seam absent" and fall through.
260
- const web = ctx && typeof ctx.get === 'function' ? ctx.get('web') : undefined
261
- if (!web || typeof web.fetch !== 'function') return null
262
- const timeoutSignal = AbortSignal.timeout(cfg.timeoutMs)
263
- const signal = externalSignal ? AbortSignal.any([externalSignal, timeoutSignal]) : timeoutSignal
264
- const res = await web.fetch(url, { signal })
265
- if (!res || typeof res.content !== 'string') return null
266
- const finalUrl = res.url || res.finalUrl || url
267
- return { html: res.content, finalUrl }
268
- } catch {
269
- return null
270
- }
271
- }
272
-
273
- // Decode common HTML entities in already-stripped text. Runs AFTER tag
274
- // stripping so escaped tags stay visible to the stripper and only remaining
275
- // text is decoded. Covers the numeric/hex forms plus the named entities that
276
- // actually show up in CJK web text (spaces, dashes, quotes, symbols) —
277
- // undecoded leftovers would both waste tokens and read as garbage.
278
- const ENTITIES = {
279
- nbsp: ' ', ensp: ' ', emsp: ' ', thinsp: ' ', zwnj: '', zwj: '',
280
- amp: '&', lt: '<', gt: '>', quot: '"', apos: "'",
281
- ndash: '–', mdash: '—', hellip: '…', middot: '·', bull: '•',
282
- lsquo: '\u2018', rsquo: '\u2019', ldquo: '\u201C', rdquo: '\u201D',
283
- laquo: '«', raquo: '»', copy: '©', reg: '®', trade: '™',
284
- deg: '°', plusmn: '±', times: '×', divide: '÷', micro: 'µ',
285
- sect: '§', para: '', dagger: '†', prime: '′', Prime: '″',
286
- permil: '‰', euro: '€', pound: '£', yen: '¥', cent: '¢',
287
- sup2: '²', sup3: '³', frac12: '½', frac14: '¼', frac34: '¾',
288
- }
289
- export function decodeTextEntities(text) {
290
- if (!text.includes('&')) return text
291
- return text.replace(/&(?:#(\d+)|#x([0-9a-fA-F]+)|([a-z][a-z0-9]{1,7}));?/gi, (m, dec, hex, name) => {
292
- if (dec !== undefined) {
293
- const c = Number(dec)
294
- return c > 0x10ffff ? m : String.fromCodePoint(c)
295
- }
296
- if (hex !== undefined) {
297
- const c = Number.parseInt(hex, 16)
298
- return c > 0x10ffff ? m : String.fromCodePoint(c)
299
- }
300
- const n = (name || '').toLowerCase()
301
- return Object.prototype.hasOwnProperty.call(ENTITIES, n) ? ENTITIES[n] : m
302
- })
303
- }
304
-
305
- // Quadratic-cost guard: a '<' that has NO '>' after it can never start a real
306
- // tag, yet every "<...>"-style regex rescans to the end of the string at each
307
- // such position (measured: 10k stray '<'s in 60KB → seconds; a 3MB page would
308
- // stall the tool call). Neutralize stray '<' in the no-'>' tail — text content
309
- // is preserved, only the impossible tag-openings go away. Short tails
310
- // ("a < b" in prose) are left untouched.
311
- function defuseLt(html) {
312
- const from = html.lastIndexOf('>') + 1
313
- if (html.length - from < 512 || !html.includes('<', from)) return html
314
- return html.slice(0, from) + html.slice(from).replace(/</g, ' ')
315
- }
316
-
317
- function textOnly(html) {
318
- return decodeTextEntities(defuseLt(html)
319
- .replace(/<!--[\s\S]*?-->/g, ' ')
320
- .replace(/<script[\s>][\s\S]*?<\/script>/gi, ' ')
321
- .replace(/<style[\s>][\s\S]*?<\/style>/gi, ' ')
322
- .replace(/<textarea[\s>][\s\S]*?<\/textarea\s*>/gi, ' ')
323
- .replace(/<noscript[\s>][\s\S]*?<\/noscript>/gi, ' ')
324
- .replace(/<[^>]{1,1000}>/g, ' ')
325
- .replace(/[ \t\r\n]+/g, ' ')
326
- .trim())
327
- }
328
-
329
- // ---- JSON / RSS / Atom content dispatch ----
330
- // A URL is not always an HTML page: data APIs answer JSON and content sources
331
- // answer feeds. Both are model-readable when compacted, so read_url handles
332
- // them natively instead of erroring with "Unsupported content-type".
333
-
334
- function xmlText(s) {
335
- let t = String(s || '').replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
336
- // Feeds double-escape HTML in descriptions (&lt;a href=…&gt;查看全文&lt;/a&gt;):
337
- // one strip-then-decode pass leaves literal tags in the rendered text
338
- // (measured on sspai.com/feed). Iterate strip → decode until stable.
339
- for (let i = 0; i < 3; i++) {
340
- const prev = t
341
- t = decodeTextEntities(t.replace(/<[^>]{1,1000}>/g, ' '))
342
- if (t === prev) break
343
- }
344
- return t.replace(/\s+/g, ' ').trim()
345
- }
346
-
347
- // RSS 2.0 <item> / Atom <entry> → compact list "title — url\n summary".
348
- function parseFeed(xml, limit) {
349
- xml = defuseLt(xml) // no-'>' tail would make the lazy item scan quadratic
350
- const isAtom = /<feed[\s>]/i.test(xml)
351
- const itemRe = isAtom ? /<entry[\s>][\s\S]*?<\/entry\s*>/gi : /<item[\s>][\s\S]*?<\/item\s*>/gi
352
- // No-closer guard: unclosed items would rescan the rest of the feed from
353
- // every <item> position.
354
- const closable = isAtom ? /<\/entry\s*>/i.test(xml) : /<\/item\s*>/i.test(xml)
355
- const items = []
356
- let m
357
- while (closable && (m = itemRe.exec(xml)) && items.length < limit) {
358
- const it = m[0]
359
- const title = xmlText(/<title[^>]*>([\s\S]*?)<\/title>/i.exec(it)?.[1] || '')
360
- const linkTag = /<link[^>]*>([\s\S]*?)<\/link>/i.exec(it)
361
- const linkHref = /<link[^>]+href=["']([^"']+)["']/i.exec(it)
362
- const link = (linkTag && xmlText(linkTag[1])) || (linkHref && linkHref[1]) || ''
363
- const desc = xmlText(/<(?:description|summary|content)[^>]*>([\s\S]*?)<\/(?:description|summary|content)>/i.exec(it)?.[1] || '')
364
- if (!title && !link && !desc) continue
365
- items.push({ title: title.slice(0, 120), url: link, summary: desc.slice(0, 200) })
366
- }
367
- const feedTitle = xmlText(/<title[^>]*>([\s\S]*?)<\/title>/i.exec(xml)?.[1] || '')
368
- const lines = []
369
- for (const it of items) {
370
- lines.push(`- ${it.title}${it.url ? ` — ${it.url}` : ''}`)
371
- if (it.summary) lines.push(` ${it.summary}`)
372
- }
373
- return { title: feedTitle, count: items.length, items, text: lines.join('\n') }
374
- }
375
-
376
- // ---- Text-density fallback (body path only) ----
377
- // Pages without <article>/<main> degrade to the whole <body>: semantic noise
378
- // stripping then still leaves link-dominated blocks (related posts, category
379
- // sidebars, "hot articles" widgets) that carry little reading value. Split the
380
- // body at block-level open tags and drop segments whose text is mostly links.
381
- // Standard-container pages (article/main) never reach this code path.
382
- const BLOCK_OPEN_RE = /<(?:p|div|section|article|main|h[1-6]|table|tbody|tr|ul|ol|li|blockquote|pre|dl|figure|figcaption|fieldset)\b[^>]{0,1000}>/gi
383
-
384
- function linkTextLength(html) {
385
- if (!/<\/a/i.test(html)) return 0 // no anchor can pair — skip the scan
386
- let n = 0
387
- const re = /<a[\s>][\s\S]*?<\/a\s*>/gi
388
- let m
389
- while ((m = re.exec(html))) n += textOnly(m[0]).length
390
- return n
391
- }
392
-
393
- export function densityFilter(html) {
394
- const marks = []
395
- BLOCK_OPEN_RE.lastIndex = 0
396
- let m
397
- while ((m = BLOCK_OPEN_RE.exec(html))) marks.push(m.index)
398
- if (marks.length < 2) return html
399
- const segs = []
400
- if (marks[0] > 0) segs.push(html.slice(0, marks[0]))
401
- for (let k = 0; k < marks.length; k++) {
402
- segs.push(html.slice(marks[k], k + 1 < marks.length ? marks[k + 1] : html.length))
403
- }
404
- const kept = segs.filter((seg) => {
405
- const text = textOnly(seg)
406
- if (text.length < 2) return false
407
- // short link-dominated segment = nav item / recommendation widget entry
408
- if (text.length < 300 && linkTextLength(seg) > text.length * 0.65) return false
409
- return true
410
- })
411
- return kept.length ? kept.join('') : html
412
- }
413
-
414
- // Depth-counted block extraction: returns html from the opening tag at
415
- // openIdx through its BALANCED closing tag. A non-greedy regex stops at the
416
- // first nested </div> — measured: gnu.org "content" div cut to 2700 chars,
417
- // naver.com "container" to 115 (zero text). Container divs nest heavily.
418
- function tagBlockAt(html, openIdx, tagName) {
419
- const re = new RegExp(`</?${tagName}(?=[\\s>])`, 'gi')
420
- re.lastIndex = openIdx
421
- let depth = 0
422
- let m
423
- while ((m = re.exec(html))) {
424
- depth += m[0][1] === '/' ? -1 : 1
425
- if (depth === 0) {
426
- const close = html.indexOf('>', m.index)
427
- return close > 0 ? html.slice(openIdx, close + 1) : null
428
- }
429
- }
430
- return null
431
- }
432
-
433
- export function pickMain(html) {
434
- // Aggregation pages (e.g. blog homepages) have one <article> per item:
435
- // collect all of them instead of only the first. A tiny <article> (e.g. a
436
- // newsletter card — measured on about.gitlab.com: 71 chars of text while
437
- // the real content sits in <main>) falls through to <main> when present.
438
- const articles = /<\/article/i.test(html)
439
- ? [...html.matchAll(/<article[\s>][\s\S]*?<\/article\s*>/gi)]
440
- : [] // no closer anywhere: matchAll would rescan per open (quadratic)
441
- const hasMain = /<main[\s>]/i.test(html)
442
- if (articles.length && (!hasMain || textOnly(articles.map((a) => a[0]).join('')).length >= 200)) {
443
- return articles.map((a) => a[0]).join('\n')
444
- }
445
- const roleMain = /<(main|div)[^>]{0,1000}role=["']main["'][\s>]/i.exec(html)
446
- if (roleMain) {
447
- const block = tagBlockAt(html, roleMain.index, roleMain[1].toLowerCase())
448
- if (block) return block
449
- }
450
- // Bare <main> without role="main": doc generators (VitePress/VuePress,
451
- // MDN, MDX sites) mark the body with a plain <main> tag. Falling back to
452
- // the whole <body> would drag in the top/local nav (menu, outline, skip
453
- // link) — the extraction quality loss we saw on vuejs.org.
454
- const bareMain = /<main[\s>]/i.exec(html)
455
- if (bareMain) {
456
- const block = tagBlockAt(html, bareMain.index, 'main')
457
- if (block) return block
458
- }
459
- const body = /<body[\s>]/i.exec(html)
460
- if (body) {
461
- const after = html.slice(body.index)
462
- const end = after.search(/<\/body>/i)
463
- return densityFilter(end > 0 ? after.slice(0, end) : after)
464
- }
465
- return densityFilter(html)
466
- }
467
-
468
- // Some sites (e.g. baidu.com) ship CSS/HTML inside hidden <textarea> with
469
- // entity-escaped tags (&lt;style&gt;...). Reveal TAG-SHAPED sequences so noise
470
- // rules can strip them. A blanket &lt; → < decode would turn prose "a &lt; b"
471
- // into a raw "<", and the tag stripper would then swallow everything from it
472
- // to the next real ">" — measured content loss on pages with escaped
473
- // comparison operators. So: &lt; is revealed only when a tag name follows;
474
- // bare &gt; / &quot; are safe to reveal (they never open a strip span).
475
- function revealEscapedTags(html) {
476
- return html
477
- .replace(/&lt;(?=\/?[a-zA-Z!])/gi, '<')
478
- .replace(/&gt;/gi, '>')
479
- .replace(/&quot;/gi, '"')
480
- }
481
-
482
- function stripNoise(mainHtml) {
483
- // Attribute runs are bounded: an unbounded [^>]+ rescans toward the next
484
- // '>' at every '<' position — adversarial markup with one distant '>' made
485
- // this quadratic (measured: 40k '<style' + one '>' → 82s). Real attributes
486
- // never approach 1000 chars.
487
- // Container groups are built from closers that actually exist: absent
488
- // closing tags made the lazy body scan walk the rest of the string per
489
- // opening position for nothing.
490
- const closers = new Set()
491
- for (const c of mainHtml.matchAll(/<\/([a-z0-9]+)/gi)) closers.add(c[1].toLowerCase())
492
- const group = (names) => names.filter((n) => closers.has(n))
493
- const raw = group(['textarea', 'style', 'script', 'noscript', 'template'])
494
- const box = group(['nav', 'footer', 'header', 'aside', 'form', 'iframe', 'svg', 'canvas', 'dialog'])
495
- let out = mainHtml.replace(/<!--[\s\S]*?-->/g, ' ')
496
- if (raw.length) {
497
- out = out.replace(new RegExp(`<(${raw.join('|')})[\\s>][\\s\\S]*?<\\/\\1\\s*>`, 'gi'), ' ')
498
- }
499
- if (box.length) {
500
- out = out.replace(new RegExp(`<(${box.join('|')})[\\s>][\\s\\S]*?<\\/\\1>`, 'gi'), ' ')
501
- }
502
- return out.replace(
503
- /<([a-z][a-z0-9]*)[^>]{1,1000}class=["'][^"']{0,300}(ad-|ads|advert|banner|sidebar|social|share|comment|popup|modal|cookie)[^"']{0,300}["'][^>]{0,1000}>[\s\S]*?<\/\1>/gi,
504
- ' ',
505
- )
506
- }
507
-
508
- // ---- HTML -> Markdown (lightweight, tag-state machine) ----
509
- function escInline(s) {
510
- return s.replace(/([\\`*_[\]])/g, '\\$1')
511
- }
512
-
513
- // <img> is a void element (never matches the paired-tag regex in either md
514
- // walker): convert descriptive images to markdown BEFORE the tag walk, via a
515
- // sentinel so escInline cannot escape the generated brackets. Decorative
516
- // images (empty alt) are dropped — the model cannot see pixels anyway.
517
- const IMG_SENTINEL = '\u0001'
518
- function imgsToMarkdown(html) {
519
- const store = []
520
- const out = html.replace(/<img\b[^>]{0,1000}>/gi, (tag) => {
521
- const alt = /alt=["']([^"']*)["']/i.exec(tag)
522
- const src = /src=["']([^"']+)["']/i.exec(tag)
523
- store.push(src && alt && alt[1].trim() ? `![${alt[1].trim()}](${src[1]})` : '')
524
- return `${IMG_SENTINEL}${store.length - 1}${IMG_SENTINEL}`
525
- })
526
- return {
527
- html: out,
528
- // Unknown sentinel indexes are PRESERVED (a nested walker's restore must
529
- // not eat sentinels owned by an outer frame — blockMd recurses into itself).
530
- restore: (s) => s.replace(new RegExp(`${IMG_SENTINEL}(\\d+)${IMG_SENTINEL}`, 'g'), (m, i) => {
531
- const v = store[Number(i)]
532
- return v === undefined ? m : v
533
- }),
534
- }
535
- }
536
-
537
- // An open tag whose closing tag appears NOWHERE in the input can never pair:
538
- // the paired alternative of the walker regex would rescan the rest of the
539
- // string at every such position (measured: 10k unclosed <div>s in 60KB →
540
- // 2.2s quadratic; a 3MB page would stall the tool call for minutes). Pre-
541
- // stripping unmatched openers is behaviour-identical — they previously fell
542
- // through to the single-tag branch, which drops them — and makes the walk
543
- // linear. Depth cap: recursion on 100+-deep nesting would overflow the stack
544
- // long before real pages nest that far; degrade to plain text instead.
545
- const OPEN_TAG_RE = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">]){0,1000})>/g
546
- const MD_MAX_DEPTH = 100
547
- function stripUnmatchedOpeners(html) {
548
- const closers = new Set()
549
- for (const m of html.matchAll(/<\/([a-zA-Z0-9]+)/g)) closers.add(m[1].toLowerCase())
550
- html = defuseLt(html)
551
- return html.replace(OPEN_TAG_RE, (m, name) => (closers.has(name.toLowerCase()) ? m : ''))
552
- }
553
-
554
- export function inlineMd(html, depth = 0) {
555
- if (depth > MD_MAX_DEPTH) {
556
- return escInline(decodeTextEntities(html.replace(/<[^>]{1,1000}>/g, ' ')).replace(/\s+/g, ' ').trim())
557
- }
558
- const img = imgsToMarkdown(html)
559
- html = stripUnmatchedOpeners(img.html)
560
- let out = ''
561
- const re = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">]){0,1000})>([\s\S]*?)<\/\1>|<[^>]{1,1000}>|([^<]+)/g
562
- let m
563
- while ((m = re.exec(html))) {
564
- if (m[4] !== undefined) {
565
- out += escInline(decodeTextEntities(m[4]))
566
- continue
567
- }
568
- if (!m[1]) continue
569
- const tag = m[1].toLowerCase()
570
- if (tag === 'style' || tag === 'script' || tag === 'textarea' || tag === 'template' || tag === 'noscript') continue
571
- const inner = inlineMd(m[3], depth + 1)
572
- if (tag === 'a') {
573
- const href = /href=["']([^"']+)["']/i.exec(m[2])
574
- out += href ? `[${inner}](${href[1]})` : inner
575
- } else if (tag === 'strong' || tag === 'b') out += `**${inner}**`
576
- else if (tag === 'em' || tag === 'i') out += `*${inner}*`
577
- else if (tag === 'code') out += `\`${inner}\``
578
- else if (tag === 'br') out += ' \n'
579
- else out += inner
580
- }
581
- return img.restore(out.replace(/[ \t]+/g, ' ').trim())
582
- }
583
-
584
- export function blockMd(html, depth = 0) {
585
- if (depth > MD_MAX_DEPTH) {
586
- return decodeTextEntities(html.replace(/<[^>]{1,1000}>/g, ' ')).replace(/\s+/g, ' ').trim()
587
- }
588
- const img = imgsToMarkdown(html)
589
- html = stripUnmatchedOpeners(img.html)
590
- let out = ''
591
- const re = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">]){0,1000})>([\s\S]*?)<\/\1>|<[^>]{1,1000}>|([^<]+)/g
592
- let m
593
- while ((m = re.exec(html))) {
594
- if (m[4] !== undefined) {
595
- out += decodeTextEntities(m[4])
596
- continue
597
- }
598
- if (!m[1]) continue
599
- const tag = m[1].toLowerCase()
600
- const attrs = m[2] || ''
601
- const inner = m[3]
602
- if (tag === 'style' || tag === 'script' || tag === 'textarea' || tag === 'template' || tag === 'noscript') {
603
- continue
604
- }
605
- if (tag === 'h1' || tag === 'h2' || tag === 'h3' || tag === 'h4' || tag === 'h5' || tag === 'h6') {
606
- const level = '#'.repeat(Number(tag[1]))
607
- out += `\n\n${level} ${inlineMd(inner, depth + 1)}`
608
- } else if (tag === 'p') {
609
- const t = inlineMd(inner, depth + 1)
610
- if (t) out += `\n\n${t}`
611
- } else if (tag === 'div' || tag === 'section' || tag === 'article' || tag === 'main') {
612
- const t = blockMd(inner, depth + 1)
613
- if (t) out += `\n\n${t}`
614
- } else if (tag === 'blockquote') {
615
- const t = blockMd(inner, depth + 1).trim()
616
- out += `\n\n> ${t.replace(/\n/g, '\n> ')}`
617
- } else if (tag === 'pre') {
618
- const code = inner.replace(/<[^>]{1,1000}>/g, '').trim()
619
- // Language hint from common highlighter conventions: the model reads
620
- // ```js fenced blocks far more accurately than unlabelled ones.
621
- const lang =
622
- /<code[^>]+class=["'][^"']*(?:language|lang)-([\w#+.-]+)["']/i.exec(inner)?.[1] || ''
623
- out += `\n\n\`\`\`${lang}\n${code}\n\`\`\``
624
- } else if (tag === 'code') {
625
- out += `\`${textOnly(inner)}\``
626
- } else if (tag === 'ul' || tag === 'ol') {
627
- // No-closer guard: <li>s without a single </li> would make the lazy
628
- // scan walk the rest of inner from every open position (quadratic).
629
- const items = []
630
- if (inner.includes('</li')) {
631
- const liRe = /<li[^>]{0,1000}>([\s\S]*?)<\/li>/gi
632
- let lm, idx = 1
633
- while ((lm = liRe.exec(inner))) {
634
- const t = inlineMd(lm[1], depth + 1)
635
- items.push(tag === 'ol' ? `${idx}. ${t}` : `- ${t}`)
636
- idx++
637
- }
638
- }
639
- if (items.length) out += `\n\n${items.join('\n')}`
640
- } else if (tag === 'li') {
641
- out += `\n- ${inlineMd(inner, depth + 1)}`
642
- } else if (tag === 'br') {
643
- out += ' \n'
644
- } else if (tag === 'hr') {
645
- out += '\n\n---'
646
- } else if (tag === 'table') {
647
- const rows = []
648
- if (inner.includes('</tr')) {
649
- const trRe = /<tr[^>]{0,1000}>([\s\S]*?)<\/tr>/gi
650
- let tm
651
- while ((tm = trRe.exec(inner))) {
652
- const cells = []
653
- if (tm[1].includes('</t')) {
654
- const tdRe = /<t[dh][^>]{0,1000}>([\s\S]*?)<\/t[dh]>/gi
655
- let cm
656
- while ((cm = tdRe.exec(tm[1]))) cells.push(inlineMd(cm[1], depth + 1).replace(/\|/g, '\\|'))
657
- }
658
- if (cells.length) rows.push(`| ${cells.join(' | ')} |`)
659
- }
660
- }
661
- if (rows.length) {
662
- const header = rows[0]
663
- // unescape cells' escaped pipes before deriving the separator row,
664
- // otherwise the --- line is one char wider per escaped pipe
665
- const sep = header.replace(/\\\|/g, '|').replace(/[^|]/g, '-')
666
- out += `\n\n${rows.join('\n')}\n${sep}`
667
- }
668
- } else if (tag === 'a' || tag === 'strong' || tag === 'b' || tag === 'em' || tag === 'i') {
669
- const t = inlineMd(inner, depth + 1)
670
- if (t) out += t
671
- } else {
672
- const t = blockMd(inner, depth + 1)
673
- if (t) out += t
674
- }
675
- }
676
- return img.restore(out)
677
- }
678
-
679
- // Page-level metadata the model actually asks about (who wrote it, when) —
680
- // cheap to harvest from <meta>, previously thrown away.
681
- function extractMeta(html) {
682
- const get = (names) => {
683
- for (const n of names) {
684
- const m = new RegExp(`<meta[^>]{1,1000}(?:property|name)=["']${n}["'][^>]{1,1000}content=["']([^"']+)["']`, 'i').exec(html)
685
- if (m) return decodeTextEntities(m[1]).replace(/<[^>]{1,1000}>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 80)
686
- }
687
- return ''
688
- }
689
- return {
690
- published: get(['article:published_time', 'og:published_time', 'pubdate', 'publishdate', 'date', 'dc.date']),
691
- author: get(['author', 'article:author', 'og:article:author']),
692
- description: get(['og:description', 'description']),
693
- }
694
- }
695
-
696
- // Meta-less pages (measured: ruanyifeng.com carries ZERO author/date meta
697
- // tags) still print the byline inside the body head: "作者:阮一峰 日期:
698
- // 2026年8月21日". Fill ONLY the missing fields, and only from the first
699
- // 600 chars of body text — deeper matches are content, not bylines.
700
- const BYLINE_AUTHOR_RE = /(?:作者|責任編輯|責任主编|editor)[::\s]{1,3}([^\s,,。/|()()[\]]{2,24})/
701
- const BYLINE_DATE_RE = /(?:发表日期|发布日期|发表时间|发布时间|日期|发表于|发布于|published(?:\s+on)?)[::\s]{0,3}(\d{4}[-/.年]\s?\d{1,2}[-/.月]\s?\d{1,2}日?|\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2})?)/i
702
-
703
- function bylineFallback(text, meta) {
704
- const head = typeof text === 'string' ? text.slice(0, 600) : ''
705
- const out = { published: meta.published, author: meta.author }
706
- if (!out.author) {
707
- const m = BYLINE_AUTHOR_RE.exec(head)
708
- if (m) out.author = m[1].trim().slice(0, 40)
709
- }
710
- if (!out.published) {
711
- const m = BYLINE_DATE_RE.exec(head)
712
- if (m) out.published = m[1].replace(/\s+/g, ' ').trim().slice(0, 40)
713
- }
714
- return out
715
- }
716
-
717
- export function extract(html, mode) {
718
- html = defuseLt(html) // kills the no-'>'-tail quadratic scans before any picker runs
719
- const titleMatch = /<title[^>]{0,1000}>([\s\S]*?)<\/title>/i.exec(html) || /<meta[^>]{1,1000}property=["']og:title["'][^>]{1,1000}content=["']([^"']+)["']/i.exec(html)
720
- const siteName = /<meta[^>]{1,1000}property=["']og:site_name["'][^>]{1,1000}content=["']([^"']+)["']/i.exec(html)
721
- const langMatch = /<html[^>]{1,1000}lang=["']([\w-]+)["']/i.exec(html)
722
- let main = stripNoise(pickMain(html))
723
- main = stripNoise(revealEscapedTags(main))
724
- let bodyText
725
- if (mode === 'markdown') {
726
- bodyText = blockMd(main).replace(/\n{3,}/g, '\n\n').trim()
727
- } else {
728
- bodyText = textOnly(main).replace(/ +/g, ' ').trim()
729
- }
730
- const meta = extractMeta(html)
731
- // Empty-body pages (login walls, JS-only shells) still carry og:description
732
- // — surface it as a fallback hint instead of nothing.
733
- let text = bodyText
734
- let usedDescription = false
735
- if (!text && meta.description) {
736
- text = meta.description
737
- usedDescription = true
738
- }
739
- const byline = bylineFallback(text, meta)
740
- return {
741
- title: titleMatch ? textOnly(titleMatch[1]) || titleMatch[1].trim() : '',
742
- siteName: siteName ? siteName[1] : '',
743
- lang: langMatch ? langMatch[1] : '',
744
- published: byline.published,
745
- author: byline.author,
746
- text,
747
- usedDescription,
748
- }
749
- }
750
-
751
- // Optional enhancement: when @mozilla/readability + happy-dom are installed
752
- // inside the DSH profile (npm i @mozilla/readability happy-dom), extraction
753
- // upgrades to the Firefox Reader Mode algorithm. Falls back to the built-in
754
- // heuristic extractor when absent. MPL-2.0 / MIT libraries, used as-is.
755
- let readabilityReady = null
756
- async function getReadabilityExtractor() {
757
- if (readabilityReady !== null) return readabilityReady
758
- try {
759
- const [{ Readability }, { Window }] = await Promise.all([
760
- import('@mozilla/readability'),
761
- import('happy-dom'),
762
- ])
763
- readabilityReady = (html, url) => {
764
- const window = new Window({ url })
765
- window.document.write(html)
766
- const article = new Readability(window.document).parse()
767
- return article ? article.content : null
768
- }
769
- } catch {
770
- readabilityReady = false
771
- }
772
- return readabilityReady
773
- }
774
-
775
- export function smartTruncate(text, maxChars, offset = 0) {
776
- const total = text.length
777
- if (offset >= total) {
778
- return { text: '', truncated: false, charsTotal: total, charsReturned: 0, charsStart: offset }
779
- }
780
- if (total <= maxChars && offset === 0) {
781
- return { text, truncated: false, charsTotal: total, charsReturned: total, charsStart: 0 }
782
- }
783
- const paragraphs = text.split(/\n\n+/)
784
- let pos = 0
785
- let start = 0
786
- for (let i = 0; i < paragraphs.length; i++) {
787
- if (pos + paragraphs[i].length > offset) {
788
- start = i
789
- break
790
- }
791
- pos += paragraphs[i].length + 2
792
- }
793
- let acc = ''
794
- for (let i = start; i < paragraphs.length; i++) {
795
- const p = paragraphs[i]
796
- if (acc.length + p.length + (acc ? 2 : 0) > maxChars) break
797
- acc += (acc ? '\n\n' : '') + p
798
- }
799
- if (!acc && maxChars > 0) {
800
- // The first paragraph after the offset is itself longer than maxChars.
801
- // Keep the alignment promise at sentence level before hard-slicing.
802
- const p = paragraphs[start] || ''
803
- const rel = Math.max(0, offset - pos)
804
- const rest = p.length > rel ? p.slice(rel) : text.slice(offset)
805
- const sentences = rest.match(/[^.!?\n。!?]+[.!?\n。!?]*/g) || [rest]
806
- for (const s of sentences) {
807
- if (acc.length + s.length > maxChars) break
808
- acc += s
809
- }
810
- if (!acc) acc = rest.slice(0, maxChars)
811
- }
812
- return {
813
- text: acc,
814
- truncated: offset + acc.length < total,
815
- charsTotal: total,
816
- charsReturned: acc.length,
817
- charsStart: offset,
818
- }
819
- }
820
-
821
- function extractLinks(html, limit, baseUrl) {
822
- const links = []
823
- const seen = new Set()
824
- // Accept relative hrefs too — most real pages link internally with relative
825
- // paths — and resolve them against the page URL so the model can follow them.
826
- const re = /<a[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi
827
- let m
828
- // Iteration cap: a page whose nav repeats the same links thousands of times
829
- // would otherwise scan the whole HTML to fill `limit` unique entries. Scan
830
- // at most limit*4 anchors; dedupe keeps the result bounded regardless.
831
- let scans = 0
832
- const maxScans = limit * 4
833
- while ((m = re.exec(html)) && links.length < limit && scans++ < maxScans) {
834
- const href = m[1].trim()
835
- if (!href || /^(javascript|mailto|tel|data):/i.test(href)) continue
836
- let url
837
- try {
838
- url = baseUrl ? new URL(href, baseUrl).href : new URL(href).href
839
- } catch {
840
- continue
841
- }
842
- // Only web links are followable by the model / crawler — ftp: etc. would
843
- // pass new URL() resolution and pollute link lists and crawl queues.
844
- if (!/^https?:/i.test(url)) continue
845
- // dedupe: nav bars repeat the same links many times — one entry per URL
846
- // keeps the link list compact (tokens) without losing coverage
847
- if (seen.has(url)) continue
848
- seen.add(url)
849
- const t = textOnly(m[2])
850
- if (t) links.push({ title: t.slice(0, 80), url })
851
- }
852
- return links
853
- }
854
-
855
- function hostOf(url) {
856
- try {
857
- return new URL(url).hostname
858
- } catch {
859
- return ''
860
- }
861
- }
862
-
863
- // ---- Pagination: auto-follow "next page" ----
864
- // Long articles on novel/news/forum sites are split into pages; without this
865
- // the model had to notice the truncation and re-call the tool per page. The
866
- // recognition is deliberately conservative: rel=next (standard) or a short
867
- // anchor whose whole text is a next-page marker — no fuzzy guessing.
868
- const NEXT_TEXT_RE = /^(?:[›»>]+\s*)?(?:下一页|下页|下一篇|下一頁|下頁|后一页|後一頁|next page|next|older entries|[›»>]{1,3})(?:\s*[›»>]+)?$/i
869
-
870
- export function findNextLink(html, baseUrl) {
871
- const resolve = (href) => {
872
- try {
873
- const u = new URL(href, baseUrl)
874
- return u.protocol === 'http:' || u.protocol === 'https:' ? u.href : null
875
- } catch {
876
- return null
877
- }
878
- }
879
- const rel = /<(?:link|a)\b[^>]+rel=["']next["'][^>]*>/i.exec(html || '')
880
- if (rel) {
881
- const href = /href=["']([^"']+)["']/i.exec(rel[0])
882
- const r = href && resolve(href[1])
883
- if (r) return r
884
- }
885
- const re = /<a\b[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a\s*>/gi
886
- let m
887
- let scans = 0
888
- // Bounded like extractLinks: a nav bar with thousands of anchors must not
889
- // scan the whole DOM hunting for a next-marker.
890
- while ((m = re.exec(html || '')) && scans++ < 400) {
891
- const t = textOnly(m[2])
892
- if (t.length <= 16 && NEXT_TEXT_RE.test(t)) {
893
- const r = resolve(m[1])
894
- if (r) return r
895
- }
896
- }
897
- return null
898
- }
899
-
900
- // Join two page bodies, dropping a repeated suffix/prefix overlap (many
901
- // paginated sites repeat the last paragraph or a heading on the next page).
902
- function joinPageText(a, b) {
903
- const max = Math.min(300, a.length, b.length)
904
- for (let n = max; n >= 30; n--) {
905
- if (a.slice(a.length - n) === b.slice(0, n)) {
906
- const rest = b.slice(n).trim()
907
- return rest ? `${a}\n\n${rest}` : a
908
- }
909
- }
910
- return `${a}\n\n${b}`
911
- }
912
-
913
- function sliceFrom(full, offset, maxChars) {
914
- const s = smartTruncate(full.fullText, maxChars, offset)
915
- const out = {
916
- url: full.url,
917
- title: full.title || '',
918
- siteName: full.siteName || hostOf(full.url),
919
- lang: full.lang || '',
920
- charset: full.charset,
921
- mode: full.mode,
922
- truncated: s.truncated,
923
- charsTotal: s.charsTotal,
924
- charsReturned: s.charsReturned,
925
- charsStart: s.charsStart,
926
- text: s.text,
927
- }
928
- if (full.rendered) out.rendered = true
929
- if (full.spaHint) out.spaHint = full.spaHint
930
- if (full.paginated > 1) out.paginated = full.paginated
931
- if (full.feedCount) out.feedCount = full.feedCount
932
- if (full.published) out.published = full.published
933
- if (full.author) out.author = full.author
934
- if (Array.isArray(full.links)) out.links = full.links
935
- return out
936
- }
937
-
938
- // ---- JSON payloads: compact render ----
939
- // Indented JSON spends a newline + indent per key — pure token overhead for
940
- // the model. Render compact, and clip monster string values (embedded HTML,
941
- // base64 blobs) with a visible marker so the model knows there is more.
942
- const JSON_STR_MAX = 1500
943
- export function compactJson(value) {
944
- const clip = (s) =>
945
- s.length > JSON_STR_MAX ? s.slice(0, JSON_STR_MAX) + `…[+${s.length - JSON_STR_MAX} chars]` : s
946
- const walk = (v, depth) => {
947
- if (typeof v === 'string') return clip(v)
948
- if (Array.isArray(v)) return v.map((x) => walk(x, depth + 1))
949
- if (v && typeof v === 'object' && depth < 12) {
950
- const o = {}
951
- for (const k of Object.keys(v)) o[k] = walk(v[k], depth + 1)
952
- return o
953
- }
954
- return v
955
- }
956
- try {
957
- return JSON.stringify(walk(value, 0))
958
- } catch {
959
- // Extreme nesting/size where even native stringify exhausts the stack —
960
- // degrade to a clipped raw render instead of crashing the tool call.
961
- try {
962
- return clip(String(value))
963
- } catch {
964
- return '[unserializable json]'
965
- }
966
- }
967
- }
968
-
969
- export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
970
- const url = String((args && args.url) || '').trim()
971
- if (!/^https?:\/\//i.test(url)) return { error: 'Only http/https URLs are supported' }
972
- const maxChars = Math.max(500, Math.min(20000, Number(args.maxChars) || cfg.maxChars))
973
- const mode = args.mode === 'markdown' ? 'markdown' : 'text'
974
- const offset = Math.max(0, Number(args.offset) || 0)
975
-
976
- // Cache stores the FULL extracted text keyed by url+mode+includeLinks, so
977
- // continuation reads (offset) and different maxChars hit the same entry —
978
- // but a links-request must not be served from a links-less cached copy.
979
- // The key drops the URL fragment: #sections would otherwise split the cache.
980
- let cacheUrl = url
981
- try {
982
- const u = new URL(url)
983
- u.hash = ''
984
- cacheUrl = u.href
985
- } catch {
986
- /* keep raw url on parse failure */
987
- }
988
- const cacheKey = `${cacheUrl}|${mode}|${args.includeLinks === true ? 'links' : 'no-links'}`
989
- // Failed fetches are cached briefly so the model doesn't re-request a
990
- // broken URL in a loop (token + latency saver).
991
- const fh = failCache.get(cacheKey)
992
- if (fh) {
993
- if (Date.now() - fh.time < 30000) return { error: fh.error, cached: true }
994
- failCache.delete(cacheKey)
995
- }
996
- const hit = cache.get(cacheKey)
997
- if (hit && Date.now() - hit.time < cfg.cacheTtlMs) {
998
- const sliced = sliceFrom(hit.full, offset, maxChars)
999
- return { ...sliced, cached: true }
1000
- }
1001
-
1002
- const failWith = (error) => {
1003
- cacheFail(cacheKey, error)
1004
- return { error }
1005
- }
1006
-
1007
- // Non-HTML payloads (JSON APIs, RSS/Atom feeds) are read natively: fetch,
1008
- // dispatch by content-type / payload shape, render compact. Falls back to
1009
- // the HTML pipeline for everything else.
1010
- const dispatch = async () => {
1011
- const viaSeam = await fetchViaWebSeam(ctx, url, externalSignal, cfg)
1012
- if (viaSeam) {
1013
- const t = viaSeam.html.trim()
1014
- if (t.startsWith('{') || t.startsWith('[')) {
1015
- try {
1016
- return { kind: 'json', text: compactJson(JSON.parse(t)) }
1017
- } catch { /* not JSON — keep HTML pipeline */ }
1018
- }
1019
- return { kind: 'html', html: viaSeam.html, finalUrl: viaSeam.finalUrl, charset: 'provider-decoded' }
1020
- }
1021
- const page = await fetchPage(url, externalSignal, cfg)
1022
- if (page.error) return { error: page.error }
1023
- const ct = (page.contentType || '').split(';')[0].toLowerCase().trim()
1024
- const decoded = decodeBuffer(page.buffer, page.contentType)
1025
- if (/json$/.test(ct)) {
1026
- try {
1027
- return { kind: 'json', text: compactJson(JSON.parse(decoded.text)) }
1028
- } catch { /* invalid JSON — serve through the HTML pipeline as text */ }
1029
- }
1030
- if (/[+/]xml$/.test(ct)) {
1031
- if (/<urlset[\s>]/i.test(decoded.text) || /<sitemapindex[\s>]/i.test(decoded.text)) {
1032
- return { error: 'Unsupported content (XML sitemap)' }
1033
- }
1034
- if (/<rss[\s>]/i.test(decoded.text) || /<feed[\s>]/i.test(decoded.text)) {
1035
- return { kind: 'feed', feed: parseFeed(decoded.text, cfg.maxLinks) }
1036
- }
1037
- }
1038
- return { kind: 'html', html: decoded.text, finalUrl: page.finalUrl || url, charset: decoded.charset }
1039
- }
1040
- const payload = await dispatch()
1041
- if (payload.error) return failWith(payload.error)
1042
-
1043
- if (payload.kind === 'json') {
1044
- const full = { url, title: '', siteName: hostOf(url), lang: '', charset: 'json', mode: 'json', fullText: payload.text, paginated: 1 }
1045
- cacheStore(cacheKey, full, cfg.cacheMax)
1046
- return sliceFrom(full, offset, maxChars)
1047
- }
1048
- if (payload.kind === 'feed') {
1049
- const f = payload.feed
1050
- const full = {
1051
- url, title: f.title || '', siteName: hostOf(url), lang: '', charset: 'feed',
1052
- mode: 'feed', fullText: f.text, paginated: 1, feedCount: f.count,
1053
- links: args.includeLinks === true ? f.items : undefined,
1054
- }
1055
- cacheStore(cacheKey, full, cfg.cacheMax)
1056
- return sliceFrom(full, offset, maxChars)
1057
- }
1058
-
1059
- let html = payload.html
1060
- let finalUrl = payload.finalUrl || url
1061
- const charset = payload.charset
1062
-
1063
- let extracted = extract(html, mode)
1064
- // Optional readability upgrade: cleaner article extraction when installed.
1065
- const ext = await getReadabilityExtractor()
1066
- const upgrade = (target, htmlText) => {
1067
- if (!ext) return target
1068
- try {
1069
- const clean = ext(htmlText, url)
1070
- if (clean && clean.length > 0) {
1071
- target.text = mode === 'markdown'
1072
- ? blockMd(clean).replace(/\n{3,}/g, '\n\n').trim()
1073
- : textOnly(clean).replace(/ +/g, ' ').trim()
1074
- }
1075
- } catch {
1076
- // keep heuristic result
1077
- }
1078
- return target
1079
- }
1080
- extracted = upgrade(extracted, html)
1081
-
1082
- // Optional SPA enhancement: if the page looks client-rendered and static
1083
- // extraction found almost nothing, try headless rendering (playwright).
1084
- // Beyond script-heavy shells (looksLikeSpa, ≥5 <script> tags), a page whose
1085
- // extraction is EMPTY but carries any <script> may be a JS-redirect shell
1086
- // (measured: taptap.cn — 2 scripts, empty <body>, redirects via JS) — with
1087
- // no text to lose, rendering is always worth one bounded attempt. Pages
1088
- // with no scripts at all cannot change under rendering and are skipped.
1089
- let rendered = false
1090
- let spaHint = ''
1091
- let renderedHtml = null
1092
- const needsRender = !extracted.text || extracted.text.length < 200
1093
- const scriptShell = !extracted.text && /<script[\s>]/i.test(html)
1094
- if (cfg.spaRender !== false && needsRender && (looksLikeSpa(html) || scriptShell)) {
1095
- const rr = await renderPage(finalUrl || url, externalSignal)
1096
- if (rr.html) {
1097
- renderedHtml = rr.html
1098
- // A challenge interstitial (Cloudflare "Just a moment...") must never
1099
- // be accepted as an improvement — it can carry MORE text than the real
1100
- // static body (measured: 259 chars of verification prose vs 135 chars
1101
- // of real content on ruanyifeng.com) and would evict the latter.
1102
- if (looksLikeChallenge(rr.html)) {
1103
- renderedHtml = null // pagination/links must not read the interstitial DOM
1104
- spaHint = extracted.text
1105
- ? '已渲染但被人机验证页拦截,保留静态提取结果'
1106
- : '已渲染但被人机验证页拦截(无静态内容可用)'
1107
- } else {
1108
- const r2 = upgrade(extract(rr.html, mode), rr.html)
1109
- // Accept the rendered text when it meaningfully beats the static one.
1110
- // From an EMPTY static extraction even a short body is a real gain
1111
- // (measured: taptap-style JS shells render 30-60 chars of real content
1112
- // that the fixed +50 margin would reject).
1113
- const gain = r2.text.length - extracted.text.length
1114
- const accept = r2.text && (gain > 50 || (!extracted.text && r2.text.length >= 20))
1115
- if (accept) {
1116
- extracted = r2
1117
- finalUrl = rr.finalUrl || finalUrl
1118
- rendered = true
1119
- } else if (!extracted.text) {
1120
- spaHint = '已渲染仍无可读内容(可能登录墙或反爬拦截)'
1121
- }
1122
- }
1123
- } else {
1124
- spaHint = rr.error
1125
- }
1126
- }
1127
-
1128
- // Auto-pagination: follow the "next page" chain (bounded, same-host, no
1129
- // loops). Continuation pages go through the fast static path — a paginated
1130
- // SPA chain would cost one full render per page, out of proportion.
1131
- let paginated = 1
1132
- const paginateMax = cfg.paginate === false ? 1 : Math.max(1, Math.min(10, Number(cfg.paginateMax) || 3))
1133
- const startHost = hostOf(finalUrl || url)
1134
- const seen = new Set([normalizeUrl(finalUrl || url)])
1135
- let nextUrl = paginateMax > 1 ? findNextLink(renderedHtml || html, finalUrl || url) : null
1136
- while (
1137
- nextUrl && paginated < paginateMax &&
1138
- sameHost(nextUrl, startHost) &&
1139
- !(externalSignal && externalSignal.aborted)
1140
- ) {
1141
- const nu = normalizeUrl(nextUrl)
1142
- if (!nu || seen.has(nu)) break
1143
- seen.add(nu)
1144
- const page = await fetchPage(nextUrl, externalSignal, cfg)
1145
- if (page.error) break
1146
- const ct = (page.contentType || '').split(';')[0].toLowerCase().trim()
1147
- if (!/html|xhtml/.test(ct)) break
1148
- const nextHtml = decodeBuffer(page.buffer, page.contentType).text
1149
- const nextEx = extract(nextHtml, mode)
1150
- if (!nextEx.text || nextEx.text.length < 20) break
1151
- extracted.text = joinPageText(extracted.text, nextEx.text)
1152
- paginated++
1153
- nextUrl = findNextLink(nextHtml, page.finalUrl || nextUrl)
1154
- }
1155
-
1156
- const full = {
1157
- url: finalUrl,
1158
- title: extracted.title || '',
1159
- siteName: extracted.siteName || '',
1160
- lang: extracted.lang || '',
1161
- charset,
1162
- mode,
1163
- fullText: extracted.text,
1164
- rendered,
1165
- spaHint,
1166
- paginated,
1167
- published: extracted.published || '',
1168
- author: extracted.author || '',
1169
- // Links come from the rendered DOM when available — otherwise the SPA
1170
- // chain (content -> links -> next page) would break for JS-only pages.
1171
- links: args.includeLinks === true ? extractLinks(renderedHtml || html, cfg.maxLinks, finalUrl) : undefined,
1172
- }
1173
-
1174
- cacheStore(cacheKey, full, cfg.cacheMax)
1175
- return sliceFrom(full, offset, maxChars)
1176
- }
1177
-
1178
- function renderResult(value) {
1179
- if (typeof value === 'string') return value
1180
- const r = value || {}
1181
- if (r.error) return `Error: ${r.error}`
1182
- const lines = []
1183
- if (r.title) lines.push(`title: ${r.title}`)
1184
- const host = hostOf(r.url || '')
1185
- const meta = []
1186
- if (r.siteName && r.siteName !== host) meta.push(r.siteName)
1187
- if (r.lang) meta.push(r.lang)
1188
- if (r.charset && r.charset !== 'utf-8' && r.charset !== 'json' && r.charset !== 'feed') {
1189
- meta.push(`charset ${r.charset}`)
1190
- }
1191
- if (r.published) meta.push(r.published)
1192
- if (r.author) meta.push(`by ${r.author}`)
1193
- if (r.feedCount) meta.push(`feed · ${r.feedCount} 条`)
1194
- if (r.mode === 'json') meta.push('json')
1195
- if (meta.length) lines.push(meta.join(' · '))
1196
- // One compact status line instead of several parenthetical lines — the
1197
- // flags are hints, not content, so they share a single line of tokens.
1198
- const flags = []
1199
- if (r.charsStart > 0) flags.push(`chars ${r.charsStart}+${r.charsReturned}/${r.charsTotal}`, 'offset 续读')
1200
- else if (r.truncated) flags.push(`chars ${r.charsReturned}/${r.charsTotal}`, '截断', 'offset 续读')
1201
- else if (typeof r.charsTotal === 'number') flags.push(`chars ${r.charsTotal}`)
1202
- if (r.cached) flags.push('cached')
1203
- if (r.paginated > 1) flags.push(`已拼接${r.paginated}页`)
1204
- if (r.rendered) flags.push('rendered')
1205
- if (flags.length) lines.push(`(${flags.join(' · ')})`)
1206
- if (!r.text) {
1207
- if (r.charsStart > 0 && !r.truncated) {
1208
- lines.push('(offset 已到文末,无更多内容)')
1209
- } else {
1210
- lines.push(r.spaHint
1211
- ? `(无可读内容 — ${r.spaHint})`
1212
- : '(无可读内容 — 登录墙 / SPA 页 / 空页面;SPA 需安装 playwright)')
1213
- }
1214
- }
1215
- lines.push('', r.text || '')
1216
- if (Array.isArray(r.links) && r.links.length) {
1217
- lines.push('', 'links:')
1218
- for (const l of r.links) lines.push(`- ${l.title} — ${l.url}`)
1219
- }
1220
- return lines.join('\n')
1221
- }
1222
-
1223
- function renderLinks(value) {
1224
- if (typeof value === 'string') return value
1225
- const r = value || {}
1226
- if (r.error) return `Error: ${r.error}`
1227
- if (!Array.isArray(r.links) || r.links.length === 0) return `No links found on ${r.url}`
1228
- const lines = [`${r.count} link(s) on ${r.url}:`]
1229
- for (const l of r.links) lines.push(`- ${l.title || l.url} — ${l.url}`)
1230
- return lines.join('\n')
1231
- }
1232
-
1233
- function readLinksTool(ctx, cfg) {
1234
- return {
1235
- name: 'read_url_links',
1236
- description:
1237
- 'List links (text + URL) on a page without body text — lighter than read_url ' +
1238
- 'for mapping what a page points to. Renders SPA pages when playwright installed.',
1239
- parameters: {
1240
- type: 'object',
1241
- additionalProperties: false,
1242
- properties: {
1243
- url: { type: 'string', description: 'http(s) URL to scan for links' },
1244
- limit: { type: 'number', description: 'Links to return (1-50; default from plugin config)' },
1245
- },
1246
- required: ['url'],
1247
- },
1248
- // Covers SPA-render fallback: full fetch timeout + playwright render cap.
1249
- timeoutMs: Math.max(45000, cfg.timeoutMs + 45000),
1250
- isConcurrencySafe: () => true,
1251
- output: {
1252
- schema: {
1253
- type: 'object',
1254
- additionalProperties: true,
1255
- properties: {
1256
- url: { type: 'string' },
1257
- count: { type: 'number' },
1258
- links: { type: 'array', items: { type: 'object' } },
1259
- error: { type: 'string' },
1260
- },
1261
- },
1262
- render: (_args, value) => [{ type: 'text', text: renderLinks(value) }],
1263
- },
1264
- async execute(args, exec) {
1265
- const url = String((args && args.url) || '').trim()
1266
- if (!/^https?:\/\//i.test(url)) return { error: 'Only http/https URLs are supported' }
1267
- const limit = Math.max(1, Math.min(50, Number(args.limit) || cfg.maxLinks))
1268
- const viaSeam = await fetchViaWebSeam(ctx, url, exec && exec.signal, cfg)
1269
- let html, finalUrl
1270
- if (viaSeam) {
1271
- html = viaSeam.html
1272
- finalUrl = viaSeam.finalUrl
1273
- } else {
1274
- const page = await fetchPage(url, exec && exec.signal, cfg)
1275
- if (page.error) return { error: page.error }
1276
- html = decodeBuffer(page.buffer, page.contentType).text
1277
- finalUrl = page.finalUrl || url
1278
- }
1279
- let links = extractLinks(html, limit, finalUrl)
1280
- // SPA fallback: a JS-only page yields no links from its static HTML;
1281
- // render it and re-extract when the static result looks empty. Any
1282
- // <script> counts as renderable when there are no links at all — the
1283
- // static shell may be a JS-redirect (same rationale as read_url).
1284
- if (links.length < 3 && cfg.spaRender !== false && (looksLikeSpa(html) || (links.length === 0 && /<script[\s>]/i.test(html)))) {
1285
- const rr = await renderPage(finalUrl || url, exec && exec.signal)
1286
- // A challenge interstitial's links are the challenge's own scripts —
1287
- // never an improvement over whatever the static HTML offered.
1288
- if (rr.html && !looksLikeChallenge(rr.html)) {
1289
- const rl = extractLinks(rr.html, limit, rr.finalUrl || finalUrl)
1290
- if (rl.length > links.length) {
1291
- links = rl
1292
- finalUrl = rr.finalUrl || finalUrl
1293
- }
1294
- }
1295
- }
1296
- return { url: finalUrl, count: links.length, links }
1297
- },
1298
- }
1299
- }
1300
-
1301
- // Run async tasks with a concurrency cap (avoids hammering a site / getting
1302
- // rate-limited when reading many URLs at once).
1303
- async function mapLimit(items, limit, fn) {
1304
- const out = new Array(items.length)
1305
- let idx = 0
1306
- const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
1307
- while (idx < items.length) {
1308
- const i = idx++
1309
- out[i] = await fn(items[i], i)
1310
- }
1311
- })
1312
- await Promise.all(workers)
1313
- return out
1314
- }
1315
-
1316
- function renderBatch(value) {
1317
- if (typeof value === 'string') return value
1318
- const v = value || {}
1319
- if (v.error) return `Error: ${v.error}`
1320
- const lines = [`读取 ${v.succeeded}/${v.total} 页成功${v.failed ? `,${v.failed} 页失败` : ''}`]
1321
- for (const p of v.pages) {
1322
- if (p.error) {
1323
- lines.push('', `[失败] ${p.url} — ${p.error}`)
1324
- continue
1325
- }
1326
- const head = p.title || p.url
1327
- lines.push('', `--- ${head} (${p.chars} 字符${p.cached ? ' · cached' : ''}) ---`, p.text || '(无可读内容)')
1328
- if (Array.isArray(p.links) && p.links.length) {
1329
- const ls = p.links.map((l) => l.title + ' — ' + l.url)
1330
- lines.push(`links: ${ls.slice(0, 6).join(' | ')}${ls.length > 6 ? ` …共${ls.length}个` : ''}`)
1331
- }
1332
- }
1333
- return lines.join('\n')
1334
- }
1335
-
1336
- function readUrlBatchTool(ctx, cfg) {
1337
- return {
1338
- name: 'read_url_batch',
1339
- description:
1340
- 'Read multiple URLs in parallel: each a compact clean block (same extraction + cache as read_url). ' +
1341
- 'Failures isolated and tagged. Max 10 URLs, capped at maxChars per page.',
1342
- parameters: {
1343
- type: 'object',
1344
- additionalProperties: false,
1345
- properties: {
1346
- urls: { type: 'array', items: { type: 'string' }, description: 'URLs to read in parallel (1-10, http/s)' },
1347
- maxChars: { type: 'number', description: 'Chars per page (500-20000; default 3000)' },
1348
- mode: { type: 'string', enum: ['text', 'markdown'], description: 'text = plain (token-cheap, default); markdown = structured' },
1349
- includeLinks: { type: 'boolean', description: 'Return page links too (title+url, bounded)' },
1350
- },
1351
- required: ['urls'],
1352
- },
1353
- // Worst case: ceil(10/4)=3 concurrency waves, each (fetch + SPA render).
1354
- timeoutMs: Math.max(60000, cfg.timeoutMs * 3 + 135000),
1355
- // Concurrent batch calls multiply fetch pressure but stay isolated (each
1356
- // call owns its result array; the session cache is commutative).
1357
- isConcurrencySafe: () => true,
1358
- output: {
1359
- schema: {
1360
- type: 'object',
1361
- additionalProperties: true,
1362
- properties: {
1363
- total: { type: 'number' },
1364
- succeeded: { type: 'number' },
1365
- failed: { type: 'number' },
1366
- pages: { type: 'array', items: { type: 'object' } },
1367
- error: { type: 'string' },
1368
- },
1369
- },
1370
- render: (_args, value) => [{ type: 'text', text: renderBatch(value) }],
1371
- },
1372
- async execute(args, exec) {
1373
- const a = args || {}
1374
- const urls = (Array.isArray(a.urls) ? a.urls : []).map((u) => String(u).trim()).filter(Boolean)
1375
- if (!urls.length) return { error: 'urls array is required (1-10 http(s) URLs)' }
1376
- const list = urls.slice(0, 10)
1377
- const perMax = Math.max(500, Math.min(20000, Number(a.maxChars) || 3000))
1378
- const mode = a.mode === 'markdown' ? 'markdown' : 'text'
1379
- const signal = exec && exec.signal
1380
- const pages = await mapLimit(list, 4, (u) =>
1381
- readUrl({ url: u, maxChars: perMax, mode, includeLinks: a.includeLinks === true }, ctx, signal, cfg),
1382
- )
1383
- const ok = pages.filter((p) => !p.error)
1384
- return {
1385
- total: list.length,
1386
- succeeded: ok.length,
1387
- failed: list.length - ok.length,
1388
- pages: pages.map((p, i) => {
1389
- if (p.error) return { url: list[i], error: p.error }
1390
- const out = {
1391
- url: p.url,
1392
- title: p.title || '',
1393
- chars: p.charsReturned,
1394
- cached: p.cached === true,
1395
- text: p.text,
1396
- }
1397
- if (Array.isArray(p.links) && p.links.length) out.links = p.links
1398
- return out
1399
- }),
1400
- }
1401
- },
1402
- }
1403
- }
1404
-
1405
- function normalizeUrl(url) {
1406
- try {
1407
- const u = new URL(url)
1408
- u.hash = ''
1409
- return u.href
1410
- } catch {
1411
- return null
1412
- }
1413
- }
1414
-
1415
- // b is a bare hostname (hostOf() output) — must NOT be passed to new URL()
1416
- // as a full URL (that throws and would make every link look external).
1417
- function sameHost(a, b) {
1418
- try {
1419
- return new URL(a).hostname === b
1420
- } catch {
1421
- return false
1422
- }
1423
- }
1424
-
1425
- // URLs not worth crawling: static assets, login/auth paths, feeds, sitemaps.
1426
- const NOISE_EXT = /\.(png|jpe?g|gif|svg|webp|ico|bmp|css|js|json|xml|pdf|zip|gz|tar|7z|mp3|mp4|avi|mov|webm|woff2?|ttf|eot|map)(\?|#|$)/i
1427
- const NOISE_PATH = /(\/login|\/signin|\/register|\/logout|\/signup|\/api\/|\/admin|\/wp-admin|\/wp-login|\/feed|\/rss|\/sitemap|\/robots\.txt|\/cdn-cgi)/i
1428
- function isNoiseUrl(url) {
1429
- return NOISE_EXT.test(url) || NOISE_PATH.test(url)
1430
- }
1431
-
1432
- // BFS crawl of a single site. Only same-host http(s) pages are followed;
1433
- // static assets / auth paths are skipped; visited URLs are deduped; a per-page
1434
- // failure is recorded and does not abort the crawl. No SPA rendering here —
1435
- // that is read_url's job; crawling favors speed and breadth.
1436
- async function crawlSite(entryUrl, cfg, opts, externalSignal) {
1437
- const { maxPages, maxDepth, includeContent, perMax } = opts
1438
- const host = hostOf(entryUrl)
1439
- const visited = new Set()
1440
- const pages = []
1441
- const failures = []
1442
- const queue = [{ url: entryUrl, depth: 0 }]
1443
- // Stop once the host's cooperative signal fires (tool budget hit): without
1444
- // this the loop keeps draining the queue on instantly-aborted fetches.
1445
- while (queue.length && pages.length < maxPages && !(externalSignal && externalSignal.aborted)) {
1446
- // Never overshoot the page budget within a batch round.
1447
- const batch = queue.splice(0, Math.min(2, maxPages - pages.length, queue.length))
1448
- const results = await Promise.all(
1449
- batch.map(async ({ url, depth }) => {
1450
- const norm = normalizeUrl(url)
1451
- if (!norm || visited.has(norm)) return null
1452
- visited.add(norm)
1453
- const page = await fetchPage(url, externalSignal, cfg)
1454
- if (page.error) return { url, depth, error: page.error }
1455
- const html = decodeBuffer(page.buffer, page.contentType).text
1456
- const ex = extract(html, 'text')
1457
- const links = extractLinks(html, 100, page.finalUrl || url)
1458
- return {
1459
- url: page.finalUrl || url,
1460
- depth,
1461
- title: ex.title || '',
1462
- chars: ex.text.length,
1463
- text: includeContent ? smartTruncate(ex.text, perMax).text : undefined,
1464
- links,
1465
- }
1466
- }),
1467
- )
1468
- for (const r of results) {
1469
- if (!r) continue
1470
- if (r.error) {
1471
- failures.push({ url: r.url, error: r.error })
1472
- continue
1473
- }
1474
- pages.push(r)
1475
- if (r.depth + 1 <= maxDepth) {
1476
- // Queue cap: a huge site (millions of links) would otherwise grow the
1477
- // queue unboundedly while maxPages bounds only what we *process*.
1478
- if (queue.length >= maxPages * 20) continue
1479
- for (const l of r.links) {
1480
- if (!sameHost(l.url, host) || isNoiseUrl(l.url)) continue
1481
- const n = normalizeUrl(l.url)
1482
- if (n && !visited.has(n)) queue.push({ url: l.url, depth: r.depth + 1 })
1483
- }
1484
- }
1485
- }
1486
- }
1487
- return { host, pages, failures }
1488
- }
1489
-
1490
- function renderSite(value) {
1491
- if (typeof value === 'string') return value
1492
- const v = value || {}
1493
- if (v.error) return `Error: ${v.error}`
1494
- const lines = [`站点: ${v.host} · 爬取 ${v.succeeded}/${v.total} 页${v.failed ? ` · ${v.failed} 页失败` : ''}`]
1495
- for (const p of v.pages) {
1496
- const indent = ' '.repeat(p.depth)
1497
- const head = p.title || p.url
1498
- lines.push(`${indent}[${p.depth}] ${head} (${p.chars} 字符) ${p.url}`)
1499
- if (p.text) lines.push(`${indent} ${p.text.slice(0, 80)}`)
1500
- }
1501
- for (const f of v.failures) lines.push(`[失败] ${f.url} — ${f.error}`)
1502
- return lines.join('\n')
1503
- }
1504
-
1505
- function readUrlSiteTool(ctx, cfg) {
1506
- return {
1507
- name: 'read_url_site',
1508
- description:
1509
- 'Crawl a site BFS from one URL: dedupe same-host pages, return a compact site map ' +
1510
- '(title + depth + size). Auth/static paths skipped; failures isolated. ' +
1511
- 'includeContent=true attaches a short summary per page (default off, token-cheap). ' +
1512
- 'Does not render SPA pages (use read_url for that).',
1513
- parameters: {
1514
- type: 'object',
1515
- additionalProperties: false,
1516
- properties: {
1517
- url: { type: 'string', description: 'Entry URL of the site to crawl (http/s)' },
1518
- maxPages: { type: 'number', description: 'Pages to crawl (2-50; default 15)' },
1519
- maxDepth: { type: 'number', description: 'Link depth from entry (1-5; default 2)' },
1520
- includeContent: { type: 'boolean', description: 'Short body summary per page (default off; structure-only is token-cheap)' },
1521
- maxCharsPerPage: { type: 'number', description: 'Summary chars per page (200-2000; default 500)' },
1522
- },
1523
- required: ['url'],
1524
- },
1525
- // Worst case: ceil(maxPages/2)=25 waves of paired fetches (no SPA render
1526
- // here); the cooperative signal stops the loop early once the budget is hit.
1527
- timeoutMs: Math.max(120000, cfg.timeoutMs * 25),
1528
- // Crawl state (visited set, queue) is call-local; only the shared session
1529
- // cache is global and commutative — parallel crawls of different hosts
1530
- // are safe and useful (one task mapping several sites).
1531
- isConcurrencySafe: () => true,
1532
- output: {
1533
- schema: {
1534
- type: 'object',
1535
- additionalProperties: true,
1536
- properties: {
1537
- host: { type: 'string' },
1538
- total: { type: 'number' },
1539
- succeeded: { type: 'number' },
1540
- failed: { type: 'number' },
1541
- pages: { type: 'array', items: { type: 'object' } },
1542
- failures: { type: 'array', items: { type: 'object' } },
1543
- error: { type: 'string' },
1544
- },
1545
- },
1546
- render: (_args, value) => [{ type: 'text', text: renderSite(value) }],
1547
- },
1548
- async execute(args, exec) {
1549
- const url = String((args && args.url) || '').trim()
1550
- if (!/^https?:\/\//i.test(url)) return { error: 'Only http/https URLs are supported' }
1551
- const maxPages = Math.max(2, Math.min(50, Number(args.maxPages) || 15))
1552
- const maxDepth = Math.max(1, Math.min(5, Number(args.maxDepth) || 2))
1553
- const includeContent = args.includeContent === true
1554
- const perMax = Math.max(200, Math.min(2000, Number(args.maxCharsPerPage) || 500))
1555
- const { host, pages, failures } = await crawlSite(
1556
- url, cfg, { maxPages, maxDepth, includeContent, perMax }, exec && exec.signal,
1557
- )
1558
- return {
1559
- host,
1560
- total: pages.length + failures.length,
1561
- succeeded: pages.length,
1562
- failed: failures.length,
1563
- pages: pages.map((p) => {
1564
- const o = { url: p.url, depth: p.depth, title: p.title || '', chars: p.chars }
1565
- if (p.text) o.text = p.text
1566
- return o
1567
- }),
1568
- failures,
1569
- }
1570
- },
1571
- }
1572
- }
1573
-
1574
- export function apply(ctx, config) {
1575
- // Plugin-level config from cordis.patch.yml (config row), merged over defaults.
1576
- // YAML is free-form: numbers can arrive quoted as strings, which would poison
1577
- // arithmetic and AbortSignal timeouts downstream — coerce to numbers once
1578
- // here, clamp to sane ranges, and fall back to defaults on garbage.
1579
- const cfg = { ...DEFAULTS, ...(config || {}) }
1580
- const clamp = (v, d, min, max) => {
1581
- const n = Number(v)
1582
- return Number.isFinite(n) ? Math.min(max, Math.max(min, n)) : d
1583
- }
1584
- cfg.timeoutMs = clamp(cfg.timeoutMs, DEFAULTS.timeoutMs, 500, 120000)
1585
- cfg.maxBytes = clamp(cfg.maxBytes, DEFAULTS.maxBytes, 65536, 16 * 1024 * 1024)
1586
- cfg.maxChars = clamp(cfg.maxChars, DEFAULTS.maxChars, 500, 20000)
1587
- cfg.maxLinks = clamp(cfg.maxLinks, DEFAULTS.maxLinks, 1, 50)
1588
- cfg.cacheTtlMs = clamp(cfg.cacheTtlMs, DEFAULTS.cacheTtlMs, 0, 3600000)
1589
- cfg.cacheMax = clamp(cfg.cacheMax, DEFAULTS.cacheMax, 1, 256)
1590
- cfg.paginateMax = clamp(cfg.paginateMax, DEFAULTS.paginateMax, 1, 10)
1591
- if (typeof cfg.userAgent !== 'string' || !cfg.userAgent.trim()) cfg.userAgent = DEFAULTS.userAgent
1592
- if (typeof cfg.spaRender !== 'boolean') cfg.spaRender = DEFAULTS.spaRender
1593
- if (typeof cfg.paginate !== 'boolean') cfg.paginate = DEFAULTS.paginate
1594
-
1595
- // Temporal composability: unload must fully revert side effects.
1596
- ctx.effect(() => () => {
1597
- cache.clear()
1598
- failCache.clear()
1599
- closeBrowser().catch(() => {})
1600
- })
1601
-
1602
- console.log('[dsh-read-url] plugin loaded; tools read_url, read_url_batch, read_url_links, read_url_site registered')
1603
-
1604
- ctx.tools.register({
1605
- name: 'read_url',
1606
- description:
1607
- 'Fetch a page and return its clean main content (auto charset detect). ' +
1608
- 'text (default) = plain body capped at maxChars; markdown = headings/links/tables. ' +
1609
- 'Compact block: title, metadata, truncated body; 5-min session cache; ' +
1610
- 'SPA pages render when playwright installed; login walls are not accessible.',
1611
- parameters: {
1612
- type: 'object',
1613
- additionalProperties: false,
1614
- properties: {
1615
- url: { type: 'string', description: 'http(s) URL to read' },
1616
- maxChars: { type: 'number', description: 'Body text chars to return (500-20000; default from plugin config)' },
1617
- offset: { type: 'number', description: 'Start at this char offset (continue long pages). Default 0; served from cache.' },
1618
- mode: { type: 'string', enum: ['text', 'markdown'], description: 'text = plain (token-cheap, default); markdown = structured' },
1619
- includeLinks: { type: 'boolean', description: 'Return page links too (title+url, bounded)' },
1620
- },
1621
- required: ['url'],
1622
- },
1623
- // Pipeline budget must cover the real worst case: first page fetch +
1624
- // SPA render (goto 30s cap + DOM-stability poll 10s) + up to paginateMax
1625
- // continuation fetches (static, no render). Derived from load-time cfg —
1626
- // scale with it, never clamp it.
1627
- timeoutMs: Math.max(45000, cfg.timeoutMs * (cfg.paginate === false ? 1 : cfg.paginateMax + 1) + 45000),
1628
- // All shared state is commutative-safe under concurrent calls (session
1629
- // cache Map get/set, decoder cache, one shared browser with isolated
1630
- // pages per render), so the agent may fan out several read_url calls in
1631
- // one parallel group — a real win for multi-source reading tasks.
1632
- isConcurrencySafe: () => true,
1633
- output: {
1634
- schema: {
1635
- type: 'object',
1636
- additionalProperties: true,
1637
- properties: {
1638
- url: { type: 'string' },
1639
- title: { type: 'string' },
1640
- siteName: { type: 'string' },
1641
- lang: { type: 'string' },
1642
- charset: { type: 'string' },
1643
- mode: { type: 'string' },
1644
- truncated: { type: 'boolean' },
1645
- charsTotal: { type: 'number' },
1646
- charsReturned: { type: 'number' },
1647
- text: { type: 'string' },
1648
- links: { type: 'array', items: { type: 'object' } },
1649
- cached: { type: 'boolean' },
1650
- error: { type: 'string' },
1651
- },
1652
- },
1653
- render: (_args, value) => [{ type: 'text', text: renderResult(value) }],
1654
- },
1655
- async execute(args, exec) {
1656
- return readUrl(args, ctx, exec && exec.signal, cfg)
1657
- },
1658
- })
1659
-
1660
- ctx.tools.register(readLinksTool(ctx, cfg))
1661
- ctx.tools.register(readUrlBatchTool(ctx, cfg))
1662
- ctx.tools.register(readUrlSiteTool(ctx, cfg))
1663
- }
1
+ // dsh-read-url — DeepSeek Harness URL reader plugin
2
+ // Zero external dependencies (Node 20+ built-ins only).
3
+ // Aligned with DSH architecture:
4
+ // - all network access goes through the ctx.web capability seam (official
5
+ // docs/capability-seams.md), falling back to global fetch when absent
6
+ // - cache is registered under ctx.effect, so unload fully reverts it
7
+ // (temporal composability, docs/cordis-primer.md)
8
+ // - cooperative tool-call timeout via ToolDefinition.timeoutMs + exec.signal
9
+ // Focus: token economy + smarter extraction for DSH agents.
10
+ // - charset auto-detect (BOM UTF-16 / GBK/GB2312/UTF-8/Big5/Shift-JIS)
11
+ // - content dispatch: HTML / JSON APIs / RSS-Atom feeds natively
12
+ // - clean main-content extraction (heuristic; optional @mozilla/readability upgrade path)
13
+ // + text-density fallback on the body path
14
+ // - auto-joined pagination, page metadata (published/author)
15
+ // - HTML -> Markdown / plain text, paragraph-aligned smart truncation
16
+ // - session-level cache, compact text render (lowest token cost)
17
+ import { TextDecoder } from 'node:util'
18
+ import { looksLikeSpa, renderPage, closeBrowser, looksLikeChallenge } from './spa.js'
19
+ import { detectProxy, fetchViaCurlProxy, looksBinary } from './proxy-fallback.js'
20
+ // Re-export for tests / programmatic (PTC) cleanup.
21
+ export { closeBrowser, renderPage, looksLikeSpa, looksLikeChallenge } from './spa.js'
22
+
23
+ export const name = 'dsh-read-url'
24
+ export const inject = ['tools']
25
+
26
+ // Plugin-level configuration (overridable via cordis.patch.yml `config:` row).
27
+ const DEFAULTS = {
28
+ timeoutMs: 15000,
29
+ maxBytes: 3 * 1024 * 1024,
30
+ maxChars: 6000,
31
+ maxLinks: 20,
32
+ cacheTtlMs: 300000,
33
+ cacheMax: 32,
34
+ spaRender: true,
35
+ paginate: true,
36
+ paginateMax: 3,
37
+ userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36',
38
+ }
39
+
40
+ // Session-level cache: url -> { time, full }. Failures live in a separate
41
+ // small map so short-lived error entries never evict hot success pages.
42
+ const cache = new Map()
43
+ const failCache = new Map()
44
+ const FAIL_CACHE_MAX = 8
45
+ function cacheFail(key, error) {
46
+ if (failCache.size >= FAIL_CACHE_MAX) failCache.delete(failCache.keys().next().value)
47
+ failCache.set(key, { time: Date.now(), error })
48
+ }
49
+ function cacheStore(key, full, cacheMax) {
50
+ if (cache.size >= cacheMax) cache.delete(cache.keys().next().value)
51
+ cache.set(key, { time: Date.now(), full })
52
+ }
53
+ const decoders = new Map()
54
+
55
+ function getDecoder(enc) {
56
+ let d = decoders.get(enc)
57
+ if (!d) {
58
+ try {
59
+ d = new TextDecoder(enc, { fatal: false })
60
+ } catch {
61
+ d = new TextDecoder('utf-8')
62
+ }
63
+ decoders.set(enc, d)
64
+ }
65
+ return d
66
+ }
67
+
68
+ function sniffCharset(buffer, contentType) {
69
+ // BOM first — byte-level evidence beats any declared charset (a UTF-16 body
70
+ // with a latin1-read <meta> probe garbles the meta match anyway).
71
+ if (buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) return 'utf-8'
72
+ if (buffer[0] === 0xff && buffer[1] === 0xfe) return 'utf-16le'
73
+ if (buffer[0] === 0xfe && buffer[1] === 0xff) return 'utf-16be'
74
+ const m = /charset=["']?([\w-]+)/i.exec(contentType || '')
75
+ if (m) return m[1]
76
+ const head = buffer.subarray(0, 2048).toString('latin1').toLowerCase()
77
+ const meta = /<meta[^>]{0,1000}charset=["']?([\w-]+)/i.exec(head)
78
+ if (meta) return meta[1]
79
+ // Legacy form: <meta http-equiv="Content-Type" content="text/html; charset=gb2312">
80
+ // (no charset= attribute anywhere) — common on old GBK pages with no header charset.
81
+ const metaEquiv = /<meta[^>]{0,1000}(?:http-equiv)=["']content-type["'][^>]{0,1000}content=["'][^"']*charset=([\w-]+)/i.exec(head)
82
+ if (metaEquiv) return metaEquiv[1]
83
+ return null
84
+ }
85
+
86
+ export function decodeBuffer(buffer, contentType) {
87
+ let enc = sniffCharset(buffer, contentType)
88
+ if (!enc) enc = 'utf-8'
89
+ enc = enc.toLowerCase().replace('gb2312', 'gbk').replace('gb_2312', 'gbk')
90
+ let text
91
+ try {
92
+ text = getDecoder(enc).decode(buffer)
93
+ } catch {
94
+ text = new TextDecoder('utf-8').decode(buffer)
95
+ }
96
+ if (enc !== 'utf-8' && enc !== 'utf8' && /\uFFFD/.test(text.slice(0, 2000))) {
97
+ const fixed = new TextDecoder('utf-8').decode(buffer)
98
+ if (!/\uFFFD/.test(fixed.slice(0, 2000))) return { text: fixed, charset: 'utf-8' }
99
+ }
100
+ return { text, charset: enc }
101
+ }
102
+
103
+ // Content types this plugin can consume. Beyond HTML: JSON (data APIs) and
104
+ // XML (RSS/Atom feeds) are read natively — see readUrl's type dispatch.
105
+ // Note "application/rss+xml": the separator before xml is '+', not '/'.
106
+ const FETCHABLE_CT = /text\/html|application\/xhtml|text\/plain|\/json|[+/]xml/i
107
+
108
+ // Retry-After header (seconds form) → ms, capped; undefined when absent/invalid.
109
+ function retryAfterMs(res) {
110
+ const v = res.headers.get('retry-after')
111
+ if (!v) return undefined
112
+ const sec = Number(v)
113
+ if (!Number.isFinite(sec) || sec < 0) return undefined
114
+ return Math.min(sec * 1000, 5000)
115
+ }
116
+
117
+ // Single fetch attempt. Never throws; maps abort/timeout/network to { error }.
118
+ async function directFetchOnce(url, signal, cfg) {
119
+ try {
120
+ const res = await fetch(url, {
121
+ redirect: 'follow',
122
+ signal,
123
+ headers: { 'user-agent': cfg.userAgent, accept: 'text/html,application/xhtml+xml,*/*;q=0.8' },
124
+ })
125
+ // Early-return paths must cancel the body stream, or the connection is
126
+ // held open undrained and cannot return to the keep-alive pool.
127
+ const discardBody = () => { if (res.body) res.body.cancel().catch(() => {}) }
128
+ if (!res.ok) {
129
+ // Rate-limit / temporary-overload: honor Retry-After with one retry
130
+ // (capped at 5s so the cooperative timeout still bounds the call).
131
+ if (res.status === 429 || res.status === 503) {
132
+ const ra = retryAfterMs(res)
133
+ discardBody()
134
+ if (ra !== undefined) return { retryAfterMs: ra }
135
+ }
136
+ discardBody()
137
+ return { error: `HTTP ${res.status} ${res.statusText}` }
138
+ }
139
+ const contentType = res.headers.get('content-type') || ''
140
+ if (contentType && !FETCHABLE_CT.test(contentType)) {
141
+ discardBody()
142
+ return { error: `Unsupported content-type: ${contentType.split(';')[0]}` }
143
+ }
144
+ if (!res.body) return { buffer: Buffer.alloc(0), contentType, finalUrl: res.url }
145
+ const chunks = []
146
+ let size = 0
147
+ for await (const chunk of res.body) {
148
+ size += chunk.length
149
+ if (size > cfg.maxBytes) { discardBody(); return { error: `Page exceeds ${cfg.maxBytes} bytes` } }
150
+ chunks.push(chunk)
151
+ }
152
+ const buffer = Buffer.concat(chunks)
153
+ // Headerless responses: gate binary bodies out of the HTML pipeline.
154
+ if (!contentType && looksBinary(buffer)) {
155
+ return { error: 'Unsupported content-type (no header, binary body)' }
156
+ }
157
+ return { buffer, contentType, finalUrl: res.url }
158
+ } catch (e) {
159
+ if (e.name === 'AbortError' || e.name === 'TimeoutError') {
160
+ return { error: `Timeout after ${cfg.timeoutMs}ms or cancelled` }
161
+ }
162
+ // undici wraps the real reason in e.cause (ENOTFOUND / ECONNREFUSED /
163
+ // connect-timeout / TLS) — surface it so the model can tell "bad domain"
164
+ // from "blocked network" and act accordingly instead of retrying blindly
165
+ const cause = e.cause && e.cause.message ? String(e.cause.message).slice(0, 90) : ''
166
+ return { error: cause || (e.message || '').slice(0, 120) }
167
+ }
168
+ }
169
+
170
+ // Direct-connect fetch with a single Retry-After-aware retry on 429/503.
171
+ async function directFetch(url, signal, cfg) {
172
+ const first = await directFetchOnce(url, signal, cfg)
173
+ if (first && first.retryAfterMs !== undefined && !(signal && signal.aborted)) {
174
+ await new Promise((resolve) => {
175
+ const t = setTimeout(resolve, first.retryAfterMs)
176
+ if (signal) signal.addEventListener('abort', () => { clearTimeout(t); resolve() }, { once: true })
177
+ })
178
+ return directFetchOnce(url, signal, cfg)
179
+ }
180
+ return first
181
+ }
182
+
183
+ // Settle-all race that resolves with the FIRST SUCCESSFUL result (a value
184
+ // with a buffer and no error). If every attempt fails it resolves with
185
+ // { success:false, failures:[...] } once the slowest one has settled —
186
+ // the proxy channel (curl --max-time ~20s) is the natural ceiling, and the
187
+ // direct fetch aborts on the same total budget.
188
+ export function raceFirstSuccess(promises) {
189
+ return new Promise((resolve) => {
190
+ if (promises.length === 0) {
191
+ resolve({ success: false, failures: [] })
192
+ return
193
+ }
194
+ const failures = []
195
+ let settled = 0
196
+ promises.forEach((p, i) => {
197
+ p.then((v) => {
198
+ if (v && !v.error && v.buffer) {
199
+ resolve({ success: true, value: v })
200
+ return
201
+ }
202
+ failures[i] = v
203
+ settled += 1
204
+ if (settled === promises.length) resolve({ success: false, failures })
205
+ }).catch((e) => {
206
+ failures[i] = { error: String((e && e.message) || e) }
207
+ settled += 1
208
+ if (settled === promises.length) resolve({ success: false, failures })
209
+ })
210
+ })
211
+ })
212
+ }
213
+ async function raceFetch(url, externalSignal, cfg, proxy) {
214
+ const directCtrl = new AbortController()
215
+ const proxyCtrl = new AbortController()
216
+ const withExt = (ctrl) =>
217
+ externalSignal ? AbortSignal.any([externalSignal, ctrl.signal]) : ctrl.signal
218
+ const directSignal = AbortSignal.any([withExt(directCtrl), AbortSignal.timeout(cfg.timeoutMs)])
219
+ const direct = directFetch(url, directSignal, cfg)
220
+ const proxied = fetchViaCurlProxy(url, cfg, withExt(proxyCtrl), proxy)
221
+ const winner = await raceFirstSuccess([direct, proxied])
222
+ directCtrl.abort()
223
+ proxyCtrl.abort()
224
+ if (!winner.success) {
225
+ // Each failure is tagged with its channel so callers read errors by
226
+ // label ('direct' | 'proxy') rather than by array position.
227
+ const labels = ['direct', 'proxy']
228
+ winner.failures = winner.failures.map((f, i) => ({
229
+ label: labels[i],
230
+ error: f && f.error ? String(f.error) : 'unknown error',
231
+ }))
232
+ }
233
+ return winner
234
+ }
235
+
236
+ async function fetchPage(url, externalSignal, cfg) {
237
+ const proxy = await detectProxy()
238
+ if (proxy) {
239
+ const winner = await raceFetch(url, externalSignal, cfg, proxy)
240
+ if (winner.success) return winner.value
241
+ const dErr = winner.failures.find((f) => f.label === 'direct')
242
+ const pErr = winner.failures.find((f) => f.label === 'proxy')
243
+ const px = pErr ? `,代理返回 ${pErr.error}` : ',代理未连通'
244
+ return { error: `Fetch failed: ${dErr ? dErr.error : 'unknown error'}(直连失败;已尝试代理 ${proxy}${px})` }
245
+ }
246
+ // No proxy configured: plain direct connect (the original behaviour).
247
+ const timeoutSignal = AbortSignal.timeout(cfg.timeoutMs)
248
+ const signal = externalSignal ? AbortSignal.any([externalSignal, timeoutSignal]) : timeoutSignal
249
+ const r = await directFetch(url, signal, cfg)
250
+ if (r.error) return { error: `Fetch failed: ${r.error}(直连失败,未检测到代理配置)` }
251
+ return r
252
+ }
253
+
254
+ // Prefer the ctx.web capability seam (official docs/capability-seams.md):
255
+ // the web provider already decodes the document, so we skip our charset
256
+ // layer and only re-extract. Falls back to global fetch when absent.
257
+ // Seam calls get the same cooperative timeout as fetchPage (a hanging
258
+ // provider should not block the tool call forever).
259
+ async function fetchViaWebSeam(ctx, url, externalSignal, cfg) {
260
+ try {
261
+ // ctx.get for a non-injected service throws on strict cordis hosts (seen
262
+ // with 'settings') — treat that as "seam absent" and fall through.
263
+ const web = ctx && typeof ctx.get === 'function' ? ctx.get('web') : undefined
264
+ if (!web || typeof web.fetch !== 'function') return null
265
+ const timeoutSignal = AbortSignal.timeout(cfg.timeoutMs)
266
+ const signal = externalSignal ? AbortSignal.any([externalSignal, timeoutSignal]) : timeoutSignal
267
+ const res = await web.fetch(url, { signal })
268
+ if (!res || typeof res.content !== 'string') return null
269
+ const finalUrl = res.url || res.finalUrl || url
270
+ return { html: res.content, finalUrl }
271
+ } catch {
272
+ return null
273
+ }
274
+ }
275
+
276
+ // Decode common HTML entities in already-stripped text. Runs AFTER tag
277
+ // stripping so escaped tags stay visible to the stripper and only remaining
278
+ // text is decoded. Covers the numeric/hex forms plus the named entities that
279
+ // actually show up in CJK web text (spaces, dashes, quotes, symbols) —
280
+ // undecoded leftovers would both waste tokens and read as garbage.
281
+ const ENTITIES = {
282
+ nbsp: ' ', ensp: ' ', emsp: ' ', thinsp: ' ', zwnj: '', zwj: '',
283
+ amp: '&', lt: '<', gt: '>', quot: '"', apos: "'",
284
+ ndash: '–', mdash: '—', hellip: '…', middot: '·', bull: '•',
285
+ lsquo: '\u2018', rsquo: '\u2019', ldquo: '\u201C', rdquo: '\u201D',
286
+ laquo: '«', raquo: '»', copy: '©', reg: '®', trade: '™',
287
+ deg: '°', plusmn: '±', times: '×', divide: '÷', micro: 'µ',
288
+ sect: '§', para: '¶', dagger: '†', prime: '′', Prime: '″',
289
+ permil: '‰', euro: '€', pound: '£', yen: '¥', cent: '¢',
290
+ sup2: '²', sup3: '³', frac12: '½', frac14: '¼', frac34: '¾',
291
+ }
292
+ // &#0; / &#127; / &#x1F; decode to control characters that pollute the model
293
+ // context (NUL, DEL, C1 controls). Map non-whitespace controls to a space;
294
+ // keep \t\n\r — pages legitimately use &#10; for line breaks, and flattening
295
+ // those to spaces would destroy paragraph structure in markdown mode. Lone
296
+ // surrogates (U+D800–DFFF) decode to replacement chars instead of leaking a
297
+ // broken code unit into the rendered text.
298
+ function fromSafeCodePoint(c) {
299
+ if (c < 0x20) return (c === 9 || c === 10 || c === 13) ? String.fromCodePoint(c) : ' '
300
+ if (c >= 0x7f && c <= 0x9f) return ' '
301
+ if (c >= 0xd800 && c <= 0xdfff) return '\uFFFD'
302
+ return String.fromCodePoint(c)
303
+ }
304
+
305
+ export function decodeTextEntities(text) {
306
+ if (!text.includes('&')) return text
307
+ return text.replace(/&(?:#(\d+)|#x([0-9a-fA-F]+)|([a-z][a-z0-9]{1,7}));?/gi, (m, dec, hex, name) => {
308
+ if (dec !== undefined) {
309
+ const c = Number(dec)
310
+ return c > 0x10ffff ? m : fromSafeCodePoint(c)
311
+ }
312
+ if (hex !== undefined) {
313
+ const c = Number.parseInt(hex, 16)
314
+ return c > 0x10ffff ? m : fromSafeCodePoint(c)
315
+ }
316
+ const n = (name || '').toLowerCase()
317
+ return Object.prototype.hasOwnProperty.call(ENTITIES, n) ? ENTITIES[n] : m
318
+ })
319
+ }
320
+
321
+ // A '<' with no '>' after it can never start a real tag, but tag-scanning
322
+ // regexes still probe the rest of the string at every such position, degrading
323
+ // to quadratic time on pathological input. Neutralize stray '<' in the no-'>'
324
+ // tail: text content is preserved, impossible tag openings disappear, and
325
+ // scanning stays linear. Short tails ("a < b" in prose) are left untouched.
326
+ function defuseLt(html) {
327
+ const from = html.lastIndexOf('>') + 1
328
+ if (html.length - from < 512 || !html.includes('<', from)) return html
329
+ return html.slice(0, from) + html.slice(from).replace(/</g, ' ')
330
+ }
331
+
332
+ function textOnly(html) {
333
+ return decodeTextEntities(defuseLt(html)
334
+ .replace(/<!--[\s\S]*?-->/g, ' ')
335
+ .replace(/<script[\s>][\s\S]*?<\/script>/gi, ' ')
336
+ .replace(/<style[\s>][\s\S]*?<\/style>/gi, ' ')
337
+ .replace(/<textarea[\s>][\s\S]*?<\/textarea\s*>/gi, ' ')
338
+ .replace(/<noscript[\s>][\s\S]*?<\/noscript>/gi, ' ')
339
+ .replace(/<[^>]{1,1000}>/g, ' ')
340
+ .replace(/[ \t\r\n]+/g, ' ')
341
+ .trim())
342
+ }
343
+
344
+ // Paragraph-aware variant of textOnly for the body extraction path: block
345
+ // boundaries become paragraph breaks so offset continuation slices at real
346
+ // paragraph seams (smartTruncate aligns on /\n\n+/) instead of mid-sentence,
347
+ // and long reference pages stay readable instead of one flattened line.
348
+ // <br> keeps single newlines; every other tag runs are stripped as spaces.
349
+ function textLines(html) {
350
+ const cleaned = decodeTextEntities(defuseLt(html)
351
+ .replace(/<!--[\s\S]*?-->/g, ' ')
352
+ .replace(/<script[\s>][\s\S]*?<\/script>/gi, ' ')
353
+ .replace(/<style[\s>][\s\S]*?<\/style>/gi, ' ')
354
+ .replace(/<textarea[\s>][\s\S]*?<\/textarea\s*>/gi, ' ')
355
+ .replace(/<noscript[\s>][\s\S]*?<\/noscript>/gi, ' ')
356
+ .replace(/<br\b[^>]{0,1000}\/?>/gi, '\n')
357
+ .replace(/<\/?(?:p|div|section|article|main|h[1-6]|li|ul|ol|table|tbody|thead|tfoot|tr|blockquote|pre|figcaption|dt|dd|dl|figure|fieldset|option)\b[^>]{0,1000}>/gi, '\n')
358
+ .replace(/<[^>]{1,1000}>/g, ' '))
359
+ const lines = []
360
+ for (const rawLine of cleaned.split('\n')) {
361
+ let line = rawLine.replace(/[ \t\r\f\v]+/g, ' ').trim()
362
+ // A heading becomes its own short paragraph in this layout; a trailing
363
+ // permalink glyph on one is anchor decoration, not content.
364
+ if (line.length <= 120) line = line.replace(/\s*[¶§]$/, '')
365
+ if (line) lines.push(line)
366
+ }
367
+ return lines.join('\n\n')
368
+ }
369
+
370
+ // ---- JSON / RSS / Atom content dispatch ----
371
+ // A URL is not always an HTML page: data APIs answer JSON and content sources
372
+ // answer feeds. Both are model-readable when compacted, so read_url handles
373
+ // them natively instead of erroring with "Unsupported content-type".
374
+
375
+ function xmlText(s) {
376
+ let t = String(s || '').replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
377
+ // Feeds double-escape HTML in descriptions (&lt;a href=…&gt;查看全文&lt;/a&gt;):
378
+ // one strip-then-decode pass leaves literal tags in the rendered text.
379
+ // Iterate strip → decode until stable.
380
+ for (let i = 0; i < 3; i++) {
381
+ const prev = t
382
+ t = decodeTextEntities(t.replace(/<[^>]{1,1000}>/g, ' '))
383
+ if (t === prev) break
384
+ }
385
+ return t.replace(/\s+/g, ' ').trim()
386
+ }
387
+
388
+ // RSS 2.0 <item> / Atom <entry> → compact list "title — url\n summary".
389
+ function parseFeed(xml, limit) {
390
+ xml = defuseLt(xml) // keeps the item scan linear
391
+ const isAtom = /<feed[\s>]/i.test(xml)
392
+ const itemRe = isAtom ? /<entry[\s>][\s\S]*?<\/entry\s*>/gi : /<item[\s>][\s\S]*?<\/item\s*>/gi
393
+ // Without a single closing tag nothing can pair, so the scan is skipped.
394
+ const closable = isAtom ? /<\/entry\s*>/i.test(xml) : /<\/item\s*>/i.test(xml)
395
+ const items = []
396
+ let m
397
+ while (closable && (m = itemRe.exec(xml)) && items.length < limit) {
398
+ const it = m[0]
399
+ const title = xmlText(/<title[^>]*>([\s\S]*?)<\/title>/i.exec(it)?.[1] || '')
400
+ const linkTag = /<link[^>]*>([\s\S]*?)<\/link>/i.exec(it)
401
+ const linkHref = /<link[^>]+href=["']([^"']+)["']/i.exec(it)
402
+ const link = (linkTag && xmlText(linkTag[1])) || (linkHref && linkHref[1]) || ''
403
+ const desc = xmlText(/<(?:description|summary|content)[^>]*>([\s\S]*?)<\/(?:description|summary|content)>/i.exec(it)?.[1] || '')
404
+ if (!title && !link && !desc) continue
405
+ items.push({ title: title.slice(0, 120), url: link, summary: desc.slice(0, 200) })
406
+ }
407
+ const feedTitle = xmlText(/<title[^>]*>([\s\S]*?)<\/title>/i.exec(xml)?.[1] || '')
408
+ const lines = []
409
+ for (const it of items) {
410
+ lines.push(`- ${it.title}${it.url ? ` — ${it.url}` : ''}`)
411
+ if (it.summary) lines.push(` ${it.summary}`)
412
+ }
413
+ return { title: feedTitle, count: items.length, items, text: lines.join('\n') }
414
+ }
415
+
416
+ // ---- Text-density fallback (body path only) ----
417
+ // Pages without <article>/<main> degrade to the whole <body>: semantic noise
418
+ // stripping then still leaves link-dominated blocks (related posts, category
419
+ // sidebars, "hot articles" widgets) that carry little reading value. Split the
420
+ // body at block-level open tags and drop segments whose text is mostly links.
421
+ // Standard-container pages (article/main) never reach this code path.
422
+ const BLOCK_OPEN_RE = /<(?:p|div|section|article|main|h[1-6]|table|tbody|tr|ul|ol|li|blockquote|pre|dl|figure|figcaption|fieldset)\b[^>]{0,1000}>/gi
423
+
424
+ function linkTextLength(html) {
425
+ if (!/<\/a/i.test(html)) return 0 // no anchor can pair — skip the scan
426
+ let n = 0
427
+ const re = /<a[\s>][\s\S]*?<\/a\s*>/gi
428
+ let m
429
+ while ((m = re.exec(html))) n += textOnly(m[0]).length
430
+ return n
431
+ }
432
+
433
+ export function densityFilter(html) {
434
+ const marks = []
435
+ BLOCK_OPEN_RE.lastIndex = 0
436
+ let m
437
+ while ((m = BLOCK_OPEN_RE.exec(html))) marks.push(m.index)
438
+ if (marks.length < 2) return html
439
+ const segs = []
440
+ if (marks[0] > 0) segs.push(html.slice(0, marks[0]))
441
+ for (let k = 0; k < marks.length; k++) {
442
+ segs.push(html.slice(marks[k], k + 1 < marks.length ? marks[k + 1] : html.length))
443
+ }
444
+ const kept = segs.filter((seg) => {
445
+ const text = textOnly(seg)
446
+ if (text.length < 2) return false
447
+ // short link-dominated segment = nav item / recommendation widget entry
448
+ if (text.length < 300 && linkTextLength(seg) > text.length * 0.65) return false
449
+ return true
450
+ })
451
+ return kept.length ? kept.join('') : html
452
+ }
453
+
454
+ // Depth-counted block extraction: returns html from the opening tag at
455
+ // openIdx through its BALANCED closing tag. A non-greedy regex would stop at
456
+ // the first nested </div>, truncating heavily nested container divs.
457
+ function tagBlockAt(html, openIdx, tagName) {
458
+ const re = new RegExp(`</?${tagName}(?=[\\s>])`, 'gi')
459
+ re.lastIndex = openIdx
460
+ let depth = 0
461
+ let m
462
+ while ((m = re.exec(html))) {
463
+ depth += m[0][1] === '/' ? -1 : 1
464
+ if (depth === 0) {
465
+ const close = html.indexOf('>', m.index)
466
+ return close > 0 ? html.slice(openIdx, close + 1) : null
467
+ }
468
+ }
469
+ return null
470
+ }
471
+
472
+ export function pickMain(html) {
473
+ // Aggregation pages (e.g. blog homepages) have one <article> per item:
474
+ // collect all of them instead of only the first. A tiny <article> (e.g. a
475
+ // newsletter card while the real content sits in <main>) falls through to
476
+ // <main> when present.
477
+ const articles = /<\/article/i.test(html)
478
+ ? [...html.matchAll(/<article[\s>][\s\S]*?<\/article\s*>/gi)]
479
+ : [] // without a closer in the document nothing can pair
480
+ const hasMain = /<main[\s>]/i.test(html)
481
+ if (articles.length && (!hasMain || textOnly(articles.map((a) => a[0]).join('')).length >= 200)) {
482
+ return articles.map((a) => a[0]).join('\n')
483
+ }
484
+ const roleMain = /<(main|div)[^>]{0,1000}role=["']main["'][\s>]/i.exec(html)
485
+ if (roleMain) {
486
+ const block = tagBlockAt(html, roleMain.index, roleMain[1].toLowerCase())
487
+ if (block) return block
488
+ }
489
+ // Bare <main> without role="main": doc generators (VitePress/VuePress,
490
+ // MDN, MDX sites) mark the body with a plain <main> tag. Falling back to
491
+ // the whole <body> would drag in the top/local nav (menu, outline, skip
492
+ // link) — the extraction quality loss we saw on vuejs.org.
493
+ const bareMain = /<main[\s>]/i.exec(html)
494
+ if (bareMain) {
495
+ const block = tagBlockAt(html, bareMain.index, 'main')
496
+ if (block) return block
497
+ }
498
+ const body = /<body[\s>]/i.exec(html)
499
+ if (body) {
500
+ const after = html.slice(body.index)
501
+ const end = after.search(/<\/body>/i)
502
+ return densityFilter(end > 0 ? after.slice(0, end) : after)
503
+ }
504
+ return densityFilter(html)
505
+ }
506
+
507
+ // Some sites (e.g. baidu.com) ship CSS/HTML inside hidden <textarea> with
508
+ // entity-escaped tags (&lt;style&gt;...). Reveal TAG-SHAPED sequences so noise
509
+ // rules can strip them. &lt; is revealed only when a tag name follows: a bare
510
+ // &lt; from prose ("a &lt; b") would be swallowed by the tag stripper up to
511
+ // the next real '>'. Bare &gt; / &quot; never open a strip span and are
512
+ // revealed unconditionally.
513
+ function revealEscapedTags(html) {
514
+ return html
515
+ .replace(/&lt;(?=\/?[a-zA-Z!])/gi, '<')
516
+ .replace(/&gt;/gi, '>')
517
+ .replace(/&quot;/gi, '"')
518
+ }
519
+
520
+ function stripNoise(mainHtml) {
521
+ // Attribute runs are bounded so a '<' whose '>' is far away costs a bounded
522
+ // scan instead of a run to that '>'; real attributes never approach 1000
523
+ // chars. Container groups only include tags whose closing tag exists in the
524
+ // input — absent closers cannot pair, so scanning for their body is wasted
525
+ // work.
526
+ const closers = new Set()
527
+ for (const c of mainHtml.matchAll(/<\/([a-z0-9]+)/gi)) closers.add(c[1].toLowerCase())
528
+ const group = (names) => names.filter((n) => closers.has(n))
529
+ const raw = group(['textarea', 'style', 'script', 'noscript', 'template'])
530
+ const box = group(['nav', 'footer', 'header', 'aside', 'form', 'iframe', 'svg', 'canvas', 'dialog'])
531
+ let out = mainHtml.replace(/<!--[\s\S]*?-->/g, ' ')
532
+ if (raw.length) {
533
+ out = out.replace(new RegExp(`<(${raw.join('|')})[\\s>][\\s\\S]*?<\\/\\1\\s*>`, 'gi'), ' ')
534
+ }
535
+ if (box.length) {
536
+ out = out.replace(new RegExp(`<(${box.join('|')})[\\s>][\\s\\S]*?<\\/\\1>`, 'gi'), ' ')
537
+ }
538
+ // Elements explicitly hidden from users are invisible decoration or stateful
539
+ // UI (collapsed panels, modal templates) — their text must not leak into the
540
+ // body. The [\s"'] guard keeps `hidden`/`style` from matching inside
541
+ // compound attr names (data-hidden, aria-hidden). Consent/GDPR banners mount
542
+ // by id (onetrust, cookiebot, cybot, gdpr) rather than class, so a second
543
+ // pass keys on the id attribute. Every scan run is bounded.
544
+ const hidden = /<(div|span|section|p|ul|table)\b[^>]{0,1000}[\s"'](?:hidden\b|aria-hidden\s*=\s*["']?true|style\s*=\s*["'][^"']{0,200}(?:display:\s*none|visibility:\s*hidden))[^>]{0,1000}>[\s\S]{0,50000}?<\/\1\s*>/gi
545
+ out = out.replace(hidden, ' ')
546
+ const consent = /<([a-z][a-z0-9]*)\s+id=["'][^"']{0,120}(?:onetrust|cookiebot|cybot|gdpr|consent|cookie-law|cookie_banner|cmp-)[^"']{0,120}["'][^>]{0,1000}>[\s\S]{0,50000}?<\/\1\s*>/gi
547
+ out = out.replace(consent, ' ')
548
+ return out.replace(
549
+ /<([a-z][a-z0-9]*)[^>]{1,1000}class=["'][^"']{0,300}(ad-|ads|advert|banner|sidebar|social|share|comment|popup|modal|cookie)[^"']{0,300}["'][^>]{0,1000}>[\s\S]*?<\/\1>/gi,
550
+ ' ',
551
+ )
552
+ }
553
+
554
+ // ---- HTML -> Markdown (lightweight, tag-state machine) ----
555
+ function escInline(s) {
556
+ return s.replace(/([\\`*_[\]])/g, '\\$1')
557
+ }
558
+
559
+ // <img> is a void element (never matches the paired-tag regex in either md
560
+ // walker): convert descriptive images to markdown BEFORE the tag walk, via a
561
+ // sentinel so escInline cannot escape the generated brackets. Decorative
562
+ // images (empty alt) are dropped — the model cannot see pixels anyway.
563
+ const IMG_SENTINEL = '\u0001'
564
+ function imgsToMarkdown(html) {
565
+ const store = []
566
+ const out = html.replace(/<img\b[^>]{0,1000}>/gi, (tag) => {
567
+ const alt = /alt=["']([^"']*)["']/i.exec(tag)
568
+ const src = /src=["']([^"']+)["']/i.exec(tag)
569
+ store.push(src && alt && alt[1].trim() ? `![${alt[1].trim()}](${src[1]})` : '')
570
+ return `${IMG_SENTINEL}${store.length - 1}${IMG_SENTINEL}`
571
+ })
572
+ return {
573
+ html: out,
574
+ // Unknown sentinel indexes are PRESERVED (a nested walker's restore must
575
+ // not eat sentinels owned by an outer frame — blockMd recurses into itself).
576
+ restore: (s) => s.replace(new RegExp(`${IMG_SENTINEL}(\\d+)${IMG_SENTINEL}`, 'g'), (m, i) => {
577
+ const v = store[Number(i)]
578
+ return v === undefined ? m : v
579
+ }),
580
+ }
581
+ }
582
+
583
+ // An open tag whose closing tag appears nowhere in the input can never pair;
584
+ // removing such openers up front keeps the walker from rescanning the rest of
585
+ // the string at every position, so the walk stays linear. The walkers drop
586
+ // these tags either way, so the rendered output is unchanged. The depth cap
587
+ // bounds recursion — real pages never nest hundreds of levels; beyond the cap
588
+ // the walker degrades to plain text.
589
+ const OPEN_TAG_RE = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">]){0,1000})>/g
590
+ const MD_MAX_DEPTH = 100
591
+ function stripUnmatchedOpeners(html) {
592
+ const closers = new Set()
593
+ for (const m of html.matchAll(/<\/([a-zA-Z0-9]+)/g)) closers.add(m[1].toLowerCase())
594
+ html = defuseLt(html)
595
+ return html.replace(OPEN_TAG_RE, (m, name) => (closers.has(name.toLowerCase()) ? m : ''))
596
+ }
597
+
598
+ export function inlineMd(html, depth = 0) {
599
+ if (depth > MD_MAX_DEPTH) {
600
+ return escInline(decodeTextEntities(html.replace(/<[^>]{1,1000}>/g, ' ')).replace(/\s+/g, ' ').trim())
601
+ }
602
+ const img = imgsToMarkdown(html)
603
+ html = stripUnmatchedOpeners(img.html)
604
+ let out = ''
605
+ const re = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">]){0,1000})>([\s\S]*?)<\/\1>|<[^>]{1,1000}>|([^<]+)/g
606
+ let m
607
+ while ((m = re.exec(html))) {
608
+ if (m[4] !== undefined) {
609
+ out += escInline(decodeTextEntities(m[4]))
610
+ continue
611
+ }
612
+ if (!m[1]) continue
613
+ const tag = m[1].toLowerCase()
614
+ if (tag === 'style' || tag === 'script' || tag === 'textarea' || tag === 'template' || tag === 'noscript') continue
615
+ const inner = inlineMd(m[3], depth + 1)
616
+ if (tag === 'a') {
617
+ const href = /href=["']([^"']+)["']/i.exec(m[2])
618
+ // A raw ')' in the target would terminate the markdown link early
619
+ // (common on wiki/framework URLs) — percent-encode parens to keep the
620
+ // link parseable by the model.
621
+ out += href ? `[${inner}](${href[1].replace(/\)/g, '%29').replace(/\(/g, '%28')})` : inner
622
+ } else if (tag === 'strong' || tag === 'b') out += `**${inner}**`
623
+ else if (tag === 'em' || tag === 'i') out += `*${inner}*`
624
+ else if (tag === 'code') out += `\`${inner}\``
625
+ else if (tag === 'br') out += ' \n'
626
+ else out += inner
627
+ }
628
+ return img.restore(out.replace(/[ \t]+/g, ' ').trim())
629
+ }
630
+
631
+ export function blockMd(html, depth = 0) {
632
+ if (depth > MD_MAX_DEPTH) {
633
+ return decodeTextEntities(html.replace(/<[^>]{1,1000}>/g, ' ')).replace(/\s+/g, ' ').trim()
634
+ }
635
+ const img = imgsToMarkdown(html)
636
+ html = stripUnmatchedOpeners(img.html)
637
+ let out = ''
638
+ const re = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">]){0,1000})>([\s\S]*?)<\/\1>|<[^>]{1,1000}>|([^<]+)/g
639
+ let m
640
+ while ((m = re.exec(html))) {
641
+ if (m[4] !== undefined) {
642
+ out += decodeTextEntities(m[4])
643
+ continue
644
+ }
645
+ if (!m[1]) continue
646
+ const tag = m[1].toLowerCase()
647
+ const attrs = m[2] || ''
648
+ const inner = m[3]
649
+ if (tag === 'style' || tag === 'script' || tag === 'textarea' || tag === 'template' || tag === 'noscript') {
650
+ continue
651
+ }
652
+ if (tag === 'h1' || tag === 'h2' || tag === 'h3' || tag === 'h4' || tag === 'h5' || tag === 'h6') {
653
+ const level = '#'.repeat(Number(tag[1]))
654
+ // Trailing permalink glyphs are anchor decorations, not heading text.
655
+ out += `\n\n${level} ${inlineMd(inner, depth + 1).replace(/\s*[¶§]\s*$/, '')}`
656
+ } else if (tag === 'p') {
657
+ const t = inlineMd(inner, depth + 1)
658
+ if (t) out += `\n\n${t}`
659
+ } else if (tag === 'div' || tag === 'section' || tag === 'article' || tag === 'main') {
660
+ const t = blockMd(inner, depth + 1)
661
+ if (t) out += `\n\n${t}`
662
+ } else if (tag === 'blockquote') {
663
+ const t = blockMd(inner, depth + 1).trim()
664
+ out += `\n\n> ${t.replace(/\n/g, '\n> ')}`
665
+ } else if (tag === 'pre') {
666
+ // Strip wrapper tags FIRST, then decode entities — a literal &lt;div&gt;
667
+ // in the code must become <div> in the fenced block, but a real nested
668
+ // tag must not survive the strip to be rendered as markdown.
669
+ const code = decodeTextEntities(inner.replace(/<[^>]{1,1000}>/g, '')).trim()
670
+ // Language hint from common highlighter conventions: the model reads
671
+ // ```js fenced blocks far more accurately than unlabelled ones.
672
+ const lang =
673
+ /<code[^>]+class=["'][^"']*(?:language|lang)-([\w#+.-]+)["']/i.exec(inner)?.[1] || ''
674
+ out += `\n\n\`\`\`${lang}\n${code}\n\`\`\``
675
+ } else if (tag === 'code') {
676
+ out += `\`${textOnly(inner)}\``
677
+ } else if (tag === 'ul' || tag === 'ol') {
678
+ // Without a single </li> nothing can pair, so the item scan is skipped.
679
+ const items = []
680
+ if (inner.includes('</li')) {
681
+ const liRe = /<li[^>]{0,1000}>([\s\S]*?)<\/li>/gi
682
+ let lm, idx = 1
683
+ while ((lm = liRe.exec(inner))) {
684
+ const t = inlineMd(lm[1], depth + 1)
685
+ items.push(tag === 'ol' ? `${idx}. ${t}` : `- ${t}`)
686
+ idx++
687
+ }
688
+ }
689
+ if (items.length) out += `\n\n${items.join('\n')}`
690
+ } else if (tag === 'li') {
691
+ out += `\n- ${inlineMd(inner, depth + 1)}`
692
+ } else if (tag === 'br') {
693
+ out += ' \n'
694
+ } else if (tag === 'hr') {
695
+ out += '\n\n---'
696
+ } else if (tag === 'table') {
697
+ const rows = []
698
+ if (inner.includes('</tr')) {
699
+ const trRe = /<tr[^>]{0,1000}>([\s\S]*?)<\/tr>/gi
700
+ let tm
701
+ while ((tm = trRe.exec(inner))) {
702
+ const cells = []
703
+ if (tm[1].includes('</t')) {
704
+ const tdRe = /<t[dh][^>]{0,1000}>([\s\S]*?)<\/t[dh]>/gi
705
+ let cm
706
+ while ((cm = tdRe.exec(tm[1]))) cells.push(inlineMd(cm[1], depth + 1).replace(/\|/g, '\\|'))
707
+ }
708
+ if (cells.length) rows.push(`| ${cells.join(' | ')} |`)
709
+ }
710
+ }
711
+ if (rows.length) {
712
+ const header = rows[0]
713
+ // unescape cells' escaped pipes before deriving the separator row,
714
+ // otherwise the --- line is one char wider per escaped pipe
715
+ const sep = header.replace(/\\\|/g, '|').replace(/[^|]/g, '-')
716
+ out += `\n\n${rows.join('\n')}\n${sep}`
717
+ }
718
+ } else if (tag === 'a' || tag === 'strong' || tag === 'b' || tag === 'em' || tag === 'i') {
719
+ const t = inlineMd(inner, depth + 1)
720
+ if (t) out += t
721
+ } else {
722
+ const t = blockMd(inner, depth + 1)
723
+ if (t) out += t
724
+ }
725
+ }
726
+ return img.restore(out)
727
+ }
728
+
729
+ // Read the content attribute of the FIRST <meta> whose property/name equals
730
+ // `key`. Attribute order is arbitrary (content may precede property/name), and
731
+ // the content value may itself contain quotes — locate the tag first, then
732
+ // read content from inside it with a backreference pair.
733
+ function metaContent(html, key) {
734
+ const tagM = new RegExp(`<meta\\b[^>]{0,1000}(?:property|name)=["']${key}["'][^>]{0,1000}>`, 'i').exec(html)
735
+ if (!tagM) return ''
736
+ const m = /content=("([^"]*)"|'([^']*)')/i.exec(tagM[0])
737
+ return m ? (m[2] || m[3] || '') : ''
738
+ }
739
+
740
+ // Page-level metadata the model actually asks about (who wrote it, when) —
741
+ // cheap to harvest from <meta>. publishedExplicit records whether the date
742
+ // came from a semantic article/og key (authoritative) or a generic date/dc.date
743
+ // stamp (often last-modified, weaker than a JSON-LD datePublished).
744
+ function extractMeta(html) {
745
+ const getKey = (names) => {
746
+ for (const n of names) {
747
+ const v = metaContent(html, n)
748
+ if (v) {
749
+ return {
750
+ v: decodeTextEntities(v).replace(/<[^>]{1,1000}>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 80),
751
+ key: n,
752
+ }
753
+ }
754
+ }
755
+ return { v: '', key: '' }
756
+ }
757
+ const p = getKey(['article:published_time', 'og:published_time', 'pubdate', 'publishdate', 'date', 'dc.date'])
758
+ const a = getKey(['author', 'article:author', 'og:article:author'])
759
+ const d = getKey(['og:description', 'description'])
760
+ return {
761
+ published: p.v,
762
+ publishedExplicit: /^(article|og):/i.test(p.key),
763
+ author: a.v,
764
+ description: d.v,
765
+ }
766
+ }
767
+
768
+ // schema.org JSON-LD blocks carry publication metadata (datePublished/author)
769
+ // many news sites put nowhere else — <meta> covers only a subset and the
770
+ // byline regex cannot see ISO-8601 timestamps. Parse the first ld+json block
771
+ // that yields either field; tolerate broken JSON and every structural shape
772
+ // (plain object, @type arrays, @graph graphs, author as string/object/array).
773
+ // Bounded block size keeps a hostile giant script tag from dragging the scan.
774
+ // 100k covers real article-list JSON-LD (news sites often embed dozens of
775
+ // items); metadata sits in the first node anyway.
776
+ const LDJSON_RE = /<script[^>]{0,1000}type=["']application\/ld\+json["'][^>]{0,1000}>([\s\S]{0,100000}?)<\/script\s*>/gi
777
+ function jsonLdMeta(html) {
778
+ // Recursively collect every node: nested structures are common (ItemList →
779
+ // mainEntity → Article, WebPage → mainEntityOfPage), so a flat top-level
780
+ // scan alone misses most real payloads. Depth-capped against hostile
781
+ // nesting; arrays, @graph and plain object properties are all expanded.
782
+ const collect = (node, out, depth = 0) => {
783
+ if (!node || typeof node !== 'object' || depth > 20) return
784
+ if (Array.isArray(node)) {
785
+ for (const it of node) collect(it, out, depth + 1)
786
+ return
787
+ }
788
+ out.push(node)
789
+ if (Array.isArray(node['@graph'])) for (const it of node['@graph']) collect(it, out, depth + 1)
790
+ for (const k of Object.keys(node)) {
791
+ if (k === '@graph') continue
792
+ const v = node[k]
793
+ if (v && typeof v === 'object') collect(v, out, depth + 1)
794
+ }
795
+ }
796
+ const authorOf = (a) => {
797
+ if (typeof a === 'string') return a
798
+ if (Array.isArray(a)) return authorOf(a[0])
799
+ if (a && typeof a === 'object') return typeof a.name === 'string' ? a.name : authorOf(a.name)
800
+ return ''
801
+ }
802
+ const clean = (s) => (s ? decodeTextEntities(s).slice(0, 40) : '')
803
+ for (const m of html.matchAll(LDJSON_RE)) {
804
+ let json
805
+ try {
806
+ json = JSON.parse(m[1])
807
+ } catch {
808
+ continue
809
+ }
810
+ const nodes = []
811
+ collect(json, nodes)
812
+ for (const n of nodes) {
813
+ const published = clean(typeof n.datePublished === 'string' ? n.datePublished : '')
814
+ const author = clean(authorOf(n.author))
815
+ if (published || author) return { published, author }
816
+ }
817
+ }
818
+ return { published: '', author: '' }
819
+ }
820
+
821
+ // Meta-less pages still print the byline inside the body head: "作者:阮一峰
822
+ // 日期:2026年8月21日". Fill ONLY the missing fields, and only from the first
823
+ // 600 chars of body text — deeper matches are content, not bylines.
824
+ const BYLINE_AUTHOR_RE = /(?:作者|責任編輯|責任主编|editor)[::\s]{1,3}([^\s,,。/|()()[\]]{2,24})/
825
+ const BYLINE_DATE_RE = /(?:发表日期|发布日期|发表时间|发布时间|日期|发表于|发布于|published(?:\s+on)?)[::\s]{0,3}(\d{4}[-/.年]\s?\d{1,2}[-/.月]\s?\d{1,2}日?|\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2})?)/i
826
+
827
+ function bylineFallback(text, meta) {
828
+ const head = typeof text === 'string' ? text.slice(0, 600) : ''
829
+ const out = { published: meta.published, author: meta.author }
830
+ if (!out.author) {
831
+ const m = BYLINE_AUTHOR_RE.exec(head)
832
+ if (m) out.author = m[1].trim().slice(0, 40)
833
+ }
834
+ if (!out.published) {
835
+ const m = BYLINE_DATE_RE.exec(head)
836
+ if (m) out.published = m[1].replace(/\s+/g, ' ').trim().slice(0, 40)
837
+ }
838
+ return out
839
+ }
840
+
841
+ // Locate the element whose id/name equals `anchor` and return the html of its
842
+ // section: for container tags (section/div/dl/…) the element's own balanced
843
+ // block IS the section; for heading/inline anchors (<h2 id>, <a name>, <dt
844
+ // id>) the anchor only marks where the section starts, so everything FROM it
845
+ // onward is kept (the heading introduces the content that follows). Returns
846
+ // null when no element matches — callers fall back to full text.
847
+ // The [\s"'] guard rejects compound attr names (data-id, data-name), and the
848
+ // exact-quoted value means id="prefix-anchor" never matches anchor "prefix".
849
+ const ANCHOR_CONTAINER = /^(?:section|div|article|aside|main|dl|table|ul|ol|details|figure|blockquote|fieldset|form)$/
850
+ function sliceAtAnchor(mainHtml, anchor) {
851
+ if (!anchor) return null
852
+ const esc = anchor.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
853
+ const m = new RegExp(`<([a-z][a-z0-9]*)\\b[^>]{0,1000}[\\s"'](?:id|name)\\s*=\\s*["']${esc}["'][^>]{0,1000}>`, 'i').exec(mainHtml)
854
+ if (!m) return null
855
+ const tag = m[1].toLowerCase()
856
+ if (ANCHOR_CONTAINER.test(tag)) {
857
+ const block = tagBlockAt(mainHtml, m.index, tag)
858
+ if (block) return block
859
+ }
860
+ return mainHtml.slice(m.index)
861
+ }
862
+
863
+ // Anchors may arrive percent-encoded (CJK section names); the document can
864
+ // carry either form. Try the raw fragment first, then its decoded variant.
865
+ function anchorCandidates(anchor) {
866
+ const list = [anchor]
867
+ try {
868
+ const dec = decodeURIComponent(anchor)
869
+ if (dec !== anchor) list.push(dec)
870
+ } catch {
871
+ /* malformed percent-encoding — raw form only */
872
+ }
873
+ return list
874
+ }
875
+
876
+ export function extract(html, mode, anchor = '') {
877
+ html = defuseLt(html) // kills the no-'>'-tail quadratic scans before any picker runs
878
+ const titleTag = /<title[^>]{0,1000}>([\s\S]*?)<\/title>/i.exec(html)
879
+ const ogTitle = metaContent(html, 'og:title')
880
+ const siteName = metaContent(html, 'og:site_name')
881
+ const langMatch = /<html[^>]{1,1000}lang=["']([\w-]+)["']/i.exec(html)
882
+ let main = stripNoise(pickMain(html))
883
+ main = stripNoise(revealEscapedTags(main))
884
+ // A URL fragment scopes the read to one section of a long reference page —
885
+ // slice at the anchor so only that section is extracted. Unmatched anchors
886
+ // degrade to the full document instead of failing.
887
+ let scoped = null
888
+ for (const cand of anchorCandidates(anchor)) {
889
+ scoped = sliceAtAnchor(main, cand)
890
+ if (scoped) break
891
+ }
892
+ let bodyText
893
+ if (mode === 'markdown') {
894
+ bodyText = blockMd(scoped || main).replace(/\n{3,}/g, '\n\n').trim()
895
+ } else {
896
+ bodyText = textLines(scoped || main).replace(/ +/g, ' ').trim()
897
+ }
898
+ const meta = extractMeta(html)
899
+ const ld = jsonLdMeta(html)
900
+ // Merge chain: <meta> → JSON-LD → byline. An explicit article:/og: meta date
901
+ // stays authoritative; otherwise a JSON-LD datePublished beats the generic
902
+ // date meta key (last-modified stamps are not publication times).
903
+ const published = ld.published
904
+ ? (meta.publishedExplicit ? meta.published : ld.published)
905
+ : meta.published
906
+ const author = ld.author || meta.author
907
+ // Empty-body pages (login walls, JS-only shells) still carry og:description
908
+ // — surface it as a fallback hint instead of nothing.
909
+ let text = bodyText
910
+ let usedDescription = false
911
+ if (!text && meta.description) {
912
+ text = meta.description
913
+ usedDescription = true
914
+ }
915
+ const byline = bylineFallback(text, { published, author })
916
+ return {
917
+ title: titleTag ? textOnly(titleTag[1]) || titleTag[1].trim() : ogTitle,
918
+ siteName,
919
+ lang: langMatch ? langMatch[1] : '',
920
+ published: byline.published,
921
+ author: byline.author,
922
+ text,
923
+ usedDescription,
924
+ anchored: !!scoped,
925
+ }
926
+ }
927
+
928
+ // Optional enhancement: when @mozilla/readability + happy-dom are installed
929
+ // inside the DSH profile (npm i @mozilla/readability happy-dom), extraction
930
+ // upgrades to the Firefox Reader Mode algorithm. Falls back to the built-in
931
+ // heuristic extractor when absent. MPL-2.0 / MIT libraries, used as-is.
932
+ let readabilityReady = null
933
+ async function getReadabilityExtractor() {
934
+ if (readabilityReady !== null) return readabilityReady
935
+ try {
936
+ const [{ Readability }, { Window }] = await Promise.all([
937
+ import('@mozilla/readability'),
938
+ import('happy-dom'),
939
+ ])
940
+ readabilityReady = (html, url) => {
941
+ const window = new Window({ url })
942
+ window.document.write(html)
943
+ const article = new Readability(window.document).parse()
944
+ return article ? article.content : null
945
+ }
946
+ } catch {
947
+ readabilityReady = false
948
+ }
949
+ return readabilityReady
950
+ }
951
+
952
+ export function smartTruncate(text, maxChars, offset = 0) {
953
+ const total = text.length
954
+ if (offset >= total) {
955
+ return { text: '', truncated: false, charsTotal: total, charsReturned: 0, charsStart: offset }
956
+ }
957
+ if (total <= maxChars && offset === 0) {
958
+ return { text, truncated: false, charsTotal: total, charsReturned: total, charsStart: 0 }
959
+ }
960
+ const paragraphs = text.split(/\n\n+/)
961
+ let pos = 0
962
+ let start = 0
963
+ for (let i = 0; i < paragraphs.length; i++) {
964
+ if (pos + paragraphs[i].length > offset) {
965
+ start = i
966
+ break
967
+ }
968
+ pos += paragraphs[i].length + 2
969
+ }
970
+ let acc = ''
971
+ for (let i = start; i < paragraphs.length; i++) {
972
+ const p = paragraphs[i]
973
+ if (acc.length + p.length + (acc ? 2 : 0) > maxChars) break
974
+ acc += (acc ? '\n\n' : '') + p
975
+ }
976
+ if (!acc && maxChars > 0) {
977
+ // The first paragraph after the offset is itself longer than maxChars.
978
+ // Keep the alignment promise at sentence level before hard-slicing.
979
+ const p = paragraphs[start] || ''
980
+ const rel = Math.max(0, offset - pos)
981
+ const rest = p.length > rel ? p.slice(rel) : text.slice(offset)
982
+ const sentences = rest.match(/[^.!?\n。!?]+[.!?\n。!?]*/g) || [rest]
983
+ for (const s of sentences) {
984
+ if (acc.length + s.length > maxChars) break
985
+ acc += s
986
+ }
987
+ if (!acc) acc = rest.slice(0, maxChars)
988
+ }
989
+ return {
990
+ text: acc,
991
+ truncated: offset + acc.length < total,
992
+ charsTotal: total,
993
+ charsReturned: acc.length,
994
+ charsStart: offset,
995
+ }
996
+ }
997
+
998
+ // <base href> overrides the document base for relative URL resolution (old
999
+ // forums / frame sites). Per HTML spec only the FIRST <base> in <head> counts;
1000
+ // attribute order is arbitrary and the href may be protocol-relative. Empty
1001
+ // href (= document URL) is a no-op; non-http(s) targets are rejected. The
1002
+ // head scan is bounded — the tag cannot legally appear beyond it.
1003
+ function detectBaseHref(html, finalUrl) {
1004
+ const headEnd = html.search(/<\/head\s*>/i)
1005
+ const head = headEnd >= 0 ? html.slice(0, headEnd) : html.slice(0, 16384)
1006
+ const base = /<base\b[^>]{0,1000}href=["']([^"']+)["'][^>]{0,1000}>/i.exec(head)
1007
+ if (!base) return finalUrl
1008
+ const href = base[1].trim()
1009
+ if (!href) return finalUrl
1010
+ try {
1011
+ const u = new URL(href, finalUrl)
1012
+ return u.protocol === 'http:' || u.protocol === 'https:' ? u.href : finalUrl
1013
+ } catch {
1014
+ return finalUrl
1015
+ }
1016
+ }
1017
+
1018
+ function extractLinks(html, limit, baseUrl) {
1019
+ const links = []
1020
+ const seen = new Set()
1021
+ // Accept relative hrefs too — most real pages link internally with relative
1022
+ // paths — and resolve them against the page URL so the model can follow them.
1023
+ // A declared <base href> wins over the document URL (frame-site links would
1024
+ // otherwise land on the wrong host).
1025
+ const linkBase = detectBaseHref(html, baseUrl)
1026
+ // Attribute runs bounded: an unclosed <a whose '>' is far away (or missing)
1027
+ // must not make the regex scan the rest of the string at every anchor
1028
+ // position. Real attributes never approach 1000 chars.
1029
+ const re = /<a\b[^>]{0,1000}href=["']([^"']+)["'][^>]{0,1000}>([\s\S]*?)<\/a\s*>/gi
1030
+ let m
1031
+ // Iteration cap: a page whose nav repeats the same links thousands of times
1032
+ // would otherwise scan the whole HTML to fill `limit` unique entries. Scan
1033
+ // at most limit*4 anchors; dedupe keeps the result bounded regardless.
1034
+ let scans = 0
1035
+ const maxScans = limit * 4
1036
+ while ((m = re.exec(html)) && links.length < limit && scans++ < maxScans) {
1037
+ const href = m[1].trim()
1038
+ if (!href || /^(javascript|mailto|tel|data):/i.test(href)) continue
1039
+ let url
1040
+ try {
1041
+ url = linkBase ? new URL(href, linkBase).href : new URL(href).href
1042
+ } catch {
1043
+ continue
1044
+ }
1045
+ // Only http(s) links are followable by the model / crawler; other schemes
1046
+ // (ftp:, data:) survive URL resolution and would pollute link lists and
1047
+ // crawl queues.
1048
+ if (!/^https?:/i.test(url)) continue
1049
+ // dedupe: nav bars repeat the same links many times — one entry per URL
1050
+ // keeps the link list compact (tokens) without losing coverage
1051
+ if (seen.has(url)) continue
1052
+ seen.add(url)
1053
+ const t = textOnly(m[2])
1054
+ if (t) links.push({ title: t.slice(0, 80), url })
1055
+ }
1056
+ return links
1057
+ }
1058
+
1059
+ function hostOf(url) {
1060
+ try {
1061
+ return new URL(url).hostname
1062
+ } catch {
1063
+ return ''
1064
+ }
1065
+ }
1066
+
1067
+ // ---- Pagination: auto-follow "next page" ----
1068
+ // Long articles on novel/news/forum sites are split into pages; without this
1069
+ // the model had to notice the truncation and re-call the tool per page. The
1070
+ // recognition is deliberately conservative: rel=next (standard) or a short
1071
+ // anchor whose whole text is a next-page marker — no fuzzy guessing.
1072
+ const NEXT_TEXT_RE = /^(?:[›»>]+\s*)?(?:下一页|下页|下一篇|下一頁|下頁|后一页|後一頁|next page|next|older entries|[›»>]{1,3})(?:\s*[›»>]+)?$/i
1073
+
1074
+ export function findNextLink(html, baseUrl) {
1075
+ // Same <base href> rule as extractLinks: frame/forum sites pin their
1076
+ // document base, and a "next page" link resolves against it.
1077
+ const base = detectBaseHref(html || '', baseUrl)
1078
+ const resolve = (href) => {
1079
+ try {
1080
+ const u = new URL(href, base)
1081
+ return u.protocol === 'http:' || u.protocol === 'https:' ? u.href : null
1082
+ } catch {
1083
+ return null
1084
+ }
1085
+ }
1086
+ const rel = /<(?:link|a)\b[^>]{0,1000}rel=["']next["'][^>]{0,1000}>/i.exec(html || '')
1087
+ if (rel) {
1088
+ const href = /href=["']([^"']+)["']/i.exec(rel[0])
1089
+ const r = href && resolve(href[1])
1090
+ if (r) return r
1091
+ }
1092
+ const re = /<a\b[^>]{0,1000}href=["']([^"']+)["'][^>]{0,1000}>([\s\S]*?)<\/a\s*>/gi
1093
+ let m
1094
+ let scans = 0
1095
+ // Bounded like extractLinks: a nav bar with thousands of anchors must not
1096
+ // scan the whole DOM hunting for a next-marker.
1097
+ while ((m = re.exec(html || '')) && scans++ < 400) {
1098
+ const t = textOnly(m[2])
1099
+ if (t.length <= 16 && NEXT_TEXT_RE.test(t)) {
1100
+ const r = resolve(m[1])
1101
+ if (r) return r
1102
+ }
1103
+ }
1104
+ return null
1105
+ }
1106
+
1107
+ // Join two page bodies, dropping a repeated suffix/prefix overlap (many
1108
+ // paginated sites repeat the last paragraph or a heading on the next page).
1109
+ function joinPageText(a, b) {
1110
+ const max = Math.min(300, a.length, b.length)
1111
+ for (let n = max; n >= 30; n--) {
1112
+ if (a.slice(a.length - n) === b.slice(0, n)) {
1113
+ const rest = b.slice(n).trim()
1114
+ return rest ? `${a}\n\n${rest}` : a
1115
+ }
1116
+ }
1117
+ return `${a}\n\n${b}`
1118
+ }
1119
+
1120
+ function sliceFrom(full, offset, maxChars) {
1121
+ const s = smartTruncate(full.fullText, maxChars, offset)
1122
+ const out = {
1123
+ url: full.url,
1124
+ title: full.title || '',
1125
+ siteName: full.siteName || hostOf(full.url),
1126
+ lang: full.lang || '',
1127
+ charset: full.charset,
1128
+ mode: full.mode,
1129
+ truncated: s.truncated,
1130
+ charsTotal: s.charsTotal,
1131
+ charsReturned: s.charsReturned,
1132
+ charsStart: s.charsStart,
1133
+ text: s.text,
1134
+ }
1135
+ if (full.rendered) out.rendered = true
1136
+ if (full.spaHint) out.spaHint = full.spaHint
1137
+ if (full.anchored) {
1138
+ out.anchored = true
1139
+ out.anchor = full.anchor || ''
1140
+ }
1141
+ if (full.paginated > 1) out.paginated = full.paginated
1142
+ if (full.feedCount) out.feedCount = full.feedCount
1143
+ if (full.published) out.published = full.published
1144
+ if (full.author) out.author = full.author
1145
+ if (Array.isArray(full.links)) out.links = full.links
1146
+ return out
1147
+ }
1148
+
1149
+ // ---- JSON payloads: compact render ----
1150
+ // Indented JSON spends a newline + indent per key — pure token overhead for
1151
+ // the model. Render compact, and clip monster string values (embedded HTML,
1152
+ // base64 blobs) with a visible marker so the model knows there is more.
1153
+ const JSON_STR_MAX = 1500
1154
+ export function compactJson(value) {
1155
+ const clip = (s) =>
1156
+ s.length > JSON_STR_MAX ? s.slice(0, JSON_STR_MAX) + `…[+${s.length - JSON_STR_MAX} chars]` : s
1157
+ const walk = (v, depth) => {
1158
+ if (typeof v === 'string') return clip(v)
1159
+ if (Array.isArray(v)) return v.map((x) => walk(x, depth + 1))
1160
+ if (v && typeof v === 'object' && depth < 12) {
1161
+ const o = {}
1162
+ for (const k of Object.keys(v)) o[k] = walk(v[k], depth + 1)
1163
+ return o
1164
+ }
1165
+ return v
1166
+ }
1167
+ try {
1168
+ return JSON.stringify(walk(value, 0))
1169
+ } catch {
1170
+ // Extreme nesting/size where even native stringify exhausts the stack —
1171
+ // degrade to a clipped raw render instead of crashing the tool call.
1172
+ try {
1173
+ return clip(String(value))
1174
+ } catch {
1175
+ return '[unserializable json]'
1176
+ }
1177
+ }
1178
+ }
1179
+
1180
+ // ---- meta-refresh shells: follow to the real page ----
1181
+ // <meta http-equiv="refresh" content="0; url=..."> stubs (link hops,
1182
+ // anti-hotlink relays, no-JS SPA entries) serve only a jump page — the body
1183
+ // lives at the target. Parse the FIRST matching <meta>; attribute order and
1184
+ // case are arbitrary, content quotes are stripped, a protocol-relative target
1185
+ // resolves against the document (or <base href>). Only IMMEDIATE redirects
1186
+ // are followed: a timed auto-refresh (content="30; url=…") is a normal page
1187
+ // with content, not a shell. A bare timed refresh with no url part reloads
1188
+ // the same page — null. Non-http(s) targets are refused.
1189
+ export function metaRefreshTarget(html, baseUrl) {
1190
+ // Locate the refresh <meta> tag first (any attribute order / case), then
1191
+ // read content from inside that tag — a forward regex would demand
1192
+ // http-equiv before content and miss the reversed order.
1193
+ const re = /<meta\b[^>]{0,1000}(?:http-equiv|name)=["']refresh["'][^>]{0,1000}>/gi
1194
+ let m
1195
+ while ((m = re.exec(html || ''))) {
1196
+ const tag = m[0]
1197
+ const contentM = /content=("([^"]*)"|'([^']*)')/i.exec(tag)
1198
+ if (!contentM) continue // a refresh meta without content is inert
1199
+ const content = (contentM[2] || contentM[3] || '').trim()
1200
+ // content="5" | content="0; url=http://…" | content=";url='…'" — the url
1201
+ // part may be single/double quoted or bare; "url=" must be a real token.
1202
+ const urlM = /\burl\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s;]+))/i.exec(content)
1203
+ if (!urlM) return null
1204
+ const delayM = /^\s*(\d+(?:\.\d+)?)/.exec(content)
1205
+ if (delayM && Number(delayM[1]) > 0) return null // timed refresh: keep the page
1206
+ const target = (urlM[1] || urlM[2] || urlM[3] || '').trim()
1207
+ if (!target) return null
1208
+ try {
1209
+ const u = new URL(target, detectBaseHref(html || '', baseUrl))
1210
+ return u.protocol === 'http:' || u.protocol === 'https:' ? u.href : null
1211
+ } catch {
1212
+ return null
1213
+ }
1214
+ }
1215
+ return null
1216
+ }
1217
+
1218
+ // Bounded following: at most 3 hops, deduped by normalized URL. Any fetch
1219
+ // problem (network error, non-HTML body) FAILS OPEN — the shell's own HTML
1220
+ // is kept for static extraction instead of erroring the call. Followed pages
1221
+ // are never written to the session cache under their own URL: the caller's
1222
+ // cache key is the ORIGINAL url, and a shell re-request should re-follow
1223
+ // rather than serve a foreign page.
1224
+ const MAX_META_REFRESH_HOPS = 3
1225
+ async function followMetaRefresh(html, finalUrl, externalSignal, cfg) {
1226
+ let cur = html
1227
+ let curUrl = finalUrl
1228
+ let curCharset = '' // set only when a hop actually happened
1229
+ const seen = new Set([normalizeUrl(finalUrl)])
1230
+ for (let i = 0; i < MAX_META_REFRESH_HOPS; i++) {
1231
+ const target = metaRefreshTarget(cur, curUrl)
1232
+ if (!target) break
1233
+ const norm = normalizeUrl(target)
1234
+ if (!norm || seen.has(norm)) break
1235
+ seen.add(norm)
1236
+ const page = await fetchPage(target, externalSignal, cfg)
1237
+ if (page.error) break
1238
+ const ct = (page.contentType || '').split(';')[0].toLowerCase().trim()
1239
+ if (!/html|xhtml/.test(ct)) break
1240
+ const decoded = decodeBuffer(page.buffer, page.contentType)
1241
+ cur = decoded.text
1242
+ curCharset = decoded.charset
1243
+ curUrl = page.finalUrl || target
1244
+ }
1245
+ return { html: cur, finalUrl: curUrl, charset: curCharset }
1246
+ }
1247
+
1248
+ export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
1249
+ const url = String((args && args.url) || '').trim()
1250
+ if (!/^https?:\/\//i.test(url)) return { error: 'Only http/https URLs are supported' }
1251
+ const maxChars = Math.max(500, Math.min(20000, Number(args.maxChars) || cfg.maxChars))
1252
+ const mode = args.mode === 'markdown' ? 'markdown' : 'text'
1253
+ const offset = Math.max(0, Number(args.offset) || 0)
1254
+
1255
+ // Cache stores the FULL extracted text keyed by url+mode+includeLinks, so
1256
+ // continuation reads (offset) and different maxChars hit the same entry —
1257
+ // but a links-request must not be served from a links-less cached copy.
1258
+ // The fragment STAYS in the key: a #section read scopes the extraction to
1259
+ // that anchor, so two fragments must never share one cache entry.
1260
+ let cacheUrl = url
1261
+ try {
1262
+ cacheUrl = new URL(url).href
1263
+ } catch {
1264
+ /* keep raw url on parse failure */
1265
+ }
1266
+ // A #fragment scopes the read to one section of a long page. The browser
1267
+ // never sends it to the server, so it must be taken from the REQUEST url —
1268
+ // finalUrl (after redirects/renders) has no fragment.
1269
+ let anchor = ''
1270
+ try {
1271
+ const h = new URL(url).hash
1272
+ if (h.length > 1) anchor = h.slice(1)
1273
+ } catch {
1274
+ /* keep bare url */
1275
+ }
1276
+ const cacheKey = `${cacheUrl}|${mode}|${args.includeLinks === true ? 'links' : 'no-links'}`
1277
+ // Failed fetches are cached briefly so the model doesn't re-request a
1278
+ // broken URL in a loop (token + latency saver).
1279
+ const fh = failCache.get(cacheKey)
1280
+ if (fh) {
1281
+ if (Date.now() - fh.time < 30000) return { error: fh.error, cached: true }
1282
+ failCache.delete(cacheKey)
1283
+ }
1284
+ const hit = cache.get(cacheKey)
1285
+ if (hit && Date.now() - hit.time < cfg.cacheTtlMs) {
1286
+ const sliced = sliceFrom(hit.full, offset, maxChars)
1287
+ return { ...sliced, cached: true }
1288
+ }
1289
+
1290
+ const failWith = (error) => {
1291
+ cacheFail(cacheKey, error)
1292
+ return { error }
1293
+ }
1294
+
1295
+ // Non-HTML payloads (JSON APIs, RSS/Atom feeds) are read natively: fetch,
1296
+ // dispatch by content-type / payload shape, render compact. Falls back to
1297
+ // the HTML pipeline for everything else.
1298
+ const dispatch = async () => {
1299
+ const viaSeam = await fetchViaWebSeam(ctx, url, externalSignal, cfg)
1300
+ if (viaSeam) {
1301
+ const t = viaSeam.html.trim()
1302
+ if (t.startsWith('{') || t.startsWith('[')) {
1303
+ try {
1304
+ return { kind: 'json', text: compactJson(JSON.parse(t)) }
1305
+ } catch { /* not JSON — keep HTML pipeline */ }
1306
+ }
1307
+ return { kind: 'html', html: viaSeam.html, finalUrl: viaSeam.finalUrl, charset: 'provider-decoded' }
1308
+ }
1309
+ const page = await fetchPage(url, externalSignal, cfg)
1310
+ if (page.error) return { error: page.error }
1311
+ const ct = (page.contentType || '').split(';')[0].toLowerCase().trim()
1312
+ const decoded = decodeBuffer(page.buffer, page.contentType)
1313
+ if (/json$/.test(ct)) {
1314
+ try {
1315
+ return { kind: 'json', text: compactJson(JSON.parse(decoded.text)) }
1316
+ } catch { /* invalid JSON — serve through the HTML pipeline as text */ }
1317
+ }
1318
+ if (/[+/]xml$/.test(ct)) {
1319
+ if (/<urlset[\s>]/i.test(decoded.text) || /<sitemapindex[\s>]/i.test(decoded.text)) {
1320
+ return { error: 'Unsupported content (XML sitemap)' }
1321
+ }
1322
+ if (/<rss[\s>]/i.test(decoded.text) || /<feed[\s>]/i.test(decoded.text)) {
1323
+ return { kind: 'feed', feed: parseFeed(decoded.text, cfg.maxLinks) }
1324
+ }
1325
+ }
1326
+ return { kind: 'html', html: decoded.text, finalUrl: page.finalUrl || url, charset: decoded.charset }
1327
+ }
1328
+ const payload = await dispatch()
1329
+ if (payload.error) return failWith(payload.error)
1330
+
1331
+ if (payload.kind === 'json') {
1332
+ const full = { url, title: '', siteName: hostOf(url), lang: '', charset: 'json', mode: 'json', fullText: payload.text, paginated: 1 }
1333
+ cacheStore(cacheKey, full, cfg.cacheMax)
1334
+ return sliceFrom(full, offset, maxChars)
1335
+ }
1336
+ if (payload.kind === 'feed') {
1337
+ const f = payload.feed
1338
+ const full = {
1339
+ url, title: f.title || '', siteName: hostOf(url), lang: '', charset: 'feed',
1340
+ mode: 'feed', fullText: f.text, paginated: 1, feedCount: f.count,
1341
+ links: args.includeLinks === true ? f.items : undefined,
1342
+ }
1343
+ cacheStore(cacheKey, full, cfg.cacheMax)
1344
+ return sliceFrom(full, offset, maxChars)
1345
+ }
1346
+
1347
+ let html = payload.html
1348
+ let finalUrl = payload.finalUrl || url
1349
+ let charset = payload.charset
1350
+
1351
+ // meta-refresh shells (link hops, anti-hotlink relays, no-JS SPA entries):
1352
+ // follow immediately, BEFORE extraction and SPA rendering — a shell that
1353
+ // redirects needs no headless round-trip, and extracting the stub is wasted
1354
+ // work. Fails open on any fetch problem; the follow is hop-bounded.
1355
+ const followed = await followMetaRefresh(html, finalUrl, externalSignal, cfg)
1356
+ html = followed.html
1357
+ finalUrl = followed.finalUrl
1358
+ // The target may use a different encoding than the shell page — reflect it.
1359
+ if (followed.charset) charset = followed.charset
1360
+
1361
+ let extracted = extract(html, mode, anchor)
1362
+ // Optional readability upgrade: cleaner article extraction when installed.
1363
+ const ext = await getReadabilityExtractor()
1364
+ // The base passed to readability must be the CURRENT final URL — after a
1365
+ // meta-refresh follow or SPA render the document may live on another host,
1366
+ // and relative links/images would resolve against the original request URL.
1367
+ const upgrade = (target, htmlText, base) => {
1368
+ if (!ext) return target
1369
+ try {
1370
+ const clean = ext(htmlText, base || finalUrl)
1371
+ if (clean && clean.length > 0) {
1372
+ target.text = mode === 'markdown'
1373
+ ? blockMd(clean).replace(/\n{3,}/g, '\n\n').trim()
1374
+ : textLines(clean).replace(/ +/g, ' ').trim()
1375
+ }
1376
+ } catch {
1377
+ // keep heuristic result
1378
+ }
1379
+ return target
1380
+ }
1381
+ // Readability would replace the scoped section with the FULL article text;
1382
+ // when an anchor matched, the scoped extraction IS the answer.
1383
+ extracted = extracted.anchored ? extracted : upgrade(extracted, html)
1384
+
1385
+ // Optional SPA enhancement: if the page looks client-rendered and static
1386
+ // extraction found almost nothing, try headless rendering (playwright).
1387
+ // Beyond script-heavy shells (looksLikeSpa, ≥5 <script> tags), a page whose
1388
+ // extraction is EMPTY but carries any <script> may be a JS-redirect shell
1389
+ // (2 scripts, empty <body>, redirect issued from JS) — with no text to
1390
+ // lose, rendering is always worth one bounded attempt. Pages with no
1391
+ // scripts at all cannot change under rendering and are skipped.
1392
+ let rendered = false
1393
+ let spaHint = ''
1394
+ let renderedHtml = null
1395
+ const needsRender = !extracted.text || extracted.text.length < 200
1396
+ const scriptShell = !extracted.text && /<script[\s>]/i.test(html)
1397
+ if (cfg.spaRender !== false && needsRender && (looksLikeSpa(html) || scriptShell)) {
1398
+ const rr = await renderPage(finalUrl || url, externalSignal)
1399
+ if (rr.html) {
1400
+ renderedHtml = rr.html
1401
+ // A challenge interstitial (Cloudflare "Just a moment...") must never
1402
+ // be accepted as an improvement — it can carry MORE text than the real
1403
+ // static body and would evict the latter.
1404
+ if (looksLikeChallenge(rr.html)) {
1405
+ renderedHtml = null // pagination/links must not read the interstitial DOM
1406
+ spaHint = extracted.text
1407
+ ? '已渲染但被人机验证页拦截,保留静态提取结果'
1408
+ : '已渲染但被人机验证页拦截(无静态内容可用)'
1409
+ } else {
1410
+ const r2raw = extract(rr.html, mode, anchor)
1411
+ const r2 = r2raw.anchored ? r2raw : upgrade(r2raw, rr.html, rr.finalUrl || finalUrl)
1412
+ // Accept the rendered text when it meaningfully beats the static one.
1413
+ // From an EMPTY static extraction even a short body is a real gain
1414
+ // (JS shells can render just 30-60 chars of real content, below the
1415
+ // fixed +50 margin).
1416
+ const gain = r2.text.length - extracted.text.length
1417
+ const accept = r2.text && (gain > 50 || (!extracted.text && r2.text.length >= 20))
1418
+ if (accept) {
1419
+ extracted = r2
1420
+ finalUrl = rr.finalUrl || finalUrl
1421
+ rendered = true
1422
+ } else if (!extracted.text) {
1423
+ spaHint = '已渲染仍无可读内容(可能登录墙或反爬拦截)'
1424
+ }
1425
+ }
1426
+ } else {
1427
+ spaHint = rr.error
1428
+ }
1429
+ }
1430
+
1431
+ // Auto-pagination: follow the "next page" chain (bounded, same-host, no
1432
+ // loops). Continuation pages go through the fast static path — a paginated
1433
+ // SPA chain would cost one full render per page, out of proportion.
1434
+ let paginated = 1
1435
+ const paginateMax = cfg.paginate === false ? 1 : Math.max(1, Math.min(10, Number(cfg.paginateMax) || 3))
1436
+ const startHost = hostOf(finalUrl || url)
1437
+ const seen = new Set([normalizeUrl(finalUrl || url)])
1438
+ // An anchored read is scoped to one section — joining the "next page" chain
1439
+ // would append whole pages the fragment never asked for.
1440
+ let nextUrl = paginateMax > 1 && !extracted.anchored ? findNextLink(renderedHtml || html, finalUrl || url) : null
1441
+ while (
1442
+ nextUrl && paginated < paginateMax &&
1443
+ sameHost(nextUrl, startHost) &&
1444
+ !(externalSignal && externalSignal.aborted)
1445
+ ) {
1446
+ const nu = normalizeUrl(nextUrl)
1447
+ if (!nu || seen.has(nu)) break
1448
+ seen.add(nu)
1449
+ const page = await fetchPage(nextUrl, externalSignal, cfg)
1450
+ if (page.error) break
1451
+ const ct = (page.contentType || '').split(';')[0].toLowerCase().trim()
1452
+ if (!/html|xhtml/.test(ct)) break
1453
+ const nextHtml = decodeBuffer(page.buffer, page.contentType).text
1454
+ const nextEx = extract(nextHtml, mode)
1455
+ if (!nextEx.text || nextEx.text.length < 20) break
1456
+ extracted.text = joinPageText(extracted.text, nextEx.text)
1457
+ paginated++
1458
+ nextUrl = findNextLink(nextHtml, page.finalUrl || nextUrl)
1459
+ }
1460
+
1461
+ // A near-empty body (login wall, anti-bot shell) with no render hint yet
1462
+ // still looks like "the page content" to the model — one constant-cost line
1463
+ // tells it not to keep offset-reading a 40-char page.
1464
+ if (!spaHint && extracted.text && extracted.text.length < 60) {
1465
+ spaHint = '正文极短(可能登录墙或反爬拦截)'
1466
+ }
1467
+
1468
+ const full = {
1469
+ url: finalUrl,
1470
+ title: extracted.title || '',
1471
+ siteName: extracted.siteName || '',
1472
+ lang: extracted.lang || '',
1473
+ charset,
1474
+ mode,
1475
+ fullText: extracted.text,
1476
+ rendered,
1477
+ spaHint,
1478
+ anchored: extracted.anchored || false,
1479
+ anchor: extracted.anchored ? anchor : '',
1480
+ paginated,
1481
+ published: extracted.published || '',
1482
+ author: extracted.author || '',
1483
+ // Links come from the rendered DOM when available — otherwise the SPA
1484
+ // chain (content -> links -> next page) would break for JS-only pages.
1485
+ links: args.includeLinks === true ? extractLinks(renderedHtml || html, cfg.maxLinks, finalUrl) : undefined,
1486
+ }
1487
+
1488
+ cacheStore(cacheKey, full, cfg.cacheMax)
1489
+ return sliceFrom(full, offset, maxChars)
1490
+ }
1491
+
1492
+ export function renderResult(value) {
1493
+ if (typeof value === 'string') return value
1494
+ const r = value || {}
1495
+ if (r.error) return `Error: ${r.error}`
1496
+ const lines = []
1497
+ if (r.title) lines.push(`title: ${r.title}`)
1498
+ const host = hostOf(r.url || '')
1499
+ const meta = []
1500
+ if (r.siteName && r.siteName !== host) meta.push(r.siteName)
1501
+ if (r.lang) meta.push(r.lang)
1502
+ if (r.charset && r.charset !== 'utf-8' && r.charset !== 'json' && r.charset !== 'feed') {
1503
+ meta.push(`charset ${r.charset}`)
1504
+ }
1505
+ if (r.published) meta.push(r.published)
1506
+ if (r.author) meta.push(`by ${r.author}`)
1507
+ // Tells the model why the text starts mid-document: the read was scoped to
1508
+ // this fragment, and offset continues from the section start.
1509
+ if (r.anchored) meta.push(`#${r.anchor || ''}`)
1510
+ if (r.feedCount) meta.push(`feed · ${r.feedCount} 条`)
1511
+ if (r.mode === 'json') meta.push('json')
1512
+ if (meta.length) lines.push(meta.join(' · '))
1513
+ // One compact status line instead of several parenthetical lines — the
1514
+ // flags are hints, not content, so they share a single line of tokens.
1515
+ const flags = []
1516
+ if (r.charsStart > 0) flags.push(`chars ${r.charsStart}+${r.charsReturned}/${r.charsTotal}`, 'offset 续读')
1517
+ else if (r.truncated) flags.push(`chars ${r.charsReturned}/${r.charsTotal}`, '截断', 'offset 续读')
1518
+ else if (typeof r.charsTotal === 'number') flags.push(`chars ${r.charsTotal}`)
1519
+ if (r.cached) flags.push('cached')
1520
+ if (r.paginated > 1) flags.push(`已拼接${r.paginated}页`)
1521
+ if (r.rendered) flags.push('rendered')
1522
+ // With text present the hint rides the flags line; with empty text the
1523
+ // dedicated "无可读内容" line below already carries it.
1524
+ if (r.text && r.spaHint) flags.push(r.spaHint)
1525
+ if (flags.length) lines.push(`(${flags.join(' · ')})`)
1526
+ if (!r.text) {
1527
+ if (r.charsStart > 0 && !r.truncated) {
1528
+ lines.push('(offset 已到文末,无更多内容)')
1529
+ } else {
1530
+ lines.push(r.spaHint
1531
+ ? `(无可读内容 — ${r.spaHint})`
1532
+ : '(无可读内容 — 登录墙 / SPA 页 / 空页面;SPA 需安装 playwright)')
1533
+ }
1534
+ }
1535
+ lines.push('', r.text || '')
1536
+ if (Array.isArray(r.links) && r.links.length) {
1537
+ lines.push('', 'links:')
1538
+ for (const l of r.links) lines.push(`- ${l.title} — ${l.url}`)
1539
+ }
1540
+ return lines.join('\n')
1541
+ }
1542
+
1543
+ function renderLinks(value) {
1544
+ if (typeof value === 'string') return value
1545
+ const r = value || {}
1546
+ if (r.error) return `Error: ${r.error}`
1547
+ if (!Array.isArray(r.links) || r.links.length === 0) return `No links found on ${r.url}`
1548
+ const lines = [`${r.count} link(s) on ${r.url}:`]
1549
+ for (const l of r.links) lines.push(`- ${l.title || l.url} — ${l.url}`)
1550
+ return lines.join('\n')
1551
+ }
1552
+
1553
+ function readLinksTool(ctx, cfg) {
1554
+ return {
1555
+ name: 'read_url_links',
1556
+ description:
1557
+ 'List links (text + URL) on a page without body text — lighter than read_url ' +
1558
+ 'for mapping what a page points to. Renders SPA pages when playwright installed.',
1559
+ parameters: {
1560
+ type: 'object',
1561
+ additionalProperties: false,
1562
+ properties: {
1563
+ url: { type: 'string', description: 'http(s) URL to scan for links' },
1564
+ limit: { type: 'number', description: 'Links to return (1-50; default from plugin config)' },
1565
+ },
1566
+ required: ['url'],
1567
+ },
1568
+ // Covers SPA-render fallback: full fetch timeout + playwright render cap.
1569
+ timeoutMs: Math.max(45000, cfg.timeoutMs + 45000),
1570
+ isConcurrencySafe: () => true,
1571
+ output: {
1572
+ schema: {
1573
+ type: 'object',
1574
+ additionalProperties: true,
1575
+ properties: {
1576
+ url: { type: 'string' },
1577
+ count: { type: 'number' },
1578
+ links: { type: 'array', items: { type: 'object' } },
1579
+ error: { type: 'string' },
1580
+ },
1581
+ },
1582
+ render: (_args, value) => [{ type: 'text', text: renderLinks(value) }],
1583
+ },
1584
+ async execute(args, exec) {
1585
+ const url = String((args && args.url) || '').trim()
1586
+ if (!/^https?:\/\//i.test(url)) return { error: 'Only http/https URLs are supported' }
1587
+ const limit = Math.max(1, Math.min(50, Number(args.limit) || cfg.maxLinks))
1588
+ const viaSeam = await fetchViaWebSeam(ctx, url, exec && exec.signal, cfg)
1589
+ let html, finalUrl
1590
+ if (viaSeam) {
1591
+ html = viaSeam.html
1592
+ finalUrl = viaSeam.finalUrl
1593
+ } else {
1594
+ const page = await fetchPage(url, exec && exec.signal, cfg)
1595
+ if (page.error) return { error: page.error }
1596
+ html = decodeBuffer(page.buffer, page.contentType).text
1597
+ finalUrl = page.finalUrl || url
1598
+ }
1599
+ // Same meta-refresh shell following as read_url: a link hop serves a
1600
+ // stub whose real links live at the target. Fails open, hop-bounded.
1601
+ const followed = await followMetaRefresh(html, finalUrl, exec && exec.signal, cfg)
1602
+ html = followed.html
1603
+ finalUrl = followed.finalUrl
1604
+ let links = extractLinks(html, limit, finalUrl)
1605
+ // SPA fallback: a JS-only page yields no links from its static HTML;
1606
+ // render it and re-extract when the static result looks empty. Any
1607
+ // <script> counts as renderable when there are no links at all — the
1608
+ // static shell may be a JS-redirect (same rationale as read_url).
1609
+ if (links.length < 3 && cfg.spaRender !== false && (looksLikeSpa(html) || (links.length === 0 && /<script[\s>]/i.test(html)))) {
1610
+ const rr = await renderPage(finalUrl || url, exec && exec.signal)
1611
+ // A challenge interstitial's links are the challenge's own scripts —
1612
+ // never an improvement over whatever the static HTML offered.
1613
+ if (rr.html && !looksLikeChallenge(rr.html)) {
1614
+ const rl = extractLinks(rr.html, limit, rr.finalUrl || finalUrl)
1615
+ if (rl.length > links.length) {
1616
+ links = rl
1617
+ finalUrl = rr.finalUrl || finalUrl
1618
+ }
1619
+ }
1620
+ }
1621
+ return { url: finalUrl, count: links.length, links }
1622
+ },
1623
+ }
1624
+ }
1625
+
1626
+ // Run async tasks with a concurrency cap (avoids hammering a site / getting
1627
+ // rate-limited when reading many URLs at once).
1628
+ async function mapLimit(items, limit, fn) {
1629
+ const out = new Array(items.length)
1630
+ let idx = 0
1631
+ const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
1632
+ while (idx < items.length) {
1633
+ const i = idx++
1634
+ out[i] = await fn(items[i], i)
1635
+ }
1636
+ })
1637
+ await Promise.all(workers)
1638
+ return out
1639
+ }
1640
+
1641
+ function renderBatch(value) {
1642
+ if (typeof value === 'string') return value
1643
+ const v = value || {}
1644
+ if (v.error) return `Error: ${v.error}`
1645
+ const lines = [`读取 ${v.succeeded}/${v.total} 页成功${v.failed ? `,${v.failed} 页失败` : ''}`]
1646
+ for (const p of v.pages) {
1647
+ if (p.error) {
1648
+ lines.push('', `[失败] ${p.url} — ${p.error}`)
1649
+ continue
1650
+ }
1651
+ const head = p.title || p.url
1652
+ lines.push('', `--- ${head} (${p.chars} 字符${p.cached ? ' · cached' : ''}) ---`, p.text || '(无可读内容)')
1653
+ if (Array.isArray(p.links) && p.links.length) {
1654
+ const ls = p.links.map((l) => l.title + ' — ' + l.url)
1655
+ lines.push(`links: ${ls.slice(0, 6).join(' | ')}${ls.length > 6 ? ` …共${ls.length}个` : ''}`)
1656
+ }
1657
+ }
1658
+ return lines.join('\n')
1659
+ }
1660
+
1661
+ function readUrlBatchTool(ctx, cfg) {
1662
+ return {
1663
+ name: 'read_url_batch',
1664
+ description:
1665
+ 'Read multiple URLs in parallel: each a compact clean block (same extraction + cache as read_url). ' +
1666
+ 'Failures isolated and tagged. Max 10 URLs, capped at maxChars per page.',
1667
+ parameters: {
1668
+ type: 'object',
1669
+ additionalProperties: false,
1670
+ properties: {
1671
+ urls: { type: 'array', items: { type: 'string' }, description: 'URLs to read in parallel (1-10, http/s)' },
1672
+ maxChars: { type: 'number', description: 'Chars per page (500-20000; default 3000)' },
1673
+ mode: { type: 'string', enum: ['text', 'markdown'], description: 'text = plain (token-cheap, default); markdown = structured' },
1674
+ includeLinks: { type: 'boolean', description: 'Return page links too (title+url, bounded)' },
1675
+ },
1676
+ required: ['urls'],
1677
+ },
1678
+ // Worst case: ceil(10/4)=3 concurrency waves, each (fetch + SPA render).
1679
+ timeoutMs: Math.max(60000, cfg.timeoutMs * 3 + 135000),
1680
+ // Concurrent batch calls multiply fetch pressure but stay isolated (each
1681
+ // call owns its result array; the session cache is commutative).
1682
+ isConcurrencySafe: () => true,
1683
+ output: {
1684
+ schema: {
1685
+ type: 'object',
1686
+ additionalProperties: true,
1687
+ properties: {
1688
+ total: { type: 'number' },
1689
+ succeeded: { type: 'number' },
1690
+ failed: { type: 'number' },
1691
+ pages: { type: 'array', items: { type: 'object' } },
1692
+ error: { type: 'string' },
1693
+ },
1694
+ },
1695
+ render: (_args, value) => [{ type: 'text', text: renderBatch(value) }],
1696
+ },
1697
+ async execute(args, exec) {
1698
+ const a = args || {}
1699
+ const urls = (Array.isArray(a.urls) ? a.urls : []).map((u) => String(u).trim()).filter(Boolean)
1700
+ if (!urls.length) return { error: 'urls array is required (1-10 http(s) URLs)' }
1701
+ const list = urls.slice(0, 10)
1702
+ const perMax = Math.max(500, Math.min(20000, Number(a.maxChars) || 3000))
1703
+ const mode = a.mode === 'markdown' ? 'markdown' : 'text'
1704
+ const signal = exec && exec.signal
1705
+ const pages = await mapLimit(list, 4, (u) =>
1706
+ readUrl({ url: u, maxChars: perMax, mode, includeLinks: a.includeLinks === true }, ctx, signal, cfg),
1707
+ )
1708
+ const ok = pages.filter((p) => !p.error)
1709
+ return {
1710
+ total: list.length,
1711
+ succeeded: ok.length,
1712
+ failed: list.length - ok.length,
1713
+ pages: pages.map((p, i) => {
1714
+ if (p.error) return { url: list[i], error: p.error }
1715
+ const out = {
1716
+ url: p.url,
1717
+ title: p.title || '',
1718
+ chars: p.charsReturned,
1719
+ cached: p.cached === true,
1720
+ text: p.text,
1721
+ }
1722
+ if (Array.isArray(p.links) && p.links.length) out.links = p.links
1723
+ return out
1724
+ }),
1725
+ }
1726
+ },
1727
+ }
1728
+ }
1729
+
1730
+ function normalizeUrl(url) {
1731
+ try {
1732
+ const u = new URL(url)
1733
+ u.hash = ''
1734
+ return u.href
1735
+ } catch {
1736
+ return null
1737
+ }
1738
+ }
1739
+
1740
+ // b is a bare hostname (hostOf() output) — must NOT be passed to new URL()
1741
+ // as a full URL (that throws and would make every link look external).
1742
+ function sameHost(a, b) {
1743
+ try {
1744
+ return new URL(a).hostname === b
1745
+ } catch {
1746
+ return false
1747
+ }
1748
+ }
1749
+
1750
+ // URLs not worth crawling: static assets, login/auth paths, feeds, sitemaps.
1751
+ const NOISE_EXT = /\.(png|jpe?g|gif|svg|webp|ico|bmp|css|js|json|xml|pdf|zip|gz|tar|7z|mp3|m3u8?|mpd|flv|ts|mp4|avi|mov|webm|woff2?|ttf|eot|map)(\?|#|$)/i
1752
+ const NOISE_PATH = /(\/login|\/signin|\/register|\/logout|\/signup|\/api\/|\/admin|\/wp-admin|\/wp-login|\/feed|\/rss|\/sitemap|\/robots\.txt|\/cdn-cgi)/i
1753
+ function isNoiseUrl(url) {
1754
+ return NOISE_EXT.test(url) || NOISE_PATH.test(url)
1755
+ }
1756
+
1757
+ // BFS crawl of a single site. Only same-host http(s) pages are followed;
1758
+ // static assets / auth paths are skipped; visited URLs are deduped; a per-page
1759
+ // failure is recorded and does not abort the crawl. No SPA rendering here —
1760
+ // that is read_url's job; crawling favors speed and breadth.
1761
+ async function crawlSite(entryUrl, cfg, opts, externalSignal) {
1762
+ const { maxPages, maxDepth, includeContent, perMax } = opts
1763
+ const host = hostOf(entryUrl)
1764
+ const visited = new Set()
1765
+ const pages = []
1766
+ const failures = []
1767
+ const queue = [{ url: entryUrl, depth: 0 }]
1768
+ // Stop once the host's cooperative signal fires (tool budget hit): without
1769
+ // this the loop keeps draining the queue on instantly-aborted fetches.
1770
+ while (queue.length && pages.length < maxPages && !(externalSignal && externalSignal.aborted)) {
1771
+ // Never overshoot the page budget within a batch round.
1772
+ const batch = queue.splice(0, Math.min(2, maxPages - pages.length, queue.length))
1773
+ const results = await Promise.all(
1774
+ batch.map(async ({ url, depth }) => {
1775
+ const norm = normalizeUrl(url)
1776
+ if (!norm || visited.has(norm)) return null
1777
+ visited.add(norm)
1778
+ const page = await fetchPage(url, externalSignal, cfg)
1779
+ if (page.error) return { url, depth, error: page.error }
1780
+ const html = decodeBuffer(page.buffer, page.contentType).text
1781
+ const ex = extract(html, 'text')
1782
+ const links = extractLinks(html, 100, page.finalUrl || url)
1783
+ return {
1784
+ url: page.finalUrl || url,
1785
+ depth,
1786
+ title: ex.title || '',
1787
+ chars: ex.text.length,
1788
+ text: includeContent ? smartTruncate(ex.text, perMax).text : undefined,
1789
+ links,
1790
+ }
1791
+ }),
1792
+ )
1793
+ for (const r of results) {
1794
+ if (!r) continue
1795
+ if (r.error) {
1796
+ failures.push({ url: r.url, error: r.error })
1797
+ continue
1798
+ }
1799
+ pages.push(r)
1800
+ if (r.depth + 1 <= maxDepth) {
1801
+ // Queue cap: a huge site (millions of links) would otherwise grow the
1802
+ // queue unboundedly while maxPages bounds only what we *process*.
1803
+ if (queue.length >= maxPages * 20) continue
1804
+ for (const l of r.links) {
1805
+ if (!sameHost(l.url, host) || isNoiseUrl(l.url)) continue
1806
+ const n = normalizeUrl(l.url)
1807
+ if (n && !visited.has(n)) queue.push({ url: l.url, depth: r.depth + 1 })
1808
+ }
1809
+ }
1810
+ }
1811
+ }
1812
+ return { host, pages, failures }
1813
+ }
1814
+
1815
+ function renderSite(value) {
1816
+ if (typeof value === 'string') return value
1817
+ const v = value || {}
1818
+ if (v.error) return `Error: ${v.error}`
1819
+ const lines = [`站点: ${v.host} · 爬取 ${v.succeeded}/${v.total} 页${v.failed ? ` · ${v.failed} 页失败` : ''}`]
1820
+ for (const p of v.pages) {
1821
+ const indent = ' '.repeat(p.depth)
1822
+ const head = p.title || p.url
1823
+ lines.push(`${indent}[${p.depth}] ${head} (${p.chars} 字符) ${p.url}`)
1824
+ if (p.text) lines.push(`${indent} ${p.text.slice(0, 80)}`)
1825
+ }
1826
+ for (const f of v.failures) lines.push(`[失败] ${f.url} — ${f.error}`)
1827
+ return lines.join('\n')
1828
+ }
1829
+
1830
+ function readUrlSiteTool(ctx, cfg) {
1831
+ return {
1832
+ name: 'read_url_site',
1833
+ description:
1834
+ 'Crawl a site BFS from one URL: dedupe same-host pages, return a compact site map ' +
1835
+ '(title + depth + size). Auth/static paths skipped; failures isolated. ' +
1836
+ 'includeContent=true attaches a short summary per page (default off, token-cheap). ' +
1837
+ 'Does not render SPA pages (use read_url for that).',
1838
+ parameters: {
1839
+ type: 'object',
1840
+ additionalProperties: false,
1841
+ properties: {
1842
+ url: { type: 'string', description: 'Entry URL of the site to crawl (http/s)' },
1843
+ maxPages: { type: 'number', description: 'Pages to crawl (2-50; default 15)' },
1844
+ maxDepth: { type: 'number', description: 'Link depth from entry (1-5; default 2)' },
1845
+ includeContent: { type: 'boolean', description: 'Short body summary per page (default off; structure-only is token-cheap)' },
1846
+ maxCharsPerPage: { type: 'number', description: 'Summary chars per page (200-2000; default 500)' },
1847
+ },
1848
+ required: ['url'],
1849
+ },
1850
+ // Worst case: ceil(maxPages/2)=25 waves of paired fetches (no SPA render
1851
+ // here); the cooperative signal stops the loop early once the budget is hit.
1852
+ timeoutMs: Math.max(120000, cfg.timeoutMs * 25),
1853
+ // Crawl state (visited set, queue) is call-local; only the shared session
1854
+ // cache is global and commutative — parallel crawls of different hosts
1855
+ // are safe and useful (one task mapping several sites).
1856
+ isConcurrencySafe: () => true,
1857
+ output: {
1858
+ schema: {
1859
+ type: 'object',
1860
+ additionalProperties: true,
1861
+ properties: {
1862
+ host: { type: 'string' },
1863
+ total: { type: 'number' },
1864
+ succeeded: { type: 'number' },
1865
+ failed: { type: 'number' },
1866
+ pages: { type: 'array', items: { type: 'object' } },
1867
+ failures: { type: 'array', items: { type: 'object' } },
1868
+ error: { type: 'string' },
1869
+ },
1870
+ },
1871
+ render: (_args, value) => [{ type: 'text', text: renderSite(value) }],
1872
+ },
1873
+ async execute(args, exec) {
1874
+ const url = String((args && args.url) || '').trim()
1875
+ if (!/^https?:\/\//i.test(url)) return { error: 'Only http/https URLs are supported' }
1876
+ const maxPages = Math.max(2, Math.min(50, Number(args.maxPages) || 15))
1877
+ const maxDepth = Math.max(1, Math.min(5, Number(args.maxDepth) || 2))
1878
+ const includeContent = args.includeContent === true
1879
+ const perMax = Math.max(200, Math.min(2000, Number(args.maxCharsPerPage) || 500))
1880
+ const { host, pages, failures } = await crawlSite(
1881
+ url, cfg, { maxPages, maxDepth, includeContent, perMax }, exec && exec.signal,
1882
+ )
1883
+ return {
1884
+ host,
1885
+ total: pages.length + failures.length,
1886
+ succeeded: pages.length,
1887
+ failed: failures.length,
1888
+ pages: pages.map((p) => {
1889
+ const o = { url: p.url, depth: p.depth, title: p.title || '', chars: p.chars }
1890
+ if (p.text) o.text = p.text
1891
+ return o
1892
+ }),
1893
+ failures,
1894
+ }
1895
+ },
1896
+ }
1897
+ }
1898
+
1899
+ export function apply(ctx, config) {
1900
+ // Plugin-level config from cordis.patch.yml (config row), merged over defaults.
1901
+ // YAML is free-form: numbers can arrive quoted as strings, which would poison
1902
+ // arithmetic and AbortSignal timeouts downstream — coerce to numbers once
1903
+ // here, clamp to sane ranges, and fall back to defaults on garbage.
1904
+ const cfg = { ...DEFAULTS, ...(config || {}) }
1905
+ const clamp = (v, d, min, max) => {
1906
+ const n = Number(v)
1907
+ return Number.isFinite(n) ? Math.min(max, Math.max(min, n)) : d
1908
+ }
1909
+ cfg.timeoutMs = clamp(cfg.timeoutMs, DEFAULTS.timeoutMs, 500, 120000)
1910
+ cfg.maxBytes = clamp(cfg.maxBytes, DEFAULTS.maxBytes, 65536, 16 * 1024 * 1024)
1911
+ cfg.maxChars = clamp(cfg.maxChars, DEFAULTS.maxChars, 500, 20000)
1912
+ cfg.maxLinks = clamp(cfg.maxLinks, DEFAULTS.maxLinks, 1, 50)
1913
+ cfg.cacheTtlMs = clamp(cfg.cacheTtlMs, DEFAULTS.cacheTtlMs, 0, 3600000)
1914
+ cfg.cacheMax = clamp(cfg.cacheMax, DEFAULTS.cacheMax, 1, 256)
1915
+ cfg.paginateMax = clamp(cfg.paginateMax, DEFAULTS.paginateMax, 1, 10)
1916
+ if (typeof cfg.userAgent !== 'string' || !cfg.userAgent.trim()) cfg.userAgent = DEFAULTS.userAgent
1917
+ if (typeof cfg.spaRender !== 'boolean') cfg.spaRender = DEFAULTS.spaRender
1918
+ if (typeof cfg.paginate !== 'boolean') cfg.paginate = DEFAULTS.paginate
1919
+
1920
+ // Temporal composability: unload must fully revert side effects.
1921
+ ctx.effect(() => () => {
1922
+ cache.clear()
1923
+ failCache.clear()
1924
+ closeBrowser().catch(() => {})
1925
+ })
1926
+
1927
+ console.log('[dsh-read-url] plugin loaded; tools read_url, read_url_batch, read_url_links, read_url_site registered')
1928
+
1929
+ ctx.tools.register({
1930
+ name: 'read_url',
1931
+ description:
1932
+ 'Fetch a page and return its clean main content (auto charset detect). ' +
1933
+ 'text (default) = plain body capped at maxChars; markdown = headings/links/tables. ' +
1934
+ 'Compact block: title, metadata, truncated body; 5-min session cache; ' +
1935
+ 'SPA pages render when playwright installed; login walls are not accessible.',
1936
+ parameters: {
1937
+ type: 'object',
1938
+ additionalProperties: false,
1939
+ properties: {
1940
+ url: { type: 'string', description: 'http(s) URL to read' },
1941
+ maxChars: { type: 'number', description: 'Body text chars to return (500-20000; default from plugin config)' },
1942
+ offset: { type: 'number', description: 'Start at this char offset (continue long pages). Default 0; served from cache.' },
1943
+ mode: { type: 'string', enum: ['text', 'markdown'], description: 'text = plain (token-cheap, default); markdown = structured' },
1944
+ includeLinks: { type: 'boolean', description: 'Return page links too (title+url, bounded)' },
1945
+ },
1946
+ required: ['url'],
1947
+ },
1948
+ // Pipeline budget must cover the real worst case: first page fetch +
1949
+ // SPA render (goto 30s cap + DOM-stability poll 10s) + up to paginateMax
1950
+ // continuation fetches (static, no render). Derived from load-time cfg —
1951
+ // scale with it, never clamp it.
1952
+ timeoutMs: Math.max(45000, cfg.timeoutMs * (cfg.paginate === false ? 1 : cfg.paginateMax + 1) + 45000),
1953
+ // All shared state is commutative-safe under concurrent calls (session
1954
+ // cache Map get/set, decoder cache, one shared browser with isolated
1955
+ // pages per render), so the agent may fan out several read_url calls in
1956
+ // one parallel group — a real win for multi-source reading tasks.
1957
+ isConcurrencySafe: () => true,
1958
+ output: {
1959
+ schema: {
1960
+ type: 'object',
1961
+ additionalProperties: true,
1962
+ properties: {
1963
+ url: { type: 'string' },
1964
+ title: { type: 'string' },
1965
+ siteName: { type: 'string' },
1966
+ lang: { type: 'string' },
1967
+ charset: { type: 'string' },
1968
+ mode: { type: 'string' },
1969
+ truncated: { type: 'boolean' },
1970
+ charsTotal: { type: 'number' },
1971
+ charsReturned: { type: 'number' },
1972
+ text: { type: 'string' },
1973
+ links: { type: 'array', items: { type: 'object' } },
1974
+ cached: { type: 'boolean' },
1975
+ error: { type: 'string' },
1976
+ },
1977
+ },
1978
+ render: (_args, value) => [{ type: 'text', text: renderResult(value) }],
1979
+ },
1980
+ async execute(args, exec) {
1981
+ return readUrl(args, ctx, exec && exec.signal, cfg)
1982
+ },
1983
+ })
1984
+
1985
+ ctx.tools.register(readLinksTool(ctx, cfg))
1986
+ ctx.tools.register(readUrlBatchTool(ctx, cfg))
1987
+ ctx.tools.register(readUrlSiteTool(ctx, cfg))
1988
+ }