dsh-read-url 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/README.md +22 -12
- package/README.zh.md +23 -13
- package/index.js +1988 -1663
- package/package.json +2 -2
- package/proxy-fallback.js +3 -4
package/index.js
CHANGED
|
@@ -1,1663 +1,1988 @@
|
|
|
1
|
-
// dsh-read-url — DeepSeek Harness URL reader plugin
|
|
2
|
-
// Zero external dependencies (Node 20+ built-ins only).
|
|
3
|
-
// Aligned with DSH architecture:
|
|
4
|
-
// - all network access goes through the ctx.web capability seam (official
|
|
5
|
-
// docs/capability-seams.md), falling back to global fetch when absent
|
|
6
|
-
// - cache is registered under ctx.effect, so unload fully reverts it
|
|
7
|
-
// (temporal composability, docs/cordis-primer.md)
|
|
8
|
-
// - cooperative tool-call timeout via ToolDefinition.timeoutMs + exec.signal
|
|
9
|
-
// Focus: token economy + smarter extraction for DSH agents.
|
|
10
|
-
// - charset auto-detect (BOM UTF-16 / GBK/GB2312/UTF-8/Big5/Shift-JIS)
|
|
11
|
-
// - content dispatch: HTML / JSON APIs / RSS-Atom feeds natively
|
|
12
|
-
// - clean main-content extraction (heuristic; optional @mozilla/readability upgrade path)
|
|
13
|
-
// + text-density fallback on the body path
|
|
14
|
-
// - auto-joined pagination, page metadata (published/author)
|
|
15
|
-
// - HTML -> Markdown / plain text, paragraph-aligned smart truncation
|
|
16
|
-
// - session-level cache, compact text render (lowest token cost)
|
|
17
|
-
import { TextDecoder } from 'node:util'
|
|
18
|
-
import { looksLikeSpa, renderPage, closeBrowser, looksLikeChallenge } from './spa.js'
|
|
19
|
-
import { detectProxy, fetchViaCurlProxy, looksBinary } from './proxy-fallback.js'
|
|
20
|
-
// Re-export for tests / programmatic (PTC) cleanup.
|
|
21
|
-
export { closeBrowser, renderPage, looksLikeSpa, looksLikeChallenge } from './spa.js'
|
|
22
|
-
|
|
23
|
-
export const name = 'dsh-read-url'
|
|
24
|
-
export const inject = ['tools']
|
|
25
|
-
|
|
26
|
-
// Plugin-level configuration (overridable via cordis.patch.yml `config:` row).
|
|
27
|
-
const DEFAULTS = {
|
|
28
|
-
timeoutMs: 15000,
|
|
29
|
-
maxBytes: 3 * 1024 * 1024,
|
|
30
|
-
maxChars: 6000,
|
|
31
|
-
maxLinks: 20,
|
|
32
|
-
cacheTtlMs: 300000,
|
|
33
|
-
cacheMax: 32,
|
|
34
|
-
spaRender: true,
|
|
35
|
-
paginate: true,
|
|
36
|
-
paginateMax: 3,
|
|
37
|
-
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36',
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
// Session-level cache: url -> { time, full }. Failures live in a separate
|
|
41
|
-
// small map so
|
|
42
|
-
|
|
43
|
-
const
|
|
44
|
-
const
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
//
|
|
71
|
-
|
|
72
|
-
if (buffer[0] ===
|
|
73
|
-
if (buffer[0] ===
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
const
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
let
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
//
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
//
|
|
184
|
-
//
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
settled
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
const
|
|
216
|
-
const
|
|
217
|
-
|
|
218
|
-
const
|
|
219
|
-
|
|
220
|
-
proxyCtrl
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
const
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
//
|
|
255
|
-
// provider
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
const
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
//
|
|
277
|
-
//
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
//
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
const
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
if (
|
|
365
|
-
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
//
|
|
378
|
-
//
|
|
379
|
-
//
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
const
|
|
395
|
-
|
|
396
|
-
let m
|
|
397
|
-
while ((m =
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
//
|
|
417
|
-
//
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
return
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
export function
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
if (
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
//
|
|
474
|
-
//
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
.
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
//
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
const
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
if (
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
//
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
}),
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
//
|
|
539
|
-
//
|
|
540
|
-
//
|
|
541
|
-
//
|
|
542
|
-
//
|
|
543
|
-
//
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
const
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
}
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
}
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
}
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
//
|
|
620
|
-
//
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
if (
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
const
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
const
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
}
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
}
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
if (!
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
}
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
}
|
|
819
|
-
}
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
}
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
}
|
|
862
|
-
|
|
863
|
-
//
|
|
864
|
-
//
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
const
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
}
|
|
879
|
-
const
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
let
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
//
|
|
901
|
-
//
|
|
902
|
-
|
|
903
|
-
const
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
const
|
|
916
|
-
|
|
917
|
-
title:
|
|
918
|
-
siteName
|
|
919
|
-
lang:
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
if (
|
|
935
|
-
|
|
936
|
-
}
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
}
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
const
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
}
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
const
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
}
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
const
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
//
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
}
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
if (
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
}
|
|
1214
|
-
}
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
}
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
}
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
'
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
//
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
//
|
|
1433
|
-
//
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
const
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
}
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
if (
|
|
1494
|
-
const
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
}
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
)
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
}
|
|
1621
|
-
|
|
1622
|
-
},
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1
|
+
// dsh-read-url — DeepSeek Harness URL reader plugin
|
|
2
|
+
// Zero external dependencies (Node 20+ built-ins only).
|
|
3
|
+
// Aligned with DSH architecture:
|
|
4
|
+
// - all network access goes through the ctx.web capability seam (official
|
|
5
|
+
// docs/capability-seams.md), falling back to global fetch when absent
|
|
6
|
+
// - cache is registered under ctx.effect, so unload fully reverts it
|
|
7
|
+
// (temporal composability, docs/cordis-primer.md)
|
|
8
|
+
// - cooperative tool-call timeout via ToolDefinition.timeoutMs + exec.signal
|
|
9
|
+
// Focus: token economy + smarter extraction for DSH agents.
|
|
10
|
+
// - charset auto-detect (BOM UTF-16 / GBK/GB2312/UTF-8/Big5/Shift-JIS)
|
|
11
|
+
// - content dispatch: HTML / JSON APIs / RSS-Atom feeds natively
|
|
12
|
+
// - clean main-content extraction (heuristic; optional @mozilla/readability upgrade path)
|
|
13
|
+
// + text-density fallback on the body path
|
|
14
|
+
// - auto-joined pagination, page metadata (published/author)
|
|
15
|
+
// - HTML -> Markdown / plain text, paragraph-aligned smart truncation
|
|
16
|
+
// - session-level cache, compact text render (lowest token cost)
|
|
17
|
+
import { TextDecoder } from 'node:util'
|
|
18
|
+
import { looksLikeSpa, renderPage, closeBrowser, looksLikeChallenge } from './spa.js'
|
|
19
|
+
import { detectProxy, fetchViaCurlProxy, looksBinary } from './proxy-fallback.js'
|
|
20
|
+
// Re-export for tests / programmatic (PTC) cleanup.
|
|
21
|
+
export { closeBrowser, renderPage, looksLikeSpa, looksLikeChallenge } from './spa.js'
|
|
22
|
+
|
|
23
|
+
export const name = 'dsh-read-url'
|
|
24
|
+
export const inject = ['tools']
|
|
25
|
+
|
|
26
|
+
// Plugin-level configuration (overridable via cordis.patch.yml `config:` row).
|
|
27
|
+
const DEFAULTS = {
|
|
28
|
+
timeoutMs: 15000,
|
|
29
|
+
maxBytes: 3 * 1024 * 1024,
|
|
30
|
+
maxChars: 6000,
|
|
31
|
+
maxLinks: 20,
|
|
32
|
+
cacheTtlMs: 300000,
|
|
33
|
+
cacheMax: 32,
|
|
34
|
+
spaRender: true,
|
|
35
|
+
paginate: true,
|
|
36
|
+
paginateMax: 3,
|
|
37
|
+
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36',
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Session-level cache: url -> { time, full }. Failures live in a separate
|
|
41
|
+
// small map so short-lived error entries never evict hot success pages.
|
|
42
|
+
const cache = new Map()
|
|
43
|
+
const failCache = new Map()
|
|
44
|
+
const FAIL_CACHE_MAX = 8
|
|
45
|
+
function cacheFail(key, error) {
|
|
46
|
+
if (failCache.size >= FAIL_CACHE_MAX) failCache.delete(failCache.keys().next().value)
|
|
47
|
+
failCache.set(key, { time: Date.now(), error })
|
|
48
|
+
}
|
|
49
|
+
function cacheStore(key, full, cacheMax) {
|
|
50
|
+
if (cache.size >= cacheMax) cache.delete(cache.keys().next().value)
|
|
51
|
+
cache.set(key, { time: Date.now(), full })
|
|
52
|
+
}
|
|
53
|
+
const decoders = new Map()
|
|
54
|
+
|
|
55
|
+
function getDecoder(enc) {
|
|
56
|
+
let d = decoders.get(enc)
|
|
57
|
+
if (!d) {
|
|
58
|
+
try {
|
|
59
|
+
d = new TextDecoder(enc, { fatal: false })
|
|
60
|
+
} catch {
|
|
61
|
+
d = new TextDecoder('utf-8')
|
|
62
|
+
}
|
|
63
|
+
decoders.set(enc, d)
|
|
64
|
+
}
|
|
65
|
+
return d
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function sniffCharset(buffer, contentType) {
|
|
69
|
+
// BOM first — byte-level evidence beats any declared charset (a UTF-16 body
|
|
70
|
+
// with a latin1-read <meta> probe garbles the meta match anyway).
|
|
71
|
+
if (buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) return 'utf-8'
|
|
72
|
+
if (buffer[0] === 0xff && buffer[1] === 0xfe) return 'utf-16le'
|
|
73
|
+
if (buffer[0] === 0xfe && buffer[1] === 0xff) return 'utf-16be'
|
|
74
|
+
const m = /charset=["']?([\w-]+)/i.exec(contentType || '')
|
|
75
|
+
if (m) return m[1]
|
|
76
|
+
const head = buffer.subarray(0, 2048).toString('latin1').toLowerCase()
|
|
77
|
+
const meta = /<meta[^>]{0,1000}charset=["']?([\w-]+)/i.exec(head)
|
|
78
|
+
if (meta) return meta[1]
|
|
79
|
+
// Legacy form: <meta http-equiv="Content-Type" content="text/html; charset=gb2312">
|
|
80
|
+
// (no charset= attribute anywhere) — common on old GBK pages with no header charset.
|
|
81
|
+
const metaEquiv = /<meta[^>]{0,1000}(?:http-equiv)=["']content-type["'][^>]{0,1000}content=["'][^"']*charset=([\w-]+)/i.exec(head)
|
|
82
|
+
if (metaEquiv) return metaEquiv[1]
|
|
83
|
+
return null
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function decodeBuffer(buffer, contentType) {
|
|
87
|
+
let enc = sniffCharset(buffer, contentType)
|
|
88
|
+
if (!enc) enc = 'utf-8'
|
|
89
|
+
enc = enc.toLowerCase().replace('gb2312', 'gbk').replace('gb_2312', 'gbk')
|
|
90
|
+
let text
|
|
91
|
+
try {
|
|
92
|
+
text = getDecoder(enc).decode(buffer)
|
|
93
|
+
} catch {
|
|
94
|
+
text = new TextDecoder('utf-8').decode(buffer)
|
|
95
|
+
}
|
|
96
|
+
if (enc !== 'utf-8' && enc !== 'utf8' && /\uFFFD/.test(text.slice(0, 2000))) {
|
|
97
|
+
const fixed = new TextDecoder('utf-8').decode(buffer)
|
|
98
|
+
if (!/\uFFFD/.test(fixed.slice(0, 2000))) return { text: fixed, charset: 'utf-8' }
|
|
99
|
+
}
|
|
100
|
+
return { text, charset: enc }
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Content types this plugin can consume. Beyond HTML: JSON (data APIs) and
|
|
104
|
+
// XML (RSS/Atom feeds) are read natively — see readUrl's type dispatch.
|
|
105
|
+
// Note "application/rss+xml": the separator before xml is '+', not '/'.
|
|
106
|
+
const FETCHABLE_CT = /text\/html|application\/xhtml|text\/plain|\/json|[+/]xml/i
|
|
107
|
+
|
|
108
|
+
// Retry-After header (seconds form) → ms, capped; undefined when absent/invalid.
|
|
109
|
+
function retryAfterMs(res) {
|
|
110
|
+
const v = res.headers.get('retry-after')
|
|
111
|
+
if (!v) return undefined
|
|
112
|
+
const sec = Number(v)
|
|
113
|
+
if (!Number.isFinite(sec) || sec < 0) return undefined
|
|
114
|
+
return Math.min(sec * 1000, 5000)
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Single fetch attempt. Never throws; maps abort/timeout/network to { error }.
|
|
118
|
+
async function directFetchOnce(url, signal, cfg) {
|
|
119
|
+
try {
|
|
120
|
+
const res = await fetch(url, {
|
|
121
|
+
redirect: 'follow',
|
|
122
|
+
signal,
|
|
123
|
+
headers: { 'user-agent': cfg.userAgent, accept: 'text/html,application/xhtml+xml,*/*;q=0.8' },
|
|
124
|
+
})
|
|
125
|
+
// Early-return paths must cancel the body stream, or the connection is
|
|
126
|
+
// held open undrained and cannot return to the keep-alive pool.
|
|
127
|
+
const discardBody = () => { if (res.body) res.body.cancel().catch(() => {}) }
|
|
128
|
+
if (!res.ok) {
|
|
129
|
+
// Rate-limit / temporary-overload: honor Retry-After with one retry
|
|
130
|
+
// (capped at 5s so the cooperative timeout still bounds the call).
|
|
131
|
+
if (res.status === 429 || res.status === 503) {
|
|
132
|
+
const ra = retryAfterMs(res)
|
|
133
|
+
discardBody()
|
|
134
|
+
if (ra !== undefined) return { retryAfterMs: ra }
|
|
135
|
+
}
|
|
136
|
+
discardBody()
|
|
137
|
+
return { error: `HTTP ${res.status} ${res.statusText}` }
|
|
138
|
+
}
|
|
139
|
+
const contentType = res.headers.get('content-type') || ''
|
|
140
|
+
if (contentType && !FETCHABLE_CT.test(contentType)) {
|
|
141
|
+
discardBody()
|
|
142
|
+
return { error: `Unsupported content-type: ${contentType.split(';')[0]}` }
|
|
143
|
+
}
|
|
144
|
+
if (!res.body) return { buffer: Buffer.alloc(0), contentType, finalUrl: res.url }
|
|
145
|
+
const chunks = []
|
|
146
|
+
let size = 0
|
|
147
|
+
for await (const chunk of res.body) {
|
|
148
|
+
size += chunk.length
|
|
149
|
+
if (size > cfg.maxBytes) { discardBody(); return { error: `Page exceeds ${cfg.maxBytes} bytes` } }
|
|
150
|
+
chunks.push(chunk)
|
|
151
|
+
}
|
|
152
|
+
const buffer = Buffer.concat(chunks)
|
|
153
|
+
// Headerless responses: gate binary bodies out of the HTML pipeline.
|
|
154
|
+
if (!contentType && looksBinary(buffer)) {
|
|
155
|
+
return { error: 'Unsupported content-type (no header, binary body)' }
|
|
156
|
+
}
|
|
157
|
+
return { buffer, contentType, finalUrl: res.url }
|
|
158
|
+
} catch (e) {
|
|
159
|
+
if (e.name === 'AbortError' || e.name === 'TimeoutError') {
|
|
160
|
+
return { error: `Timeout after ${cfg.timeoutMs}ms or cancelled` }
|
|
161
|
+
}
|
|
162
|
+
// undici wraps the real reason in e.cause (ENOTFOUND / ECONNREFUSED /
|
|
163
|
+
// connect-timeout / TLS) — surface it so the model can tell "bad domain"
|
|
164
|
+
// from "blocked network" and act accordingly instead of retrying blindly
|
|
165
|
+
const cause = e.cause && e.cause.message ? String(e.cause.message).slice(0, 90) : ''
|
|
166
|
+
return { error: cause || (e.message || '').slice(0, 120) }
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// Direct-connect fetch with a single Retry-After-aware retry on 429/503.
|
|
171
|
+
async function directFetch(url, signal, cfg) {
|
|
172
|
+
const first = await directFetchOnce(url, signal, cfg)
|
|
173
|
+
if (first && first.retryAfterMs !== undefined && !(signal && signal.aborted)) {
|
|
174
|
+
await new Promise((resolve) => {
|
|
175
|
+
const t = setTimeout(resolve, first.retryAfterMs)
|
|
176
|
+
if (signal) signal.addEventListener('abort', () => { clearTimeout(t); resolve() }, { once: true })
|
|
177
|
+
})
|
|
178
|
+
return directFetchOnce(url, signal, cfg)
|
|
179
|
+
}
|
|
180
|
+
return first
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Settle-all race that resolves with the FIRST SUCCESSFUL result (a value
|
|
184
|
+
// with a buffer and no error). If every attempt fails it resolves with
|
|
185
|
+
// { success:false, failures:[...] } once the slowest one has settled —
|
|
186
|
+
// the proxy channel (curl --max-time ~20s) is the natural ceiling, and the
|
|
187
|
+
// direct fetch aborts on the same total budget.
|
|
188
|
+
export function raceFirstSuccess(promises) {
|
|
189
|
+
return new Promise((resolve) => {
|
|
190
|
+
if (promises.length === 0) {
|
|
191
|
+
resolve({ success: false, failures: [] })
|
|
192
|
+
return
|
|
193
|
+
}
|
|
194
|
+
const failures = []
|
|
195
|
+
let settled = 0
|
|
196
|
+
promises.forEach((p, i) => {
|
|
197
|
+
p.then((v) => {
|
|
198
|
+
if (v && !v.error && v.buffer) {
|
|
199
|
+
resolve({ success: true, value: v })
|
|
200
|
+
return
|
|
201
|
+
}
|
|
202
|
+
failures[i] = v
|
|
203
|
+
settled += 1
|
|
204
|
+
if (settled === promises.length) resolve({ success: false, failures })
|
|
205
|
+
}).catch((e) => {
|
|
206
|
+
failures[i] = { error: String((e && e.message) || e) }
|
|
207
|
+
settled += 1
|
|
208
|
+
if (settled === promises.length) resolve({ success: false, failures })
|
|
209
|
+
})
|
|
210
|
+
})
|
|
211
|
+
})
|
|
212
|
+
}
|
|
213
|
+
async function raceFetch(url, externalSignal, cfg, proxy) {
|
|
214
|
+
const directCtrl = new AbortController()
|
|
215
|
+
const proxyCtrl = new AbortController()
|
|
216
|
+
const withExt = (ctrl) =>
|
|
217
|
+
externalSignal ? AbortSignal.any([externalSignal, ctrl.signal]) : ctrl.signal
|
|
218
|
+
const directSignal = AbortSignal.any([withExt(directCtrl), AbortSignal.timeout(cfg.timeoutMs)])
|
|
219
|
+
const direct = directFetch(url, directSignal, cfg)
|
|
220
|
+
const proxied = fetchViaCurlProxy(url, cfg, withExt(proxyCtrl), proxy)
|
|
221
|
+
const winner = await raceFirstSuccess([direct, proxied])
|
|
222
|
+
directCtrl.abort()
|
|
223
|
+
proxyCtrl.abort()
|
|
224
|
+
if (!winner.success) {
|
|
225
|
+
// Each failure is tagged with its channel so callers read errors by
|
|
226
|
+
// label ('direct' | 'proxy') rather than by array position.
|
|
227
|
+
const labels = ['direct', 'proxy']
|
|
228
|
+
winner.failures = winner.failures.map((f, i) => ({
|
|
229
|
+
label: labels[i],
|
|
230
|
+
error: f && f.error ? String(f.error) : 'unknown error',
|
|
231
|
+
}))
|
|
232
|
+
}
|
|
233
|
+
return winner
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
async function fetchPage(url, externalSignal, cfg) {
|
|
237
|
+
const proxy = await detectProxy()
|
|
238
|
+
if (proxy) {
|
|
239
|
+
const winner = await raceFetch(url, externalSignal, cfg, proxy)
|
|
240
|
+
if (winner.success) return winner.value
|
|
241
|
+
const dErr = winner.failures.find((f) => f.label === 'direct')
|
|
242
|
+
const pErr = winner.failures.find((f) => f.label === 'proxy')
|
|
243
|
+
const px = pErr ? `,代理返回 ${pErr.error}` : ',代理未连通'
|
|
244
|
+
return { error: `Fetch failed: ${dErr ? dErr.error : 'unknown error'}(直连失败;已尝试代理 ${proxy}${px})` }
|
|
245
|
+
}
|
|
246
|
+
// No proxy configured: plain direct connect (the original behaviour).
|
|
247
|
+
const timeoutSignal = AbortSignal.timeout(cfg.timeoutMs)
|
|
248
|
+
const signal = externalSignal ? AbortSignal.any([externalSignal, timeoutSignal]) : timeoutSignal
|
|
249
|
+
const r = await directFetch(url, signal, cfg)
|
|
250
|
+
if (r.error) return { error: `Fetch failed: ${r.error}(直连失败,未检测到代理配置)` }
|
|
251
|
+
return r
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// Prefer the ctx.web capability seam (official docs/capability-seams.md):
|
|
255
|
+
// the web provider already decodes the document, so we skip our charset
|
|
256
|
+
// layer and only re-extract. Falls back to global fetch when absent.
|
|
257
|
+
// Seam calls get the same cooperative timeout as fetchPage (a hanging
|
|
258
|
+
// provider should not block the tool call forever).
|
|
259
|
+
async function fetchViaWebSeam(ctx, url, externalSignal, cfg) {
|
|
260
|
+
try {
|
|
261
|
+
// ctx.get for a non-injected service throws on strict cordis hosts (seen
|
|
262
|
+
// with 'settings') — treat that as "seam absent" and fall through.
|
|
263
|
+
const web = ctx && typeof ctx.get === 'function' ? ctx.get('web') : undefined
|
|
264
|
+
if (!web || typeof web.fetch !== 'function') return null
|
|
265
|
+
const timeoutSignal = AbortSignal.timeout(cfg.timeoutMs)
|
|
266
|
+
const signal = externalSignal ? AbortSignal.any([externalSignal, timeoutSignal]) : timeoutSignal
|
|
267
|
+
const res = await web.fetch(url, { signal })
|
|
268
|
+
if (!res || typeof res.content !== 'string') return null
|
|
269
|
+
const finalUrl = res.url || res.finalUrl || url
|
|
270
|
+
return { html: res.content, finalUrl }
|
|
271
|
+
} catch {
|
|
272
|
+
return null
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// Decode common HTML entities in already-stripped text. Runs AFTER tag
|
|
277
|
+
// stripping so escaped tags stay visible to the stripper and only remaining
|
|
278
|
+
// text is decoded. Covers the numeric/hex forms plus the named entities that
|
|
279
|
+
// actually show up in CJK web text (spaces, dashes, quotes, symbols) —
|
|
280
|
+
// undecoded leftovers would both waste tokens and read as garbage.
|
|
281
|
+
const ENTITIES = {
|
|
282
|
+
nbsp: ' ', ensp: ' ', emsp: ' ', thinsp: ' ', zwnj: '', zwj: '',
|
|
283
|
+
amp: '&', lt: '<', gt: '>', quot: '"', apos: "'",
|
|
284
|
+
ndash: '–', mdash: '—', hellip: '…', middot: '·', bull: '•',
|
|
285
|
+
lsquo: '\u2018', rsquo: '\u2019', ldquo: '\u201C', rdquo: '\u201D',
|
|
286
|
+
laquo: '«', raquo: '»', copy: '©', reg: '®', trade: '™',
|
|
287
|
+
deg: '°', plusmn: '±', times: '×', divide: '÷', micro: 'µ',
|
|
288
|
+
sect: '§', para: '¶', dagger: '†', prime: '′', Prime: '″',
|
|
289
|
+
permil: '‰', euro: '€', pound: '£', yen: '¥', cent: '¢',
|
|
290
|
+
sup2: '²', sup3: '³', frac12: '½', frac14: '¼', frac34: '¾',
|
|
291
|
+
}
|
|
292
|
+
// � /  /  decode to control characters that pollute the model
|
|
293
|
+
// context (NUL, DEL, C1 controls). Map non-whitespace controls to a space;
|
|
294
|
+
// keep \t\n\r — pages legitimately use for line breaks, and flattening
|
|
295
|
+
// those to spaces would destroy paragraph structure in markdown mode. Lone
|
|
296
|
+
// surrogates (U+D800–DFFF) decode to replacement chars instead of leaking a
|
|
297
|
+
// broken code unit into the rendered text.
|
|
298
|
+
function fromSafeCodePoint(c) {
|
|
299
|
+
if (c < 0x20) return (c === 9 || c === 10 || c === 13) ? String.fromCodePoint(c) : ' '
|
|
300
|
+
if (c >= 0x7f && c <= 0x9f) return ' '
|
|
301
|
+
if (c >= 0xd800 && c <= 0xdfff) return '\uFFFD'
|
|
302
|
+
return String.fromCodePoint(c)
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
export function decodeTextEntities(text) {
|
|
306
|
+
if (!text.includes('&')) return text
|
|
307
|
+
return text.replace(/&(?:#(\d+)|#x([0-9a-fA-F]+)|([a-z][a-z0-9]{1,7}));?/gi, (m, dec, hex, name) => {
|
|
308
|
+
if (dec !== undefined) {
|
|
309
|
+
const c = Number(dec)
|
|
310
|
+
return c > 0x10ffff ? m : fromSafeCodePoint(c)
|
|
311
|
+
}
|
|
312
|
+
if (hex !== undefined) {
|
|
313
|
+
const c = Number.parseInt(hex, 16)
|
|
314
|
+
return c > 0x10ffff ? m : fromSafeCodePoint(c)
|
|
315
|
+
}
|
|
316
|
+
const n = (name || '').toLowerCase()
|
|
317
|
+
return Object.prototype.hasOwnProperty.call(ENTITIES, n) ? ENTITIES[n] : m
|
|
318
|
+
})
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// A '<' with no '>' after it can never start a real tag, but tag-scanning
|
|
322
|
+
// regexes still probe the rest of the string at every such position, degrading
|
|
323
|
+
// to quadratic time on pathological input. Neutralize stray '<' in the no-'>'
|
|
324
|
+
// tail: text content is preserved, impossible tag openings disappear, and
|
|
325
|
+
// scanning stays linear. Short tails ("a < b" in prose) are left untouched.
|
|
326
|
+
function defuseLt(html) {
|
|
327
|
+
const from = html.lastIndexOf('>') + 1
|
|
328
|
+
if (html.length - from < 512 || !html.includes('<', from)) return html
|
|
329
|
+
return html.slice(0, from) + html.slice(from).replace(/</g, ' ')
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
function textOnly(html) {
|
|
333
|
+
return decodeTextEntities(defuseLt(html)
|
|
334
|
+
.replace(/<!--[\s\S]*?-->/g, ' ')
|
|
335
|
+
.replace(/<script[\s>][\s\S]*?<\/script>/gi, ' ')
|
|
336
|
+
.replace(/<style[\s>][\s\S]*?<\/style>/gi, ' ')
|
|
337
|
+
.replace(/<textarea[\s>][\s\S]*?<\/textarea\s*>/gi, ' ')
|
|
338
|
+
.replace(/<noscript[\s>][\s\S]*?<\/noscript>/gi, ' ')
|
|
339
|
+
.replace(/<[^>]{1,1000}>/g, ' ')
|
|
340
|
+
.replace(/[ \t\r\n]+/g, ' ')
|
|
341
|
+
.trim())
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// Paragraph-aware variant of textOnly for the body extraction path: block
|
|
345
|
+
// boundaries become paragraph breaks so offset continuation slices at real
|
|
346
|
+
// paragraph seams (smartTruncate aligns on /\n\n+/) instead of mid-sentence,
|
|
347
|
+
// and long reference pages stay readable instead of one flattened line.
|
|
348
|
+
// <br> keeps single newlines; every other tag runs are stripped as spaces.
|
|
349
|
+
function textLines(html) {
|
|
350
|
+
const cleaned = decodeTextEntities(defuseLt(html)
|
|
351
|
+
.replace(/<!--[\s\S]*?-->/g, ' ')
|
|
352
|
+
.replace(/<script[\s>][\s\S]*?<\/script>/gi, ' ')
|
|
353
|
+
.replace(/<style[\s>][\s\S]*?<\/style>/gi, ' ')
|
|
354
|
+
.replace(/<textarea[\s>][\s\S]*?<\/textarea\s*>/gi, ' ')
|
|
355
|
+
.replace(/<noscript[\s>][\s\S]*?<\/noscript>/gi, ' ')
|
|
356
|
+
.replace(/<br\b[^>]{0,1000}\/?>/gi, '\n')
|
|
357
|
+
.replace(/<\/?(?:p|div|section|article|main|h[1-6]|li|ul|ol|table|tbody|thead|tfoot|tr|blockquote|pre|figcaption|dt|dd|dl|figure|fieldset|option)\b[^>]{0,1000}>/gi, '\n')
|
|
358
|
+
.replace(/<[^>]{1,1000}>/g, ' '))
|
|
359
|
+
const lines = []
|
|
360
|
+
for (const rawLine of cleaned.split('\n')) {
|
|
361
|
+
let line = rawLine.replace(/[ \t\r\f\v]+/g, ' ').trim()
|
|
362
|
+
// A heading becomes its own short paragraph in this layout; a trailing
|
|
363
|
+
// permalink glyph on one is anchor decoration, not content.
|
|
364
|
+
if (line.length <= 120) line = line.replace(/\s*[¶§]$/, '')
|
|
365
|
+
if (line) lines.push(line)
|
|
366
|
+
}
|
|
367
|
+
return lines.join('\n\n')
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// ---- JSON / RSS / Atom content dispatch ----
|
|
371
|
+
// A URL is not always an HTML page: data APIs answer JSON and content sources
|
|
372
|
+
// answer feeds. Both are model-readable when compacted, so read_url handles
|
|
373
|
+
// them natively instead of erroring with "Unsupported content-type".
|
|
374
|
+
|
|
375
|
+
function xmlText(s) {
|
|
376
|
+
let t = String(s || '').replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
|
|
377
|
+
// Feeds double-escape HTML in descriptions (<a href=…>查看全文</a>):
|
|
378
|
+
// one strip-then-decode pass leaves literal tags in the rendered text.
|
|
379
|
+
// Iterate strip → decode until stable.
|
|
380
|
+
for (let i = 0; i < 3; i++) {
|
|
381
|
+
const prev = t
|
|
382
|
+
t = decodeTextEntities(t.replace(/<[^>]{1,1000}>/g, ' '))
|
|
383
|
+
if (t === prev) break
|
|
384
|
+
}
|
|
385
|
+
return t.replace(/\s+/g, ' ').trim()
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
// RSS 2.0 <item> / Atom <entry> → compact list "title — url\n summary".
|
|
389
|
+
function parseFeed(xml, limit) {
|
|
390
|
+
xml = defuseLt(xml) // keeps the item scan linear
|
|
391
|
+
const isAtom = /<feed[\s>]/i.test(xml)
|
|
392
|
+
const itemRe = isAtom ? /<entry[\s>][\s\S]*?<\/entry\s*>/gi : /<item[\s>][\s\S]*?<\/item\s*>/gi
|
|
393
|
+
// Without a single closing tag nothing can pair, so the scan is skipped.
|
|
394
|
+
const closable = isAtom ? /<\/entry\s*>/i.test(xml) : /<\/item\s*>/i.test(xml)
|
|
395
|
+
const items = []
|
|
396
|
+
let m
|
|
397
|
+
while (closable && (m = itemRe.exec(xml)) && items.length < limit) {
|
|
398
|
+
const it = m[0]
|
|
399
|
+
const title = xmlText(/<title[^>]*>([\s\S]*?)<\/title>/i.exec(it)?.[1] || '')
|
|
400
|
+
const linkTag = /<link[^>]*>([\s\S]*?)<\/link>/i.exec(it)
|
|
401
|
+
const linkHref = /<link[^>]+href=["']([^"']+)["']/i.exec(it)
|
|
402
|
+
const link = (linkTag && xmlText(linkTag[1])) || (linkHref && linkHref[1]) || ''
|
|
403
|
+
const desc = xmlText(/<(?:description|summary|content)[^>]*>([\s\S]*?)<\/(?:description|summary|content)>/i.exec(it)?.[1] || '')
|
|
404
|
+
if (!title && !link && !desc) continue
|
|
405
|
+
items.push({ title: title.slice(0, 120), url: link, summary: desc.slice(0, 200) })
|
|
406
|
+
}
|
|
407
|
+
const feedTitle = xmlText(/<title[^>]*>([\s\S]*?)<\/title>/i.exec(xml)?.[1] || '')
|
|
408
|
+
const lines = []
|
|
409
|
+
for (const it of items) {
|
|
410
|
+
lines.push(`- ${it.title}${it.url ? ` — ${it.url}` : ''}`)
|
|
411
|
+
if (it.summary) lines.push(` ${it.summary}`)
|
|
412
|
+
}
|
|
413
|
+
return { title: feedTitle, count: items.length, items, text: lines.join('\n') }
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
// ---- Text-density fallback (body path only) ----
|
|
417
|
+
// Pages without <article>/<main> degrade to the whole <body>: semantic noise
|
|
418
|
+
// stripping then still leaves link-dominated blocks (related posts, category
|
|
419
|
+
// sidebars, "hot articles" widgets) that carry little reading value. Split the
|
|
420
|
+
// body at block-level open tags and drop segments whose text is mostly links.
|
|
421
|
+
// Standard-container pages (article/main) never reach this code path.
|
|
422
|
+
const BLOCK_OPEN_RE = /<(?:p|div|section|article|main|h[1-6]|table|tbody|tr|ul|ol|li|blockquote|pre|dl|figure|figcaption|fieldset)\b[^>]{0,1000}>/gi
|
|
423
|
+
|
|
424
|
+
function linkTextLength(html) {
|
|
425
|
+
if (!/<\/a/i.test(html)) return 0 // no anchor can pair — skip the scan
|
|
426
|
+
let n = 0
|
|
427
|
+
const re = /<a[\s>][\s\S]*?<\/a\s*>/gi
|
|
428
|
+
let m
|
|
429
|
+
while ((m = re.exec(html))) n += textOnly(m[0]).length
|
|
430
|
+
return n
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
export function densityFilter(html) {
|
|
434
|
+
const marks = []
|
|
435
|
+
BLOCK_OPEN_RE.lastIndex = 0
|
|
436
|
+
let m
|
|
437
|
+
while ((m = BLOCK_OPEN_RE.exec(html))) marks.push(m.index)
|
|
438
|
+
if (marks.length < 2) return html
|
|
439
|
+
const segs = []
|
|
440
|
+
if (marks[0] > 0) segs.push(html.slice(0, marks[0]))
|
|
441
|
+
for (let k = 0; k < marks.length; k++) {
|
|
442
|
+
segs.push(html.slice(marks[k], k + 1 < marks.length ? marks[k + 1] : html.length))
|
|
443
|
+
}
|
|
444
|
+
const kept = segs.filter((seg) => {
|
|
445
|
+
const text = textOnly(seg)
|
|
446
|
+
if (text.length < 2) return false
|
|
447
|
+
// short link-dominated segment = nav item / recommendation widget entry
|
|
448
|
+
if (text.length < 300 && linkTextLength(seg) > text.length * 0.65) return false
|
|
449
|
+
return true
|
|
450
|
+
})
|
|
451
|
+
return kept.length ? kept.join('') : html
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// Depth-counted block extraction: returns html from the opening tag at
|
|
455
|
+
// openIdx through its BALANCED closing tag. A non-greedy regex would stop at
|
|
456
|
+
// the first nested </div>, truncating heavily nested container divs.
|
|
457
|
+
function tagBlockAt(html, openIdx, tagName) {
|
|
458
|
+
const re = new RegExp(`</?${tagName}(?=[\\s>])`, 'gi')
|
|
459
|
+
re.lastIndex = openIdx
|
|
460
|
+
let depth = 0
|
|
461
|
+
let m
|
|
462
|
+
while ((m = re.exec(html))) {
|
|
463
|
+
depth += m[0][1] === '/' ? -1 : 1
|
|
464
|
+
if (depth === 0) {
|
|
465
|
+
const close = html.indexOf('>', m.index)
|
|
466
|
+
return close > 0 ? html.slice(openIdx, close + 1) : null
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
return null
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
export function pickMain(html) {
|
|
473
|
+
// Aggregation pages (e.g. blog homepages) have one <article> per item:
|
|
474
|
+
// collect all of them instead of only the first. A tiny <article> (e.g. a
|
|
475
|
+
// newsletter card while the real content sits in <main>) falls through to
|
|
476
|
+
// <main> when present.
|
|
477
|
+
const articles = /<\/article/i.test(html)
|
|
478
|
+
? [...html.matchAll(/<article[\s>][\s\S]*?<\/article\s*>/gi)]
|
|
479
|
+
: [] // without a closer in the document nothing can pair
|
|
480
|
+
const hasMain = /<main[\s>]/i.test(html)
|
|
481
|
+
if (articles.length && (!hasMain || textOnly(articles.map((a) => a[0]).join('')).length >= 200)) {
|
|
482
|
+
return articles.map((a) => a[0]).join('\n')
|
|
483
|
+
}
|
|
484
|
+
const roleMain = /<(main|div)[^>]{0,1000}role=["']main["'][\s>]/i.exec(html)
|
|
485
|
+
if (roleMain) {
|
|
486
|
+
const block = tagBlockAt(html, roleMain.index, roleMain[1].toLowerCase())
|
|
487
|
+
if (block) return block
|
|
488
|
+
}
|
|
489
|
+
// Bare <main> without role="main": doc generators (VitePress/VuePress,
|
|
490
|
+
// MDN, MDX sites) mark the body with a plain <main> tag. Falling back to
|
|
491
|
+
// the whole <body> would drag in the top/local nav (menu, outline, skip
|
|
492
|
+
// link) — the extraction quality loss we saw on vuejs.org.
|
|
493
|
+
const bareMain = /<main[\s>]/i.exec(html)
|
|
494
|
+
if (bareMain) {
|
|
495
|
+
const block = tagBlockAt(html, bareMain.index, 'main')
|
|
496
|
+
if (block) return block
|
|
497
|
+
}
|
|
498
|
+
const body = /<body[\s>]/i.exec(html)
|
|
499
|
+
if (body) {
|
|
500
|
+
const after = html.slice(body.index)
|
|
501
|
+
const end = after.search(/<\/body>/i)
|
|
502
|
+
return densityFilter(end > 0 ? after.slice(0, end) : after)
|
|
503
|
+
}
|
|
504
|
+
return densityFilter(html)
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
// Some sites (e.g. baidu.com) ship CSS/HTML inside hidden <textarea> with
|
|
508
|
+
// entity-escaped tags (<style>...). Reveal TAG-SHAPED sequences so noise
|
|
509
|
+
// rules can strip them. < is revealed only when a tag name follows: a bare
|
|
510
|
+
// < from prose ("a < b") would be swallowed by the tag stripper up to
|
|
511
|
+
// the next real '>'. Bare > / " never open a strip span and are
|
|
512
|
+
// revealed unconditionally.
|
|
513
|
+
function revealEscapedTags(html) {
|
|
514
|
+
return html
|
|
515
|
+
.replace(/<(?=\/?[a-zA-Z!])/gi, '<')
|
|
516
|
+
.replace(/>/gi, '>')
|
|
517
|
+
.replace(/"/gi, '"')
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
function stripNoise(mainHtml) {
|
|
521
|
+
// Attribute runs are bounded so a '<' whose '>' is far away costs a bounded
|
|
522
|
+
// scan instead of a run to that '>'; real attributes never approach 1000
|
|
523
|
+
// chars. Container groups only include tags whose closing tag exists in the
|
|
524
|
+
// input — absent closers cannot pair, so scanning for their body is wasted
|
|
525
|
+
// work.
|
|
526
|
+
const closers = new Set()
|
|
527
|
+
for (const c of mainHtml.matchAll(/<\/([a-z0-9]+)/gi)) closers.add(c[1].toLowerCase())
|
|
528
|
+
const group = (names) => names.filter((n) => closers.has(n))
|
|
529
|
+
const raw = group(['textarea', 'style', 'script', 'noscript', 'template'])
|
|
530
|
+
const box = group(['nav', 'footer', 'header', 'aside', 'form', 'iframe', 'svg', 'canvas', 'dialog'])
|
|
531
|
+
let out = mainHtml.replace(/<!--[\s\S]*?-->/g, ' ')
|
|
532
|
+
if (raw.length) {
|
|
533
|
+
out = out.replace(new RegExp(`<(${raw.join('|')})[\\s>][\\s\\S]*?<\\/\\1\\s*>`, 'gi'), ' ')
|
|
534
|
+
}
|
|
535
|
+
if (box.length) {
|
|
536
|
+
out = out.replace(new RegExp(`<(${box.join('|')})[\\s>][\\s\\S]*?<\\/\\1>`, 'gi'), ' ')
|
|
537
|
+
}
|
|
538
|
+
// Elements explicitly hidden from users are invisible decoration or stateful
|
|
539
|
+
// UI (collapsed panels, modal templates) — their text must not leak into the
|
|
540
|
+
// body. The [\s"'] guard keeps `hidden`/`style` from matching inside
|
|
541
|
+
// compound attr names (data-hidden, aria-hidden). Consent/GDPR banners mount
|
|
542
|
+
// by id (onetrust, cookiebot, cybot, gdpr) rather than class, so a second
|
|
543
|
+
// pass keys on the id attribute. Every scan run is bounded.
|
|
544
|
+
const hidden = /<(div|span|section|p|ul|table)\b[^>]{0,1000}[\s"'](?:hidden\b|aria-hidden\s*=\s*["']?true|style\s*=\s*["'][^"']{0,200}(?:display:\s*none|visibility:\s*hidden))[^>]{0,1000}>[\s\S]{0,50000}?<\/\1\s*>/gi
|
|
545
|
+
out = out.replace(hidden, ' ')
|
|
546
|
+
const consent = /<([a-z][a-z0-9]*)\s+id=["'][^"']{0,120}(?:onetrust|cookiebot|cybot|gdpr|consent|cookie-law|cookie_banner|cmp-)[^"']{0,120}["'][^>]{0,1000}>[\s\S]{0,50000}?<\/\1\s*>/gi
|
|
547
|
+
out = out.replace(consent, ' ')
|
|
548
|
+
return out.replace(
|
|
549
|
+
/<([a-z][a-z0-9]*)[^>]{1,1000}class=["'][^"']{0,300}(ad-|ads|advert|banner|sidebar|social|share|comment|popup|modal|cookie)[^"']{0,300}["'][^>]{0,1000}>[\s\S]*?<\/\1>/gi,
|
|
550
|
+
' ',
|
|
551
|
+
)
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
// ---- HTML -> Markdown (lightweight, tag-state machine) ----
|
|
555
|
+
function escInline(s) {
|
|
556
|
+
return s.replace(/([\\`*_[\]])/g, '\\$1')
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
// <img> is a void element (never matches the paired-tag regex in either md
|
|
560
|
+
// walker): convert descriptive images to markdown BEFORE the tag walk, via a
|
|
561
|
+
// sentinel so escInline cannot escape the generated brackets. Decorative
|
|
562
|
+
// images (empty alt) are dropped — the model cannot see pixels anyway.
|
|
563
|
+
const IMG_SENTINEL = '\u0001'
|
|
564
|
+
function imgsToMarkdown(html) {
|
|
565
|
+
const store = []
|
|
566
|
+
const out = html.replace(/<img\b[^>]{0,1000}>/gi, (tag) => {
|
|
567
|
+
const alt = /alt=["']([^"']*)["']/i.exec(tag)
|
|
568
|
+
const src = /src=["']([^"']+)["']/i.exec(tag)
|
|
569
|
+
store.push(src && alt && alt[1].trim() ? `![${alt[1].trim()}](${src[1]})` : '')
|
|
570
|
+
return `${IMG_SENTINEL}${store.length - 1}${IMG_SENTINEL}`
|
|
571
|
+
})
|
|
572
|
+
return {
|
|
573
|
+
html: out,
|
|
574
|
+
// Unknown sentinel indexes are PRESERVED (a nested walker's restore must
|
|
575
|
+
// not eat sentinels owned by an outer frame — blockMd recurses into itself).
|
|
576
|
+
restore: (s) => s.replace(new RegExp(`${IMG_SENTINEL}(\\d+)${IMG_SENTINEL}`, 'g'), (m, i) => {
|
|
577
|
+
const v = store[Number(i)]
|
|
578
|
+
return v === undefined ? m : v
|
|
579
|
+
}),
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
// An open tag whose closing tag appears nowhere in the input can never pair;
|
|
584
|
+
// removing such openers up front keeps the walker from rescanning the rest of
|
|
585
|
+
// the string at every position, so the walk stays linear. The walkers drop
|
|
586
|
+
// these tags either way, so the rendered output is unchanged. The depth cap
|
|
587
|
+
// bounds recursion — real pages never nest hundreds of levels; beyond the cap
|
|
588
|
+
// the walker degrades to plain text.
|
|
589
|
+
const OPEN_TAG_RE = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">]){0,1000})>/g
|
|
590
|
+
const MD_MAX_DEPTH = 100
|
|
591
|
+
function stripUnmatchedOpeners(html) {
|
|
592
|
+
const closers = new Set()
|
|
593
|
+
for (const m of html.matchAll(/<\/([a-zA-Z0-9]+)/g)) closers.add(m[1].toLowerCase())
|
|
594
|
+
html = defuseLt(html)
|
|
595
|
+
return html.replace(OPEN_TAG_RE, (m, name) => (closers.has(name.toLowerCase()) ? m : ''))
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
export function inlineMd(html, depth = 0) {
|
|
599
|
+
if (depth > MD_MAX_DEPTH) {
|
|
600
|
+
return escInline(decodeTextEntities(html.replace(/<[^>]{1,1000}>/g, ' ')).replace(/\s+/g, ' ').trim())
|
|
601
|
+
}
|
|
602
|
+
const img = imgsToMarkdown(html)
|
|
603
|
+
html = stripUnmatchedOpeners(img.html)
|
|
604
|
+
let out = ''
|
|
605
|
+
const re = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">]){0,1000})>([\s\S]*?)<\/\1>|<[^>]{1,1000}>|([^<]+)/g
|
|
606
|
+
let m
|
|
607
|
+
while ((m = re.exec(html))) {
|
|
608
|
+
if (m[4] !== undefined) {
|
|
609
|
+
out += escInline(decodeTextEntities(m[4]))
|
|
610
|
+
continue
|
|
611
|
+
}
|
|
612
|
+
if (!m[1]) continue
|
|
613
|
+
const tag = m[1].toLowerCase()
|
|
614
|
+
if (tag === 'style' || tag === 'script' || tag === 'textarea' || tag === 'template' || tag === 'noscript') continue
|
|
615
|
+
const inner = inlineMd(m[3], depth + 1)
|
|
616
|
+
if (tag === 'a') {
|
|
617
|
+
const href = /href=["']([^"']+)["']/i.exec(m[2])
|
|
618
|
+
// A raw ')' in the target would terminate the markdown link early
|
|
619
|
+
// (common on wiki/framework URLs) — percent-encode parens to keep the
|
|
620
|
+
// link parseable by the model.
|
|
621
|
+
out += href ? `[${inner}](${href[1].replace(/\)/g, '%29').replace(/\(/g, '%28')})` : inner
|
|
622
|
+
} else if (tag === 'strong' || tag === 'b') out += `**${inner}**`
|
|
623
|
+
else if (tag === 'em' || tag === 'i') out += `*${inner}*`
|
|
624
|
+
else if (tag === 'code') out += `\`${inner}\``
|
|
625
|
+
else if (tag === 'br') out += ' \n'
|
|
626
|
+
else out += inner
|
|
627
|
+
}
|
|
628
|
+
return img.restore(out.replace(/[ \t]+/g, ' ').trim())
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
export function blockMd(html, depth = 0) {
|
|
632
|
+
if (depth > MD_MAX_DEPTH) {
|
|
633
|
+
return decodeTextEntities(html.replace(/<[^>]{1,1000}>/g, ' ')).replace(/\s+/g, ' ').trim()
|
|
634
|
+
}
|
|
635
|
+
const img = imgsToMarkdown(html)
|
|
636
|
+
html = stripUnmatchedOpeners(img.html)
|
|
637
|
+
let out = ''
|
|
638
|
+
const re = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">]){0,1000})>([\s\S]*?)<\/\1>|<[^>]{1,1000}>|([^<]+)/g
|
|
639
|
+
let m
|
|
640
|
+
while ((m = re.exec(html))) {
|
|
641
|
+
if (m[4] !== undefined) {
|
|
642
|
+
out += decodeTextEntities(m[4])
|
|
643
|
+
continue
|
|
644
|
+
}
|
|
645
|
+
if (!m[1]) continue
|
|
646
|
+
const tag = m[1].toLowerCase()
|
|
647
|
+
const attrs = m[2] || ''
|
|
648
|
+
const inner = m[3]
|
|
649
|
+
if (tag === 'style' || tag === 'script' || tag === 'textarea' || tag === 'template' || tag === 'noscript') {
|
|
650
|
+
continue
|
|
651
|
+
}
|
|
652
|
+
if (tag === 'h1' || tag === 'h2' || tag === 'h3' || tag === 'h4' || tag === 'h5' || tag === 'h6') {
|
|
653
|
+
const level = '#'.repeat(Number(tag[1]))
|
|
654
|
+
// Trailing permalink glyphs are anchor decorations, not heading text.
|
|
655
|
+
out += `\n\n${level} ${inlineMd(inner, depth + 1).replace(/\s*[¶§]\s*$/, '')}`
|
|
656
|
+
} else if (tag === 'p') {
|
|
657
|
+
const t = inlineMd(inner, depth + 1)
|
|
658
|
+
if (t) out += `\n\n${t}`
|
|
659
|
+
} else if (tag === 'div' || tag === 'section' || tag === 'article' || tag === 'main') {
|
|
660
|
+
const t = blockMd(inner, depth + 1)
|
|
661
|
+
if (t) out += `\n\n${t}`
|
|
662
|
+
} else if (tag === 'blockquote') {
|
|
663
|
+
const t = blockMd(inner, depth + 1).trim()
|
|
664
|
+
out += `\n\n> ${t.replace(/\n/g, '\n> ')}`
|
|
665
|
+
} else if (tag === 'pre') {
|
|
666
|
+
// Strip wrapper tags FIRST, then decode entities — a literal <div>
|
|
667
|
+
// in the code must become <div> in the fenced block, but a real nested
|
|
668
|
+
// tag must not survive the strip to be rendered as markdown.
|
|
669
|
+
const code = decodeTextEntities(inner.replace(/<[^>]{1,1000}>/g, '')).trim()
|
|
670
|
+
// Language hint from common highlighter conventions: the model reads
|
|
671
|
+
// ```js fenced blocks far more accurately than unlabelled ones.
|
|
672
|
+
const lang =
|
|
673
|
+
/<code[^>]+class=["'][^"']*(?:language|lang)-([\w#+.-]+)["']/i.exec(inner)?.[1] || ''
|
|
674
|
+
out += `\n\n\`\`\`${lang}\n${code}\n\`\`\``
|
|
675
|
+
} else if (tag === 'code') {
|
|
676
|
+
out += `\`${textOnly(inner)}\``
|
|
677
|
+
} else if (tag === 'ul' || tag === 'ol') {
|
|
678
|
+
// Without a single </li> nothing can pair, so the item scan is skipped.
|
|
679
|
+
const items = []
|
|
680
|
+
if (inner.includes('</li')) {
|
|
681
|
+
const liRe = /<li[^>]{0,1000}>([\s\S]*?)<\/li>/gi
|
|
682
|
+
let lm, idx = 1
|
|
683
|
+
while ((lm = liRe.exec(inner))) {
|
|
684
|
+
const t = inlineMd(lm[1], depth + 1)
|
|
685
|
+
items.push(tag === 'ol' ? `${idx}. ${t}` : `- ${t}`)
|
|
686
|
+
idx++
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
if (items.length) out += `\n\n${items.join('\n')}`
|
|
690
|
+
} else if (tag === 'li') {
|
|
691
|
+
out += `\n- ${inlineMd(inner, depth + 1)}`
|
|
692
|
+
} else if (tag === 'br') {
|
|
693
|
+
out += ' \n'
|
|
694
|
+
} else if (tag === 'hr') {
|
|
695
|
+
out += '\n\n---'
|
|
696
|
+
} else if (tag === 'table') {
|
|
697
|
+
const rows = []
|
|
698
|
+
if (inner.includes('</tr')) {
|
|
699
|
+
const trRe = /<tr[^>]{0,1000}>([\s\S]*?)<\/tr>/gi
|
|
700
|
+
let tm
|
|
701
|
+
while ((tm = trRe.exec(inner))) {
|
|
702
|
+
const cells = []
|
|
703
|
+
if (tm[1].includes('</t')) {
|
|
704
|
+
const tdRe = /<t[dh][^>]{0,1000}>([\s\S]*?)<\/t[dh]>/gi
|
|
705
|
+
let cm
|
|
706
|
+
while ((cm = tdRe.exec(tm[1]))) cells.push(inlineMd(cm[1], depth + 1).replace(/\|/g, '\\|'))
|
|
707
|
+
}
|
|
708
|
+
if (cells.length) rows.push(`| ${cells.join(' | ')} |`)
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
if (rows.length) {
|
|
712
|
+
const header = rows[0]
|
|
713
|
+
// unescape cells' escaped pipes before deriving the separator row,
|
|
714
|
+
// otherwise the --- line is one char wider per escaped pipe
|
|
715
|
+
const sep = header.replace(/\\\|/g, '|').replace(/[^|]/g, '-')
|
|
716
|
+
out += `\n\n${rows.join('\n')}\n${sep}`
|
|
717
|
+
}
|
|
718
|
+
} else if (tag === 'a' || tag === 'strong' || tag === 'b' || tag === 'em' || tag === 'i') {
|
|
719
|
+
const t = inlineMd(inner, depth + 1)
|
|
720
|
+
if (t) out += t
|
|
721
|
+
} else {
|
|
722
|
+
const t = blockMd(inner, depth + 1)
|
|
723
|
+
if (t) out += t
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
return img.restore(out)
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
// Read the content attribute of the FIRST <meta> whose property/name equals
|
|
730
|
+
// `key`. Attribute order is arbitrary (content may precede property/name), and
|
|
731
|
+
// the content value may itself contain quotes — locate the tag first, then
|
|
732
|
+
// read content from inside it with a backreference pair.
|
|
733
|
+
function metaContent(html, key) {
|
|
734
|
+
const tagM = new RegExp(`<meta\\b[^>]{0,1000}(?:property|name)=["']${key}["'][^>]{0,1000}>`, 'i').exec(html)
|
|
735
|
+
if (!tagM) return ''
|
|
736
|
+
const m = /content=("([^"]*)"|'([^']*)')/i.exec(tagM[0])
|
|
737
|
+
return m ? (m[2] || m[3] || '') : ''
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
// Page-level metadata the model actually asks about (who wrote it, when) —
|
|
741
|
+
// cheap to harvest from <meta>. publishedExplicit records whether the date
|
|
742
|
+
// came from a semantic article/og key (authoritative) or a generic date/dc.date
|
|
743
|
+
// stamp (often last-modified, weaker than a JSON-LD datePublished).
|
|
744
|
+
function extractMeta(html) {
|
|
745
|
+
const getKey = (names) => {
|
|
746
|
+
for (const n of names) {
|
|
747
|
+
const v = metaContent(html, n)
|
|
748
|
+
if (v) {
|
|
749
|
+
return {
|
|
750
|
+
v: decodeTextEntities(v).replace(/<[^>]{1,1000}>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 80),
|
|
751
|
+
key: n,
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
return { v: '', key: '' }
|
|
756
|
+
}
|
|
757
|
+
const p = getKey(['article:published_time', 'og:published_time', 'pubdate', 'publishdate', 'date', 'dc.date'])
|
|
758
|
+
const a = getKey(['author', 'article:author', 'og:article:author'])
|
|
759
|
+
const d = getKey(['og:description', 'description'])
|
|
760
|
+
return {
|
|
761
|
+
published: p.v,
|
|
762
|
+
publishedExplicit: /^(article|og):/i.test(p.key),
|
|
763
|
+
author: a.v,
|
|
764
|
+
description: d.v,
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
// schema.org JSON-LD blocks carry publication metadata (datePublished/author)
|
|
769
|
+
// many news sites put nowhere else — <meta> covers only a subset and the
|
|
770
|
+
// byline regex cannot see ISO-8601 timestamps. Parse the first ld+json block
|
|
771
|
+
// that yields either field; tolerate broken JSON and every structural shape
|
|
772
|
+
// (plain object, @type arrays, @graph graphs, author as string/object/array).
|
|
773
|
+
// Bounded block size keeps a hostile giant script tag from dragging the scan.
|
|
774
|
+
// 100k covers real article-list JSON-LD (news sites often embed dozens of
|
|
775
|
+
// items); metadata sits in the first node anyway.
|
|
776
|
+
const LDJSON_RE = /<script[^>]{0,1000}type=["']application\/ld\+json["'][^>]{0,1000}>([\s\S]{0,100000}?)<\/script\s*>/gi
|
|
777
|
+
function jsonLdMeta(html) {
|
|
778
|
+
// Recursively collect every node: nested structures are common (ItemList →
|
|
779
|
+
// mainEntity → Article, WebPage → mainEntityOfPage), so a flat top-level
|
|
780
|
+
// scan alone misses most real payloads. Depth-capped against hostile
|
|
781
|
+
// nesting; arrays, @graph and plain object properties are all expanded.
|
|
782
|
+
const collect = (node, out, depth = 0) => {
|
|
783
|
+
if (!node || typeof node !== 'object' || depth > 20) return
|
|
784
|
+
if (Array.isArray(node)) {
|
|
785
|
+
for (const it of node) collect(it, out, depth + 1)
|
|
786
|
+
return
|
|
787
|
+
}
|
|
788
|
+
out.push(node)
|
|
789
|
+
if (Array.isArray(node['@graph'])) for (const it of node['@graph']) collect(it, out, depth + 1)
|
|
790
|
+
for (const k of Object.keys(node)) {
|
|
791
|
+
if (k === '@graph') continue
|
|
792
|
+
const v = node[k]
|
|
793
|
+
if (v && typeof v === 'object') collect(v, out, depth + 1)
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
const authorOf = (a) => {
|
|
797
|
+
if (typeof a === 'string') return a
|
|
798
|
+
if (Array.isArray(a)) return authorOf(a[0])
|
|
799
|
+
if (a && typeof a === 'object') return typeof a.name === 'string' ? a.name : authorOf(a.name)
|
|
800
|
+
return ''
|
|
801
|
+
}
|
|
802
|
+
const clean = (s) => (s ? decodeTextEntities(s).slice(0, 40) : '')
|
|
803
|
+
for (const m of html.matchAll(LDJSON_RE)) {
|
|
804
|
+
let json
|
|
805
|
+
try {
|
|
806
|
+
json = JSON.parse(m[1])
|
|
807
|
+
} catch {
|
|
808
|
+
continue
|
|
809
|
+
}
|
|
810
|
+
const nodes = []
|
|
811
|
+
collect(json, nodes)
|
|
812
|
+
for (const n of nodes) {
|
|
813
|
+
const published = clean(typeof n.datePublished === 'string' ? n.datePublished : '')
|
|
814
|
+
const author = clean(authorOf(n.author))
|
|
815
|
+
if (published || author) return { published, author }
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
return { published: '', author: '' }
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
// Meta-less pages still print the byline inside the body head: "作者:阮一峰
|
|
822
|
+
// 日期:2026年8月21日". Fill ONLY the missing fields, and only from the first
|
|
823
|
+
// 600 chars of body text — deeper matches are content, not bylines.
|
|
824
|
+
const BYLINE_AUTHOR_RE = /(?:作者|責任編輯|責任主编|editor)[::\s]{1,3}([^\s,,。/|()()[\]]{2,24})/
|
|
825
|
+
const BYLINE_DATE_RE = /(?:发表日期|发布日期|发表时间|发布时间|日期|发表于|发布于|published(?:\s+on)?)[::\s]{0,3}(\d{4}[-/.年]\s?\d{1,2}[-/.月]\s?\d{1,2}日?|\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2})?)/i
|
|
826
|
+
|
|
827
|
+
function bylineFallback(text, meta) {
|
|
828
|
+
const head = typeof text === 'string' ? text.slice(0, 600) : ''
|
|
829
|
+
const out = { published: meta.published, author: meta.author }
|
|
830
|
+
if (!out.author) {
|
|
831
|
+
const m = BYLINE_AUTHOR_RE.exec(head)
|
|
832
|
+
if (m) out.author = m[1].trim().slice(0, 40)
|
|
833
|
+
}
|
|
834
|
+
if (!out.published) {
|
|
835
|
+
const m = BYLINE_DATE_RE.exec(head)
|
|
836
|
+
if (m) out.published = m[1].replace(/\s+/g, ' ').trim().slice(0, 40)
|
|
837
|
+
}
|
|
838
|
+
return out
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
// Locate the element whose id/name equals `anchor` and return the html of its
|
|
842
|
+
// section: for container tags (section/div/dl/…) the element's own balanced
|
|
843
|
+
// block IS the section; for heading/inline anchors (<h2 id>, <a name>, <dt
|
|
844
|
+
// id>) the anchor only marks where the section starts, so everything FROM it
|
|
845
|
+
// onward is kept (the heading introduces the content that follows). Returns
|
|
846
|
+
// null when no element matches — callers fall back to full text.
|
|
847
|
+
// The [\s"'] guard rejects compound attr names (data-id, data-name), and the
|
|
848
|
+
// exact-quoted value means id="prefix-anchor" never matches anchor "prefix".
|
|
849
|
+
const ANCHOR_CONTAINER = /^(?:section|div|article|aside|main|dl|table|ul|ol|details|figure|blockquote|fieldset|form)$/
|
|
850
|
+
function sliceAtAnchor(mainHtml, anchor) {
|
|
851
|
+
if (!anchor) return null
|
|
852
|
+
const esc = anchor.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
|
853
|
+
const m = new RegExp(`<([a-z][a-z0-9]*)\\b[^>]{0,1000}[\\s"'](?:id|name)\\s*=\\s*["']${esc}["'][^>]{0,1000}>`, 'i').exec(mainHtml)
|
|
854
|
+
if (!m) return null
|
|
855
|
+
const tag = m[1].toLowerCase()
|
|
856
|
+
if (ANCHOR_CONTAINER.test(tag)) {
|
|
857
|
+
const block = tagBlockAt(mainHtml, m.index, tag)
|
|
858
|
+
if (block) return block
|
|
859
|
+
}
|
|
860
|
+
return mainHtml.slice(m.index)
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
// Anchors may arrive percent-encoded (CJK section names); the document can
|
|
864
|
+
// carry either form. Try the raw fragment first, then its decoded variant.
|
|
865
|
+
function anchorCandidates(anchor) {
|
|
866
|
+
const list = [anchor]
|
|
867
|
+
try {
|
|
868
|
+
const dec = decodeURIComponent(anchor)
|
|
869
|
+
if (dec !== anchor) list.push(dec)
|
|
870
|
+
} catch {
|
|
871
|
+
/* malformed percent-encoding — raw form only */
|
|
872
|
+
}
|
|
873
|
+
return list
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
export function extract(html, mode, anchor = '') {
|
|
877
|
+
html = defuseLt(html) // kills the no-'>'-tail quadratic scans before any picker runs
|
|
878
|
+
const titleTag = /<title[^>]{0,1000}>([\s\S]*?)<\/title>/i.exec(html)
|
|
879
|
+
const ogTitle = metaContent(html, 'og:title')
|
|
880
|
+
const siteName = metaContent(html, 'og:site_name')
|
|
881
|
+
const langMatch = /<html[^>]{1,1000}lang=["']([\w-]+)["']/i.exec(html)
|
|
882
|
+
let main = stripNoise(pickMain(html))
|
|
883
|
+
main = stripNoise(revealEscapedTags(main))
|
|
884
|
+
// A URL fragment scopes the read to one section of a long reference page —
|
|
885
|
+
// slice at the anchor so only that section is extracted. Unmatched anchors
|
|
886
|
+
// degrade to the full document instead of failing.
|
|
887
|
+
let scoped = null
|
|
888
|
+
for (const cand of anchorCandidates(anchor)) {
|
|
889
|
+
scoped = sliceAtAnchor(main, cand)
|
|
890
|
+
if (scoped) break
|
|
891
|
+
}
|
|
892
|
+
let bodyText
|
|
893
|
+
if (mode === 'markdown') {
|
|
894
|
+
bodyText = blockMd(scoped || main).replace(/\n{3,}/g, '\n\n').trim()
|
|
895
|
+
} else {
|
|
896
|
+
bodyText = textLines(scoped || main).replace(/ +/g, ' ').trim()
|
|
897
|
+
}
|
|
898
|
+
const meta = extractMeta(html)
|
|
899
|
+
const ld = jsonLdMeta(html)
|
|
900
|
+
// Merge chain: <meta> → JSON-LD → byline. An explicit article:/og: meta date
|
|
901
|
+
// stays authoritative; otherwise a JSON-LD datePublished beats the generic
|
|
902
|
+
// date meta key (last-modified stamps are not publication times).
|
|
903
|
+
const published = ld.published
|
|
904
|
+
? (meta.publishedExplicit ? meta.published : ld.published)
|
|
905
|
+
: meta.published
|
|
906
|
+
const author = ld.author || meta.author
|
|
907
|
+
// Empty-body pages (login walls, JS-only shells) still carry og:description
|
|
908
|
+
// — surface it as a fallback hint instead of nothing.
|
|
909
|
+
let text = bodyText
|
|
910
|
+
let usedDescription = false
|
|
911
|
+
if (!text && meta.description) {
|
|
912
|
+
text = meta.description
|
|
913
|
+
usedDescription = true
|
|
914
|
+
}
|
|
915
|
+
const byline = bylineFallback(text, { published, author })
|
|
916
|
+
return {
|
|
917
|
+
title: titleTag ? textOnly(titleTag[1]) || titleTag[1].trim() : ogTitle,
|
|
918
|
+
siteName,
|
|
919
|
+
lang: langMatch ? langMatch[1] : '',
|
|
920
|
+
published: byline.published,
|
|
921
|
+
author: byline.author,
|
|
922
|
+
text,
|
|
923
|
+
usedDescription,
|
|
924
|
+
anchored: !!scoped,
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
// Optional enhancement: when @mozilla/readability + happy-dom are installed
|
|
929
|
+
// inside the DSH profile (npm i @mozilla/readability happy-dom), extraction
|
|
930
|
+
// upgrades to the Firefox Reader Mode algorithm. Falls back to the built-in
|
|
931
|
+
// heuristic extractor when absent. MPL-2.0 / MIT libraries, used as-is.
|
|
932
|
+
let readabilityReady = null
|
|
933
|
+
async function getReadabilityExtractor() {
|
|
934
|
+
if (readabilityReady !== null) return readabilityReady
|
|
935
|
+
try {
|
|
936
|
+
const [{ Readability }, { Window }] = await Promise.all([
|
|
937
|
+
import('@mozilla/readability'),
|
|
938
|
+
import('happy-dom'),
|
|
939
|
+
])
|
|
940
|
+
readabilityReady = (html, url) => {
|
|
941
|
+
const window = new Window({ url })
|
|
942
|
+
window.document.write(html)
|
|
943
|
+
const article = new Readability(window.document).parse()
|
|
944
|
+
return article ? article.content : null
|
|
945
|
+
}
|
|
946
|
+
} catch {
|
|
947
|
+
readabilityReady = false
|
|
948
|
+
}
|
|
949
|
+
return readabilityReady
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
export function smartTruncate(text, maxChars, offset = 0) {
|
|
953
|
+
const total = text.length
|
|
954
|
+
if (offset >= total) {
|
|
955
|
+
return { text: '', truncated: false, charsTotal: total, charsReturned: 0, charsStart: offset }
|
|
956
|
+
}
|
|
957
|
+
if (total <= maxChars && offset === 0) {
|
|
958
|
+
return { text, truncated: false, charsTotal: total, charsReturned: total, charsStart: 0 }
|
|
959
|
+
}
|
|
960
|
+
const paragraphs = text.split(/\n\n+/)
|
|
961
|
+
let pos = 0
|
|
962
|
+
let start = 0
|
|
963
|
+
for (let i = 0; i < paragraphs.length; i++) {
|
|
964
|
+
if (pos + paragraphs[i].length > offset) {
|
|
965
|
+
start = i
|
|
966
|
+
break
|
|
967
|
+
}
|
|
968
|
+
pos += paragraphs[i].length + 2
|
|
969
|
+
}
|
|
970
|
+
let acc = ''
|
|
971
|
+
for (let i = start; i < paragraphs.length; i++) {
|
|
972
|
+
const p = paragraphs[i]
|
|
973
|
+
if (acc.length + p.length + (acc ? 2 : 0) > maxChars) break
|
|
974
|
+
acc += (acc ? '\n\n' : '') + p
|
|
975
|
+
}
|
|
976
|
+
if (!acc && maxChars > 0) {
|
|
977
|
+
// The first paragraph after the offset is itself longer than maxChars.
|
|
978
|
+
// Keep the alignment promise at sentence level before hard-slicing.
|
|
979
|
+
const p = paragraphs[start] || ''
|
|
980
|
+
const rel = Math.max(0, offset - pos)
|
|
981
|
+
const rest = p.length > rel ? p.slice(rel) : text.slice(offset)
|
|
982
|
+
const sentences = rest.match(/[^.!?\n。!?]+[.!?\n。!?]*/g) || [rest]
|
|
983
|
+
for (const s of sentences) {
|
|
984
|
+
if (acc.length + s.length > maxChars) break
|
|
985
|
+
acc += s
|
|
986
|
+
}
|
|
987
|
+
if (!acc) acc = rest.slice(0, maxChars)
|
|
988
|
+
}
|
|
989
|
+
return {
|
|
990
|
+
text: acc,
|
|
991
|
+
truncated: offset + acc.length < total,
|
|
992
|
+
charsTotal: total,
|
|
993
|
+
charsReturned: acc.length,
|
|
994
|
+
charsStart: offset,
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
// <base href> overrides the document base for relative URL resolution (old
|
|
999
|
+
// forums / frame sites). Per HTML spec only the FIRST <base> in <head> counts;
|
|
1000
|
+
// attribute order is arbitrary and the href may be protocol-relative. Empty
|
|
1001
|
+
// href (= document URL) is a no-op; non-http(s) targets are rejected. The
|
|
1002
|
+
// head scan is bounded — the tag cannot legally appear beyond it.
|
|
1003
|
+
function detectBaseHref(html, finalUrl) {
|
|
1004
|
+
const headEnd = html.search(/<\/head\s*>/i)
|
|
1005
|
+
const head = headEnd >= 0 ? html.slice(0, headEnd) : html.slice(0, 16384)
|
|
1006
|
+
const base = /<base\b[^>]{0,1000}href=["']([^"']+)["'][^>]{0,1000}>/i.exec(head)
|
|
1007
|
+
if (!base) return finalUrl
|
|
1008
|
+
const href = base[1].trim()
|
|
1009
|
+
if (!href) return finalUrl
|
|
1010
|
+
try {
|
|
1011
|
+
const u = new URL(href, finalUrl)
|
|
1012
|
+
return u.protocol === 'http:' || u.protocol === 'https:' ? u.href : finalUrl
|
|
1013
|
+
} catch {
|
|
1014
|
+
return finalUrl
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
function extractLinks(html, limit, baseUrl) {
|
|
1019
|
+
const links = []
|
|
1020
|
+
const seen = new Set()
|
|
1021
|
+
// Accept relative hrefs too — most real pages link internally with relative
|
|
1022
|
+
// paths — and resolve them against the page URL so the model can follow them.
|
|
1023
|
+
// A declared <base href> wins over the document URL (frame-site links would
|
|
1024
|
+
// otherwise land on the wrong host).
|
|
1025
|
+
const linkBase = detectBaseHref(html, baseUrl)
|
|
1026
|
+
// Attribute runs bounded: an unclosed <a whose '>' is far away (or missing)
|
|
1027
|
+
// must not make the regex scan the rest of the string at every anchor
|
|
1028
|
+
// position. Real attributes never approach 1000 chars.
|
|
1029
|
+
const re = /<a\b[^>]{0,1000}href=["']([^"']+)["'][^>]{0,1000}>([\s\S]*?)<\/a\s*>/gi
|
|
1030
|
+
let m
|
|
1031
|
+
// Iteration cap: a page whose nav repeats the same links thousands of times
|
|
1032
|
+
// would otherwise scan the whole HTML to fill `limit` unique entries. Scan
|
|
1033
|
+
// at most limit*4 anchors; dedupe keeps the result bounded regardless.
|
|
1034
|
+
let scans = 0
|
|
1035
|
+
const maxScans = limit * 4
|
|
1036
|
+
while ((m = re.exec(html)) && links.length < limit && scans++ < maxScans) {
|
|
1037
|
+
const href = m[1].trim()
|
|
1038
|
+
if (!href || /^(javascript|mailto|tel|data):/i.test(href)) continue
|
|
1039
|
+
let url
|
|
1040
|
+
try {
|
|
1041
|
+
url = linkBase ? new URL(href, linkBase).href : new URL(href).href
|
|
1042
|
+
} catch {
|
|
1043
|
+
continue
|
|
1044
|
+
}
|
|
1045
|
+
// Only http(s) links are followable by the model / crawler; other schemes
|
|
1046
|
+
// (ftp:, data:) survive URL resolution and would pollute link lists and
|
|
1047
|
+
// crawl queues.
|
|
1048
|
+
if (!/^https?:/i.test(url)) continue
|
|
1049
|
+
// dedupe: nav bars repeat the same links many times — one entry per URL
|
|
1050
|
+
// keeps the link list compact (tokens) without losing coverage
|
|
1051
|
+
if (seen.has(url)) continue
|
|
1052
|
+
seen.add(url)
|
|
1053
|
+
const t = textOnly(m[2])
|
|
1054
|
+
if (t) links.push({ title: t.slice(0, 80), url })
|
|
1055
|
+
}
|
|
1056
|
+
return links
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
function hostOf(url) {
|
|
1060
|
+
try {
|
|
1061
|
+
return new URL(url).hostname
|
|
1062
|
+
} catch {
|
|
1063
|
+
return ''
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
// ---- Pagination: auto-follow "next page" ----
|
|
1068
|
+
// Long articles on novel/news/forum sites are split into pages; without this
|
|
1069
|
+
// the model had to notice the truncation and re-call the tool per page. The
|
|
1070
|
+
// recognition is deliberately conservative: rel=next (standard) or a short
|
|
1071
|
+
// anchor whose whole text is a next-page marker — no fuzzy guessing.
|
|
1072
|
+
const NEXT_TEXT_RE = /^(?:[›»>]+\s*)?(?:下一页|下页|下一篇|下一頁|下頁|后一页|後一頁|next page|next|older entries|[›»>]{1,3})(?:\s*[›»>]+)?$/i
|
|
1073
|
+
|
|
1074
|
+
export function findNextLink(html, baseUrl) {
|
|
1075
|
+
// Same <base href> rule as extractLinks: frame/forum sites pin their
|
|
1076
|
+
// document base, and a "next page" link resolves against it.
|
|
1077
|
+
const base = detectBaseHref(html || '', baseUrl)
|
|
1078
|
+
const resolve = (href) => {
|
|
1079
|
+
try {
|
|
1080
|
+
const u = new URL(href, base)
|
|
1081
|
+
return u.protocol === 'http:' || u.protocol === 'https:' ? u.href : null
|
|
1082
|
+
} catch {
|
|
1083
|
+
return null
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
const rel = /<(?:link|a)\b[^>]{0,1000}rel=["']next["'][^>]{0,1000}>/i.exec(html || '')
|
|
1087
|
+
if (rel) {
|
|
1088
|
+
const href = /href=["']([^"']+)["']/i.exec(rel[0])
|
|
1089
|
+
const r = href && resolve(href[1])
|
|
1090
|
+
if (r) return r
|
|
1091
|
+
}
|
|
1092
|
+
const re = /<a\b[^>]{0,1000}href=["']([^"']+)["'][^>]{0,1000}>([\s\S]*?)<\/a\s*>/gi
|
|
1093
|
+
let m
|
|
1094
|
+
let scans = 0
|
|
1095
|
+
// Bounded like extractLinks: a nav bar with thousands of anchors must not
|
|
1096
|
+
// scan the whole DOM hunting for a next-marker.
|
|
1097
|
+
while ((m = re.exec(html || '')) && scans++ < 400) {
|
|
1098
|
+
const t = textOnly(m[2])
|
|
1099
|
+
if (t.length <= 16 && NEXT_TEXT_RE.test(t)) {
|
|
1100
|
+
const r = resolve(m[1])
|
|
1101
|
+
if (r) return r
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
return null
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
// Join two page bodies, dropping a repeated suffix/prefix overlap (many
|
|
1108
|
+
// paginated sites repeat the last paragraph or a heading on the next page).
|
|
1109
|
+
function joinPageText(a, b) {
|
|
1110
|
+
const max = Math.min(300, a.length, b.length)
|
|
1111
|
+
for (let n = max; n >= 30; n--) {
|
|
1112
|
+
if (a.slice(a.length - n) === b.slice(0, n)) {
|
|
1113
|
+
const rest = b.slice(n).trim()
|
|
1114
|
+
return rest ? `${a}\n\n${rest}` : a
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
return `${a}\n\n${b}`
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
function sliceFrom(full, offset, maxChars) {
|
|
1121
|
+
const s = smartTruncate(full.fullText, maxChars, offset)
|
|
1122
|
+
const out = {
|
|
1123
|
+
url: full.url,
|
|
1124
|
+
title: full.title || '',
|
|
1125
|
+
siteName: full.siteName || hostOf(full.url),
|
|
1126
|
+
lang: full.lang || '',
|
|
1127
|
+
charset: full.charset,
|
|
1128
|
+
mode: full.mode,
|
|
1129
|
+
truncated: s.truncated,
|
|
1130
|
+
charsTotal: s.charsTotal,
|
|
1131
|
+
charsReturned: s.charsReturned,
|
|
1132
|
+
charsStart: s.charsStart,
|
|
1133
|
+
text: s.text,
|
|
1134
|
+
}
|
|
1135
|
+
if (full.rendered) out.rendered = true
|
|
1136
|
+
if (full.spaHint) out.spaHint = full.spaHint
|
|
1137
|
+
if (full.anchored) {
|
|
1138
|
+
out.anchored = true
|
|
1139
|
+
out.anchor = full.anchor || ''
|
|
1140
|
+
}
|
|
1141
|
+
if (full.paginated > 1) out.paginated = full.paginated
|
|
1142
|
+
if (full.feedCount) out.feedCount = full.feedCount
|
|
1143
|
+
if (full.published) out.published = full.published
|
|
1144
|
+
if (full.author) out.author = full.author
|
|
1145
|
+
if (Array.isArray(full.links)) out.links = full.links
|
|
1146
|
+
return out
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
// ---- JSON payloads: compact render ----
|
|
1150
|
+
// Indented JSON spends a newline + indent per key — pure token overhead for
|
|
1151
|
+
// the model. Render compact, and clip monster string values (embedded HTML,
|
|
1152
|
+
// base64 blobs) with a visible marker so the model knows there is more.
|
|
1153
|
+
const JSON_STR_MAX = 1500
|
|
1154
|
+
export function compactJson(value) {
|
|
1155
|
+
const clip = (s) =>
|
|
1156
|
+
s.length > JSON_STR_MAX ? s.slice(0, JSON_STR_MAX) + `…[+${s.length - JSON_STR_MAX} chars]` : s
|
|
1157
|
+
const walk = (v, depth) => {
|
|
1158
|
+
if (typeof v === 'string') return clip(v)
|
|
1159
|
+
if (Array.isArray(v)) return v.map((x) => walk(x, depth + 1))
|
|
1160
|
+
if (v && typeof v === 'object' && depth < 12) {
|
|
1161
|
+
const o = {}
|
|
1162
|
+
for (const k of Object.keys(v)) o[k] = walk(v[k], depth + 1)
|
|
1163
|
+
return o
|
|
1164
|
+
}
|
|
1165
|
+
return v
|
|
1166
|
+
}
|
|
1167
|
+
try {
|
|
1168
|
+
return JSON.stringify(walk(value, 0))
|
|
1169
|
+
} catch {
|
|
1170
|
+
// Extreme nesting/size where even native stringify exhausts the stack —
|
|
1171
|
+
// degrade to a clipped raw render instead of crashing the tool call.
|
|
1172
|
+
try {
|
|
1173
|
+
return clip(String(value))
|
|
1174
|
+
} catch {
|
|
1175
|
+
return '[unserializable json]'
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
// ---- meta-refresh shells: follow to the real page ----
|
|
1181
|
+
// <meta http-equiv="refresh" content="0; url=..."> stubs (link hops,
|
|
1182
|
+
// anti-hotlink relays, no-JS SPA entries) serve only a jump page — the body
|
|
1183
|
+
// lives at the target. Parse the FIRST matching <meta>; attribute order and
|
|
1184
|
+
// case are arbitrary, content quotes are stripped, a protocol-relative target
|
|
1185
|
+
// resolves against the document (or <base href>). Only IMMEDIATE redirects
|
|
1186
|
+
// are followed: a timed auto-refresh (content="30; url=…") is a normal page
|
|
1187
|
+
// with content, not a shell. A bare timed refresh with no url part reloads
|
|
1188
|
+
// the same page — null. Non-http(s) targets are refused.
|
|
1189
|
+
export function metaRefreshTarget(html, baseUrl) {
|
|
1190
|
+
// Locate the refresh <meta> tag first (any attribute order / case), then
|
|
1191
|
+
// read content from inside that tag — a forward regex would demand
|
|
1192
|
+
// http-equiv before content and miss the reversed order.
|
|
1193
|
+
const re = /<meta\b[^>]{0,1000}(?:http-equiv|name)=["']refresh["'][^>]{0,1000}>/gi
|
|
1194
|
+
let m
|
|
1195
|
+
while ((m = re.exec(html || ''))) {
|
|
1196
|
+
const tag = m[0]
|
|
1197
|
+
const contentM = /content=("([^"]*)"|'([^']*)')/i.exec(tag)
|
|
1198
|
+
if (!contentM) continue // a refresh meta without content is inert
|
|
1199
|
+
const content = (contentM[2] || contentM[3] || '').trim()
|
|
1200
|
+
// content="5" | content="0; url=http://…" | content=";url='…'" — the url
|
|
1201
|
+
// part may be single/double quoted or bare; "url=" must be a real token.
|
|
1202
|
+
const urlM = /\burl\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s;]+))/i.exec(content)
|
|
1203
|
+
if (!urlM) return null
|
|
1204
|
+
const delayM = /^\s*(\d+(?:\.\d+)?)/.exec(content)
|
|
1205
|
+
if (delayM && Number(delayM[1]) > 0) return null // timed refresh: keep the page
|
|
1206
|
+
const target = (urlM[1] || urlM[2] || urlM[3] || '').trim()
|
|
1207
|
+
if (!target) return null
|
|
1208
|
+
try {
|
|
1209
|
+
const u = new URL(target, detectBaseHref(html || '', baseUrl))
|
|
1210
|
+
return u.protocol === 'http:' || u.protocol === 'https:' ? u.href : null
|
|
1211
|
+
} catch {
|
|
1212
|
+
return null
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
return null
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
// Bounded following: at most 3 hops, deduped by normalized URL. Any fetch
|
|
1219
|
+
// problem (network error, non-HTML body) FAILS OPEN — the shell's own HTML
|
|
1220
|
+
// is kept for static extraction instead of erroring the call. Followed pages
|
|
1221
|
+
// are never written to the session cache under their own URL: the caller's
|
|
1222
|
+
// cache key is the ORIGINAL url, and a shell re-request should re-follow
|
|
1223
|
+
// rather than serve a foreign page.
|
|
1224
|
+
const MAX_META_REFRESH_HOPS = 3
|
|
1225
|
+
async function followMetaRefresh(html, finalUrl, externalSignal, cfg) {
|
|
1226
|
+
let cur = html
|
|
1227
|
+
let curUrl = finalUrl
|
|
1228
|
+
let curCharset = '' // set only when a hop actually happened
|
|
1229
|
+
const seen = new Set([normalizeUrl(finalUrl)])
|
|
1230
|
+
for (let i = 0; i < MAX_META_REFRESH_HOPS; i++) {
|
|
1231
|
+
const target = metaRefreshTarget(cur, curUrl)
|
|
1232
|
+
if (!target) break
|
|
1233
|
+
const norm = normalizeUrl(target)
|
|
1234
|
+
if (!norm || seen.has(norm)) break
|
|
1235
|
+
seen.add(norm)
|
|
1236
|
+
const page = await fetchPage(target, externalSignal, cfg)
|
|
1237
|
+
if (page.error) break
|
|
1238
|
+
const ct = (page.contentType || '').split(';')[0].toLowerCase().trim()
|
|
1239
|
+
if (!/html|xhtml/.test(ct)) break
|
|
1240
|
+
const decoded = decodeBuffer(page.buffer, page.contentType)
|
|
1241
|
+
cur = decoded.text
|
|
1242
|
+
curCharset = decoded.charset
|
|
1243
|
+
curUrl = page.finalUrl || target
|
|
1244
|
+
}
|
|
1245
|
+
return { html: cur, finalUrl: curUrl, charset: curCharset }
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
|
|
1249
|
+
const url = String((args && args.url) || '').trim()
|
|
1250
|
+
if (!/^https?:\/\//i.test(url)) return { error: 'Only http/https URLs are supported' }
|
|
1251
|
+
const maxChars = Math.max(500, Math.min(20000, Number(args.maxChars) || cfg.maxChars))
|
|
1252
|
+
const mode = args.mode === 'markdown' ? 'markdown' : 'text'
|
|
1253
|
+
const offset = Math.max(0, Number(args.offset) || 0)
|
|
1254
|
+
|
|
1255
|
+
// Cache stores the FULL extracted text keyed by url+mode+includeLinks, so
|
|
1256
|
+
// continuation reads (offset) and different maxChars hit the same entry —
|
|
1257
|
+
// but a links-request must not be served from a links-less cached copy.
|
|
1258
|
+
// The fragment STAYS in the key: a #section read scopes the extraction to
|
|
1259
|
+
// that anchor, so two fragments must never share one cache entry.
|
|
1260
|
+
let cacheUrl = url
|
|
1261
|
+
try {
|
|
1262
|
+
cacheUrl = new URL(url).href
|
|
1263
|
+
} catch {
|
|
1264
|
+
/* keep raw url on parse failure */
|
|
1265
|
+
}
|
|
1266
|
+
// A #fragment scopes the read to one section of a long page. The browser
|
|
1267
|
+
// never sends it to the server, so it must be taken from the REQUEST url —
|
|
1268
|
+
// finalUrl (after redirects/renders) has no fragment.
|
|
1269
|
+
let anchor = ''
|
|
1270
|
+
try {
|
|
1271
|
+
const h = new URL(url).hash
|
|
1272
|
+
if (h.length > 1) anchor = h.slice(1)
|
|
1273
|
+
} catch {
|
|
1274
|
+
/* keep bare url */
|
|
1275
|
+
}
|
|
1276
|
+
const cacheKey = `${cacheUrl}|${mode}|${args.includeLinks === true ? 'links' : 'no-links'}`
|
|
1277
|
+
// Failed fetches are cached briefly so the model doesn't re-request a
|
|
1278
|
+
// broken URL in a loop (token + latency saver).
|
|
1279
|
+
const fh = failCache.get(cacheKey)
|
|
1280
|
+
if (fh) {
|
|
1281
|
+
if (Date.now() - fh.time < 30000) return { error: fh.error, cached: true }
|
|
1282
|
+
failCache.delete(cacheKey)
|
|
1283
|
+
}
|
|
1284
|
+
const hit = cache.get(cacheKey)
|
|
1285
|
+
if (hit && Date.now() - hit.time < cfg.cacheTtlMs) {
|
|
1286
|
+
const sliced = sliceFrom(hit.full, offset, maxChars)
|
|
1287
|
+
return { ...sliced, cached: true }
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
const failWith = (error) => {
|
|
1291
|
+
cacheFail(cacheKey, error)
|
|
1292
|
+
return { error }
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1295
|
+
// Non-HTML payloads (JSON APIs, RSS/Atom feeds) are read natively: fetch,
|
|
1296
|
+
// dispatch by content-type / payload shape, render compact. Falls back to
|
|
1297
|
+
// the HTML pipeline for everything else.
|
|
1298
|
+
const dispatch = async () => {
|
|
1299
|
+
const viaSeam = await fetchViaWebSeam(ctx, url, externalSignal, cfg)
|
|
1300
|
+
if (viaSeam) {
|
|
1301
|
+
const t = viaSeam.html.trim()
|
|
1302
|
+
if (t.startsWith('{') || t.startsWith('[')) {
|
|
1303
|
+
try {
|
|
1304
|
+
return { kind: 'json', text: compactJson(JSON.parse(t)) }
|
|
1305
|
+
} catch { /* not JSON — keep HTML pipeline */ }
|
|
1306
|
+
}
|
|
1307
|
+
return { kind: 'html', html: viaSeam.html, finalUrl: viaSeam.finalUrl, charset: 'provider-decoded' }
|
|
1308
|
+
}
|
|
1309
|
+
const page = await fetchPage(url, externalSignal, cfg)
|
|
1310
|
+
if (page.error) return { error: page.error }
|
|
1311
|
+
const ct = (page.contentType || '').split(';')[0].toLowerCase().trim()
|
|
1312
|
+
const decoded = decodeBuffer(page.buffer, page.contentType)
|
|
1313
|
+
if (/json$/.test(ct)) {
|
|
1314
|
+
try {
|
|
1315
|
+
return { kind: 'json', text: compactJson(JSON.parse(decoded.text)) }
|
|
1316
|
+
} catch { /* invalid JSON — serve through the HTML pipeline as text */ }
|
|
1317
|
+
}
|
|
1318
|
+
if (/[+/]xml$/.test(ct)) {
|
|
1319
|
+
if (/<urlset[\s>]/i.test(decoded.text) || /<sitemapindex[\s>]/i.test(decoded.text)) {
|
|
1320
|
+
return { error: 'Unsupported content (XML sitemap)' }
|
|
1321
|
+
}
|
|
1322
|
+
if (/<rss[\s>]/i.test(decoded.text) || /<feed[\s>]/i.test(decoded.text)) {
|
|
1323
|
+
return { kind: 'feed', feed: parseFeed(decoded.text, cfg.maxLinks) }
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1326
|
+
return { kind: 'html', html: decoded.text, finalUrl: page.finalUrl || url, charset: decoded.charset }
|
|
1327
|
+
}
|
|
1328
|
+
const payload = await dispatch()
|
|
1329
|
+
if (payload.error) return failWith(payload.error)
|
|
1330
|
+
|
|
1331
|
+
if (payload.kind === 'json') {
|
|
1332
|
+
const full = { url, title: '', siteName: hostOf(url), lang: '', charset: 'json', mode: 'json', fullText: payload.text, paginated: 1 }
|
|
1333
|
+
cacheStore(cacheKey, full, cfg.cacheMax)
|
|
1334
|
+
return sliceFrom(full, offset, maxChars)
|
|
1335
|
+
}
|
|
1336
|
+
if (payload.kind === 'feed') {
|
|
1337
|
+
const f = payload.feed
|
|
1338
|
+
const full = {
|
|
1339
|
+
url, title: f.title || '', siteName: hostOf(url), lang: '', charset: 'feed',
|
|
1340
|
+
mode: 'feed', fullText: f.text, paginated: 1, feedCount: f.count,
|
|
1341
|
+
links: args.includeLinks === true ? f.items : undefined,
|
|
1342
|
+
}
|
|
1343
|
+
cacheStore(cacheKey, full, cfg.cacheMax)
|
|
1344
|
+
return sliceFrom(full, offset, maxChars)
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
let html = payload.html
|
|
1348
|
+
let finalUrl = payload.finalUrl || url
|
|
1349
|
+
let charset = payload.charset
|
|
1350
|
+
|
|
1351
|
+
// meta-refresh shells (link hops, anti-hotlink relays, no-JS SPA entries):
|
|
1352
|
+
// follow immediately, BEFORE extraction and SPA rendering — a shell that
|
|
1353
|
+
// redirects needs no headless round-trip, and extracting the stub is wasted
|
|
1354
|
+
// work. Fails open on any fetch problem; the follow is hop-bounded.
|
|
1355
|
+
const followed = await followMetaRefresh(html, finalUrl, externalSignal, cfg)
|
|
1356
|
+
html = followed.html
|
|
1357
|
+
finalUrl = followed.finalUrl
|
|
1358
|
+
// The target may use a different encoding than the shell page — reflect it.
|
|
1359
|
+
if (followed.charset) charset = followed.charset
|
|
1360
|
+
|
|
1361
|
+
let extracted = extract(html, mode, anchor)
|
|
1362
|
+
// Optional readability upgrade: cleaner article extraction when installed.
|
|
1363
|
+
const ext = await getReadabilityExtractor()
|
|
1364
|
+
// The base passed to readability must be the CURRENT final URL — after a
|
|
1365
|
+
// meta-refresh follow or SPA render the document may live on another host,
|
|
1366
|
+
// and relative links/images would resolve against the original request URL.
|
|
1367
|
+
const upgrade = (target, htmlText, base) => {
|
|
1368
|
+
if (!ext) return target
|
|
1369
|
+
try {
|
|
1370
|
+
const clean = ext(htmlText, base || finalUrl)
|
|
1371
|
+
if (clean && clean.length > 0) {
|
|
1372
|
+
target.text = mode === 'markdown'
|
|
1373
|
+
? blockMd(clean).replace(/\n{3,}/g, '\n\n').trim()
|
|
1374
|
+
: textLines(clean).replace(/ +/g, ' ').trim()
|
|
1375
|
+
}
|
|
1376
|
+
} catch {
|
|
1377
|
+
// keep heuristic result
|
|
1378
|
+
}
|
|
1379
|
+
return target
|
|
1380
|
+
}
|
|
1381
|
+
// Readability would replace the scoped section with the FULL article text;
|
|
1382
|
+
// when an anchor matched, the scoped extraction IS the answer.
|
|
1383
|
+
extracted = extracted.anchored ? extracted : upgrade(extracted, html)
|
|
1384
|
+
|
|
1385
|
+
// Optional SPA enhancement: if the page looks client-rendered and static
|
|
1386
|
+
// extraction found almost nothing, try headless rendering (playwright).
|
|
1387
|
+
// Beyond script-heavy shells (looksLikeSpa, ≥5 <script> tags), a page whose
|
|
1388
|
+
// extraction is EMPTY but carries any <script> may be a JS-redirect shell
|
|
1389
|
+
// (2 scripts, empty <body>, redirect issued from JS) — with no text to
|
|
1390
|
+
// lose, rendering is always worth one bounded attempt. Pages with no
|
|
1391
|
+
// scripts at all cannot change under rendering and are skipped.
|
|
1392
|
+
let rendered = false
|
|
1393
|
+
let spaHint = ''
|
|
1394
|
+
let renderedHtml = null
|
|
1395
|
+
const needsRender = !extracted.text || extracted.text.length < 200
|
|
1396
|
+
const scriptShell = !extracted.text && /<script[\s>]/i.test(html)
|
|
1397
|
+
if (cfg.spaRender !== false && needsRender && (looksLikeSpa(html) || scriptShell)) {
|
|
1398
|
+
const rr = await renderPage(finalUrl || url, externalSignal)
|
|
1399
|
+
if (rr.html) {
|
|
1400
|
+
renderedHtml = rr.html
|
|
1401
|
+
// A challenge interstitial (Cloudflare "Just a moment...") must never
|
|
1402
|
+
// be accepted as an improvement — it can carry MORE text than the real
|
|
1403
|
+
// static body and would evict the latter.
|
|
1404
|
+
if (looksLikeChallenge(rr.html)) {
|
|
1405
|
+
renderedHtml = null // pagination/links must not read the interstitial DOM
|
|
1406
|
+
spaHint = extracted.text
|
|
1407
|
+
? '已渲染但被人机验证页拦截,保留静态提取结果'
|
|
1408
|
+
: '已渲染但被人机验证页拦截(无静态内容可用)'
|
|
1409
|
+
} else {
|
|
1410
|
+
const r2raw = extract(rr.html, mode, anchor)
|
|
1411
|
+
const r2 = r2raw.anchored ? r2raw : upgrade(r2raw, rr.html, rr.finalUrl || finalUrl)
|
|
1412
|
+
// Accept the rendered text when it meaningfully beats the static one.
|
|
1413
|
+
// From an EMPTY static extraction even a short body is a real gain
|
|
1414
|
+
// (JS shells can render just 30-60 chars of real content, below the
|
|
1415
|
+
// fixed +50 margin).
|
|
1416
|
+
const gain = r2.text.length - extracted.text.length
|
|
1417
|
+
const accept = r2.text && (gain > 50 || (!extracted.text && r2.text.length >= 20))
|
|
1418
|
+
if (accept) {
|
|
1419
|
+
extracted = r2
|
|
1420
|
+
finalUrl = rr.finalUrl || finalUrl
|
|
1421
|
+
rendered = true
|
|
1422
|
+
} else if (!extracted.text) {
|
|
1423
|
+
spaHint = '已渲染仍无可读内容(可能登录墙或反爬拦截)'
|
|
1424
|
+
}
|
|
1425
|
+
}
|
|
1426
|
+
} else {
|
|
1427
|
+
spaHint = rr.error
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
// Auto-pagination: follow the "next page" chain (bounded, same-host, no
|
|
1432
|
+
// loops). Continuation pages go through the fast static path — a paginated
|
|
1433
|
+
// SPA chain would cost one full render per page, out of proportion.
|
|
1434
|
+
let paginated = 1
|
|
1435
|
+
const paginateMax = cfg.paginate === false ? 1 : Math.max(1, Math.min(10, Number(cfg.paginateMax) || 3))
|
|
1436
|
+
const startHost = hostOf(finalUrl || url)
|
|
1437
|
+
const seen = new Set([normalizeUrl(finalUrl || url)])
|
|
1438
|
+
// An anchored read is scoped to one section — joining the "next page" chain
|
|
1439
|
+
// would append whole pages the fragment never asked for.
|
|
1440
|
+
let nextUrl = paginateMax > 1 && !extracted.anchored ? findNextLink(renderedHtml || html, finalUrl || url) : null
|
|
1441
|
+
while (
|
|
1442
|
+
nextUrl && paginated < paginateMax &&
|
|
1443
|
+
sameHost(nextUrl, startHost) &&
|
|
1444
|
+
!(externalSignal && externalSignal.aborted)
|
|
1445
|
+
) {
|
|
1446
|
+
const nu = normalizeUrl(nextUrl)
|
|
1447
|
+
if (!nu || seen.has(nu)) break
|
|
1448
|
+
seen.add(nu)
|
|
1449
|
+
const page = await fetchPage(nextUrl, externalSignal, cfg)
|
|
1450
|
+
if (page.error) break
|
|
1451
|
+
const ct = (page.contentType || '').split(';')[0].toLowerCase().trim()
|
|
1452
|
+
if (!/html|xhtml/.test(ct)) break
|
|
1453
|
+
const nextHtml = decodeBuffer(page.buffer, page.contentType).text
|
|
1454
|
+
const nextEx = extract(nextHtml, mode)
|
|
1455
|
+
if (!nextEx.text || nextEx.text.length < 20) break
|
|
1456
|
+
extracted.text = joinPageText(extracted.text, nextEx.text)
|
|
1457
|
+
paginated++
|
|
1458
|
+
nextUrl = findNextLink(nextHtml, page.finalUrl || nextUrl)
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
// A near-empty body (login wall, anti-bot shell) with no render hint yet
|
|
1462
|
+
// still looks like "the page content" to the model — one constant-cost line
|
|
1463
|
+
// tells it not to keep offset-reading a 40-char page.
|
|
1464
|
+
if (!spaHint && extracted.text && extracted.text.length < 60) {
|
|
1465
|
+
spaHint = '正文极短(可能登录墙或反爬拦截)'
|
|
1466
|
+
}
|
|
1467
|
+
|
|
1468
|
+
const full = {
|
|
1469
|
+
url: finalUrl,
|
|
1470
|
+
title: extracted.title || '',
|
|
1471
|
+
siteName: extracted.siteName || '',
|
|
1472
|
+
lang: extracted.lang || '',
|
|
1473
|
+
charset,
|
|
1474
|
+
mode,
|
|
1475
|
+
fullText: extracted.text,
|
|
1476
|
+
rendered,
|
|
1477
|
+
spaHint,
|
|
1478
|
+
anchored: extracted.anchored || false,
|
|
1479
|
+
anchor: extracted.anchored ? anchor : '',
|
|
1480
|
+
paginated,
|
|
1481
|
+
published: extracted.published || '',
|
|
1482
|
+
author: extracted.author || '',
|
|
1483
|
+
// Links come from the rendered DOM when available — otherwise the SPA
|
|
1484
|
+
// chain (content -> links -> next page) would break for JS-only pages.
|
|
1485
|
+
links: args.includeLinks === true ? extractLinks(renderedHtml || html, cfg.maxLinks, finalUrl) : undefined,
|
|
1486
|
+
}
|
|
1487
|
+
|
|
1488
|
+
cacheStore(cacheKey, full, cfg.cacheMax)
|
|
1489
|
+
return sliceFrom(full, offset, maxChars)
|
|
1490
|
+
}
|
|
1491
|
+
|
|
1492
|
+
export function renderResult(value) {
|
|
1493
|
+
if (typeof value === 'string') return value
|
|
1494
|
+
const r = value || {}
|
|
1495
|
+
if (r.error) return `Error: ${r.error}`
|
|
1496
|
+
const lines = []
|
|
1497
|
+
if (r.title) lines.push(`title: ${r.title}`)
|
|
1498
|
+
const host = hostOf(r.url || '')
|
|
1499
|
+
const meta = []
|
|
1500
|
+
if (r.siteName && r.siteName !== host) meta.push(r.siteName)
|
|
1501
|
+
if (r.lang) meta.push(r.lang)
|
|
1502
|
+
if (r.charset && r.charset !== 'utf-8' && r.charset !== 'json' && r.charset !== 'feed') {
|
|
1503
|
+
meta.push(`charset ${r.charset}`)
|
|
1504
|
+
}
|
|
1505
|
+
if (r.published) meta.push(r.published)
|
|
1506
|
+
if (r.author) meta.push(`by ${r.author}`)
|
|
1507
|
+
// Tells the model why the text starts mid-document: the read was scoped to
|
|
1508
|
+
// this fragment, and offset continues from the section start.
|
|
1509
|
+
if (r.anchored) meta.push(`#${r.anchor || ''}`)
|
|
1510
|
+
if (r.feedCount) meta.push(`feed · ${r.feedCount} 条`)
|
|
1511
|
+
if (r.mode === 'json') meta.push('json')
|
|
1512
|
+
if (meta.length) lines.push(meta.join(' · '))
|
|
1513
|
+
// One compact status line instead of several parenthetical lines — the
|
|
1514
|
+
// flags are hints, not content, so they share a single line of tokens.
|
|
1515
|
+
const flags = []
|
|
1516
|
+
if (r.charsStart > 0) flags.push(`chars ${r.charsStart}+${r.charsReturned}/${r.charsTotal}`, 'offset 续读')
|
|
1517
|
+
else if (r.truncated) flags.push(`chars ${r.charsReturned}/${r.charsTotal}`, '截断', 'offset 续读')
|
|
1518
|
+
else if (typeof r.charsTotal === 'number') flags.push(`chars ${r.charsTotal}`)
|
|
1519
|
+
if (r.cached) flags.push('cached')
|
|
1520
|
+
if (r.paginated > 1) flags.push(`已拼接${r.paginated}页`)
|
|
1521
|
+
if (r.rendered) flags.push('rendered')
|
|
1522
|
+
// With text present the hint rides the flags line; with empty text the
|
|
1523
|
+
// dedicated "无可读内容" line below already carries it.
|
|
1524
|
+
if (r.text && r.spaHint) flags.push(r.spaHint)
|
|
1525
|
+
if (flags.length) lines.push(`(${flags.join(' · ')})`)
|
|
1526
|
+
if (!r.text) {
|
|
1527
|
+
if (r.charsStart > 0 && !r.truncated) {
|
|
1528
|
+
lines.push('(offset 已到文末,无更多内容)')
|
|
1529
|
+
} else {
|
|
1530
|
+
lines.push(r.spaHint
|
|
1531
|
+
? `(无可读内容 — ${r.spaHint})`
|
|
1532
|
+
: '(无可读内容 — 登录墙 / SPA 页 / 空页面;SPA 需安装 playwright)')
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
lines.push('', r.text || '')
|
|
1536
|
+
if (Array.isArray(r.links) && r.links.length) {
|
|
1537
|
+
lines.push('', 'links:')
|
|
1538
|
+
for (const l of r.links) lines.push(`- ${l.title} — ${l.url}`)
|
|
1539
|
+
}
|
|
1540
|
+
return lines.join('\n')
|
|
1541
|
+
}
|
|
1542
|
+
|
|
1543
|
+
function renderLinks(value) {
|
|
1544
|
+
if (typeof value === 'string') return value
|
|
1545
|
+
const r = value || {}
|
|
1546
|
+
if (r.error) return `Error: ${r.error}`
|
|
1547
|
+
if (!Array.isArray(r.links) || r.links.length === 0) return `No links found on ${r.url}`
|
|
1548
|
+
const lines = [`${r.count} link(s) on ${r.url}:`]
|
|
1549
|
+
for (const l of r.links) lines.push(`- ${l.title || l.url} — ${l.url}`)
|
|
1550
|
+
return lines.join('\n')
|
|
1551
|
+
}
|
|
1552
|
+
|
|
1553
|
+
function readLinksTool(ctx, cfg) {
|
|
1554
|
+
return {
|
|
1555
|
+
name: 'read_url_links',
|
|
1556
|
+
description:
|
|
1557
|
+
'List links (text + URL) on a page without body text — lighter than read_url ' +
|
|
1558
|
+
'for mapping what a page points to. Renders SPA pages when playwright installed.',
|
|
1559
|
+
parameters: {
|
|
1560
|
+
type: 'object',
|
|
1561
|
+
additionalProperties: false,
|
|
1562
|
+
properties: {
|
|
1563
|
+
url: { type: 'string', description: 'http(s) URL to scan for links' },
|
|
1564
|
+
limit: { type: 'number', description: 'Links to return (1-50; default from plugin config)' },
|
|
1565
|
+
},
|
|
1566
|
+
required: ['url'],
|
|
1567
|
+
},
|
|
1568
|
+
// Covers SPA-render fallback: full fetch timeout + playwright render cap.
|
|
1569
|
+
timeoutMs: Math.max(45000, cfg.timeoutMs + 45000),
|
|
1570
|
+
isConcurrencySafe: () => true,
|
|
1571
|
+
output: {
|
|
1572
|
+
schema: {
|
|
1573
|
+
type: 'object',
|
|
1574
|
+
additionalProperties: true,
|
|
1575
|
+
properties: {
|
|
1576
|
+
url: { type: 'string' },
|
|
1577
|
+
count: { type: 'number' },
|
|
1578
|
+
links: { type: 'array', items: { type: 'object' } },
|
|
1579
|
+
error: { type: 'string' },
|
|
1580
|
+
},
|
|
1581
|
+
},
|
|
1582
|
+
render: (_args, value) => [{ type: 'text', text: renderLinks(value) }],
|
|
1583
|
+
},
|
|
1584
|
+
async execute(args, exec) {
|
|
1585
|
+
const url = String((args && args.url) || '').trim()
|
|
1586
|
+
if (!/^https?:\/\//i.test(url)) return { error: 'Only http/https URLs are supported' }
|
|
1587
|
+
const limit = Math.max(1, Math.min(50, Number(args.limit) || cfg.maxLinks))
|
|
1588
|
+
const viaSeam = await fetchViaWebSeam(ctx, url, exec && exec.signal, cfg)
|
|
1589
|
+
let html, finalUrl
|
|
1590
|
+
if (viaSeam) {
|
|
1591
|
+
html = viaSeam.html
|
|
1592
|
+
finalUrl = viaSeam.finalUrl
|
|
1593
|
+
} else {
|
|
1594
|
+
const page = await fetchPage(url, exec && exec.signal, cfg)
|
|
1595
|
+
if (page.error) return { error: page.error }
|
|
1596
|
+
html = decodeBuffer(page.buffer, page.contentType).text
|
|
1597
|
+
finalUrl = page.finalUrl || url
|
|
1598
|
+
}
|
|
1599
|
+
// Same meta-refresh shell following as read_url: a link hop serves a
|
|
1600
|
+
// stub whose real links live at the target. Fails open, hop-bounded.
|
|
1601
|
+
const followed = await followMetaRefresh(html, finalUrl, exec && exec.signal, cfg)
|
|
1602
|
+
html = followed.html
|
|
1603
|
+
finalUrl = followed.finalUrl
|
|
1604
|
+
let links = extractLinks(html, limit, finalUrl)
|
|
1605
|
+
// SPA fallback: a JS-only page yields no links from its static HTML;
|
|
1606
|
+
// render it and re-extract when the static result looks empty. Any
|
|
1607
|
+
// <script> counts as renderable when there are no links at all — the
|
|
1608
|
+
// static shell may be a JS-redirect (same rationale as read_url).
|
|
1609
|
+
if (links.length < 3 && cfg.spaRender !== false && (looksLikeSpa(html) || (links.length === 0 && /<script[\s>]/i.test(html)))) {
|
|
1610
|
+
const rr = await renderPage(finalUrl || url, exec && exec.signal)
|
|
1611
|
+
// A challenge interstitial's links are the challenge's own scripts —
|
|
1612
|
+
// never an improvement over whatever the static HTML offered.
|
|
1613
|
+
if (rr.html && !looksLikeChallenge(rr.html)) {
|
|
1614
|
+
const rl = extractLinks(rr.html, limit, rr.finalUrl || finalUrl)
|
|
1615
|
+
if (rl.length > links.length) {
|
|
1616
|
+
links = rl
|
|
1617
|
+
finalUrl = rr.finalUrl || finalUrl
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
return { url: finalUrl, count: links.length, links }
|
|
1622
|
+
},
|
|
1623
|
+
}
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
// Run async tasks with a concurrency cap (avoids hammering a site / getting
|
|
1627
|
+
// rate-limited when reading many URLs at once).
|
|
1628
|
+
async function mapLimit(items, limit, fn) {
|
|
1629
|
+
const out = new Array(items.length)
|
|
1630
|
+
let idx = 0
|
|
1631
|
+
const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
|
1632
|
+
while (idx < items.length) {
|
|
1633
|
+
const i = idx++
|
|
1634
|
+
out[i] = await fn(items[i], i)
|
|
1635
|
+
}
|
|
1636
|
+
})
|
|
1637
|
+
await Promise.all(workers)
|
|
1638
|
+
return out
|
|
1639
|
+
}
|
|
1640
|
+
|
|
1641
|
+
function renderBatch(value) {
|
|
1642
|
+
if (typeof value === 'string') return value
|
|
1643
|
+
const v = value || {}
|
|
1644
|
+
if (v.error) return `Error: ${v.error}`
|
|
1645
|
+
const lines = [`读取 ${v.succeeded}/${v.total} 页成功${v.failed ? `,${v.failed} 页失败` : ''}`]
|
|
1646
|
+
for (const p of v.pages) {
|
|
1647
|
+
if (p.error) {
|
|
1648
|
+
lines.push('', `[失败] ${p.url} — ${p.error}`)
|
|
1649
|
+
continue
|
|
1650
|
+
}
|
|
1651
|
+
const head = p.title || p.url
|
|
1652
|
+
lines.push('', `--- ${head} (${p.chars} 字符${p.cached ? ' · cached' : ''}) ---`, p.text || '(无可读内容)')
|
|
1653
|
+
if (Array.isArray(p.links) && p.links.length) {
|
|
1654
|
+
const ls = p.links.map((l) => l.title + ' — ' + l.url)
|
|
1655
|
+
lines.push(`links: ${ls.slice(0, 6).join(' | ')}${ls.length > 6 ? ` …共${ls.length}个` : ''}`)
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
return lines.join('\n')
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
function readUrlBatchTool(ctx, cfg) {
|
|
1662
|
+
return {
|
|
1663
|
+
name: 'read_url_batch',
|
|
1664
|
+
description:
|
|
1665
|
+
'Read multiple URLs in parallel: each a compact clean block (same extraction + cache as read_url). ' +
|
|
1666
|
+
'Failures isolated and tagged. Max 10 URLs, capped at maxChars per page.',
|
|
1667
|
+
parameters: {
|
|
1668
|
+
type: 'object',
|
|
1669
|
+
additionalProperties: false,
|
|
1670
|
+
properties: {
|
|
1671
|
+
urls: { type: 'array', items: { type: 'string' }, description: 'URLs to read in parallel (1-10, http/s)' },
|
|
1672
|
+
maxChars: { type: 'number', description: 'Chars per page (500-20000; default 3000)' },
|
|
1673
|
+
mode: { type: 'string', enum: ['text', 'markdown'], description: 'text = plain (token-cheap, default); markdown = structured' },
|
|
1674
|
+
includeLinks: { type: 'boolean', description: 'Return page links too (title+url, bounded)' },
|
|
1675
|
+
},
|
|
1676
|
+
required: ['urls'],
|
|
1677
|
+
},
|
|
1678
|
+
// Worst case: ceil(10/4)=3 concurrency waves, each (fetch + SPA render).
|
|
1679
|
+
timeoutMs: Math.max(60000, cfg.timeoutMs * 3 + 135000),
|
|
1680
|
+
// Concurrent batch calls multiply fetch pressure but stay isolated (each
|
|
1681
|
+
// call owns its result array; the session cache is commutative).
|
|
1682
|
+
isConcurrencySafe: () => true,
|
|
1683
|
+
output: {
|
|
1684
|
+
schema: {
|
|
1685
|
+
type: 'object',
|
|
1686
|
+
additionalProperties: true,
|
|
1687
|
+
properties: {
|
|
1688
|
+
total: { type: 'number' },
|
|
1689
|
+
succeeded: { type: 'number' },
|
|
1690
|
+
failed: { type: 'number' },
|
|
1691
|
+
pages: { type: 'array', items: { type: 'object' } },
|
|
1692
|
+
error: { type: 'string' },
|
|
1693
|
+
},
|
|
1694
|
+
},
|
|
1695
|
+
render: (_args, value) => [{ type: 'text', text: renderBatch(value) }],
|
|
1696
|
+
},
|
|
1697
|
+
async execute(args, exec) {
|
|
1698
|
+
const a = args || {}
|
|
1699
|
+
const urls = (Array.isArray(a.urls) ? a.urls : []).map((u) => String(u).trim()).filter(Boolean)
|
|
1700
|
+
if (!urls.length) return { error: 'urls array is required (1-10 http(s) URLs)' }
|
|
1701
|
+
const list = urls.slice(0, 10)
|
|
1702
|
+
const perMax = Math.max(500, Math.min(20000, Number(a.maxChars) || 3000))
|
|
1703
|
+
const mode = a.mode === 'markdown' ? 'markdown' : 'text'
|
|
1704
|
+
const signal = exec && exec.signal
|
|
1705
|
+
const pages = await mapLimit(list, 4, (u) =>
|
|
1706
|
+
readUrl({ url: u, maxChars: perMax, mode, includeLinks: a.includeLinks === true }, ctx, signal, cfg),
|
|
1707
|
+
)
|
|
1708
|
+
const ok = pages.filter((p) => !p.error)
|
|
1709
|
+
return {
|
|
1710
|
+
total: list.length,
|
|
1711
|
+
succeeded: ok.length,
|
|
1712
|
+
failed: list.length - ok.length,
|
|
1713
|
+
pages: pages.map((p, i) => {
|
|
1714
|
+
if (p.error) return { url: list[i], error: p.error }
|
|
1715
|
+
const out = {
|
|
1716
|
+
url: p.url,
|
|
1717
|
+
title: p.title || '',
|
|
1718
|
+
chars: p.charsReturned,
|
|
1719
|
+
cached: p.cached === true,
|
|
1720
|
+
text: p.text,
|
|
1721
|
+
}
|
|
1722
|
+
if (Array.isArray(p.links) && p.links.length) out.links = p.links
|
|
1723
|
+
return out
|
|
1724
|
+
}),
|
|
1725
|
+
}
|
|
1726
|
+
},
|
|
1727
|
+
}
|
|
1728
|
+
}
|
|
1729
|
+
|
|
1730
|
+
function normalizeUrl(url) {
|
|
1731
|
+
try {
|
|
1732
|
+
const u = new URL(url)
|
|
1733
|
+
u.hash = ''
|
|
1734
|
+
return u.href
|
|
1735
|
+
} catch {
|
|
1736
|
+
return null
|
|
1737
|
+
}
|
|
1738
|
+
}
|
|
1739
|
+
|
|
1740
|
+
// b is a bare hostname (hostOf() output) — must NOT be passed to new URL()
|
|
1741
|
+
// as a full URL (that throws and would make every link look external).
|
|
1742
|
+
function sameHost(a, b) {
|
|
1743
|
+
try {
|
|
1744
|
+
return new URL(a).hostname === b
|
|
1745
|
+
} catch {
|
|
1746
|
+
return false
|
|
1747
|
+
}
|
|
1748
|
+
}
|
|
1749
|
+
|
|
1750
|
+
// URLs not worth crawling: static assets, login/auth paths, feeds, sitemaps.
|
|
1751
|
+
const NOISE_EXT = /\.(png|jpe?g|gif|svg|webp|ico|bmp|css|js|json|xml|pdf|zip|gz|tar|7z|mp3|m3u8?|mpd|flv|ts|mp4|avi|mov|webm|woff2?|ttf|eot|map)(\?|#|$)/i
|
|
1752
|
+
const NOISE_PATH = /(\/login|\/signin|\/register|\/logout|\/signup|\/api\/|\/admin|\/wp-admin|\/wp-login|\/feed|\/rss|\/sitemap|\/robots\.txt|\/cdn-cgi)/i
|
|
1753
|
+
function isNoiseUrl(url) {
|
|
1754
|
+
return NOISE_EXT.test(url) || NOISE_PATH.test(url)
|
|
1755
|
+
}
|
|
1756
|
+
|
|
1757
|
+
// BFS crawl of a single site. Only same-host http(s) pages are followed;
|
|
1758
|
+
// static assets / auth paths are skipped; visited URLs are deduped; a per-page
|
|
1759
|
+
// failure is recorded and does not abort the crawl. No SPA rendering here —
|
|
1760
|
+
// that is read_url's job; crawling favors speed and breadth.
|
|
1761
|
+
async function crawlSite(entryUrl, cfg, opts, externalSignal) {
|
|
1762
|
+
const { maxPages, maxDepth, includeContent, perMax } = opts
|
|
1763
|
+
const host = hostOf(entryUrl)
|
|
1764
|
+
const visited = new Set()
|
|
1765
|
+
const pages = []
|
|
1766
|
+
const failures = []
|
|
1767
|
+
const queue = [{ url: entryUrl, depth: 0 }]
|
|
1768
|
+
// Stop once the host's cooperative signal fires (tool budget hit): without
|
|
1769
|
+
// this the loop keeps draining the queue on instantly-aborted fetches.
|
|
1770
|
+
while (queue.length && pages.length < maxPages && !(externalSignal && externalSignal.aborted)) {
|
|
1771
|
+
// Never overshoot the page budget within a batch round.
|
|
1772
|
+
const batch = queue.splice(0, Math.min(2, maxPages - pages.length, queue.length))
|
|
1773
|
+
const results = await Promise.all(
|
|
1774
|
+
batch.map(async ({ url, depth }) => {
|
|
1775
|
+
const norm = normalizeUrl(url)
|
|
1776
|
+
if (!norm || visited.has(norm)) return null
|
|
1777
|
+
visited.add(norm)
|
|
1778
|
+
const page = await fetchPage(url, externalSignal, cfg)
|
|
1779
|
+
if (page.error) return { url, depth, error: page.error }
|
|
1780
|
+
const html = decodeBuffer(page.buffer, page.contentType).text
|
|
1781
|
+
const ex = extract(html, 'text')
|
|
1782
|
+
const links = extractLinks(html, 100, page.finalUrl || url)
|
|
1783
|
+
return {
|
|
1784
|
+
url: page.finalUrl || url,
|
|
1785
|
+
depth,
|
|
1786
|
+
title: ex.title || '',
|
|
1787
|
+
chars: ex.text.length,
|
|
1788
|
+
text: includeContent ? smartTruncate(ex.text, perMax).text : undefined,
|
|
1789
|
+
links,
|
|
1790
|
+
}
|
|
1791
|
+
}),
|
|
1792
|
+
)
|
|
1793
|
+
for (const r of results) {
|
|
1794
|
+
if (!r) continue
|
|
1795
|
+
if (r.error) {
|
|
1796
|
+
failures.push({ url: r.url, error: r.error })
|
|
1797
|
+
continue
|
|
1798
|
+
}
|
|
1799
|
+
pages.push(r)
|
|
1800
|
+
if (r.depth + 1 <= maxDepth) {
|
|
1801
|
+
// Queue cap: a huge site (millions of links) would otherwise grow the
|
|
1802
|
+
// queue unboundedly while maxPages bounds only what we *process*.
|
|
1803
|
+
if (queue.length >= maxPages * 20) continue
|
|
1804
|
+
for (const l of r.links) {
|
|
1805
|
+
if (!sameHost(l.url, host) || isNoiseUrl(l.url)) continue
|
|
1806
|
+
const n = normalizeUrl(l.url)
|
|
1807
|
+
if (n && !visited.has(n)) queue.push({ url: l.url, depth: r.depth + 1 })
|
|
1808
|
+
}
|
|
1809
|
+
}
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
return { host, pages, failures }
|
|
1813
|
+
}
|
|
1814
|
+
|
|
1815
|
+
function renderSite(value) {
|
|
1816
|
+
if (typeof value === 'string') return value
|
|
1817
|
+
const v = value || {}
|
|
1818
|
+
if (v.error) return `Error: ${v.error}`
|
|
1819
|
+
const lines = [`站点: ${v.host} · 爬取 ${v.succeeded}/${v.total} 页${v.failed ? ` · ${v.failed} 页失败` : ''}`]
|
|
1820
|
+
for (const p of v.pages) {
|
|
1821
|
+
const indent = ' '.repeat(p.depth)
|
|
1822
|
+
const head = p.title || p.url
|
|
1823
|
+
lines.push(`${indent}[${p.depth}] ${head} (${p.chars} 字符) ${p.url}`)
|
|
1824
|
+
if (p.text) lines.push(`${indent} ${p.text.slice(0, 80)}`)
|
|
1825
|
+
}
|
|
1826
|
+
for (const f of v.failures) lines.push(`[失败] ${f.url} — ${f.error}`)
|
|
1827
|
+
return lines.join('\n')
|
|
1828
|
+
}
|
|
1829
|
+
|
|
1830
|
+
function readUrlSiteTool(ctx, cfg) {
|
|
1831
|
+
return {
|
|
1832
|
+
name: 'read_url_site',
|
|
1833
|
+
description:
|
|
1834
|
+
'Crawl a site BFS from one URL: dedupe same-host pages, return a compact site map ' +
|
|
1835
|
+
'(title + depth + size). Auth/static paths skipped; failures isolated. ' +
|
|
1836
|
+
'includeContent=true attaches a short summary per page (default off, token-cheap). ' +
|
|
1837
|
+
'Does not render SPA pages (use read_url for that).',
|
|
1838
|
+
parameters: {
|
|
1839
|
+
type: 'object',
|
|
1840
|
+
additionalProperties: false,
|
|
1841
|
+
properties: {
|
|
1842
|
+
url: { type: 'string', description: 'Entry URL of the site to crawl (http/s)' },
|
|
1843
|
+
maxPages: { type: 'number', description: 'Pages to crawl (2-50; default 15)' },
|
|
1844
|
+
maxDepth: { type: 'number', description: 'Link depth from entry (1-5; default 2)' },
|
|
1845
|
+
includeContent: { type: 'boolean', description: 'Short body summary per page (default off; structure-only is token-cheap)' },
|
|
1846
|
+
maxCharsPerPage: { type: 'number', description: 'Summary chars per page (200-2000; default 500)' },
|
|
1847
|
+
},
|
|
1848
|
+
required: ['url'],
|
|
1849
|
+
},
|
|
1850
|
+
// Worst case: ceil(maxPages/2)=25 waves of paired fetches (no SPA render
|
|
1851
|
+
// here); the cooperative signal stops the loop early once the budget is hit.
|
|
1852
|
+
timeoutMs: Math.max(120000, cfg.timeoutMs * 25),
|
|
1853
|
+
// Crawl state (visited set, queue) is call-local; only the shared session
|
|
1854
|
+
// cache is global and commutative — parallel crawls of different hosts
|
|
1855
|
+
// are safe and useful (one task mapping several sites).
|
|
1856
|
+
isConcurrencySafe: () => true,
|
|
1857
|
+
output: {
|
|
1858
|
+
schema: {
|
|
1859
|
+
type: 'object',
|
|
1860
|
+
additionalProperties: true,
|
|
1861
|
+
properties: {
|
|
1862
|
+
host: { type: 'string' },
|
|
1863
|
+
total: { type: 'number' },
|
|
1864
|
+
succeeded: { type: 'number' },
|
|
1865
|
+
failed: { type: 'number' },
|
|
1866
|
+
pages: { type: 'array', items: { type: 'object' } },
|
|
1867
|
+
failures: { type: 'array', items: { type: 'object' } },
|
|
1868
|
+
error: { type: 'string' },
|
|
1869
|
+
},
|
|
1870
|
+
},
|
|
1871
|
+
render: (_args, value) => [{ type: 'text', text: renderSite(value) }],
|
|
1872
|
+
},
|
|
1873
|
+
async execute(args, exec) {
|
|
1874
|
+
const url = String((args && args.url) || '').trim()
|
|
1875
|
+
if (!/^https?:\/\//i.test(url)) return { error: 'Only http/https URLs are supported' }
|
|
1876
|
+
const maxPages = Math.max(2, Math.min(50, Number(args.maxPages) || 15))
|
|
1877
|
+
const maxDepth = Math.max(1, Math.min(5, Number(args.maxDepth) || 2))
|
|
1878
|
+
const includeContent = args.includeContent === true
|
|
1879
|
+
const perMax = Math.max(200, Math.min(2000, Number(args.maxCharsPerPage) || 500))
|
|
1880
|
+
const { host, pages, failures } = await crawlSite(
|
|
1881
|
+
url, cfg, { maxPages, maxDepth, includeContent, perMax }, exec && exec.signal,
|
|
1882
|
+
)
|
|
1883
|
+
return {
|
|
1884
|
+
host,
|
|
1885
|
+
total: pages.length + failures.length,
|
|
1886
|
+
succeeded: pages.length,
|
|
1887
|
+
failed: failures.length,
|
|
1888
|
+
pages: pages.map((p) => {
|
|
1889
|
+
const o = { url: p.url, depth: p.depth, title: p.title || '', chars: p.chars }
|
|
1890
|
+
if (p.text) o.text = p.text
|
|
1891
|
+
return o
|
|
1892
|
+
}),
|
|
1893
|
+
failures,
|
|
1894
|
+
}
|
|
1895
|
+
},
|
|
1896
|
+
}
|
|
1897
|
+
}
|
|
1898
|
+
|
|
1899
|
+
export function apply(ctx, config) {
|
|
1900
|
+
// Plugin-level config from cordis.patch.yml (config row), merged over defaults.
|
|
1901
|
+
// YAML is free-form: numbers can arrive quoted as strings, which would poison
|
|
1902
|
+
// arithmetic and AbortSignal timeouts downstream — coerce to numbers once
|
|
1903
|
+
// here, clamp to sane ranges, and fall back to defaults on garbage.
|
|
1904
|
+
const cfg = { ...DEFAULTS, ...(config || {}) }
|
|
1905
|
+
const clamp = (v, d, min, max) => {
|
|
1906
|
+
const n = Number(v)
|
|
1907
|
+
return Number.isFinite(n) ? Math.min(max, Math.max(min, n)) : d
|
|
1908
|
+
}
|
|
1909
|
+
cfg.timeoutMs = clamp(cfg.timeoutMs, DEFAULTS.timeoutMs, 500, 120000)
|
|
1910
|
+
cfg.maxBytes = clamp(cfg.maxBytes, DEFAULTS.maxBytes, 65536, 16 * 1024 * 1024)
|
|
1911
|
+
cfg.maxChars = clamp(cfg.maxChars, DEFAULTS.maxChars, 500, 20000)
|
|
1912
|
+
cfg.maxLinks = clamp(cfg.maxLinks, DEFAULTS.maxLinks, 1, 50)
|
|
1913
|
+
cfg.cacheTtlMs = clamp(cfg.cacheTtlMs, DEFAULTS.cacheTtlMs, 0, 3600000)
|
|
1914
|
+
cfg.cacheMax = clamp(cfg.cacheMax, DEFAULTS.cacheMax, 1, 256)
|
|
1915
|
+
cfg.paginateMax = clamp(cfg.paginateMax, DEFAULTS.paginateMax, 1, 10)
|
|
1916
|
+
if (typeof cfg.userAgent !== 'string' || !cfg.userAgent.trim()) cfg.userAgent = DEFAULTS.userAgent
|
|
1917
|
+
if (typeof cfg.spaRender !== 'boolean') cfg.spaRender = DEFAULTS.spaRender
|
|
1918
|
+
if (typeof cfg.paginate !== 'boolean') cfg.paginate = DEFAULTS.paginate
|
|
1919
|
+
|
|
1920
|
+
// Temporal composability: unload must fully revert side effects.
|
|
1921
|
+
ctx.effect(() => () => {
|
|
1922
|
+
cache.clear()
|
|
1923
|
+
failCache.clear()
|
|
1924
|
+
closeBrowser().catch(() => {})
|
|
1925
|
+
})
|
|
1926
|
+
|
|
1927
|
+
console.log('[dsh-read-url] plugin loaded; tools read_url, read_url_batch, read_url_links, read_url_site registered')
|
|
1928
|
+
|
|
1929
|
+
ctx.tools.register({
|
|
1930
|
+
name: 'read_url',
|
|
1931
|
+
description:
|
|
1932
|
+
'Fetch a page and return its clean main content (auto charset detect). ' +
|
|
1933
|
+
'text (default) = plain body capped at maxChars; markdown = headings/links/tables. ' +
|
|
1934
|
+
'Compact block: title, metadata, truncated body; 5-min session cache; ' +
|
|
1935
|
+
'SPA pages render when playwright installed; login walls are not accessible.',
|
|
1936
|
+
parameters: {
|
|
1937
|
+
type: 'object',
|
|
1938
|
+
additionalProperties: false,
|
|
1939
|
+
properties: {
|
|
1940
|
+
url: { type: 'string', description: 'http(s) URL to read' },
|
|
1941
|
+
maxChars: { type: 'number', description: 'Body text chars to return (500-20000; default from plugin config)' },
|
|
1942
|
+
offset: { type: 'number', description: 'Start at this char offset (continue long pages). Default 0; served from cache.' },
|
|
1943
|
+
mode: { type: 'string', enum: ['text', 'markdown'], description: 'text = plain (token-cheap, default); markdown = structured' },
|
|
1944
|
+
includeLinks: { type: 'boolean', description: 'Return page links too (title+url, bounded)' },
|
|
1945
|
+
},
|
|
1946
|
+
required: ['url'],
|
|
1947
|
+
},
|
|
1948
|
+
// Pipeline budget must cover the real worst case: first page fetch +
|
|
1949
|
+
// SPA render (goto 30s cap + DOM-stability poll 10s) + up to paginateMax
|
|
1950
|
+
// continuation fetches (static, no render). Derived from load-time cfg —
|
|
1951
|
+
// scale with it, never clamp it.
|
|
1952
|
+
timeoutMs: Math.max(45000, cfg.timeoutMs * (cfg.paginate === false ? 1 : cfg.paginateMax + 1) + 45000),
|
|
1953
|
+
// All shared state is commutative-safe under concurrent calls (session
|
|
1954
|
+
// cache Map get/set, decoder cache, one shared browser with isolated
|
|
1955
|
+
// pages per render), so the agent may fan out several read_url calls in
|
|
1956
|
+
// one parallel group — a real win for multi-source reading tasks.
|
|
1957
|
+
isConcurrencySafe: () => true,
|
|
1958
|
+
output: {
|
|
1959
|
+
schema: {
|
|
1960
|
+
type: 'object',
|
|
1961
|
+
additionalProperties: true,
|
|
1962
|
+
properties: {
|
|
1963
|
+
url: { type: 'string' },
|
|
1964
|
+
title: { type: 'string' },
|
|
1965
|
+
siteName: { type: 'string' },
|
|
1966
|
+
lang: { type: 'string' },
|
|
1967
|
+
charset: { type: 'string' },
|
|
1968
|
+
mode: { type: 'string' },
|
|
1969
|
+
truncated: { type: 'boolean' },
|
|
1970
|
+
charsTotal: { type: 'number' },
|
|
1971
|
+
charsReturned: { type: 'number' },
|
|
1972
|
+
text: { type: 'string' },
|
|
1973
|
+
links: { type: 'array', items: { type: 'object' } },
|
|
1974
|
+
cached: { type: 'boolean' },
|
|
1975
|
+
error: { type: 'string' },
|
|
1976
|
+
},
|
|
1977
|
+
},
|
|
1978
|
+
render: (_args, value) => [{ type: 'text', text: renderResult(value) }],
|
|
1979
|
+
},
|
|
1980
|
+
async execute(args, exec) {
|
|
1981
|
+
return readUrl(args, ctx, exec && exec.signal, cfg)
|
|
1982
|
+
},
|
|
1983
|
+
})
|
|
1984
|
+
|
|
1985
|
+
ctx.tools.register(readLinksTool(ctx, cfg))
|
|
1986
|
+
ctx.tools.register(readUrlBatchTool(ctx, cfg))
|
|
1987
|
+
ctx.tools.register(readUrlSiteTool(ctx, cfg))
|
|
1988
|
+
}
|