@danypops/pi-web-spider 0.10.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts ADDED
@@ -0,0 +1,635 @@
1
+ /**
2
+ * @danypops/pi-web-spider — Pi extension exposing web_fetch.
3
+ *
4
+ * Install: pi install git:github.com/DanyPops/web-spider
5
+ */
6
+ import { existsSync, mkdirSync, appendFileSync } from "node:fs"
7
+ import { homedir } from "node:os"
8
+ import { dirname, join } from "node:path"
9
+ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"
10
+ import { Type } from "typebox"
11
+ import type { Static } from "typebox"
12
+ import type { SpideredPage, DOMNode } from "@danypops/web-spider"
13
+ import { bodyLinks, highlightHit, leanOutput, linksOutput, markdownOutput, navLinksCount, omitEmpty } from "./format.js"
14
+
15
+ // ---------------------------------------------------------------------------
16
+ // Tool
17
+ // ---------------------------------------------------------------------------
18
+
19
+ export default async function (pi: ExtensionAPI) {
20
+ // Dynamic import bypasses jiti/Bun CJS interop, which can silently lose
21
+ // class constructors when require()-ing ESM packages with "type":"module".
22
+ // Native import() always uses the "import" condition and returns proper ESM.
23
+ const lib = await import("@danypops/web-spider")
24
+ const { spider, crawl, searchPages, SpiderCache, PageGraph, PlaywrightHttpClient,
25
+ queryTree, navigateTree, defaultSearchEngine, DomainThrottle, RobotsCache } = lib
26
+
27
+ // Browser processes are expensive — one shared instance per session.
28
+ let playwrightClient: InstanceType<typeof PlaywrightHttpClient> | null = null
29
+ const getPlaywrightClient = () => {
30
+ if (!playwrightClient) {
31
+ // /nonexistent in tests forces an immediate launch failure rather than a hang.
32
+ const executablePath = process.env["WEB_SPIDER_PLAYWRIGHT_EXECUTABLE"]
33
+ playwrightClient = new PlaywrightHttpClient(executablePath ? { executablePath } : undefined)
34
+ }
35
+ return playwrightClient
36
+ }
37
+
38
+ // Diagnostics go only to a file — never to stdout/stderr, which belong to Pi's TUI.
39
+ const diagPath = process.env["WEB_SPIDER_DIAG_PATH"] ?? join(homedir(), ".cache", "web-spider", "diag.log")
40
+ const diag = (entry: Record<string, unknown>) => {
41
+ const line = JSON.stringify({ ts: new Date().toISOString(), ...entry })
42
+ try { appendFileSync(diagPath, line + "\n") } catch { /* best-effort */ }
43
+ }
44
+ const log = (level: "info" | "warn" | "error", msg: string, extra?: unknown) => {
45
+ diag({ level, msg, ...extra !== undefined ? { extra } : {} })
46
+ }
47
+
48
+ // throttle.js and robots.js become undefined via the barrel under jiti
49
+ // tryNative:false (Bun binary mode) — they load as side-effects of crawl.js
50
+ // before index.js finishes its own re-exports. crawl() creates its own
51
+ // DomainThrottle / RobotsCache internally, so we don't need to import them.
52
+ //
53
+ // WEB_SPIDER_CACHE_PATH — pages JSON index (default: ~/.cache/web-spider/pages.json)
54
+ // WEB_SPIDER_IMAGES_PATH — large image files; DiskCache derives this from
55
+ // dirname(cachePath)/images, so set only to override.
56
+ // Falls back to in-memory SpiderCache if the cache path is not writable.
57
+ const cache = (() => {
58
+ const cachePath = process.env["WEB_SPIDER_CACHE_PATH"]
59
+ ?? join(homedir(), ".cache", "web-spider", "pages.json")
60
+ const imagesDir = process.env["WEB_SPIDER_IMAGES_PATH"]
61
+ ?? join(homedir(), ".cache", "web-spider", "images")
62
+ try {
63
+ const dir = dirname(cachePath)
64
+ if (!existsSync(dir)) mkdirSync(dir, { recursive: true })
65
+ if (!existsSync(imagesDir)) mkdirSync(imagesDir, { recursive: true })
66
+ return new lib.DiskCache(cachePath, { maxSize: 500, ttlMs: 30 * 60 * 1000 })
67
+ } catch {
68
+ return new SpiderCache({ maxSize: 200, ttlMs: 30 * 60 * 1000 })
69
+ }
70
+ })()
71
+ const graph = new PageGraph()
72
+
73
+ {
74
+ const storeRaw = (cache as unknown as Record<string, unknown>)["store"] ??
75
+ (cache as unknown as Record<string, unknown>)["map"]
76
+ diag({
77
+ tag: "boot-probe",
78
+ cacheClass: cache.constructor.name,
79
+ storeTag: Object.prototype.toString.call(storeRaw),
80
+ storeIsMap: storeRaw instanceof Map,
81
+ })
82
+ }
83
+ const corpus: SpideredPage[] = []
84
+
85
+ // ---------------------------------------------------------------------------
86
+ // Per-request helpers
87
+ // ---------------------------------------------------------------------------
88
+
89
+ type Params = Static<typeof paramsSchema>
90
+
91
+ // Shared throttle + robots checker for single-page spider() calls.
92
+ // (crawl() creates its own internally; these cover depth=0 fetches.)
93
+ const sharedThrottle = new DomainThrottle({ minDelayMs: 500 })
94
+ const sharedRobots = new RobotsCache()
95
+
96
+ function buildSpiderOpts(params: Params) {
97
+ return {
98
+ rootSelector: params.rootSelector,
99
+ excludeSelectors: params.excludeSelectors,
100
+ tokenBudget: params.tokenBudget,
101
+ timeoutMs: params.timeoutMs,
102
+ httpClient: params.enhanced ? getPlaywrightClient() : undefined,
103
+ throttle: sharedThrottle,
104
+ robotsCache: sharedRobots,
105
+ }
106
+ }
107
+
108
+ function buildFetchPage(params: Params) {
109
+ const spiderOpts = buildSpiderOpts(params)
110
+ return async (url: string): Promise<SpideredPage> => {
111
+ let probe = "cache.get"
112
+ try {
113
+ const hit = cache.get(url)
114
+ if (hit) return hit
115
+
116
+ log("info", "fetching", { url, enhanced: params.enhanced ?? false })
117
+ probe = "spider"
118
+ let page = await spider(url, spiderOpts)
119
+ log("info", "plain fetch done", { url, wordCount: page.wordCount, jsRendered: page.jsRendered })
120
+
121
+ if (page.jsRendered && !params.enhanced) {
122
+ log("info", "jsRendered detected, retrying with Playwright", { url })
123
+ probe = "spider(playwright)"
124
+ try {
125
+ page = await spider(url, { ...spiderOpts, httpClient: getPlaywrightClient() })
126
+ log("info", "Playwright fetch done", { url, wordCount: page.wordCount })
127
+ } catch (err) {
128
+ const msg = err instanceof Error ? err.message : String(err)
129
+ log("error", "Playwright fallback failed", { url, error: msg })
130
+ throw new Error(`Playwright fallback failed: ${msg}`)
131
+ }
132
+ }
133
+
134
+ probe = "cache.set"
135
+ cache.set(url, page)
136
+ corpus.push(page)
137
+ probe = "graph.addPage"
138
+ graph.addPage(page)
139
+ return page
140
+ } catch (err) {
141
+ diag({
142
+ tag: "call-site-throw",
143
+ probe,
144
+ error: err instanceof Error ? err.message : String(err),
145
+ stack: err instanceof Error ? err.stack?.split("\n").slice(0, 6).join(" | ") : undefined,
146
+ })
147
+ throw err
148
+ }
149
+ }
150
+ }
151
+
152
+ // ---------------------------------------------------------------------------
153
+ // Local materialized view helpers
154
+ // ---------------------------------------------------------------------------
155
+
156
+ function cachedPages(): SpideredPage[] {
157
+ return cache.values()
158
+ }
159
+
160
+ // ---------------------------------------------------------------------------
161
+ // Path handlers — each owns one execution branch. SRP: one reason to change.
162
+ // ---------------------------------------------------------------------------
163
+
164
+ async function handleCrawl(params: Params) {
165
+ const spiderOpts = buildSpiderOpts(params)
166
+ const fmt = params.format ?? "markdown"
167
+ const depth = params.depth ?? 0
168
+ const url = params.url ?? ""
169
+
170
+ const result = await crawl(url, {
171
+ maxDepth: depth,
172
+ maxPages: params.maxPages ?? 10,
173
+ sameDomainOnly: params.sameDomain ?? true,
174
+ cache,
175
+ graph,
176
+ onPage: (page) => corpus.push(page),
177
+ ...spiderOpts,
178
+ })
179
+
180
+ const pages = [...result.pages.values()]
181
+ const errorsObj = result.errors.size
182
+ ? { errors: result.errors.size, errorUrls: [...result.errors.keys()] }
183
+ : {}
184
+
185
+ if (fmt === "highlights") {
186
+ if (!params.query?.trim()) {
187
+ return {
188
+ content: [{ type: "text" as const, text: JSON.stringify({ error: "highlights format requires a query." }) }],
189
+ details: {},
190
+ }
191
+ }
192
+ const hits = searchPages(pages, params.query, { topN: 8, snippetRadius: 150 })
193
+ return {
194
+ content: [{ type: "text" as const, text: JSON.stringify({
195
+ query: params.query,
196
+ pagesSearched: pages.length,
197
+ hits: hits.map((h) => ({
198
+ url: h.url,
199
+ ...highlightHit(h, pages.find((p) => p.url === h.url)?.chunks ?? []),
200
+ })),
201
+ }) }],
202
+ details: { format: "highlights", pagesSearched: pages.length, hits: hits.length },
203
+ }
204
+ }
205
+
206
+ const summary = fmt === "lean"
207
+ ? { pagesFound: result.pages.size, ...errorsObj, pages: pages.map(leanOutput) }
208
+ : {
209
+ pagesFound: result.pages.size,
210
+ ...errorsObj,
211
+ note: "All pages cached — use web_fetch(depth=0, format=highlights, query=...) to search them.",
212
+ pages: pages.map((p) => omitEmpty({ url: p.url, title: p.title, description: p.description, wordCount: p.wordCount, tags: p.tags })),
213
+ }
214
+
215
+ return {
216
+ content: [{ type: "text" as const, text: JSON.stringify(summary) }],
217
+ details: { depth, pagesFound: result.pages.size },
218
+ }
219
+ }
220
+
221
+ // Trees are too large and volatile for DiskCache — session-scoped only.
222
+ const treeCache = new Map<string, DOMNode>()
223
+
224
+ async function fetchTree(url: string, timeoutMs?: number): Promise<DOMNode> {
225
+ const hit = treeCache.get(url)
226
+ if (hit) return hit
227
+ const page = await spider(url, { view: "tree", timeoutMs })
228
+ treeCache.set(url, page.tree)
229
+ return page.tree
230
+ }
231
+
232
+ function handleCacheListing(params: Params) {
233
+ let pages = cachedPages()
234
+ const total = pages.length
235
+
236
+ if (params.grep?.trim()) {
237
+ const pat = params.grep.toLowerCase()
238
+ pages = pages.filter(
239
+ (p) =>
240
+ p.url.toLowerCase().includes(pat) ||
241
+ p.title.toLowerCase().includes(pat) ||
242
+ p.domain.toLowerCase().includes(pat) ||
243
+ (p.description ?? "").toLowerCase().includes(pat),
244
+ )
245
+ }
246
+
247
+ const filtered = pages.length
248
+ const offset = params.offset ?? 0
249
+ const limit = Math.min(params.limit ?? 20, 100) // hard cap at 100
250
+ const slice = pages.slice(offset, offset + limit)
251
+ const remaining = filtered - offset - slice.length
252
+
253
+ const meta = omitEmpty({
254
+ total,
255
+ filtered: filtered !== total ? filtered : undefined,
256
+ offset: offset || undefined,
257
+ limit,
258
+ remaining: remaining > 0 ? remaining : undefined,
259
+ })
260
+
261
+ return {
262
+ content: [{ type: "text" as const, text: JSON.stringify({ ...meta, pages: slice.map(leanOutput) }) }],
263
+ details: { format: "lean", total, filtered, offset, limit },
264
+ }
265
+ }
266
+
267
+ function handleCacheSearch(params: Params) {
268
+ const pages = cachedPages()
269
+ if (pages.length === 0) {
270
+ return {
271
+ content: [{ type: "text" as const, text: JSON.stringify({ error: "Local cache is empty. Fetch some pages first with depth=0 or depth>0." }) }],
272
+ details: {},
273
+ }
274
+ }
275
+
276
+ const topN = params.limit ?? 10
277
+ const hits = searchPages(pages, params.query ?? "", { topN, snippetRadius: 150 })
278
+ return {
279
+ content: [{ type: "text" as const, text: JSON.stringify({
280
+ ...omitEmpty({ query: params.query, pagesSearched: pages.length }),
281
+ hits: hits.map((h) => ({
282
+ url: h.url,
283
+ ...highlightHit(h, pages.find((p) => p.url === h.url)?.chunks ?? []),
284
+ })),
285
+ ...(hits.length === 0 ? { hint: "No matches. Try broader terms, or list cached pages with web_fetch(format=lean) and no url." } : {}),
286
+ }) }],
287
+ details: { format: "highlights", pagesSearched: pages.length, hits: hits.length },
288
+ }
289
+ }
290
+
291
+ async function handleSinglePage(params: Params, fetchPage: ReturnType<typeof buildFetchPage>) {
292
+ const fmt = params.format ?? "markdown"
293
+ const url = params.url ?? ""
294
+
295
+ // ── Tree formats ───────────────────────────────────────────────────────────────
296
+ if (fmt === "tree") {
297
+ try {
298
+ const tree = await fetchTree(url, params.timeoutMs)
299
+
300
+ // path= → navigate to a specific node
301
+ if (params.path) {
302
+ const node = navigateTree(tree, params.path)
303
+ if (!node) {
304
+ return {
305
+ content: [{ type: "text" as const, text: JSON.stringify({ error: `Path not found: ${params.path}` }) }],
306
+ details: {},
307
+ }
308
+ }
309
+ return {
310
+ content: [{ type: "text" as const, text: JSON.stringify(node) }],
311
+ details: { format: "tree", mode: "navigate", tag: node.tag, path: node.path },
312
+ }
313
+ }
314
+
315
+ // query= → search the tree (atomic hits — whole code blocks, whole table rows)
316
+ if (params.query?.trim()) {
317
+ const hits = queryTree(tree, params.query, { topN: params.topN ?? 5 })
318
+ return {
319
+ content: [{ type: "text" as const, text: JSON.stringify(omitEmpty({
320
+ url: params.url,
321
+ query: params.query,
322
+ hits: hits.map((h) => omitEmpty({
323
+ path: h.path,
324
+ tag: h.node.tag,
325
+ score: Math.round(h.score * 100) / 100,
326
+ snippet: h.snippet,
327
+ text: h.node.text,
328
+ childCount: h.node.children?.length,
329
+ })),
330
+ })) }],
331
+ details: { format: "tree", mode: "query", hits: hits.length },
332
+ }
333
+ }
334
+
335
+ // no query, no path → return the full tree
336
+ return {
337
+ content: [{ type: "text" as const, text: JSON.stringify(tree) }],
338
+ details: { format: "tree", mode: "full" },
339
+ }
340
+ } catch (err) {
341
+ const message = err instanceof Error ? err.message : String(err)
342
+ return {
343
+ content: [{ type: "text" as const, text: JSON.stringify({ error: message }) }],
344
+ details: {},
345
+ }
346
+ }
347
+ }
348
+
349
+ const page = await fetchPage(url)
350
+
351
+ if (fmt === "lean") {
352
+ return {
353
+ content: [{ type: "text" as const, text: JSON.stringify(leanOutput(page)) }],
354
+ details: { format: "lean", wordCount: page.wordCount },
355
+ }
356
+ }
357
+
358
+ if (fmt === "links") {
359
+ return {
360
+ content: [{ type: "text" as const, text: JSON.stringify(linksOutput(page)) }],
361
+ details: { format: "links", bodyLinks: bodyLinks(page).length, navLinksCount: navLinksCount(page) },
362
+ }
363
+ }
364
+
365
+ if (fmt === "highlights") {
366
+ if (!params.query?.trim()) {
367
+ return {
368
+ content: [{ type: "text" as const, text: JSON.stringify({
369
+ error: "highlights format requires a query.",
370
+ hint: "Pass query='what you are looking for', or use format=markdown to read the full page.",
371
+ }) }],
372
+ details: {},
373
+ }
374
+ }
375
+ const hits = searchPages([page], params.query, { topN: 5, snippetRadius: 150 })
376
+ return {
377
+ content: [{ type: "text" as const, text: JSON.stringify({
378
+ ...omitEmpty({ url: page.url, title: page.title, query: params.query }),
379
+ hits: hits.map((h) => highlightHit(h, page.chunks)),
380
+ ...(hits.length === 0 ? { hint: "No matches. Try broader terms or use format=markdown." } : {}),
381
+ }) }],
382
+ details: { format: "highlights", hits: hits.length },
383
+ }
384
+ }
385
+
386
+ // markdown (default)
387
+ return {
388
+ content: [{ type: "text" as const, text: JSON.stringify(markdownOutput(page)) }],
389
+ details: { format: "markdown", wordCount: page.wordCount },
390
+ }
391
+ }
392
+
393
+ // ---------------------------------------------------------------------------
394
+ // Tool registration
395
+ // ---------------------------------------------------------------------------
396
+
397
+ // Defined here so Params = Static<typeof paramsSchema> resolves concretely
398
+ // rather than being derived through registerTool's unresolved generic.
399
+ const paramsSchema = Type.Object({
400
+ url: Type.Optional(Type.String({ description: "Fully-qualified http(s) URL to fetch or crawl from" })),
401
+
402
+ depth: Type.Optional(
403
+ Type.Number({
404
+ description:
405
+ "BFS depth. 0=single page (default). 1=page + all its links. N=N hops deep.",
406
+ })
407
+ ),
408
+ maxPages: Type.Optional(
409
+ Type.Number({
410
+ description: "Hard cap on total pages when depth>0 (default 10).",
411
+ })
412
+ ),
413
+ sameDomain: Type.Optional(
414
+ Type.Boolean({
415
+ description: "Only follow links on the same domain when depth>0 (default true).",
416
+ })
417
+ ),
418
+
419
+ enhanced: Type.Optional(
420
+ Type.Boolean({
421
+ description:
422
+ "When true, always uses a headless browser (playwright-core + system Chrome, stealth mode). " +
423
+ "When false (default), direct fetch is used and Playwright kicks in automatically " +
424
+ "only if the page is detected as JS-rendered.",
425
+ })
426
+ ),
427
+
428
+ format: Type.Optional(
429
+ Type.Union(
430
+ [
431
+ Type.Literal("markdown"),
432
+ Type.Literal("lean"),
433
+ Type.Literal("links"),
434
+ Type.Literal("highlights"),
435
+ Type.Literal("tree"),
436
+ ],
437
+ {
438
+ description:
439
+ "markdown=full body (default), lean=outline only, links=link list, highlights=BM25F chunks, tree=semantic DOM tree.",
440
+ }
441
+ )
442
+ ),
443
+ query: Type.Optional(
444
+ Type.String({
445
+ description: "Search phrase. Required for format=highlights. Optional for format=tree (searches the tree).",
446
+ })
447
+ ),
448
+ path: Type.Optional(
449
+ Type.String({
450
+ description: "Dot-bracket path for format=tree navigation, e.g. article.section[1].pre[0].code",
451
+ })
452
+ ),
453
+ topN: Type.Optional(
454
+ Type.Number({
455
+ description: "Max hits to return for format=tree with query (default 5).",
456
+ })
457
+ ),
458
+
459
+ grep: Type.Optional(
460
+ Type.String({
461
+ description:
462
+ "Filter cached pages by substring match on url, title, domain, or description. Only applies when url is omitted (local cache listing).",
463
+ })
464
+ ),
465
+ offset: Type.Optional(
466
+ Type.Number({
467
+ description: "Skip first N results when listing or searching the local cache (pagination).",
468
+ })
469
+ ),
470
+ limit: Type.Optional(
471
+ Type.Number({
472
+ description: "Max results to return from cache listing or search (default 20, hard cap 100 for listing, 10 for search).",
473
+ })
474
+ ),
475
+
476
+ rootSelector: Type.Optional(
477
+ Type.String({
478
+ description:
479
+ "CSS selector to scope extraction (e.g. \"article\"). Discards everything outside.",
480
+ })
481
+ ),
482
+ excludeSelectors: Type.Optional(
483
+ Type.String({
484
+ description:
485
+ "Comma-separated CSS selectors to remove before extraction (e.g. \"nav, footer, .sidebar\").",
486
+ })
487
+ ),
488
+ tokenBudget: Type.Optional(
489
+ Type.Number({
490
+ description:
491
+ "Approximate max tokens to return (~4 chars/token). Truncated at a line boundary.",
492
+ })
493
+ ),
494
+ searchQuery: Type.Optional(
495
+ Type.String({
496
+ description:
497
+ "Web search query. Pass instead of url when you don't know the exact URL. " +
498
+ "Returns ranked results (url, title, snippet) from Brave/Tavily/Exa/DDG. " +
499
+ "Use the returned URLs to fetch the actual page content.",
500
+ })
501
+ ),
502
+ timeoutMs: Type.Optional(
503
+ Type.Number({
504
+ description:
505
+ "Per-request fetch timeout in milliseconds (default 30 000). " +
506
+ "Increase for slow sites; decrease to fail fast in latency-sensitive loops.",
507
+ })
508
+ ),
509
+ })
510
+
511
+ pi.registerTool({
512
+ name: "web_fetch",
513
+ label: "Web Fetch",
514
+ description: [
515
+ "Fetch a URL and return its content. Optionally crawl to a given depth.",
516
+ "Can also search the web when searchQuery is provided instead of a URL.",
517
+ "",
518
+ "SEARCH FIRST — avoid hallucinated URLs",
519
+ " If you are not certain the URL exists, pass searchQuery instead of url.",
520
+ " The tool will run a web search and return ranked results with real URLs.",
521
+ " Then fetch the result URL you want. Never guess article slugs or paths.",
522
+ " Example wrong: web_fetch(url='martinfowler.com/articles/agent-as-platform.html')",
523
+ " Example right: web_fetch(searchQuery='Martin Fowler agent as platform')",
524
+ "",
525
+ "LOCAL MATERIALIZED VIEW (no url)",
526
+ " Omit url to query the local page cache (disk-backed, survives restarts).",
527
+ " No url, no query — list all cached pages in lean format.",
528
+ " No url, query=X — BM25F full-text search across all cached pages.",
529
+ " grep=X — filter list by url/title/domain/description substring.",
530
+ " offset/limit — paginate results (default limit 20, hard cap 100).",
531
+ "",
532
+ "DEPTH",
533
+ " depth=0 (default) — fetch the single URL.",
534
+ " depth=1 — fetch the URL and every page it links to (same domain).",
535
+ " depth=N — BFS crawl N hops deep, up to maxPages total.",
536
+ " When depth>0, returns a crawl summary and caches all pages.",
537
+ " Subsequent calls with depth=0 to any cached URL are free (no network).",
538
+ "",
539
+ "FORMAT",
540
+ " markdown — clean markdown body + metadata. Default.",
541
+ " lean — metadata + headings + links, no body text. ~10-20x fewer tokens.",
542
+ " Best for deciding whether to read a page, or crawl triage.",
543
+ " links — outbound links only (href + anchor text + rel).",
544
+ " highlights — BM25F search the page and return matching text blocks.",
545
+ " Requires `query`. Returns up to 5 scored chunks with context.",
546
+ " Use instead of reading full markdown when you know what to find.",
547
+ " Works across all cached pages when depth>0.",
548
+ " tree — collapsed semantic DOM tree (div/span stripped, only meaningful tags).",
549
+ " Add query= to search the tree (atomic hits: whole code blocks, whole tables).",
550
+ " Add path= to navigate to one node (e.g. article.section[1].pre[0].code).",
551
+ " Tree is cached — tree then tree+query then tree+path costs one network request.",
552
+ "",
553
+ "SCOPING",
554
+ " rootSelector — CSS selector to scope to (e.g. \"article\"). Ignores everything else.",
555
+ " excludeSelectors — comma-separated selectors to strip (e.g. \"nav, footer, .ads\").",
556
+ " tokenBudget — max ~tokens returned (~4 chars/token). Truncates at line boundary.",
557
+ "",
558
+ "ENHANCED MODE (JS rendering)",
559
+ " enhanced=true — use a headless browser with stealth (playwright-core + system Chrome).",
560
+ " Use for SPAs, JS-heavy pages, or sites with basic bot detection.",
561
+ " enhanced=false — use direct fetch (default). Playwright auto-fallback kicks in",
562
+ " when the page is detected as JS-rendered.",
563
+ "",
564
+ "THROTTLING",
565
+ " Requests are automatically rate-limited per domain (500ms min delay).",
566
+ " On 429/503, backs off exponentially and respects Retry-After headers.",
567
+ " robots.txt is checked and respected before each fetch (depth=0 and depth>0).",
568
+ ].join("\n"),
569
+ promptSnippet:
570
+ "Fetch URL: format=markdown/lean/links/highlights, depth, rootSelector, tokenBudget",
571
+ parameters: paramsSchema,
572
+
573
+
574
+ // -------------------------------------------------------------------------
575
+ // Router — routes to the correct path handler. One reason to change: routing
576
+ // logic. Business logic lives in the handlers above.
577
+ // -------------------------------------------------------------------------
578
+ async execute(_id, params, _signal, _onUpdate, _ctx) {
579
+ try {
580
+ const fetchPage = buildFetchPage(params)
581
+
582
+ // Web search path — searchQuery instead of url.
583
+ if (params.searchQuery?.trim()) {
584
+ try {
585
+ const engine = defaultSearchEngine()
586
+ const results = await engine.search({ query: params.searchQuery, numResults: params.limit ?? 10 })
587
+ log("info", "web search done", { query: params.searchQuery, hits: results.length })
588
+ return {
589
+ content: [{ type: "text" as const, text: JSON.stringify({
590
+ query: params.searchQuery,
591
+ results,
592
+ hint: "Use the url field from a result to fetch its full content with web_fetch(url=...).",
593
+ }) }],
594
+ details: { searchQuery: params.searchQuery, hits: results.length },
595
+ }
596
+ } catch (err) {
597
+ const message = err instanceof Error ? err.message : String(err)
598
+ return {
599
+ content: [{ type: "text" as const, text: JSON.stringify({ error: `Web search failed: ${message}` }) }],
600
+ details: {},
601
+ }
602
+ }
603
+ }
604
+
605
+ // Local materialized view path — no url: query the cache directly.
606
+ if (!params.url) {
607
+ if (params.query?.trim()) return handleCacheSearch(params)
608
+ return handleCacheListing(params)
609
+ }
610
+
611
+ if ((params.depth ?? 0) > 0) return await handleCrawl(params)
612
+
613
+ return await handleSinglePage(params, fetchPage)
614
+ } catch (err) {
615
+ if (err instanceof Error && err.message.startsWith("Blocked by robots.txt:")) {
616
+ return {
617
+ content: [{ type: "text" as const, text: JSON.stringify({
618
+ blocked: true,
619
+ url: params.url,
620
+ reason: "robots.txt",
621
+ hint: "The site's robots.txt disallows crawling this URL. Try a different path or domain.",
622
+ }) }],
623
+ details: { blocked: true },
624
+ }
625
+ }
626
+ const message = err instanceof Error ? err.message : String(err)
627
+ return {
628
+ content: [{ type: "text" as const, text: JSON.stringify({ error: message }) }],
629
+ details: {},
630
+ isError: true,
631
+ }
632
+ }
633
+ },
634
+ })
635
+ }