@iflow-mcp/jakeliume-webpeel 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +313 -0
- package/dist/cache.d.ts +30 -0
- package/dist/cache.js +139 -0
- package/dist/cli/commands/auth.d.ts +5 -0
- package/dist/cli/commands/auth.js +411 -0
- package/dist/cli/commands/doctor.d.ts +37 -0
- package/dist/cli/commands/doctor.js +371 -0
- package/dist/cli/commands/fetch.d.ts +6 -0
- package/dist/cli/commands/fetch.js +1345 -0
- package/dist/cli/commands/guide.d.ts +2 -0
- package/dist/cli/commands/guide.js +183 -0
- package/dist/cli/commands/interact.d.ts +5 -0
- package/dist/cli/commands/interact.js +840 -0
- package/dist/cli/commands/jobs.d.ts +5 -0
- package/dist/cli/commands/jobs.js +997 -0
- package/dist/cli/commands/monitor.d.ts +12 -0
- package/dist/cli/commands/monitor.js +197 -0
- package/dist/cli/commands/observe.d.ts +12 -0
- package/dist/cli/commands/observe.js +158 -0
- package/dist/cli/commands/screenshot.d.ts +5 -0
- package/dist/cli/commands/screenshot.js +282 -0
- package/dist/cli/commands/search.d.ts +5 -0
- package/dist/cli/commands/search.js +1021 -0
- package/dist/cli/commands/setup.d.ts +13 -0
- package/dist/cli/commands/setup.js +244 -0
- package/dist/cli/commands/skill.d.ts +15 -0
- package/dist/cli/commands/skill.js +195 -0
- package/dist/cli/utils.d.ts +84 -0
- package/dist/cli/utils.js +806 -0
- package/dist/cli-auth.d.ts +75 -0
- package/dist/cli-auth.js +369 -0
- package/dist/cli.d.ts +17 -0
- package/dist/cli.js +99 -0
- package/dist/core/actions.d.ts +69 -0
- package/dist/core/actions.js +495 -0
- package/dist/core/agent.d.ts +98 -0
- package/dist/core/agent.js +558 -0
- package/dist/core/answer.d.ts +42 -0
- package/dist/core/answer.js +395 -0
- package/dist/core/application-tracker.d.ts +84 -0
- package/dist/core/application-tracker.js +184 -0
- package/dist/core/apply.d.ts +162 -0
- package/dist/core/apply.js +816 -0
- package/dist/core/auth-detection.d.ts +35 -0
- package/dist/core/auth-detection.js +358 -0
- package/dist/core/auto-extract.d.ts +82 -0
- package/dist/core/auto-extract.js +604 -0
- package/dist/core/auto-interact.d.ts +23 -0
- package/dist/core/auto-interact.js +246 -0
- package/dist/core/bm25-filter.d.ts +66 -0
- package/dist/core/bm25-filter.js +288 -0
- package/dist/core/branding.d.ts +54 -0
- package/dist/core/branding.js +234 -0
- package/dist/core/browser-fetch.d.ts +323 -0
- package/dist/core/browser-fetch.js +1600 -0
- package/dist/core/browser-pool.d.ts +91 -0
- package/dist/core/browser-pool.js +550 -0
- package/dist/core/budget.d.ts +42 -0
- package/dist/core/budget.js +324 -0
- package/dist/core/business-intel.d.ts +47 -0
- package/dist/core/business-intel.js +279 -0
- package/dist/core/cache.d.ts +13 -0
- package/dist/core/cache.js +121 -0
- package/dist/core/cf-worker-proxy.d.ts +32 -0
- package/dist/core/cf-worker-proxy.js +87 -0
- package/dist/core/challenge-detection.d.ts +26 -0
- package/dist/core/challenge-detection.js +468 -0
- package/dist/core/change-tracking.d.ts +75 -0
- package/dist/core/change-tracking.js +276 -0
- package/dist/core/chunker.d.ts +46 -0
- package/dist/core/chunker.js +249 -0
- package/dist/core/chunking.d.ts +42 -0
- package/dist/core/chunking.js +181 -0
- package/dist/core/circuit-breaker.d.ts +44 -0
- package/dist/core/circuit-breaker.js +85 -0
- package/dist/core/content-pruner.d.ts +47 -0
- package/dist/core/content-pruner.js +425 -0
- package/dist/core/cookie-cache.d.ts +60 -0
- package/dist/core/cookie-cache.js +163 -0
- package/dist/core/crawl-checkpoint.d.ts +54 -0
- package/dist/core/crawl-checkpoint.js +104 -0
- package/dist/core/crawler.d.ts +84 -0
- package/dist/core/crawler.js +349 -0
- package/dist/core/cross-verify.d.ts +27 -0
- package/dist/core/cross-verify.js +93 -0
- package/dist/core/deep-fetch.d.ts +74 -0
- package/dist/core/deep-fetch.js +405 -0
- package/dist/core/deep-research.d.ts +141 -0
- package/dist/core/deep-research.js +972 -0
- package/dist/core/design-analysis.d.ts +70 -0
- package/dist/core/design-analysis.js +490 -0
- package/dist/core/design-compare.d.ts +38 -0
- package/dist/core/design-compare.js +264 -0
- package/dist/core/diff.d.ts +61 -0
- package/dist/core/diff.js +289 -0
- package/dist/core/dns-cache.d.ts +20 -0
- package/dist/core/dns-cache.js +198 -0
- package/dist/core/documents.d.ts +23 -0
- package/dist/core/documents.js +123 -0
- package/dist/core/domain-memory.d.ts +66 -0
- package/dist/core/domain-memory.js +163 -0
- package/dist/core/domain-verify.d.ts +40 -0
- package/dist/core/domain-verify.js +379 -0
- package/dist/core/engine-ranker.d.ts +112 -0
- package/dist/core/engine-ranker.js +395 -0
- package/dist/core/extract-inline.d.ts +38 -0
- package/dist/core/extract-inline.js +215 -0
- package/dist/core/extract-listings.d.ts +38 -0
- package/dist/core/extract-listings.js +461 -0
- package/dist/core/extract.d.ts +9 -0
- package/dist/core/extract.js +139 -0
- package/dist/core/fetch-cache.d.ts +57 -0
- package/dist/core/fetch-cache.js +95 -0
- package/dist/core/fetcher.d.ts +13 -0
- package/dist/core/fetcher.js +12 -0
- package/dist/core/google-cache.d.ts +29 -0
- package/dist/core/google-cache.js +180 -0
- package/dist/core/google-serp-parser.d.ts +82 -0
- package/dist/core/google-serp-parser.js +287 -0
- package/dist/core/hotel-search.d.ts +122 -0
- package/dist/core/hotel-search.js +382 -0
- package/dist/core/http-fetch.d.ts +72 -0
- package/dist/core/http-fetch.js +820 -0
- package/dist/core/human.d.ts +175 -0
- package/dist/core/human.js +680 -0
- package/dist/core/image-caption.d.ts +44 -0
- package/dist/core/image-caption.js +271 -0
- package/dist/core/jobs.d.ts +75 -0
- package/dist/core/jobs.js +634 -0
- package/dist/core/json-ld.d.ts +15 -0
- package/dist/core/json-ld.js +617 -0
- package/dist/core/language-detect.d.ts +18 -0
- package/dist/core/language-detect.js +135 -0
- package/dist/core/links.d.ts +10 -0
- package/dist/core/links.js +44 -0
- package/dist/core/llm-extract.d.ts +71 -0
- package/dist/core/llm-extract.js +507 -0
- package/dist/core/llm-provider.d.ts +100 -0
- package/dist/core/llm-provider.js +702 -0
- package/dist/core/local-search.d.ts +60 -0
- package/dist/core/local-search.js +308 -0
- package/dist/core/logger.d.ts +28 -0
- package/dist/core/logger.js +104 -0
- package/dist/core/map.d.ts +33 -0
- package/dist/core/map.js +127 -0
- package/dist/core/markdown.d.ts +92 -0
- package/dist/core/markdown.js +809 -0
- package/dist/core/metadata.d.ts +34 -0
- package/dist/core/metadata.js +422 -0
- package/dist/core/observe.d.ts +113 -0
- package/dist/core/observe.js +395 -0
- package/dist/core/ocr.d.ts +12 -0
- package/dist/core/ocr.js +33 -0
- package/dist/core/paginate.d.ts +31 -0
- package/dist/core/paginate.js +106 -0
- package/dist/core/pdf.d.ts +8 -0
- package/dist/core/pdf.js +25 -0
- package/dist/core/peel-tls.d.ts +25 -0
- package/dist/core/peel-tls.js +220 -0
- package/dist/core/pipeline.d.ts +132 -0
- package/dist/core/pipeline.js +1666 -0
- package/dist/core/profiles.d.ts +61 -0
- package/dist/core/profiles.js +350 -0
- package/dist/core/prompt-guard.d.ts +30 -0
- package/dist/core/prompt-guard.js +119 -0
- package/dist/core/proxy-config.d.ts +90 -0
- package/dist/core/proxy-config.js +172 -0
- package/dist/core/quick-answer.d.ts +53 -0
- package/dist/core/quick-answer.js +833 -0
- package/dist/core/rate-governor.d.ts +80 -0
- package/dist/core/rate-governor.js +238 -0
- package/dist/core/readability.d.ts +57 -0
- package/dist/core/readability.js +533 -0
- package/dist/core/research.d.ts +66 -0
- package/dist/core/research.js +270 -0
- package/dist/core/retry.d.ts +60 -0
- package/dist/core/retry.js +119 -0
- package/dist/core/safe-browsing.d.ts +30 -0
- package/dist/core/safe-browsing.js +206 -0
- package/dist/core/schema-extraction.d.ts +66 -0
- package/dist/core/schema-extraction.js +352 -0
- package/dist/core/schema-postprocess.d.ts +32 -0
- package/dist/core/schema-postprocess.js +469 -0
- package/dist/core/schema-templates.d.ts +19 -0
- package/dist/core/schema-templates.js +143 -0
- package/dist/core/screenshot.d.ts +224 -0
- package/dist/core/screenshot.js +207 -0
- package/dist/core/search-engines.d.ts +25 -0
- package/dist/core/search-engines.js +182 -0
- package/dist/core/search-provider.d.ts +243 -0
- package/dist/core/search-provider.js +1629 -0
- package/dist/core/searxng-provider.d.ts +35 -0
- package/dist/core/searxng-provider.js +105 -0
- package/dist/core/selective-evidence.d.ts +151 -0
- package/dist/core/selective-evidence.js +389 -0
- package/dist/core/site-search.d.ts +44 -0
- package/dist/core/site-search.js +252 -0
- package/dist/core/sitemap.d.ts +23 -0
- package/dist/core/sitemap.js +105 -0
- package/dist/core/source-credibility.d.ts +29 -0
- package/dist/core/source-credibility.js +584 -0
- package/dist/core/source-scoring.d.ts +166 -0
- package/dist/core/source-scoring.js +396 -0
- package/dist/core/stemmer.d.ts +38 -0
- package/dist/core/stemmer.js +509 -0
- package/dist/core/strategies.d.ts +104 -0
- package/dist/core/strategies.js +1044 -0
- package/dist/core/strategy-hooks.d.ts +145 -0
- package/dist/core/strategy-hooks.js +74 -0
- package/dist/core/structured-extract.d.ts +43 -0
- package/dist/core/structured-extract.js +550 -0
- package/dist/core/summarize.d.ts +17 -0
- package/dist/core/summarize.js +78 -0
- package/dist/core/synonyms.d.ts +42 -0
- package/dist/core/synonyms.js +184 -0
- package/dist/core/system-monitor.d.ts +61 -0
- package/dist/core/system-monitor.js +133 -0
- package/dist/core/table-format.d.ts +30 -0
- package/dist/core/table-format.js +146 -0
- package/dist/core/threat-feeds.d.ts +23 -0
- package/dist/core/threat-feeds.js +104 -0
- package/dist/core/timing.d.ts +21 -0
- package/dist/core/timing.js +33 -0
- package/dist/core/transcript-export.d.ts +47 -0
- package/dist/core/transcript-export.js +107 -0
- package/dist/core/user-agents.d.ts +82 -0
- package/dist/core/user-agents.js +239 -0
- package/dist/core/vertical-search.d.ts +54 -0
- package/dist/core/vertical-search.js +158 -0
- package/dist/core/watch-manager.d.ts +175 -0
- package/dist/core/watch-manager.js +416 -0
- package/dist/core/watch.d.ts +101 -0
- package/dist/core/watch.js +389 -0
- package/dist/core/youtube.d.ts +130 -0
- package/dist/core/youtube.js +1175 -0
- package/dist/ee/challenge-re-export.d.ts +1 -0
- package/dist/ee/challenge-re-export.js +1 -0
- package/dist/ee/challenge-solver.d.ts +72 -0
- package/dist/ee/challenge-solver.js +720 -0
- package/dist/ee/domain-extractors.d.ts +8 -0
- package/dist/ee/domain-extractors.js +8 -0
- package/dist/ee/domain-intel.d.ts +16 -0
- package/dist/ee/domain-intel.js +133 -0
- package/dist/ee/extractors/allrecipes.d.ts +2 -0
- package/dist/ee/extractors/allrecipes.js +120 -0
- package/dist/ee/extractors/amazon.d.ts +2 -0
- package/dist/ee/extractors/amazon.js +78 -0
- package/dist/ee/extractors/arxiv.d.ts +2 -0
- package/dist/ee/extractors/arxiv.js +137 -0
- package/dist/ee/extractors/bestbuy.d.ts +2 -0
- package/dist/ee/extractors/bestbuy.js +78 -0
- package/dist/ee/extractors/carscom.d.ts +2 -0
- package/dist/ee/extractors/carscom.js +121 -0
- package/dist/ee/extractors/coingecko.d.ts +2 -0
- package/dist/ee/extractors/coingecko.js +134 -0
- package/dist/ee/extractors/craigslist.d.ts +2 -0
- package/dist/ee/extractors/craigslist.js +92 -0
- package/dist/ee/extractors/devto.d.ts +2 -0
- package/dist/ee/extractors/devto.js +135 -0
- package/dist/ee/extractors/ebay.d.ts +2 -0
- package/dist/ee/extractors/ebay.js +90 -0
- package/dist/ee/extractors/espn.d.ts +2 -0
- package/dist/ee/extractors/espn.js +260 -0
- package/dist/ee/extractors/etsy.d.ts +2 -0
- package/dist/ee/extractors/etsy.js +52 -0
- package/dist/ee/extractors/facebook.d.ts +2 -0
- package/dist/ee/extractors/facebook.js +46 -0
- package/dist/ee/extractors/github.d.ts +2 -0
- package/dist/ee/extractors/github.js +196 -0
- package/dist/ee/extractors/google-flights.d.ts +2 -0
- package/dist/ee/extractors/google-flights.js +176 -0
- package/dist/ee/extractors/hackernews.d.ts +2 -0
- package/dist/ee/extractors/hackernews.js +147 -0
- package/dist/ee/extractors/imdb.d.ts +2 -0
- package/dist/ee/extractors/imdb.js +172 -0
- package/dist/ee/extractors/index.d.ts +26 -0
- package/dist/ee/extractors/index.js +247 -0
- package/dist/ee/extractors/instagram.d.ts +2 -0
- package/dist/ee/extractors/instagram.js +102 -0
- package/dist/ee/extractors/kalshi.d.ts +2 -0
- package/dist/ee/extractors/kalshi.js +121 -0
- package/dist/ee/extractors/kayak-cars.d.ts +2 -0
- package/dist/ee/extractors/kayak-cars.js +270 -0
- package/dist/ee/extractors/linkedin.d.ts +2 -0
- package/dist/ee/extractors/linkedin.js +113 -0
- package/dist/ee/extractors/medium.d.ts +2 -0
- package/dist/ee/extractors/medium.js +130 -0
- package/dist/ee/extractors/news.d.ts +4 -0
- package/dist/ee/extractors/news.js +173 -0
- package/dist/ee/extractors/npm.d.ts +2 -0
- package/dist/ee/extractors/npm.js +86 -0
- package/dist/ee/extractors/pdf.d.ts +2 -0
- package/dist/ee/extractors/pdf.js +108 -0
- package/dist/ee/extractors/pinterest.d.ts +2 -0
- package/dist/ee/extractors/pinterest.js +34 -0
- package/dist/ee/extractors/polymarket.d.ts +2 -0
- package/dist/ee/extractors/polymarket.js +358 -0
- package/dist/ee/extractors/producthunt.d.ts +2 -0
- package/dist/ee/extractors/producthunt.js +88 -0
- package/dist/ee/extractors/pubmed.d.ts +2 -0
- package/dist/ee/extractors/pubmed.js +162 -0
- package/dist/ee/extractors/pypi.d.ts +2 -0
- package/dist/ee/extractors/pypi.js +80 -0
- package/dist/ee/extractors/reddit.d.ts +2 -0
- package/dist/ee/extractors/reddit.js +438 -0
- package/dist/ee/extractors/redfin.d.ts +2 -0
- package/dist/ee/extractors/redfin.js +156 -0
- package/dist/ee/extractors/semanticscholar.d.ts +2 -0
- package/dist/ee/extractors/semanticscholar.js +131 -0
- package/dist/ee/extractors/shared.d.ts +12 -0
- package/dist/ee/extractors/shared.js +76 -0
- package/dist/ee/extractors/soundcloud.d.ts +2 -0
- package/dist/ee/extractors/soundcloud.js +34 -0
- package/dist/ee/extractors/sportsbetting.d.ts +2 -0
- package/dist/ee/extractors/sportsbetting.js +37 -0
- package/dist/ee/extractors/spotify.d.ts +2 -0
- package/dist/ee/extractors/spotify.js +34 -0
- package/dist/ee/extractors/stackoverflow.d.ts +2 -0
- package/dist/ee/extractors/stackoverflow.js +61 -0
- package/dist/ee/extractors/substack.d.ts +2 -0
- package/dist/ee/extractors/substack.js +115 -0
- package/dist/ee/extractors/substackroot.d.ts +2 -0
- package/dist/ee/extractors/substackroot.js +46 -0
- package/dist/ee/extractors/tiktok.d.ts +2 -0
- package/dist/ee/extractors/tiktok.js +29 -0
- package/dist/ee/extractors/tradingview.d.ts +2 -0
- package/dist/ee/extractors/tradingview.js +182 -0
- package/dist/ee/extractors/twitch.d.ts +2 -0
- package/dist/ee/extractors/twitch.js +36 -0
- package/dist/ee/extractors/twitter.d.ts +2 -0
- package/dist/ee/extractors/twitter.js +327 -0
- package/dist/ee/extractors/types.d.ts +14 -0
- package/dist/ee/extractors/types.js +1 -0
- package/dist/ee/extractors/walmart.d.ts +2 -0
- package/dist/ee/extractors/walmart.js +50 -0
- package/dist/ee/extractors/weather.d.ts +2 -0
- package/dist/ee/extractors/weather.js +133 -0
- package/dist/ee/extractors/wikipedia.d.ts +4 -0
- package/dist/ee/extractors/wikipedia.js +235 -0
- package/dist/ee/extractors/yelp.d.ts +2 -0
- package/dist/ee/extractors/yelp.js +216 -0
- package/dist/ee/extractors/youtube.d.ts +2 -0
- package/dist/ee/extractors/youtube.js +189 -0
- package/dist/ee/extractors/zillow.d.ts +54 -0
- package/dist/ee/extractors/zillow.js +247 -0
- package/dist/ee/extractors-re-export.d.ts +1 -0
- package/dist/ee/extractors-re-export.js +1 -0
- package/dist/ee/premium-hooks.d.ts +20 -0
- package/dist/ee/premium-hooks.js +50 -0
- package/dist/ee/spa-detection.d.ts +2 -0
- package/dist/ee/spa-detection.js +2 -0
- package/dist/ee/stability.d.ts +4 -0
- package/dist/ee/stability.js +29 -0
- package/dist/ee/swr-cache.d.ts +14 -0
- package/dist/ee/swr-cache.js +34 -0
- package/dist/index.d.ts +143 -0
- package/dist/index.js +291 -0
- package/dist/integrations/index.d.ts +2 -0
- package/dist/integrations/index.js +2 -0
- package/dist/integrations/langchain.d.ts +64 -0
- package/dist/integrations/langchain.js +115 -0
- package/dist/integrations/llamaindex.d.ts +50 -0
- package/dist/integrations/llamaindex.js +91 -0
- package/dist/mcp/handlers/act.d.ts +5 -0
- package/dist/mcp/handlers/act.js +34 -0
- package/dist/mcp/handlers/definitions.d.ts +6 -0
- package/dist/mcp/handlers/definitions.js +395 -0
- package/dist/mcp/handlers/extract.d.ts +7 -0
- package/dist/mcp/handlers/extract.js +135 -0
- package/dist/mcp/handlers/fetch.d.ts +6 -0
- package/dist/mcp/handlers/fetch.js +98 -0
- package/dist/mcp/handlers/find.d.ts +5 -0
- package/dist/mcp/handlers/find.js +137 -0
- package/dist/mcp/handlers/index.d.ts +13 -0
- package/dist/mcp/handlers/index.js +63 -0
- package/dist/mcp/handlers/legacy.d.ts +25 -0
- package/dist/mcp/handlers/legacy.js +450 -0
- package/dist/mcp/handlers/meta.d.ts +6 -0
- package/dist/mcp/handlers/meta.js +40 -0
- package/dist/mcp/handlers/monitor.d.ts +5 -0
- package/dist/mcp/handlers/monitor.js +41 -0
- package/dist/mcp/handlers/observe.d.ts +8 -0
- package/dist/mcp/handlers/observe.js +37 -0
- package/dist/mcp/handlers/read.d.ts +6 -0
- package/dist/mcp/handlers/read.js +78 -0
- package/dist/mcp/handlers/see.d.ts +5 -0
- package/dist/mcp/handlers/see.js +75 -0
- package/dist/mcp/handlers/types.d.ts +29 -0
- package/dist/mcp/handlers/types.js +28 -0
- package/dist/mcp/server.d.ts +7 -0
- package/dist/mcp/server.js +108 -0
- package/dist/mcp/smart-router.d.ts +23 -0
- package/dist/mcp/smart-router.js +178 -0
- package/dist/server/app.d.ts +14 -0
- package/dist/server/app.js +632 -0
- package/dist/server/auth-store.d.ts +28 -0
- package/dist/server/auth-store.js +88 -0
- package/dist/server/bull-queues.d.ts +60 -0
- package/dist/server/bull-queues.js +90 -0
- package/dist/server/email-service.d.ts +55 -0
- package/dist/server/email-service.js +291 -0
- package/dist/server/job-queue.d.ts +100 -0
- package/dist/server/job-queue.js +145 -0
- package/dist/server/logger.d.ts +10 -0
- package/dist/server/logger.js +37 -0
- package/dist/server/middleware/audit-log.d.ts +14 -0
- package/dist/server/middleware/audit-log.js +73 -0
- package/dist/server/middleware/auth.d.ts +35 -0
- package/dist/server/middleware/auth.js +225 -0
- package/dist/server/middleware/rate-limit.d.ts +50 -0
- package/dist/server/middleware/rate-limit.js +270 -0
- package/dist/server/middleware/scope-guard.d.ts +25 -0
- package/dist/server/middleware/scope-guard.js +45 -0
- package/dist/server/middleware/url-validator.d.ts +15 -0
- package/dist/server/middleware/url-validator.js +201 -0
- package/dist/server/openapi.yaml +6418 -0
- package/dist/server/pg-auth-store.d.ts +146 -0
- package/dist/server/pg-auth-store.js +576 -0
- package/dist/server/pg-job-queue.d.ts +59 -0
- package/dist/server/pg-job-queue.js +375 -0
- package/dist/server/routes/activity.d.ts +6 -0
- package/dist/server/routes/activity.js +79 -0
- package/dist/server/routes/admin-active.d.ts +7 -0
- package/dist/server/routes/admin-active.js +120 -0
- package/dist/server/routes/admin-stats.d.ts +7 -0
- package/dist/server/routes/admin-stats.js +176 -0
- package/dist/server/routes/agent.d.ts +24 -0
- package/dist/server/routes/agent.js +480 -0
- package/dist/server/routes/answer.d.ts +5 -0
- package/dist/server/routes/answer.js +125 -0
- package/dist/server/routes/ask.d.ts +28 -0
- package/dist/server/routes/ask.js +295 -0
- package/dist/server/routes/batch.d.ts +6 -0
- package/dist/server/routes/batch.js +493 -0
- package/dist/server/routes/cache-warm.d.ts +25 -0
- package/dist/server/routes/cache-warm.js +212 -0
- package/dist/server/routes/cli-usage.d.ts +6 -0
- package/dist/server/routes/cli-usage.js +127 -0
- package/dist/server/routes/compat.d.ts +23 -0
- package/dist/server/routes/compat.js +652 -0
- package/dist/server/routes/crawl.d.ts +13 -0
- package/dist/server/routes/crawl.js +287 -0
- package/dist/server/routes/deep-fetch.d.ts +8 -0
- package/dist/server/routes/deep-fetch.js +57 -0
- package/dist/server/routes/deep-research.d.ts +11 -0
- package/dist/server/routes/deep-research.js +232 -0
- package/dist/server/routes/demo.d.ts +24 -0
- package/dist/server/routes/demo.js +517 -0
- package/dist/server/routes/do.d.ts +8 -0
- package/dist/server/routes/do.js +72 -0
- package/dist/server/routes/extract.d.ts +14 -0
- package/dist/server/routes/extract.js +325 -0
- package/dist/server/routes/feed.d.ts +15 -0
- package/dist/server/routes/feed.js +311 -0
- package/dist/server/routes/fetch-queue.d.ts +13 -0
- package/dist/server/routes/fetch-queue.js +357 -0
- package/dist/server/routes/fetch.d.ts +7 -0
- package/dist/server/routes/fetch.js +1274 -0
- package/dist/server/routes/go.d.ts +14 -0
- package/dist/server/routes/go.js +81 -0
- package/dist/server/routes/health.d.ts +11 -0
- package/dist/server/routes/health.js +141 -0
- package/dist/server/routes/jobs.d.ts +7 -0
- package/dist/server/routes/jobs.js +574 -0
- package/dist/server/routes/map.d.ts +11 -0
- package/dist/server/routes/map.js +116 -0
- package/dist/server/routes/mcp.d.ts +14 -0
- package/dist/server/routes/mcp.js +197 -0
- package/dist/server/routes/metrics.d.ts +37 -0
- package/dist/server/routes/metrics.js +149 -0
- package/dist/server/routes/oauth.d.ts +9 -0
- package/dist/server/routes/oauth.js +396 -0
- package/dist/server/routes/playground.d.ts +17 -0
- package/dist/server/routes/playground.js +283 -0
- package/dist/server/routes/reader.d.ts +18 -0
- package/dist/server/routes/reader.js +192 -0
- package/dist/server/routes/research.d.ts +14 -0
- package/dist/server/routes/research.js +482 -0
- package/dist/server/routes/screenshot.d.ts +22 -0
- package/dist/server/routes/screenshot.js +820 -0
- package/dist/server/routes/search.d.ts +6 -0
- package/dist/server/routes/search.js +874 -0
- package/dist/server/routes/session.d.ts +17 -0
- package/dist/server/routes/session.js +548 -0
- package/dist/server/routes/share.d.ts +18 -0
- package/dist/server/routes/share.js +462 -0
- package/dist/server/routes/smart-search/handlers/cars.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/cars.js +102 -0
- package/dist/server/routes/smart-search/handlers/flights.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/flights.js +72 -0
- package/dist/server/routes/smart-search/handlers/general.d.ts +13 -0
- package/dist/server/routes/smart-search/handlers/general.js +717 -0
- package/dist/server/routes/smart-search/handlers/hotels.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/hotels.js +88 -0
- package/dist/server/routes/smart-search/handlers/products.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/products.js +1309 -0
- package/dist/server/routes/smart-search/handlers/rental.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/rental.js +154 -0
- package/dist/server/routes/smart-search/handlers/restaurants.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/restaurants.js +225 -0
- package/dist/server/routes/smart-search/handlers/transit-verdict.d.ts +41 -0
- package/dist/server/routes/smart-search/handlers/transit-verdict.js +224 -0
- package/dist/server/routes/smart-search/index.d.ts +19 -0
- package/dist/server/routes/smart-search/index.js +546 -0
- package/dist/server/routes/smart-search/intent.d.ts +3 -0
- package/dist/server/routes/smart-search/intent.js +264 -0
- package/dist/server/routes/smart-search/llm.d.ts +16 -0
- package/dist/server/routes/smart-search/llm.js +70 -0
- package/dist/server/routes/smart-search/sources/reddit.d.ts +18 -0
- package/dist/server/routes/smart-search/sources/reddit.js +34 -0
- package/dist/server/routes/smart-search/sources/yelp.d.ts +25 -0
- package/dist/server/routes/smart-search/sources/yelp.js +171 -0
- package/dist/server/routes/smart-search/sources/youtube.d.ts +8 -0
- package/dist/server/routes/smart-search/sources/youtube.js +9 -0
- package/dist/server/routes/smart-search/types.d.ts +81 -0
- package/dist/server/routes/smart-search/types.js +1 -0
- package/dist/server/routes/smart-search/utils.d.ts +20 -0
- package/dist/server/routes/smart-search/utils.js +146 -0
- package/dist/server/routes/stats.d.ts +6 -0
- package/dist/server/routes/stats.js +71 -0
- package/dist/server/routes/stripe.d.ts +15 -0
- package/dist/server/routes/stripe.js +296 -0
- package/dist/server/routes/transcript-export.d.ts +10 -0
- package/dist/server/routes/transcript-export.js +178 -0
- package/dist/server/routes/usage.d.ts +9 -0
- package/dist/server/routes/usage.js +279 -0
- package/dist/server/routes/users.d.ts +8 -0
- package/dist/server/routes/users.js +1867 -0
- package/dist/server/routes/watch.d.ts +15 -0
- package/dist/server/routes/watch.js +309 -0
- package/dist/server/routes/webhooks.d.ts +26 -0
- package/dist/server/routes/webhooks.js +170 -0
- package/dist/server/routes/youtube.d.ts +6 -0
- package/dist/server/routes/youtube.js +130 -0
- package/dist/server/sentry.d.ts +14 -0
- package/dist/server/sentry.js +104 -0
- package/dist/server/types.d.ts +15 -0
- package/dist/server/types.js +7 -0
- package/dist/server/utils/response.d.ts +44 -0
- package/dist/server/utils/response.js +69 -0
- package/dist/server/utils/sse.d.ts +22 -0
- package/dist/server/utils/sse.js +38 -0
- package/dist/types.d.ts +552 -0
- package/dist/types.js +39 -0
- package/llms.txt +105 -0
- package/package.json +189 -0
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool schema definitions — single source of truth for both transports.
|
|
3
|
+
* Imported by the standalone MCP server and the HTTP MCP route for tools/list.
|
|
4
|
+
*/
|
|
5
|
+
export const toolDefinitions = [
|
|
6
|
+
{
|
|
7
|
+
name: 'webpeel',
|
|
8
|
+
description: "Your complete web toolkit — fetch, search, screenshot, extract, monitor, and interact with any website. " +
|
|
9
|
+
"Handles JS rendering, Cloudflare, CAPTCHAs, and 55+ domain extractors automatically. 65-98% token savings. " +
|
|
10
|
+
"Describe what you want in plain English. " +
|
|
11
|
+
"Examples: 'read https://stripe.com/pricing', 'screenshot bbc.com on mobile', " +
|
|
12
|
+
"'search for best AI frameworks 2024', 'extract product prices from amazon.com/dp/...', " +
|
|
13
|
+
"'watch stripe.com/pricing for price changes', 'get YouTube transcript from youtu.be/...'. " +
|
|
14
|
+
"For JavaScript-heavy SPAs (React, Vue, Next.js, Polymarket, Airbnb, etc.), mention 'render' or 'use browser' in your task. " +
|
|
15
|
+
"For infinite scroll or lazy-loaded content, say 'scroll to bottom'. " +
|
|
16
|
+
"For bot-protected sites (Cloudflare), say 'stealth mode'. " +
|
|
17
|
+
"If you get sparse or empty content, retry and mention 'render' — the site likely requires JavaScript.",
|
|
18
|
+
annotations: {
|
|
19
|
+
title: 'WebPeel Smart Web Tool',
|
|
20
|
+
readOnlyHint: true,
|
|
21
|
+
destructiveHint: false,
|
|
22
|
+
idempotentHint: true,
|
|
23
|
+
openWorldHint: true,
|
|
24
|
+
},
|
|
25
|
+
inputSchema: {
|
|
26
|
+
type: 'object',
|
|
27
|
+
properties: {
|
|
28
|
+
task: {
|
|
29
|
+
type: 'string',
|
|
30
|
+
description: 'Plain English description of what you want to do with the web.',
|
|
31
|
+
},
|
|
32
|
+
llmProvider: {
|
|
33
|
+
type: 'string',
|
|
34
|
+
enum: ['openai', 'anthropic', 'google'],
|
|
35
|
+
description: 'LLM provider for schema extraction (BYOK). Pass when asking for structured data extraction.',
|
|
36
|
+
},
|
|
37
|
+
llmApiKey: {
|
|
38
|
+
type: 'string',
|
|
39
|
+
description: 'Your LLM API key (BYOK). Pass when asking for structured data extraction with llmProvider.',
|
|
40
|
+
},
|
|
41
|
+
llmModel: {
|
|
42
|
+
type: 'string',
|
|
43
|
+
description: 'LLM model name (optional). Defaults: gpt-4o-mini (OpenAI), claude-haiku-4-5 (Anthropic), gemini-2.0-flash (Google).',
|
|
44
|
+
},
|
|
45
|
+
llmBaseUrl: {
|
|
46
|
+
type: 'string',
|
|
47
|
+
description: 'Custom OpenAI-compatible API base URL. Use for OpenRouter, Glama, or self-hosted models.',
|
|
48
|
+
},
|
|
49
|
+
},
|
|
50
|
+
required: ['task'],
|
|
51
|
+
},
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
name: 'webpeel_read',
|
|
55
|
+
description: 'Fetch any URL and return clean, LLM-optimized markdown. 65-98% fewer tokens than raw HTML. ' +
|
|
56
|
+
'Automatically handles: web pages, YouTube transcripts (with timestamps), PDFs, ' +
|
|
57
|
+
'JS-rendered SPAs, Cloudflare-protected sites, and 55+ domain-specific extractors (Amazon, Reddit, GitHub, etc.). ' +
|
|
58
|
+
'IMPORTANT: Use render=true for ANY JavaScript-heavy site (React, Vue, Angular, Svelte, Next.js, SPAs). ' +
|
|
59
|
+
'Known SPAs that need render=true: Polymarket, Airbnb, Booking.com, Expedia, Indeed, Zillow, Google, and more. ' +
|
|
60
|
+
'If content is sparse or empty, ALWAYS retry with render=true before concluding the page has no content. ' +
|
|
61
|
+
'Use actions= to interact with the page before extraction (scroll, click, type, wait). ' +
|
|
62
|
+
'Use stealth=true for Cloudflare-protected or bot-blocked sites (auto-enables render). ' +
|
|
63
|
+
'Use question= for instant Q&A (no LLM needed). ' +
|
|
64
|
+
'Use summary=true for a short summary. Use budget=N to distill to N tokens.',
|
|
65
|
+
annotations: {
|
|
66
|
+
title: 'Read Web Page',
|
|
67
|
+
readOnlyHint: true,
|
|
68
|
+
destructiveHint: false,
|
|
69
|
+
idempotentHint: true,
|
|
70
|
+
openWorldHint: true,
|
|
71
|
+
},
|
|
72
|
+
inputSchema: {
|
|
73
|
+
type: 'object',
|
|
74
|
+
properties: {
|
|
75
|
+
url: { type: 'string', description: 'URL to fetch' },
|
|
76
|
+
format: {
|
|
77
|
+
type: 'string',
|
|
78
|
+
enum: ['markdown', 'text', 'html'],
|
|
79
|
+
description: 'Output format (default: markdown)',
|
|
80
|
+
default: 'markdown',
|
|
81
|
+
},
|
|
82
|
+
render: {
|
|
83
|
+
type: 'boolean',
|
|
84
|
+
description: 'Force browser rendering for JS-heavy sites',
|
|
85
|
+
default: false,
|
|
86
|
+
},
|
|
87
|
+
question: {
|
|
88
|
+
type: 'string',
|
|
89
|
+
description: 'Ask a question about the page content (BM25, no LLM needed)',
|
|
90
|
+
},
|
|
91
|
+
summary: {
|
|
92
|
+
type: 'boolean',
|
|
93
|
+
description: 'Return a summary instead of full content',
|
|
94
|
+
default: false,
|
|
95
|
+
},
|
|
96
|
+
budget: {
|
|
97
|
+
type: 'number',
|
|
98
|
+
description: 'Smart token budget — distill content to N tokens',
|
|
99
|
+
},
|
|
100
|
+
readable: {
|
|
101
|
+
type: 'boolean',
|
|
102
|
+
description: 'Reader mode — extract only article content',
|
|
103
|
+
default: false,
|
|
104
|
+
},
|
|
105
|
+
actions: {
|
|
106
|
+
type: 'array',
|
|
107
|
+
description: 'Browser actions to perform before extracting content. Requires render=true (auto-enabled). ' +
|
|
108
|
+
'Each action is a string: "scroll:bottom" (infinite scroll), "wait:2000" (wait 2s), ' +
|
|
109
|
+
'"click:.selector" (click element), "type:#input:text" (type into field), ' +
|
|
110
|
+
'"waitFor:.selector" (wait for element), "hover:.element" (hover), ' +
|
|
111
|
+
'"scroll:down:500" (scroll 500px), "scroll:0,1500" (scroll to coords). ' +
|
|
112
|
+
'Chain multiple actions: ["scroll:bottom", "wait:2000"] to load lazy content.',
|
|
113
|
+
items: { type: 'string' },
|
|
114
|
+
},
|
|
115
|
+
stealth: {
|
|
116
|
+
type: 'boolean',
|
|
117
|
+
description: 'Stealth mode for bot-protected sites (Cloudflare, fingerprinting, rate limiting). ' +
|
|
118
|
+
'Use when render=true still returns a challenge page or access denied. Auto-enables render.',
|
|
119
|
+
default: false,
|
|
120
|
+
},
|
|
121
|
+
},
|
|
122
|
+
required: ['url'],
|
|
123
|
+
},
|
|
124
|
+
},
|
|
125
|
+
{
|
|
126
|
+
name: 'webpeel_see',
|
|
127
|
+
description: "Capture a screenshot of any web page. Returns the page as an image for visual inspection. " +
|
|
128
|
+
"Supports mobile, tablet, and desktop viewports. " +
|
|
129
|
+
"Use mode='design' for AI-powered design analysis and suggestions. " +
|
|
130
|
+
"Use mode='compare' with compare_url to diff two pages visually. " +
|
|
131
|
+
"Use full_page=true to capture the entire scrollable page.",
|
|
132
|
+
annotations: {
|
|
133
|
+
title: 'See Page Visually',
|
|
134
|
+
readOnlyHint: true,
|
|
135
|
+
destructiveHint: false,
|
|
136
|
+
idempotentHint: true,
|
|
137
|
+
openWorldHint: true,
|
|
138
|
+
},
|
|
139
|
+
inputSchema: {
|
|
140
|
+
type: 'object',
|
|
141
|
+
properties: {
|
|
142
|
+
url: { type: 'string', description: 'URL to screenshot' },
|
|
143
|
+
mode: {
|
|
144
|
+
type: 'string',
|
|
145
|
+
enum: ['screenshot', 'design', 'compare'],
|
|
146
|
+
description: "Mode: 'screenshot' (default), 'design' (analysis), 'compare' (visual diff)",
|
|
147
|
+
default: 'screenshot',
|
|
148
|
+
},
|
|
149
|
+
compare_url: {
|
|
150
|
+
type: 'string',
|
|
151
|
+
description: "Second URL to compare against (for mode='compare')",
|
|
152
|
+
},
|
|
153
|
+
viewport: {
|
|
154
|
+
description: "Viewport size: 'mobile' | 'tablet' | {width, height}",
|
|
155
|
+
oneOf: [
|
|
156
|
+
{ type: 'string', enum: ['mobile', 'tablet', 'desktop'] },
|
|
157
|
+
{
|
|
158
|
+
type: 'object',
|
|
159
|
+
properties: {
|
|
160
|
+
width: { type: 'number' },
|
|
161
|
+
height: { type: 'number' },
|
|
162
|
+
},
|
|
163
|
+
required: ['width', 'height'],
|
|
164
|
+
},
|
|
165
|
+
],
|
|
166
|
+
},
|
|
167
|
+
full_page: {
|
|
168
|
+
type: 'boolean',
|
|
169
|
+
description: 'Capture the full scrollable page',
|
|
170
|
+
default: false,
|
|
171
|
+
},
|
|
172
|
+
},
|
|
173
|
+
required: ['url'],
|
|
174
|
+
},
|
|
175
|
+
},
|
|
176
|
+
{
|
|
177
|
+
name: 'webpeel_find',
|
|
178
|
+
description: "Search the web or discover all pages on a site. " +
|
|
179
|
+
"Pass query= to search the web and get ranked results with titles, URLs, and snippets. " +
|
|
180
|
+
"Pass url= to map/crawl a domain and discover all its pages. " +
|
|
181
|
+
"Use depth='deep' for multi-source research that synthesizes answers from multiple pages. " +
|
|
182
|
+
"Smart search detects intent for restaurants, products, flights, hotels, and more.",
|
|
183
|
+
annotations: {
|
|
184
|
+
title: 'Find on the Web',
|
|
185
|
+
readOnlyHint: true,
|
|
186
|
+
destructiveHint: false,
|
|
187
|
+
idempotentHint: true,
|
|
188
|
+
openWorldHint: true,
|
|
189
|
+
},
|
|
190
|
+
inputSchema: {
|
|
191
|
+
type: 'object',
|
|
192
|
+
properties: {
|
|
193
|
+
query: { type: 'string', description: 'Search query' },
|
|
194
|
+
url: { type: 'string', description: 'Domain URL to map/discover all pages' },
|
|
195
|
+
depth: {
|
|
196
|
+
type: 'string',
|
|
197
|
+
enum: ['quick', 'deep'],
|
|
198
|
+
description: "Search depth: 'quick' = single search, 'deep' = multi-source research",
|
|
199
|
+
default: 'quick',
|
|
200
|
+
},
|
|
201
|
+
limit: {
|
|
202
|
+
type: 'number',
|
|
203
|
+
description: 'Max results to return (default: 5)',
|
|
204
|
+
default: 5,
|
|
205
|
+
},
|
|
206
|
+
},
|
|
207
|
+
},
|
|
208
|
+
},
|
|
209
|
+
{
|
|
210
|
+
name: 'webpeel_extract',
|
|
211
|
+
description: "Extract structured JSON data from any URL. No LLM needed for built-in schemas. " +
|
|
212
|
+
"Pass fields=['price','title','description'] to extract specific named fields. " +
|
|
213
|
+
"Pass schema={...} with a full JSON schema for custom structured output. " +
|
|
214
|
+
"Built-in schemas: product, article, recipe, job, event, contact, business, review, listing. " +
|
|
215
|
+
"Works on Amazon, Yelp, LinkedIn, job boards, e-commerce sites, and any web page.",
|
|
216
|
+
annotations: {
|
|
217
|
+
title: 'Extract Structured Data',
|
|
218
|
+
readOnlyHint: true,
|
|
219
|
+
destructiveHint: false,
|
|
220
|
+
idempotentHint: true,
|
|
221
|
+
openWorldHint: true,
|
|
222
|
+
},
|
|
223
|
+
inputSchema: {
|
|
224
|
+
type: 'object',
|
|
225
|
+
properties: {
|
|
226
|
+
url: { type: 'string', description: 'URL to extract from' },
|
|
227
|
+
schema: { type: 'object', description: 'JSON schema describing desired output structure' },
|
|
228
|
+
fields: {
|
|
229
|
+
type: 'array',
|
|
230
|
+
items: { type: 'string' },
|
|
231
|
+
description: "Specific fields to extract, e.g. ['price', 'title', 'description']",
|
|
232
|
+
},
|
|
233
|
+
format: {
|
|
234
|
+
type: 'string',
|
|
235
|
+
enum: ['json', 'markdown'],
|
|
236
|
+
description: 'Output format (default: json)',
|
|
237
|
+
default: 'json',
|
|
238
|
+
},
|
|
239
|
+
llmProvider: {
|
|
240
|
+
type: 'string',
|
|
241
|
+
enum: ['openai', 'anthropic', 'google'],
|
|
242
|
+
description: 'LLM provider for schema extraction (BYOK). Required when using custom schema.',
|
|
243
|
+
},
|
|
244
|
+
llmApiKey: {
|
|
245
|
+
type: 'string',
|
|
246
|
+
description: 'Your LLM API key (BYOK). Required when using custom schema with llmProvider.',
|
|
247
|
+
},
|
|
248
|
+
llmModel: {
|
|
249
|
+
type: 'string',
|
|
250
|
+
description: 'LLM model name (optional). Defaults: gpt-4o-mini (OpenAI), claude-haiku-4-5 (Anthropic), gemini-2.0-flash (Google).',
|
|
251
|
+
},
|
|
252
|
+
llmBaseUrl: {
|
|
253
|
+
type: 'string',
|
|
254
|
+
description: 'Custom OpenAI-compatible API base URL. Use for OpenRouter, Glama, or self-hosted models.',
|
|
255
|
+
},
|
|
256
|
+
},
|
|
257
|
+
required: ['url'],
|
|
258
|
+
},
|
|
259
|
+
},
|
|
260
|
+
{
|
|
261
|
+
name: 'webpeel_monitor',
|
|
262
|
+
description: "Track a web page for content changes over time. Call once to take a snapshot, call again to get a diff. " +
|
|
263
|
+
"Use selector= to monitor a specific CSS element (e.g. a price, a status badge). " +
|
|
264
|
+
"Use webhook= for persistent monitoring with automatic notifications when content changes. " +
|
|
265
|
+
"Use interval= to set how frequently to check ('1h', '30m', '1d'). " +
|
|
266
|
+
"Ideal for: price tracking, job listing changes, release monitoring, competitor updates.",
|
|
267
|
+
annotations: {
|
|
268
|
+
title: 'Monitor URL for Changes',
|
|
269
|
+
readOnlyHint: false,
|
|
270
|
+
destructiveHint: false,
|
|
271
|
+
idempotentHint: false,
|
|
272
|
+
openWorldHint: true,
|
|
273
|
+
},
|
|
274
|
+
inputSchema: {
|
|
275
|
+
type: 'object',
|
|
276
|
+
properties: {
|
|
277
|
+
url: { type: 'string', description: 'URL to monitor' },
|
|
278
|
+
webhook: {
|
|
279
|
+
type: 'string',
|
|
280
|
+
description: 'Webhook URL to notify when content changes',
|
|
281
|
+
},
|
|
282
|
+
interval: {
|
|
283
|
+
type: 'string',
|
|
284
|
+
description: "Check interval, e.g. '1h', '30m', '1d'",
|
|
285
|
+
default: '1h',
|
|
286
|
+
},
|
|
287
|
+
selector: {
|
|
288
|
+
type: 'string',
|
|
289
|
+
description: 'CSS selector to monitor a specific part of the page',
|
|
290
|
+
},
|
|
291
|
+
},
|
|
292
|
+
required: ['url'],
|
|
293
|
+
},
|
|
294
|
+
},
|
|
295
|
+
{
|
|
296
|
+
name: 'webpeel_observe',
|
|
297
|
+
description: "Give yourself eyes — discover all interactive elements on a web page without needing a vision model. " +
|
|
298
|
+
"Returns structured data: links, buttons, inputs, forms, selects, and media with CSS selectors and text labels. " +
|
|
299
|
+
"Each element has a ref (e.g. 'button-0', 'input-2') for easy reference in follow-up actions. " +
|
|
300
|
+
"Use this BEFORE webpeel_act to discover what's on the page. " +
|
|
301
|
+
"The observe → decide → act → observe loop gives agents full web autonomy. " +
|
|
302
|
+
"Use viewport='mobile' or 'tablet' to see responsive layouts. " +
|
|
303
|
+
"Use selector= to scope observation to a specific part of the page.",
|
|
304
|
+
annotations: {
|
|
305
|
+
title: 'Observe Page Elements',
|
|
306
|
+
readOnlyHint: true,
|
|
307
|
+
destructiveHint: false,
|
|
308
|
+
idempotentHint: true,
|
|
309
|
+
openWorldHint: true,
|
|
310
|
+
},
|
|
311
|
+
inputSchema: {
|
|
312
|
+
type: 'object',
|
|
313
|
+
properties: {
|
|
314
|
+
url: { type: 'string', description: 'URL to observe' },
|
|
315
|
+
selector: {
|
|
316
|
+
type: 'string',
|
|
317
|
+
description: 'CSS selector to scope observation (e.g. "main", "#content")',
|
|
318
|
+
},
|
|
319
|
+
viewport: {
|
|
320
|
+
type: 'string',
|
|
321
|
+
enum: ['desktop', 'mobile', 'tablet'],
|
|
322
|
+
description: "Viewport to use (default: 'desktop')",
|
|
323
|
+
default: 'desktop',
|
|
324
|
+
},
|
|
325
|
+
screenshot: {
|
|
326
|
+
type: 'boolean',
|
|
327
|
+
description: 'Include a screenshot alongside structured element data',
|
|
328
|
+
default: false,
|
|
329
|
+
},
|
|
330
|
+
max_elements: {
|
|
331
|
+
type: 'number',
|
|
332
|
+
description: 'Max elements per category (default: 50)',
|
|
333
|
+
default: 50,
|
|
334
|
+
},
|
|
335
|
+
stealth: {
|
|
336
|
+
type: 'boolean',
|
|
337
|
+
description: 'Use stealth mode for bot-protected sites',
|
|
338
|
+
default: false,
|
|
339
|
+
},
|
|
340
|
+
},
|
|
341
|
+
required: ['url'],
|
|
342
|
+
},
|
|
343
|
+
},
|
|
344
|
+
{
|
|
345
|
+
name: 'webpeel_act',
|
|
346
|
+
description: "Automate interactions with any web page using a real browser. " +
|
|
347
|
+
"Click buttons, fill forms, select dropdowns, scroll, wait for elements, and press keys. " +
|
|
348
|
+
"Returns extracted content and optionally a screenshot after all actions complete. " +
|
|
349
|
+
"Use for: logging into sites, submitting forms, navigating multi-step flows, " +
|
|
350
|
+
"interacting with dynamic content that requires user input. " +
|
|
351
|
+
"Actions: click, type, fill, scroll, wait, press, hover, select.",
|
|
352
|
+
annotations: {
|
|
353
|
+
title: 'Act on Web Page',
|
|
354
|
+
readOnlyHint: false,
|
|
355
|
+
destructiveHint: false,
|
|
356
|
+
idempotentHint: false,
|
|
357
|
+
openWorldHint: true,
|
|
358
|
+
},
|
|
359
|
+
inputSchema: {
|
|
360
|
+
type: 'object',
|
|
361
|
+
properties: {
|
|
362
|
+
url: { type: 'string', description: 'URL to interact with' },
|
|
363
|
+
actions: {
|
|
364
|
+
type: 'array',
|
|
365
|
+
description: 'Actions to perform, e.g. [{type:"click",selector:".btn"}, {type:"type",selector:"#q",value:"hello"}]',
|
|
366
|
+
items: {
|
|
367
|
+
type: 'object',
|
|
368
|
+
properties: {
|
|
369
|
+
type: {
|
|
370
|
+
type: 'string',
|
|
371
|
+
enum: ['click', 'type', 'fill', 'scroll', 'wait', 'press', 'hover', 'select'],
|
|
372
|
+
},
|
|
373
|
+
selector: { type: 'string' },
|
|
374
|
+
value: { type: 'string' },
|
|
375
|
+
key: { type: 'string' },
|
|
376
|
+
milliseconds: { type: 'number' },
|
|
377
|
+
},
|
|
378
|
+
required: ['type'],
|
|
379
|
+
},
|
|
380
|
+
},
|
|
381
|
+
extract_after: {
|
|
382
|
+
type: 'boolean',
|
|
383
|
+
description: 'Extract content after actions complete',
|
|
384
|
+
default: true,
|
|
385
|
+
},
|
|
386
|
+
screenshot_after: {
|
|
387
|
+
type: 'boolean',
|
|
388
|
+
description: 'Take screenshot after actions complete',
|
|
389
|
+
default: false,
|
|
390
|
+
},
|
|
391
|
+
},
|
|
392
|
+
required: ['url', 'actions'],
|
|
393
|
+
},
|
|
394
|
+
},
|
|
395
|
+
];
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* handleExtract — extract structured data from a URL.
|
|
3
|
+
* Supports auto-detection, field lists, schema, and brand presets.
|
|
4
|
+
* Supports LLM-based extraction via llmProvider + llmApiKey.
|
|
5
|
+
*/
|
|
6
|
+
import { type McpHandler } from './types.js';
|
|
7
|
+
export declare const handleExtract: McpHandler;
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* handleExtract — extract structured data from a URL.
|
|
3
|
+
* Supports auto-detection, field lists, schema, and brand presets.
|
|
4
|
+
* Supports LLM-based extraction via llmProvider + llmApiKey.
|
|
5
|
+
*/
|
|
6
|
+
import { peel } from '../../index.js';
|
|
7
|
+
import { textResult, safeStringify, timeout } from './types.js';
|
|
8
|
+
import { extractWithLLM } from '../../core/llm-extract.js';
|
|
9
|
+
function extractColorsFromContent(content) {
|
|
10
|
+
const hexRegex = /#[0-9A-Fa-f]{6}|#[0-9A-Fa-f]{3}/g;
|
|
11
|
+
const matches = content.match(hexRegex);
|
|
12
|
+
return matches ? [...new Set(matches)].slice(0, 10) : [];
|
|
13
|
+
}
|
|
14
|
+
function extractFontsFromContent(content) {
|
|
15
|
+
const fontRegex = /font-family:\s*([^;}"'\n]+)/gi;
|
|
16
|
+
const fonts = [];
|
|
17
|
+
let match;
|
|
18
|
+
while ((match = fontRegex.exec(content)) !== null) {
|
|
19
|
+
fonts.push(match[1].trim());
|
|
20
|
+
}
|
|
21
|
+
return [...new Set(fonts)].slice(0, 5);
|
|
22
|
+
}
|
|
23
|
+
export const handleExtract = async (args, _ctx) => {
|
|
24
|
+
const url = args['url'];
|
|
25
|
+
if (!url || typeof url !== 'string')
|
|
26
|
+
throw new Error('url is required');
|
|
27
|
+
if (url.length > 2048)
|
|
28
|
+
throw new Error('URL too long (max 2048 characters)');
|
|
29
|
+
const schema = args['schema'];
|
|
30
|
+
const fields = args['fields'];
|
|
31
|
+
const render = args['render'] || false;
|
|
32
|
+
const llmApiKey = args['llmApiKey'];
|
|
33
|
+
const llmProvider = args['llmProvider'];
|
|
34
|
+
const llmModel = args['llmModel'];
|
|
35
|
+
const llmBaseUrl = args['llmBaseUrl'];
|
|
36
|
+
const prompt = args['prompt'];
|
|
37
|
+
// LLM-based extraction: when llmApiKey (and optionally llmProvider) are provided
|
|
38
|
+
if (llmApiKey && (schema || prompt)) {
|
|
39
|
+
const peelResult = await Promise.race([
|
|
40
|
+
peel(url, { format: 'markdown', render }),
|
|
41
|
+
timeout(60000, 'LLM extract fetch'),
|
|
42
|
+
]);
|
|
43
|
+
const extractResult = await extractWithLLM({
|
|
44
|
+
content: peelResult.content,
|
|
45
|
+
schema: schema,
|
|
46
|
+
prompt,
|
|
47
|
+
llmApiKey,
|
|
48
|
+
llmProvider: llmProvider || 'openai',
|
|
49
|
+
llmModel,
|
|
50
|
+
baseUrl: llmBaseUrl,
|
|
51
|
+
});
|
|
52
|
+
return textResult(safeStringify({
|
|
53
|
+
success: true,
|
|
54
|
+
url: peelResult.url,
|
|
55
|
+
data: extractResult.items.length === 1 ? extractResult.items[0] : extractResult.items,
|
|
56
|
+
llm: {
|
|
57
|
+
provider: extractResult.provider || llmProvider || 'openai',
|
|
58
|
+
model: extractResult.model,
|
|
59
|
+
tokens: extractResult.tokensUsed,
|
|
60
|
+
},
|
|
61
|
+
}));
|
|
62
|
+
}
|
|
63
|
+
// Brand preset: fields=['name','logo','colors','fonts','socials'] or _brand flag
|
|
64
|
+
const isBrandPreset = args['_brand'] ||
|
|
65
|
+
(Array.isArray(fields) &&
|
|
66
|
+
['name', 'logo', 'colors', 'fonts', 'socials'].every((f) => fields.includes(f)));
|
|
67
|
+
if (isBrandPreset) {
|
|
68
|
+
const options = {
|
|
69
|
+
render,
|
|
70
|
+
extract: {
|
|
71
|
+
selectors: {
|
|
72
|
+
primaryColor: 'meta[name="theme-color"]',
|
|
73
|
+
title: 'title',
|
|
74
|
+
logo: 'img[class*="logo"], img[alt*="logo"]',
|
|
75
|
+
},
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
const result = await Promise.race([
|
|
79
|
+
peel(url, options),
|
|
80
|
+
timeout(60000, 'Brand extraction'),
|
|
81
|
+
]);
|
|
82
|
+
return textResult(safeStringify({
|
|
83
|
+
url: result.url,
|
|
84
|
+
title: result.title,
|
|
85
|
+
extracted: result.extracted,
|
|
86
|
+
metadata: result.metadata,
|
|
87
|
+
colors: extractColorsFromContent(result.content || ''),
|
|
88
|
+
fonts: extractFontsFromContent(result.content || ''),
|
|
89
|
+
}));
|
|
90
|
+
}
|
|
91
|
+
// Auto-extract when no schema provided
|
|
92
|
+
if (!schema && (!fields || fields.length === 0)) {
|
|
93
|
+
const htmlResult = await Promise.race([
|
|
94
|
+
peel(url, { format: 'html', render }),
|
|
95
|
+
timeout(60000, 'Auto-extract fetch'),
|
|
96
|
+
]);
|
|
97
|
+
const { autoExtract } = await import('../../core/auto-extract.js');
|
|
98
|
+
const extracted = autoExtract(htmlResult.content || '', url);
|
|
99
|
+
return textResult(safeStringify({ url, pageType: extracted.type, structured: extracted }));
|
|
100
|
+
}
|
|
101
|
+
// Field-based extraction (CSS selectors from field names)
|
|
102
|
+
if (fields && fields.length > 0 && !schema) {
|
|
103
|
+
const fieldSelectorMap = {
|
|
104
|
+
price: '[class*="price"], [data-price]',
|
|
105
|
+
title: 'h1, title',
|
|
106
|
+
description: '[class*="description"], [class*="summary"]',
|
|
107
|
+
image: 'img[class*="main"], img[class*="hero"]',
|
|
108
|
+
name: 'h1, [class*="name"]',
|
|
109
|
+
logo: 'img[class*="logo"], img[alt*="logo"]',
|
|
110
|
+
colors: 'meta[name="theme-color"]',
|
|
111
|
+
fonts: 'link[rel="stylesheet"]',
|
|
112
|
+
socials: 'a[href*="twitter.com"], a[href*="linkedin.com"], a[href*="github.com"]',
|
|
113
|
+
};
|
|
114
|
+
const selectors = {};
|
|
115
|
+
for (const field of fields) {
|
|
116
|
+
selectors[field] = fieldSelectorMap[field] || `[class*="${field}"], [id*="${field}"]`;
|
|
117
|
+
}
|
|
118
|
+
const options = { render, extract: { selectors } };
|
|
119
|
+
const result = await Promise.race([
|
|
120
|
+
peel(url, options),
|
|
121
|
+
timeout(60000, 'Field extraction'),
|
|
122
|
+
]);
|
|
123
|
+
return textResult(safeStringify(result));
|
|
124
|
+
}
|
|
125
|
+
// Schema-based extraction
|
|
126
|
+
const options = {
|
|
127
|
+
render,
|
|
128
|
+
extract: { schema },
|
|
129
|
+
};
|
|
130
|
+
const result = await Promise.race([
|
|
131
|
+
peel(url, options),
|
|
132
|
+
timeout(60000, 'Schema extraction'),
|
|
133
|
+
]);
|
|
134
|
+
return textResult(safeStringify(result));
|
|
135
|
+
};
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* handleFetch — the full-featured webpeel_fetch handler.
|
|
3
|
+
* Supports all options: render, stealth, actions, inline LLM extraction, etc.
|
|
4
|
+
*/
|
|
5
|
+
import { peel } from '../../index.js';
|
|
6
|
+
import { normalizeActions } from '../../core/actions.js';
|
|
7
|
+
import { extractInlineJson } from '../../core/extract-inline.js';
|
|
8
|
+
import { textResult, safeStringify, timeout } from './types.js';
|
|
9
|
+
export const handleFetch = async (args, _ctx) => {
|
|
10
|
+
const { url, render, stealth, wait, format, screenshot: ssFlag, screenshotFullPage, selector, exclude, includeTags, excludeTags, images, location, headers, actions: rawActions, autoScroll: autoScrollParam, maxTokens, extract, inlineExtract, llmProvider, llmApiKey, llmModel, question, budget: budgetArg, readable, } = args;
|
|
11
|
+
if (!url || typeof url !== 'string')
|
|
12
|
+
throw new Error('Invalid URL parameter');
|
|
13
|
+
if (url.length > 2048)
|
|
14
|
+
throw new Error('URL too long (max 2048 characters)');
|
|
15
|
+
const normalizedActions = rawActions ? normalizeActions(rawActions) : undefined;
|
|
16
|
+
const hasActions = normalizedActions && normalizedActions.length > 0;
|
|
17
|
+
const options = {
|
|
18
|
+
render: render || hasActions || !!autoScrollParam || false,
|
|
19
|
+
stealth: stealth || false,
|
|
20
|
+
wait: wait || 0,
|
|
21
|
+
format: format || 'markdown',
|
|
22
|
+
screenshot: ssFlag || false,
|
|
23
|
+
screenshotFullPage: screenshotFullPage || false,
|
|
24
|
+
selector,
|
|
25
|
+
exclude,
|
|
26
|
+
includeTags,
|
|
27
|
+
excludeTags,
|
|
28
|
+
images,
|
|
29
|
+
location: location ? { country: location } : undefined,
|
|
30
|
+
headers,
|
|
31
|
+
actions: normalizedActions,
|
|
32
|
+
autoScroll: autoScrollParam,
|
|
33
|
+
maxTokens,
|
|
34
|
+
extract,
|
|
35
|
+
readable: readable || false,
|
|
36
|
+
lite: args['lite'] || false,
|
|
37
|
+
question,
|
|
38
|
+
budget: args['lite'] ? undefined : (budgetArg ?? (maxTokens === undefined ? 4000 : undefined)),
|
|
39
|
+
};
|
|
40
|
+
const peeled = await Promise.race([
|
|
41
|
+
peel(url, options),
|
|
42
|
+
timeout(60000, 'MCP operation'),
|
|
43
|
+
]);
|
|
44
|
+
// Allow mutable result for inline extraction annotations
|
|
45
|
+
const result = peeled;
|
|
46
|
+
// Inline LLM extraction (BYOK)
|
|
47
|
+
if (inlineExtract && (inlineExtract.schema || inlineExtract.prompt) && llmApiKey && llmProvider) {
|
|
48
|
+
const validProviders = ['openai', 'anthropic', 'google'];
|
|
49
|
+
if (validProviders.includes(llmProvider)) {
|
|
50
|
+
const extractResult = await extractInlineJson(result.content, {
|
|
51
|
+
schema: inlineExtract.schema,
|
|
52
|
+
prompt: inlineExtract.prompt,
|
|
53
|
+
llmProvider: llmProvider,
|
|
54
|
+
llmApiKey,
|
|
55
|
+
llmModel,
|
|
56
|
+
});
|
|
57
|
+
result.json = extractResult.data;
|
|
58
|
+
result.extractTokensUsed = extractResult.tokensUsed;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
const out = {
|
|
62
|
+
url: result.url || url,
|
|
63
|
+
title: result.title || result.metadata?.title || '',
|
|
64
|
+
tokens: result.tokens || 0,
|
|
65
|
+
content: result.content,
|
|
66
|
+
};
|
|
67
|
+
if (result.metadata)
|
|
68
|
+
out['metadata'] = result.metadata;
|
|
69
|
+
if (result.domainData)
|
|
70
|
+
out['domainData'] = result.domainData;
|
|
71
|
+
if (result.readability) {
|
|
72
|
+
out['readability'] = {
|
|
73
|
+
readingTime: result.readability.readingTime,
|
|
74
|
+
wordCount: result.readability.wordCount,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
if (result.quickAnswer)
|
|
78
|
+
out['quickAnswer'] = result.quickAnswer;
|
|
79
|
+
if (result.json)
|
|
80
|
+
out['json'] = result.json;
|
|
81
|
+
if (result.extracted)
|
|
82
|
+
out['extracted'] = result.extracted;
|
|
83
|
+
if (result.images?.length)
|
|
84
|
+
out['images'] = result.images;
|
|
85
|
+
if (result.screenshot)
|
|
86
|
+
out['screenshot'] = result.screenshot;
|
|
87
|
+
if (result.fingerprint)
|
|
88
|
+
out['fingerprint'] = result.fingerprint;
|
|
89
|
+
if (result.extractTokensUsed)
|
|
90
|
+
out['extractTokensUsed'] = result.extractTokensUsed;
|
|
91
|
+
if (result.quality !== undefined)
|
|
92
|
+
out['quality'] = result.quality;
|
|
93
|
+
if (result.timing)
|
|
94
|
+
out['timing'] = result.timing;
|
|
95
|
+
if (result.method)
|
|
96
|
+
out['method'] = result.method;
|
|
97
|
+
return textResult(safeStringify(out));
|
|
98
|
+
};
|