@iflow-mcp/jakeliume-webpeel 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +313 -0
- package/dist/cache.d.ts +30 -0
- package/dist/cache.js +139 -0
- package/dist/cli/commands/auth.d.ts +5 -0
- package/dist/cli/commands/auth.js +411 -0
- package/dist/cli/commands/doctor.d.ts +37 -0
- package/dist/cli/commands/doctor.js +371 -0
- package/dist/cli/commands/fetch.d.ts +6 -0
- package/dist/cli/commands/fetch.js +1345 -0
- package/dist/cli/commands/guide.d.ts +2 -0
- package/dist/cli/commands/guide.js +183 -0
- package/dist/cli/commands/interact.d.ts +5 -0
- package/dist/cli/commands/interact.js +840 -0
- package/dist/cli/commands/jobs.d.ts +5 -0
- package/dist/cli/commands/jobs.js +997 -0
- package/dist/cli/commands/monitor.d.ts +12 -0
- package/dist/cli/commands/monitor.js +197 -0
- package/dist/cli/commands/observe.d.ts +12 -0
- package/dist/cli/commands/observe.js +158 -0
- package/dist/cli/commands/screenshot.d.ts +5 -0
- package/dist/cli/commands/screenshot.js +282 -0
- package/dist/cli/commands/search.d.ts +5 -0
- package/dist/cli/commands/search.js +1021 -0
- package/dist/cli/commands/setup.d.ts +13 -0
- package/dist/cli/commands/setup.js +244 -0
- package/dist/cli/commands/skill.d.ts +15 -0
- package/dist/cli/commands/skill.js +195 -0
- package/dist/cli/utils.d.ts +84 -0
- package/dist/cli/utils.js +806 -0
- package/dist/cli-auth.d.ts +75 -0
- package/dist/cli-auth.js +369 -0
- package/dist/cli.d.ts +17 -0
- package/dist/cli.js +99 -0
- package/dist/core/actions.d.ts +69 -0
- package/dist/core/actions.js +495 -0
- package/dist/core/agent.d.ts +98 -0
- package/dist/core/agent.js +558 -0
- package/dist/core/answer.d.ts +42 -0
- package/dist/core/answer.js +395 -0
- package/dist/core/application-tracker.d.ts +84 -0
- package/dist/core/application-tracker.js +184 -0
- package/dist/core/apply.d.ts +162 -0
- package/dist/core/apply.js +816 -0
- package/dist/core/auth-detection.d.ts +35 -0
- package/dist/core/auth-detection.js +358 -0
- package/dist/core/auto-extract.d.ts +82 -0
- package/dist/core/auto-extract.js +604 -0
- package/dist/core/auto-interact.d.ts +23 -0
- package/dist/core/auto-interact.js +246 -0
- package/dist/core/bm25-filter.d.ts +66 -0
- package/dist/core/bm25-filter.js +288 -0
- package/dist/core/branding.d.ts +54 -0
- package/dist/core/branding.js +234 -0
- package/dist/core/browser-fetch.d.ts +323 -0
- package/dist/core/browser-fetch.js +1600 -0
- package/dist/core/browser-pool.d.ts +91 -0
- package/dist/core/browser-pool.js +550 -0
- package/dist/core/budget.d.ts +42 -0
- package/dist/core/budget.js +324 -0
- package/dist/core/business-intel.d.ts +47 -0
- package/dist/core/business-intel.js +279 -0
- package/dist/core/cache.d.ts +13 -0
- package/dist/core/cache.js +121 -0
- package/dist/core/cf-worker-proxy.d.ts +32 -0
- package/dist/core/cf-worker-proxy.js +87 -0
- package/dist/core/challenge-detection.d.ts +26 -0
- package/dist/core/challenge-detection.js +468 -0
- package/dist/core/change-tracking.d.ts +75 -0
- package/dist/core/change-tracking.js +276 -0
- package/dist/core/chunker.d.ts +46 -0
- package/dist/core/chunker.js +249 -0
- package/dist/core/chunking.d.ts +42 -0
- package/dist/core/chunking.js +181 -0
- package/dist/core/circuit-breaker.d.ts +44 -0
- package/dist/core/circuit-breaker.js +85 -0
- package/dist/core/content-pruner.d.ts +47 -0
- package/dist/core/content-pruner.js +425 -0
- package/dist/core/cookie-cache.d.ts +60 -0
- package/dist/core/cookie-cache.js +163 -0
- package/dist/core/crawl-checkpoint.d.ts +54 -0
- package/dist/core/crawl-checkpoint.js +104 -0
- package/dist/core/crawler.d.ts +84 -0
- package/dist/core/crawler.js +349 -0
- package/dist/core/cross-verify.d.ts +27 -0
- package/dist/core/cross-verify.js +93 -0
- package/dist/core/deep-fetch.d.ts +74 -0
- package/dist/core/deep-fetch.js +405 -0
- package/dist/core/deep-research.d.ts +141 -0
- package/dist/core/deep-research.js +972 -0
- package/dist/core/design-analysis.d.ts +70 -0
- package/dist/core/design-analysis.js +490 -0
- package/dist/core/design-compare.d.ts +38 -0
- package/dist/core/design-compare.js +264 -0
- package/dist/core/diff.d.ts +61 -0
- package/dist/core/diff.js +289 -0
- package/dist/core/dns-cache.d.ts +20 -0
- package/dist/core/dns-cache.js +198 -0
- package/dist/core/documents.d.ts +23 -0
- package/dist/core/documents.js +123 -0
- package/dist/core/domain-memory.d.ts +66 -0
- package/dist/core/domain-memory.js +163 -0
- package/dist/core/domain-verify.d.ts +40 -0
- package/dist/core/domain-verify.js +379 -0
- package/dist/core/engine-ranker.d.ts +112 -0
- package/dist/core/engine-ranker.js +395 -0
- package/dist/core/extract-inline.d.ts +38 -0
- package/dist/core/extract-inline.js +215 -0
- package/dist/core/extract-listings.d.ts +38 -0
- package/dist/core/extract-listings.js +461 -0
- package/dist/core/extract.d.ts +9 -0
- package/dist/core/extract.js +139 -0
- package/dist/core/fetch-cache.d.ts +57 -0
- package/dist/core/fetch-cache.js +95 -0
- package/dist/core/fetcher.d.ts +13 -0
- package/dist/core/fetcher.js +12 -0
- package/dist/core/google-cache.d.ts +29 -0
- package/dist/core/google-cache.js +180 -0
- package/dist/core/google-serp-parser.d.ts +82 -0
- package/dist/core/google-serp-parser.js +287 -0
- package/dist/core/hotel-search.d.ts +122 -0
- package/dist/core/hotel-search.js +382 -0
- package/dist/core/http-fetch.d.ts +72 -0
- package/dist/core/http-fetch.js +820 -0
- package/dist/core/human.d.ts +175 -0
- package/dist/core/human.js +680 -0
- package/dist/core/image-caption.d.ts +44 -0
- package/dist/core/image-caption.js +271 -0
- package/dist/core/jobs.d.ts +75 -0
- package/dist/core/jobs.js +634 -0
- package/dist/core/json-ld.d.ts +15 -0
- package/dist/core/json-ld.js +617 -0
- package/dist/core/language-detect.d.ts +18 -0
- package/dist/core/language-detect.js +135 -0
- package/dist/core/links.d.ts +10 -0
- package/dist/core/links.js +44 -0
- package/dist/core/llm-extract.d.ts +71 -0
- package/dist/core/llm-extract.js +507 -0
- package/dist/core/llm-provider.d.ts +100 -0
- package/dist/core/llm-provider.js +702 -0
- package/dist/core/local-search.d.ts +60 -0
- package/dist/core/local-search.js +308 -0
- package/dist/core/logger.d.ts +28 -0
- package/dist/core/logger.js +104 -0
- package/dist/core/map.d.ts +33 -0
- package/dist/core/map.js +127 -0
- package/dist/core/markdown.d.ts +92 -0
- package/dist/core/markdown.js +809 -0
- package/dist/core/metadata.d.ts +34 -0
- package/dist/core/metadata.js +422 -0
- package/dist/core/observe.d.ts +113 -0
- package/dist/core/observe.js +395 -0
- package/dist/core/ocr.d.ts +12 -0
- package/dist/core/ocr.js +33 -0
- package/dist/core/paginate.d.ts +31 -0
- package/dist/core/paginate.js +106 -0
- package/dist/core/pdf.d.ts +8 -0
- package/dist/core/pdf.js +25 -0
- package/dist/core/peel-tls.d.ts +25 -0
- package/dist/core/peel-tls.js +220 -0
- package/dist/core/pipeline.d.ts +132 -0
- package/dist/core/pipeline.js +1666 -0
- package/dist/core/profiles.d.ts +61 -0
- package/dist/core/profiles.js +350 -0
- package/dist/core/prompt-guard.d.ts +30 -0
- package/dist/core/prompt-guard.js +119 -0
- package/dist/core/proxy-config.d.ts +90 -0
- package/dist/core/proxy-config.js +172 -0
- package/dist/core/quick-answer.d.ts +53 -0
- package/dist/core/quick-answer.js +833 -0
- package/dist/core/rate-governor.d.ts +80 -0
- package/dist/core/rate-governor.js +238 -0
- package/dist/core/readability.d.ts +57 -0
- package/dist/core/readability.js +533 -0
- package/dist/core/research.d.ts +66 -0
- package/dist/core/research.js +270 -0
- package/dist/core/retry.d.ts +60 -0
- package/dist/core/retry.js +119 -0
- package/dist/core/safe-browsing.d.ts +30 -0
- package/dist/core/safe-browsing.js +206 -0
- package/dist/core/schema-extraction.d.ts +66 -0
- package/dist/core/schema-extraction.js +352 -0
- package/dist/core/schema-postprocess.d.ts +32 -0
- package/dist/core/schema-postprocess.js +469 -0
- package/dist/core/schema-templates.d.ts +19 -0
- package/dist/core/schema-templates.js +143 -0
- package/dist/core/screenshot.d.ts +224 -0
- package/dist/core/screenshot.js +207 -0
- package/dist/core/search-engines.d.ts +25 -0
- package/dist/core/search-engines.js +182 -0
- package/dist/core/search-provider.d.ts +243 -0
- package/dist/core/search-provider.js +1629 -0
- package/dist/core/searxng-provider.d.ts +35 -0
- package/dist/core/searxng-provider.js +105 -0
- package/dist/core/selective-evidence.d.ts +151 -0
- package/dist/core/selective-evidence.js +389 -0
- package/dist/core/site-search.d.ts +44 -0
- package/dist/core/site-search.js +252 -0
- package/dist/core/sitemap.d.ts +23 -0
- package/dist/core/sitemap.js +105 -0
- package/dist/core/source-credibility.d.ts +29 -0
- package/dist/core/source-credibility.js +584 -0
- package/dist/core/source-scoring.d.ts +166 -0
- package/dist/core/source-scoring.js +396 -0
- package/dist/core/stemmer.d.ts +38 -0
- package/dist/core/stemmer.js +509 -0
- package/dist/core/strategies.d.ts +104 -0
- package/dist/core/strategies.js +1044 -0
- package/dist/core/strategy-hooks.d.ts +145 -0
- package/dist/core/strategy-hooks.js +74 -0
- package/dist/core/structured-extract.d.ts +43 -0
- package/dist/core/structured-extract.js +550 -0
- package/dist/core/summarize.d.ts +17 -0
- package/dist/core/summarize.js +78 -0
- package/dist/core/synonyms.d.ts +42 -0
- package/dist/core/synonyms.js +184 -0
- package/dist/core/system-monitor.d.ts +61 -0
- package/dist/core/system-monitor.js +133 -0
- package/dist/core/table-format.d.ts +30 -0
- package/dist/core/table-format.js +146 -0
- package/dist/core/threat-feeds.d.ts +23 -0
- package/dist/core/threat-feeds.js +104 -0
- package/dist/core/timing.d.ts +21 -0
- package/dist/core/timing.js +33 -0
- package/dist/core/transcript-export.d.ts +47 -0
- package/dist/core/transcript-export.js +107 -0
- package/dist/core/user-agents.d.ts +82 -0
- package/dist/core/user-agents.js +239 -0
- package/dist/core/vertical-search.d.ts +54 -0
- package/dist/core/vertical-search.js +158 -0
- package/dist/core/watch-manager.d.ts +175 -0
- package/dist/core/watch-manager.js +416 -0
- package/dist/core/watch.d.ts +101 -0
- package/dist/core/watch.js +389 -0
- package/dist/core/youtube.d.ts +130 -0
- package/dist/core/youtube.js +1175 -0
- package/dist/ee/challenge-re-export.d.ts +1 -0
- package/dist/ee/challenge-re-export.js +1 -0
- package/dist/ee/challenge-solver.d.ts +72 -0
- package/dist/ee/challenge-solver.js +720 -0
- package/dist/ee/domain-extractors.d.ts +8 -0
- package/dist/ee/domain-extractors.js +8 -0
- package/dist/ee/domain-intel.d.ts +16 -0
- package/dist/ee/domain-intel.js +133 -0
- package/dist/ee/extractors/allrecipes.d.ts +2 -0
- package/dist/ee/extractors/allrecipes.js +120 -0
- package/dist/ee/extractors/amazon.d.ts +2 -0
- package/dist/ee/extractors/amazon.js +78 -0
- package/dist/ee/extractors/arxiv.d.ts +2 -0
- package/dist/ee/extractors/arxiv.js +137 -0
- package/dist/ee/extractors/bestbuy.d.ts +2 -0
- package/dist/ee/extractors/bestbuy.js +78 -0
- package/dist/ee/extractors/carscom.d.ts +2 -0
- package/dist/ee/extractors/carscom.js +121 -0
- package/dist/ee/extractors/coingecko.d.ts +2 -0
- package/dist/ee/extractors/coingecko.js +134 -0
- package/dist/ee/extractors/craigslist.d.ts +2 -0
- package/dist/ee/extractors/craigslist.js +92 -0
- package/dist/ee/extractors/devto.d.ts +2 -0
- package/dist/ee/extractors/devto.js +135 -0
- package/dist/ee/extractors/ebay.d.ts +2 -0
- package/dist/ee/extractors/ebay.js +90 -0
- package/dist/ee/extractors/espn.d.ts +2 -0
- package/dist/ee/extractors/espn.js +260 -0
- package/dist/ee/extractors/etsy.d.ts +2 -0
- package/dist/ee/extractors/etsy.js +52 -0
- package/dist/ee/extractors/facebook.d.ts +2 -0
- package/dist/ee/extractors/facebook.js +46 -0
- package/dist/ee/extractors/github.d.ts +2 -0
- package/dist/ee/extractors/github.js +196 -0
- package/dist/ee/extractors/google-flights.d.ts +2 -0
- package/dist/ee/extractors/google-flights.js +176 -0
- package/dist/ee/extractors/hackernews.d.ts +2 -0
- package/dist/ee/extractors/hackernews.js +147 -0
- package/dist/ee/extractors/imdb.d.ts +2 -0
- package/dist/ee/extractors/imdb.js +172 -0
- package/dist/ee/extractors/index.d.ts +26 -0
- package/dist/ee/extractors/index.js +247 -0
- package/dist/ee/extractors/instagram.d.ts +2 -0
- package/dist/ee/extractors/instagram.js +102 -0
- package/dist/ee/extractors/kalshi.d.ts +2 -0
- package/dist/ee/extractors/kalshi.js +121 -0
- package/dist/ee/extractors/kayak-cars.d.ts +2 -0
- package/dist/ee/extractors/kayak-cars.js +270 -0
- package/dist/ee/extractors/linkedin.d.ts +2 -0
- package/dist/ee/extractors/linkedin.js +113 -0
- package/dist/ee/extractors/medium.d.ts +2 -0
- package/dist/ee/extractors/medium.js +130 -0
- package/dist/ee/extractors/news.d.ts +4 -0
- package/dist/ee/extractors/news.js +173 -0
- package/dist/ee/extractors/npm.d.ts +2 -0
- package/dist/ee/extractors/npm.js +86 -0
- package/dist/ee/extractors/pdf.d.ts +2 -0
- package/dist/ee/extractors/pdf.js +108 -0
- package/dist/ee/extractors/pinterest.d.ts +2 -0
- package/dist/ee/extractors/pinterest.js +34 -0
- package/dist/ee/extractors/polymarket.d.ts +2 -0
- package/dist/ee/extractors/polymarket.js +358 -0
- package/dist/ee/extractors/producthunt.d.ts +2 -0
- package/dist/ee/extractors/producthunt.js +88 -0
- package/dist/ee/extractors/pubmed.d.ts +2 -0
- package/dist/ee/extractors/pubmed.js +162 -0
- package/dist/ee/extractors/pypi.d.ts +2 -0
- package/dist/ee/extractors/pypi.js +80 -0
- package/dist/ee/extractors/reddit.d.ts +2 -0
- package/dist/ee/extractors/reddit.js +438 -0
- package/dist/ee/extractors/redfin.d.ts +2 -0
- package/dist/ee/extractors/redfin.js +156 -0
- package/dist/ee/extractors/semanticscholar.d.ts +2 -0
- package/dist/ee/extractors/semanticscholar.js +131 -0
- package/dist/ee/extractors/shared.d.ts +12 -0
- package/dist/ee/extractors/shared.js +76 -0
- package/dist/ee/extractors/soundcloud.d.ts +2 -0
- package/dist/ee/extractors/soundcloud.js +34 -0
- package/dist/ee/extractors/sportsbetting.d.ts +2 -0
- package/dist/ee/extractors/sportsbetting.js +37 -0
- package/dist/ee/extractors/spotify.d.ts +2 -0
- package/dist/ee/extractors/spotify.js +34 -0
- package/dist/ee/extractors/stackoverflow.d.ts +2 -0
- package/dist/ee/extractors/stackoverflow.js +61 -0
- package/dist/ee/extractors/substack.d.ts +2 -0
- package/dist/ee/extractors/substack.js +115 -0
- package/dist/ee/extractors/substackroot.d.ts +2 -0
- package/dist/ee/extractors/substackroot.js +46 -0
- package/dist/ee/extractors/tiktok.d.ts +2 -0
- package/dist/ee/extractors/tiktok.js +29 -0
- package/dist/ee/extractors/tradingview.d.ts +2 -0
- package/dist/ee/extractors/tradingview.js +182 -0
- package/dist/ee/extractors/twitch.d.ts +2 -0
- package/dist/ee/extractors/twitch.js +36 -0
- package/dist/ee/extractors/twitter.d.ts +2 -0
- package/dist/ee/extractors/twitter.js +327 -0
- package/dist/ee/extractors/types.d.ts +14 -0
- package/dist/ee/extractors/types.js +1 -0
- package/dist/ee/extractors/walmart.d.ts +2 -0
- package/dist/ee/extractors/walmart.js +50 -0
- package/dist/ee/extractors/weather.d.ts +2 -0
- package/dist/ee/extractors/weather.js +133 -0
- package/dist/ee/extractors/wikipedia.d.ts +4 -0
- package/dist/ee/extractors/wikipedia.js +235 -0
- package/dist/ee/extractors/yelp.d.ts +2 -0
- package/dist/ee/extractors/yelp.js +216 -0
- package/dist/ee/extractors/youtube.d.ts +2 -0
- package/dist/ee/extractors/youtube.js +189 -0
- package/dist/ee/extractors/zillow.d.ts +54 -0
- package/dist/ee/extractors/zillow.js +247 -0
- package/dist/ee/extractors-re-export.d.ts +1 -0
- package/dist/ee/extractors-re-export.js +1 -0
- package/dist/ee/premium-hooks.d.ts +20 -0
- package/dist/ee/premium-hooks.js +50 -0
- package/dist/ee/spa-detection.d.ts +2 -0
- package/dist/ee/spa-detection.js +2 -0
- package/dist/ee/stability.d.ts +4 -0
- package/dist/ee/stability.js +29 -0
- package/dist/ee/swr-cache.d.ts +14 -0
- package/dist/ee/swr-cache.js +34 -0
- package/dist/index.d.ts +143 -0
- package/dist/index.js +291 -0
- package/dist/integrations/index.d.ts +2 -0
- package/dist/integrations/index.js +2 -0
- package/dist/integrations/langchain.d.ts +64 -0
- package/dist/integrations/langchain.js +115 -0
- package/dist/integrations/llamaindex.d.ts +50 -0
- package/dist/integrations/llamaindex.js +91 -0
- package/dist/mcp/handlers/act.d.ts +5 -0
- package/dist/mcp/handlers/act.js +34 -0
- package/dist/mcp/handlers/definitions.d.ts +6 -0
- package/dist/mcp/handlers/definitions.js +395 -0
- package/dist/mcp/handlers/extract.d.ts +7 -0
- package/dist/mcp/handlers/extract.js +135 -0
- package/dist/mcp/handlers/fetch.d.ts +6 -0
- package/dist/mcp/handlers/fetch.js +98 -0
- package/dist/mcp/handlers/find.d.ts +5 -0
- package/dist/mcp/handlers/find.js +137 -0
- package/dist/mcp/handlers/index.d.ts +13 -0
- package/dist/mcp/handlers/index.js +63 -0
- package/dist/mcp/handlers/legacy.d.ts +25 -0
- package/dist/mcp/handlers/legacy.js +450 -0
- package/dist/mcp/handlers/meta.d.ts +6 -0
- package/dist/mcp/handlers/meta.js +40 -0
- package/dist/mcp/handlers/monitor.d.ts +5 -0
- package/dist/mcp/handlers/monitor.js +41 -0
- package/dist/mcp/handlers/observe.d.ts +8 -0
- package/dist/mcp/handlers/observe.js +37 -0
- package/dist/mcp/handlers/read.d.ts +6 -0
- package/dist/mcp/handlers/read.js +78 -0
- package/dist/mcp/handlers/see.d.ts +5 -0
- package/dist/mcp/handlers/see.js +75 -0
- package/dist/mcp/handlers/types.d.ts +29 -0
- package/dist/mcp/handlers/types.js +28 -0
- package/dist/mcp/server.d.ts +7 -0
- package/dist/mcp/server.js +108 -0
- package/dist/mcp/smart-router.d.ts +23 -0
- package/dist/mcp/smart-router.js +178 -0
- package/dist/server/app.d.ts +14 -0
- package/dist/server/app.js +632 -0
- package/dist/server/auth-store.d.ts +28 -0
- package/dist/server/auth-store.js +88 -0
- package/dist/server/bull-queues.d.ts +60 -0
- package/dist/server/bull-queues.js +90 -0
- package/dist/server/email-service.d.ts +55 -0
- package/dist/server/email-service.js +291 -0
- package/dist/server/job-queue.d.ts +100 -0
- package/dist/server/job-queue.js +145 -0
- package/dist/server/logger.d.ts +10 -0
- package/dist/server/logger.js +37 -0
- package/dist/server/middleware/audit-log.d.ts +14 -0
- package/dist/server/middleware/audit-log.js +73 -0
- package/dist/server/middleware/auth.d.ts +35 -0
- package/dist/server/middleware/auth.js +225 -0
- package/dist/server/middleware/rate-limit.d.ts +50 -0
- package/dist/server/middleware/rate-limit.js +270 -0
- package/dist/server/middleware/scope-guard.d.ts +25 -0
- package/dist/server/middleware/scope-guard.js +45 -0
- package/dist/server/middleware/url-validator.d.ts +15 -0
- package/dist/server/middleware/url-validator.js +201 -0
- package/dist/server/openapi.yaml +6418 -0
- package/dist/server/pg-auth-store.d.ts +146 -0
- package/dist/server/pg-auth-store.js +576 -0
- package/dist/server/pg-job-queue.d.ts +59 -0
- package/dist/server/pg-job-queue.js +375 -0
- package/dist/server/routes/activity.d.ts +6 -0
- package/dist/server/routes/activity.js +79 -0
- package/dist/server/routes/admin-active.d.ts +7 -0
- package/dist/server/routes/admin-active.js +120 -0
- package/dist/server/routes/admin-stats.d.ts +7 -0
- package/dist/server/routes/admin-stats.js +176 -0
- package/dist/server/routes/agent.d.ts +24 -0
- package/dist/server/routes/agent.js +480 -0
- package/dist/server/routes/answer.d.ts +5 -0
- package/dist/server/routes/answer.js +125 -0
- package/dist/server/routes/ask.d.ts +28 -0
- package/dist/server/routes/ask.js +295 -0
- package/dist/server/routes/batch.d.ts +6 -0
- package/dist/server/routes/batch.js +493 -0
- package/dist/server/routes/cache-warm.d.ts +25 -0
- package/dist/server/routes/cache-warm.js +212 -0
- package/dist/server/routes/cli-usage.d.ts +6 -0
- package/dist/server/routes/cli-usage.js +127 -0
- package/dist/server/routes/compat.d.ts +23 -0
- package/dist/server/routes/compat.js +652 -0
- package/dist/server/routes/crawl.d.ts +13 -0
- package/dist/server/routes/crawl.js +287 -0
- package/dist/server/routes/deep-fetch.d.ts +8 -0
- package/dist/server/routes/deep-fetch.js +57 -0
- package/dist/server/routes/deep-research.d.ts +11 -0
- package/dist/server/routes/deep-research.js +232 -0
- package/dist/server/routes/demo.d.ts +24 -0
- package/dist/server/routes/demo.js +517 -0
- package/dist/server/routes/do.d.ts +8 -0
- package/dist/server/routes/do.js +72 -0
- package/dist/server/routes/extract.d.ts +14 -0
- package/dist/server/routes/extract.js +325 -0
- package/dist/server/routes/feed.d.ts +15 -0
- package/dist/server/routes/feed.js +311 -0
- package/dist/server/routes/fetch-queue.d.ts +13 -0
- package/dist/server/routes/fetch-queue.js +357 -0
- package/dist/server/routes/fetch.d.ts +7 -0
- package/dist/server/routes/fetch.js +1274 -0
- package/dist/server/routes/go.d.ts +14 -0
- package/dist/server/routes/go.js +81 -0
- package/dist/server/routes/health.d.ts +11 -0
- package/dist/server/routes/health.js +141 -0
- package/dist/server/routes/jobs.d.ts +7 -0
- package/dist/server/routes/jobs.js +574 -0
- package/dist/server/routes/map.d.ts +11 -0
- package/dist/server/routes/map.js +116 -0
- package/dist/server/routes/mcp.d.ts +14 -0
- package/dist/server/routes/mcp.js +197 -0
- package/dist/server/routes/metrics.d.ts +37 -0
- package/dist/server/routes/metrics.js +149 -0
- package/dist/server/routes/oauth.d.ts +9 -0
- package/dist/server/routes/oauth.js +396 -0
- package/dist/server/routes/playground.d.ts +17 -0
- package/dist/server/routes/playground.js +283 -0
- package/dist/server/routes/reader.d.ts +18 -0
- package/dist/server/routes/reader.js +192 -0
- package/dist/server/routes/research.d.ts +14 -0
- package/dist/server/routes/research.js +482 -0
- package/dist/server/routes/screenshot.d.ts +22 -0
- package/dist/server/routes/screenshot.js +820 -0
- package/dist/server/routes/search.d.ts +6 -0
- package/dist/server/routes/search.js +874 -0
- package/dist/server/routes/session.d.ts +17 -0
- package/dist/server/routes/session.js +548 -0
- package/dist/server/routes/share.d.ts +18 -0
- package/dist/server/routes/share.js +462 -0
- package/dist/server/routes/smart-search/handlers/cars.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/cars.js +102 -0
- package/dist/server/routes/smart-search/handlers/flights.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/flights.js +72 -0
- package/dist/server/routes/smart-search/handlers/general.d.ts +13 -0
- package/dist/server/routes/smart-search/handlers/general.js +717 -0
- package/dist/server/routes/smart-search/handlers/hotels.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/hotels.js +88 -0
- package/dist/server/routes/smart-search/handlers/products.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/products.js +1309 -0
- package/dist/server/routes/smart-search/handlers/rental.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/rental.js +154 -0
- package/dist/server/routes/smart-search/handlers/restaurants.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/restaurants.js +225 -0
- package/dist/server/routes/smart-search/handlers/transit-verdict.d.ts +41 -0
- package/dist/server/routes/smart-search/handlers/transit-verdict.js +224 -0
- package/dist/server/routes/smart-search/index.d.ts +19 -0
- package/dist/server/routes/smart-search/index.js +546 -0
- package/dist/server/routes/smart-search/intent.d.ts +3 -0
- package/dist/server/routes/smart-search/intent.js +264 -0
- package/dist/server/routes/smart-search/llm.d.ts +16 -0
- package/dist/server/routes/smart-search/llm.js +70 -0
- package/dist/server/routes/smart-search/sources/reddit.d.ts +18 -0
- package/dist/server/routes/smart-search/sources/reddit.js +34 -0
- package/dist/server/routes/smart-search/sources/yelp.d.ts +25 -0
- package/dist/server/routes/smart-search/sources/yelp.js +171 -0
- package/dist/server/routes/smart-search/sources/youtube.d.ts +8 -0
- package/dist/server/routes/smart-search/sources/youtube.js +9 -0
- package/dist/server/routes/smart-search/types.d.ts +81 -0
- package/dist/server/routes/smart-search/types.js +1 -0
- package/dist/server/routes/smart-search/utils.d.ts +20 -0
- package/dist/server/routes/smart-search/utils.js +146 -0
- package/dist/server/routes/stats.d.ts +6 -0
- package/dist/server/routes/stats.js +71 -0
- package/dist/server/routes/stripe.d.ts +15 -0
- package/dist/server/routes/stripe.js +296 -0
- package/dist/server/routes/transcript-export.d.ts +10 -0
- package/dist/server/routes/transcript-export.js +178 -0
- package/dist/server/routes/usage.d.ts +9 -0
- package/dist/server/routes/usage.js +279 -0
- package/dist/server/routes/users.d.ts +8 -0
- package/dist/server/routes/users.js +1867 -0
- package/dist/server/routes/watch.d.ts +15 -0
- package/dist/server/routes/watch.js +309 -0
- package/dist/server/routes/webhooks.d.ts +26 -0
- package/dist/server/routes/webhooks.js +170 -0
- package/dist/server/routes/youtube.d.ts +6 -0
- package/dist/server/routes/youtube.js +130 -0
- package/dist/server/sentry.d.ts +14 -0
- package/dist/server/sentry.js +104 -0
- package/dist/server/types.d.ts +15 -0
- package/dist/server/types.js +7 -0
- package/dist/server/utils/response.d.ts +44 -0
- package/dist/server/utils/response.js +69 -0
- package/dist/server/utils/sse.d.ts +22 -0
- package/dist/server/utils/sse.js +38 -0
- package/dist/types.d.ts +552 -0
- package/dist/types.js +39 -0
- package/llms.txt +105 -0
- package/package.json +189 -0
|
@@ -0,0 +1,450 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Legacy tool handlers — backward-compatible implementations for all
|
|
3
|
+
* pre-consolidated tool names. Each handler delegates to the appropriate
|
|
4
|
+
* core module or consolidated handler.
|
|
5
|
+
*/
|
|
6
|
+
import { peel, peelBatch } from '../../index.js';
|
|
7
|
+
import { getBestSearchProvider } from '../../core/search-provider.js';
|
|
8
|
+
import { quickAnswer } from '../../core/quick-answer.js';
|
|
9
|
+
import { answerQuestion } from '../../core/answer.js';
|
|
10
|
+
import { runAgent } from '../../core/agent.js';
|
|
11
|
+
import { textResult, safeStringify, timeout } from './types.js';
|
|
12
|
+
import { handleExtract } from './extract.js';
|
|
13
|
+
import { handleMonitor } from './monitor.js';
|
|
14
|
+
import { handleFind } from './find.js';
|
|
15
|
+
// ── webpeel_youtube ──────────────────────────────────────────────────────────
|
|
16
|
+
export const handleYoutube = async (args, _ctx) => {
|
|
17
|
+
const { getYouTubeTranscript } = await import('../../core/youtube.js');
|
|
18
|
+
const url = args['url'];
|
|
19
|
+
if (!url || typeof url !== 'string')
|
|
20
|
+
throw new Error('url is required');
|
|
21
|
+
const language = args['language'] || 'en';
|
|
22
|
+
const transcript = await Promise.race([
|
|
23
|
+
getYouTubeTranscript(url, { language }),
|
|
24
|
+
timeout(60000, 'YouTube transcript'),
|
|
25
|
+
]);
|
|
26
|
+
return textResult(safeStringify(transcript));
|
|
27
|
+
};
|
|
28
|
+
// ── webpeel_screenshot ───────────────────────────────────────────────────────
|
|
29
|
+
export const handleScreenshot = async (args, _ctx) => {
|
|
30
|
+
const { takeScreenshot } = await import('../../core/screenshot.js');
|
|
31
|
+
const { url, fullPage, width, height, format, quality, waitFor, stealth, actions, } = args;
|
|
32
|
+
if (!url)
|
|
33
|
+
throw new Error('url is required');
|
|
34
|
+
const result = await Promise.race([
|
|
35
|
+
takeScreenshot(url, {
|
|
36
|
+
fullPage: fullPage || false,
|
|
37
|
+
width,
|
|
38
|
+
height,
|
|
39
|
+
format: format || 'png',
|
|
40
|
+
quality,
|
|
41
|
+
waitFor,
|
|
42
|
+
stealth: stealth || false,
|
|
43
|
+
actions,
|
|
44
|
+
}),
|
|
45
|
+
timeout(60000, 'Screenshot'),
|
|
46
|
+
]);
|
|
47
|
+
return textResult(safeStringify({
|
|
48
|
+
url: result.url,
|
|
49
|
+
format: result.format,
|
|
50
|
+
contentType: result.contentType,
|
|
51
|
+
screenshot: result.screenshot,
|
|
52
|
+
}));
|
|
53
|
+
};
|
|
54
|
+
// ── webpeel_search ───────────────────────────────────────────────────────────
|
|
55
|
+
export const handleSearch = async (args, ctx) => {
|
|
56
|
+
return handleFind({ ...args, depth: 'quick' }, ctx);
|
|
57
|
+
};
|
|
58
|
+
// ── webpeel_research ─────────────────────────────────────────────────────────
|
|
59
|
+
export const handleResearch = async (args, _ctx) => {
|
|
60
|
+
const { query, maxSources, maxDepth, llmApiKey, llmModel, llmBaseUrl, outputFormat, timeout: resTimeout, } = args;
|
|
61
|
+
if (!query)
|
|
62
|
+
throw new Error('query is required');
|
|
63
|
+
const { research } = await import('../../core/research.js');
|
|
64
|
+
const result = await Promise.race([
|
|
65
|
+
research({
|
|
66
|
+
query,
|
|
67
|
+
maxSources: maxSources ?? 5,
|
|
68
|
+
maxDepth: maxDepth ?? 1,
|
|
69
|
+
apiKey: llmApiKey,
|
|
70
|
+
model: llmModel,
|
|
71
|
+
baseUrl: llmBaseUrl,
|
|
72
|
+
outputFormat: outputFormat ?? 'report',
|
|
73
|
+
timeout: resTimeout ?? 60000,
|
|
74
|
+
}),
|
|
75
|
+
timeout(180000, 'Research'),
|
|
76
|
+
]);
|
|
77
|
+
return textResult(safeStringify({
|
|
78
|
+
report: result.report,
|
|
79
|
+
sources: result.sources,
|
|
80
|
+
totalSourcesFound: result.totalSourcesFound,
|
|
81
|
+
sourcesConsulted: result.sourcesConsulted,
|
|
82
|
+
elapsed: result.elapsed,
|
|
83
|
+
tokensUsed: result.tokensUsed,
|
|
84
|
+
cost: result.cost,
|
|
85
|
+
}));
|
|
86
|
+
};
|
|
87
|
+
// ── webpeel_crawl ────────────────────────────────────────────────────────────
|
|
88
|
+
export const handleCrawl = async (args, _ctx) => {
|
|
89
|
+
const { crawl } = await import('../../core/crawler.js');
|
|
90
|
+
const { url, maxPages, maxDepth, allowedDomains, excludePatterns, respectRobotsTxt, rateLimitMs, sitemapFirst, render, stealth, } = args;
|
|
91
|
+
if (!url)
|
|
92
|
+
throw new Error('url is required');
|
|
93
|
+
const results = await Promise.race([
|
|
94
|
+
crawl(url, {
|
|
95
|
+
maxPages,
|
|
96
|
+
maxDepth,
|
|
97
|
+
allowedDomains,
|
|
98
|
+
excludePatterns,
|
|
99
|
+
respectRobotsTxt,
|
|
100
|
+
rateLimitMs,
|
|
101
|
+
sitemapFirst,
|
|
102
|
+
render,
|
|
103
|
+
stealth,
|
|
104
|
+
}),
|
|
105
|
+
timeout(600000, 'Crawl'),
|
|
106
|
+
]);
|
|
107
|
+
return textResult(safeStringify(results));
|
|
108
|
+
};
|
|
109
|
+
// ── webpeel_map ──────────────────────────────────────────────────────────────
|
|
110
|
+
export const handleMap = async (args, _ctx) => {
|
|
111
|
+
const { mapDomain } = await import('../../core/map.js');
|
|
112
|
+
const { url, maxUrls, includePatterns, excludePatterns } = args;
|
|
113
|
+
if (!url)
|
|
114
|
+
throw new Error('url is required');
|
|
115
|
+
const results = await Promise.race([
|
|
116
|
+
mapDomain(url, { maxUrls, includePatterns, excludePatterns }),
|
|
117
|
+
timeout(600000, 'Map'),
|
|
118
|
+
]);
|
|
119
|
+
return textResult(safeStringify(results));
|
|
120
|
+
};
|
|
121
|
+
// ── webpeel_batch ────────────────────────────────────────────────────────────
|
|
122
|
+
export const handleBatch = async (args, _ctx) => {
|
|
123
|
+
const { urls, concurrency, render, format, selector } = args;
|
|
124
|
+
if (!urls || !Array.isArray(urls))
|
|
125
|
+
throw new Error('urls must be an array');
|
|
126
|
+
if (urls.length === 0)
|
|
127
|
+
throw new Error('urls cannot be empty');
|
|
128
|
+
if (urls.length > 50)
|
|
129
|
+
throw new Error('Too many URLs (max 50)');
|
|
130
|
+
const options = {
|
|
131
|
+
concurrency: concurrency || 3,
|
|
132
|
+
render: render || false,
|
|
133
|
+
format: format || 'markdown',
|
|
134
|
+
selector,
|
|
135
|
+
};
|
|
136
|
+
const results = await Promise.race([
|
|
137
|
+
peelBatch(urls, options),
|
|
138
|
+
timeout(300000, 'Batch'),
|
|
139
|
+
]);
|
|
140
|
+
return textResult(safeStringify(results));
|
|
141
|
+
};
|
|
142
|
+
// ── webpeel_deep_fetch ───────────────────────────────────────────────────────
|
|
143
|
+
export const handleDeepFetch = async (args, ctx) => {
|
|
144
|
+
const { query, count: countArg, format: formatArg } = args;
|
|
145
|
+
if (!query)
|
|
146
|
+
throw new Error('query is required');
|
|
147
|
+
return handleFind({ query, depth: 'deep', limit: Math.min(Math.max(countArg ?? 5, 1), 10), format: formatArg }, ctx);
|
|
148
|
+
};
|
|
149
|
+
// ── webpeel_summarize ────────────────────────────────────────────────────────
|
|
150
|
+
export const handleSummarize = async (args, _ctx) => {
|
|
151
|
+
const { url, llmApiKey, prompt, llmModel, llmBaseUrl, render } = args;
|
|
152
|
+
if (!url)
|
|
153
|
+
throw new Error('url is required');
|
|
154
|
+
if (!llmApiKey)
|
|
155
|
+
throw new Error('llmApiKey is required');
|
|
156
|
+
const options = {
|
|
157
|
+
render: render || false,
|
|
158
|
+
extract: {
|
|
159
|
+
prompt: prompt || 'Summarize this webpage in 2-3 sentences.',
|
|
160
|
+
llmApiKey,
|
|
161
|
+
llmModel: llmModel || 'gpt-4o-mini',
|
|
162
|
+
llmBaseUrl: llmBaseUrl || 'https://api.openai.com/v1',
|
|
163
|
+
},
|
|
164
|
+
};
|
|
165
|
+
const result = await Promise.race([
|
|
166
|
+
peel(url, options),
|
|
167
|
+
timeout(60000, 'Summarize'),
|
|
168
|
+
]);
|
|
169
|
+
return textResult(safeStringify({ url: result.url, title: result.title, summary: result.extracted }));
|
|
170
|
+
};
|
|
171
|
+
// ── webpeel_answer ───────────────────────────────────────────────────────────
|
|
172
|
+
export const handleAnswer = async (args, _ctx) => {
|
|
173
|
+
const { question: q, searchProvider: sp, searchApiKey: sak, llmProvider: lp, llmApiKey: lak, llmModel: lm, maxSources: ms, } = args;
|
|
174
|
+
if (!q)
|
|
175
|
+
throw new Error('question is required');
|
|
176
|
+
if (!lp)
|
|
177
|
+
throw new Error('llmProvider is required');
|
|
178
|
+
if (!lak)
|
|
179
|
+
throw new Error('llmApiKey is required');
|
|
180
|
+
const validLlm = ['openai', 'anthropic', 'google'];
|
|
181
|
+
if (!validLlm.includes(lp))
|
|
182
|
+
throw new Error('Invalid llmProvider');
|
|
183
|
+
const validSp = ['duckduckgo', 'brave', 'stealth', 'google'];
|
|
184
|
+
const resolvedSp = (sp && validSp.includes(sp) ? sp : 'duckduckgo');
|
|
185
|
+
const result = await Promise.race([
|
|
186
|
+
answerQuestion({
|
|
187
|
+
question: q,
|
|
188
|
+
searchProvider: resolvedSp,
|
|
189
|
+
searchApiKey: sak,
|
|
190
|
+
llmProvider: lp,
|
|
191
|
+
llmApiKey: lak,
|
|
192
|
+
llmModel: lm,
|
|
193
|
+
maxSources: Math.min(Math.max(ms ?? 5, 1), 10),
|
|
194
|
+
stream: false,
|
|
195
|
+
}),
|
|
196
|
+
timeout(180000, 'Answer'),
|
|
197
|
+
]);
|
|
198
|
+
return textResult(safeStringify(result));
|
|
199
|
+
};
|
|
200
|
+
// ── webpeel_quick_answer ─────────────────────────────────────────────────────
|
|
201
|
+
export const handleQuickAnswer = async (args, _ctx) => {
|
|
202
|
+
const { url, question: q, maxPassages: mpArg, render } = args;
|
|
203
|
+
if (!url)
|
|
204
|
+
throw new Error('url is required');
|
|
205
|
+
if (!q)
|
|
206
|
+
throw new Error('question is required');
|
|
207
|
+
const maxPassages = typeof mpArg === 'number' ? Math.min(Math.max(mpArg, 1), 10) : 3;
|
|
208
|
+
const peelResult = await Promise.race([
|
|
209
|
+
peel(url, { render: render || false, format: 'markdown', budget: 8000 }),
|
|
210
|
+
timeout(60000, 'Quick answer fetch'),
|
|
211
|
+
]);
|
|
212
|
+
const qa = quickAnswer({
|
|
213
|
+
question: q,
|
|
214
|
+
content: peelResult.content || '',
|
|
215
|
+
url: peelResult.url || url,
|
|
216
|
+
maxPassages,
|
|
217
|
+
});
|
|
218
|
+
return textResult(safeStringify({
|
|
219
|
+
url: peelResult.url || url,
|
|
220
|
+
title: peelResult.title,
|
|
221
|
+
question: qa.question,
|
|
222
|
+
answer: qa.answer,
|
|
223
|
+
confidence: qa.confidence,
|
|
224
|
+
passages: qa.passages,
|
|
225
|
+
method: qa.method,
|
|
226
|
+
}));
|
|
227
|
+
};
|
|
228
|
+
// ── webpeel_brand ────────────────────────────────────────────────────────────
|
|
229
|
+
export const handleBrand = async (args, ctx) => {
|
|
230
|
+
return handleExtract({ ...args, _brand: true }, ctx);
|
|
231
|
+
};
|
|
232
|
+
// ── webpeel_change_track ─────────────────────────────────────────────────────
|
|
233
|
+
export const handleChangeTrack = async (args, ctx) => {
|
|
234
|
+
return handleMonitor(args, ctx);
|
|
235
|
+
};
|
|
236
|
+
// ── webpeel_watch ────────────────────────────────────────────────────────────
|
|
237
|
+
export const handleWatch = async (args, ctx) => {
|
|
238
|
+
const action = args['action'];
|
|
239
|
+
const pool = ctx?.pool;
|
|
240
|
+
// If pool is available (HTTP route), use WatchManager
|
|
241
|
+
if (pool) {
|
|
242
|
+
const { WatchManager } = await import('../../core/watch-manager.js');
|
|
243
|
+
const wm = new WatchManager(pool);
|
|
244
|
+
const accountId = ctx?.accountId || 'anonymous';
|
|
245
|
+
if (action === 'create') {
|
|
246
|
+
const watch = await wm.create(accountId, args['url'], {
|
|
247
|
+
webhookUrl: args['webhookUrl'],
|
|
248
|
+
checkIntervalMinutes: args['intervalMinutes'] || 60,
|
|
249
|
+
selector: args['selector'],
|
|
250
|
+
});
|
|
251
|
+
return textResult(safeStringify(watch));
|
|
252
|
+
}
|
|
253
|
+
if (action === 'list') {
|
|
254
|
+
const watches = await wm.list(accountId);
|
|
255
|
+
return textResult(safeStringify(watches));
|
|
256
|
+
}
|
|
257
|
+
if (action === 'check') {
|
|
258
|
+
const result = await wm.check(args['id']);
|
|
259
|
+
return textResult(safeStringify(result));
|
|
260
|
+
}
|
|
261
|
+
if (action === 'delete') {
|
|
262
|
+
await wm.delete(args['id']);
|
|
263
|
+
return textResult(safeStringify({ success: true }));
|
|
264
|
+
}
|
|
265
|
+
return textResult(safeStringify({ error: `Unknown watch action: ${action}` }));
|
|
266
|
+
}
|
|
267
|
+
// Standalone fallback (no pool)
|
|
268
|
+
if (!action || action === 'list' || action === 'check' || action === 'delete') {
|
|
269
|
+
return textResult(safeStringify({
|
|
270
|
+
message: 'URL watching requires the hosted API (api.webpeel.dev). ' +
|
|
271
|
+
'Use webpeel_monitor for one-time change detection.',
|
|
272
|
+
}));
|
|
273
|
+
}
|
|
274
|
+
if (action === 'create') {
|
|
275
|
+
return handleMonitor({
|
|
276
|
+
url: args['url'],
|
|
277
|
+
webhook: args['webhookUrl'],
|
|
278
|
+
}, ctx);
|
|
279
|
+
}
|
|
280
|
+
return textResult(safeStringify({
|
|
281
|
+
message: 'URL watching requires the hosted API (api.webpeel.dev).',
|
|
282
|
+
}));
|
|
283
|
+
};
|
|
284
|
+
// ── webpeel_hotels ───────────────────────────────────────────────────────────
|
|
285
|
+
export const handleHotels = async (args, _ctx) => {
|
|
286
|
+
const { searchHotels, parseDate, addDays } = await import('../../core/hotel-search.js');
|
|
287
|
+
const destination = args['destination'];
|
|
288
|
+
if (!destination) {
|
|
289
|
+
return textResult(safeStringify({ error: 'Missing destination' }));
|
|
290
|
+
}
|
|
291
|
+
const checkin = args['checkin']
|
|
292
|
+
? parseDate(args['checkin'])
|
|
293
|
+
: parseDate('tomorrow');
|
|
294
|
+
const checkout = args['checkout']
|
|
295
|
+
? parseDate(args['checkout'])
|
|
296
|
+
: addDays(checkin, 1);
|
|
297
|
+
const sort = (['price', 'rating', 'value'].includes(args['sort'])
|
|
298
|
+
? args['sort']
|
|
299
|
+
: 'price');
|
|
300
|
+
const limit = Math.max(1, Math.min(50, args['limit'] || 20));
|
|
301
|
+
const result = await searchHotels({ destination, checkin, checkout, sort, limit, stealth: true });
|
|
302
|
+
return textResult(safeStringify({
|
|
303
|
+
destination,
|
|
304
|
+
checkin,
|
|
305
|
+
checkout,
|
|
306
|
+
sources: result.sources,
|
|
307
|
+
count: result.results.length,
|
|
308
|
+
results: result.results.slice(0, limit),
|
|
309
|
+
}));
|
|
310
|
+
};
|
|
311
|
+
// ── webpeel_design_analysis ──────────────────────────────────────────────────
|
|
312
|
+
export const handleDesignAnalysis = async (args, _ctx) => {
|
|
313
|
+
const { takeDesignAnalysis } = await import('../../core/screenshot.js');
|
|
314
|
+
const url = args['url'];
|
|
315
|
+
if (!url || typeof url !== 'string')
|
|
316
|
+
throw new Error('url is required');
|
|
317
|
+
if (url.length > 2048)
|
|
318
|
+
throw new Error('URL too long (max 2048 characters)');
|
|
319
|
+
const result = await Promise.race([
|
|
320
|
+
takeDesignAnalysis(url, {}),
|
|
321
|
+
timeout(90000, 'Design analysis'),
|
|
322
|
+
]);
|
|
323
|
+
return textResult(safeStringify({ url: result.url, analysis: result.analysis }));
|
|
324
|
+
};
|
|
325
|
+
// ── webpeel_design_compare ───────────────────────────────────────────────────
|
|
326
|
+
export const handleDesignCompare = async (args, _ctx) => {
|
|
327
|
+
const { takeDesignComparison } = await import('../../core/screenshot.js');
|
|
328
|
+
const url1 = args['url1'];
|
|
329
|
+
const url2 = args['url2'];
|
|
330
|
+
if (!url1 || typeof url1 !== 'string')
|
|
331
|
+
throw new Error('url1 is required');
|
|
332
|
+
if (!url2 || typeof url2 !== 'string')
|
|
333
|
+
throw new Error('url2 is required');
|
|
334
|
+
if (url1.length > 2048)
|
|
335
|
+
throw new Error('url1 too long');
|
|
336
|
+
if (url2.length > 2048)
|
|
337
|
+
throw new Error('url2 too long');
|
|
338
|
+
if (url1 === url2)
|
|
339
|
+
throw new Error('url1 and url2 must be different URLs');
|
|
340
|
+
const result = await Promise.race([
|
|
341
|
+
takeDesignComparison(url1, url2, {}),
|
|
342
|
+
timeout(120000, 'Design comparison'),
|
|
343
|
+
]);
|
|
344
|
+
return textResult(safeStringify({
|
|
345
|
+
subjectUrl: result.subjectUrl,
|
|
346
|
+
referenceUrl: result.referenceUrl,
|
|
347
|
+
score: result.comparison.score,
|
|
348
|
+
summary: result.comparison.summary,
|
|
349
|
+
gaps: result.comparison.gaps,
|
|
350
|
+
subjectAnalysis: result.comparison.subjectAnalysis,
|
|
351
|
+
referenceAnalysis: result.comparison.referenceAnalysis,
|
|
352
|
+
}));
|
|
353
|
+
};
|
|
354
|
+
// ── webpeel_auto_extract ─────────────────────────────────────────────────────
|
|
355
|
+
export const handleAutoExtract = async (args, ctx) => {
|
|
356
|
+
return handleExtract(args, ctx);
|
|
357
|
+
};
|
|
358
|
+
// ── webpeel_agent / agent ────────────────────────────────────────────────────
|
|
359
|
+
export const handleAgent = async (args, _ctx) => {
|
|
360
|
+
const llmApiKey = args['llmApiKey'];
|
|
361
|
+
// LLM mode — delegate to runAgent
|
|
362
|
+
if (llmApiKey) {
|
|
363
|
+
const prompt = args['prompt'];
|
|
364
|
+
if (!prompt)
|
|
365
|
+
throw new Error('prompt is required');
|
|
366
|
+
const result = await Promise.race([
|
|
367
|
+
runAgent({
|
|
368
|
+
prompt,
|
|
369
|
+
llmApiKey,
|
|
370
|
+
llmModel: args['llmModel'],
|
|
371
|
+
depth: (args['depth'] || 'basic'),
|
|
372
|
+
topic: (args['topic'] || 'general'),
|
|
373
|
+
urls: args['urls'],
|
|
374
|
+
maxSources: args['maxSources'] || args['maxResults'] || undefined,
|
|
375
|
+
outputSchema: args['outputSchema'],
|
|
376
|
+
}),
|
|
377
|
+
timeout(180000, 'Agent'),
|
|
378
|
+
]);
|
|
379
|
+
return textResult(safeStringify(result));
|
|
380
|
+
}
|
|
381
|
+
// LLM-free mode — search + fetch + BM25 extraction
|
|
382
|
+
const urlsArg = args['urls'] || [];
|
|
383
|
+
const searchArg = args['search'];
|
|
384
|
+
if (urlsArg.length === 0 && !searchArg) {
|
|
385
|
+
return textResult(safeStringify({
|
|
386
|
+
message: 'This tool has been deprecated. Use the webpeel tool instead.',
|
|
387
|
+
}));
|
|
388
|
+
}
|
|
389
|
+
const promptArg = args['prompt'];
|
|
390
|
+
const schema = args['schema'];
|
|
391
|
+
const budgetArg = args['budget'] || 4000;
|
|
392
|
+
const maxResults = Math.min(args['maxResults'] || 5, 20);
|
|
393
|
+
const targetUrls = [...urlsArg];
|
|
394
|
+
if (searchArg) {
|
|
395
|
+
try {
|
|
396
|
+
const { provider, apiKey } = getBestSearchProvider();
|
|
397
|
+
const searchResults = await provider.searchWeb(searchArg, {
|
|
398
|
+
count: Math.max(maxResults, 5),
|
|
399
|
+
apiKey,
|
|
400
|
+
});
|
|
401
|
+
for (const r of searchResults) {
|
|
402
|
+
if (!targetUrls.includes(r.url))
|
|
403
|
+
targetUrls.push(r.url);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
catch { /* continue with provided URLs */ }
|
|
407
|
+
}
|
|
408
|
+
const urlsToFetch = targetUrls.slice(0, maxResults);
|
|
409
|
+
const agentResults = [];
|
|
410
|
+
await Promise.all(urlsToFetch.map(async (u) => {
|
|
411
|
+
try {
|
|
412
|
+
const page = (await peel(u, { budget: budgetArg, format: 'markdown' }));
|
|
413
|
+
const content = page.content || '';
|
|
414
|
+
const title = page.title || u;
|
|
415
|
+
let extracted = null;
|
|
416
|
+
let confidence = 0;
|
|
417
|
+
if (schema && Object.keys(schema).length > 0) {
|
|
418
|
+
extracted = {};
|
|
419
|
+
let total = 0;
|
|
420
|
+
for (const [field] of Object.entries(schema)) {
|
|
421
|
+
const question = promptArg
|
|
422
|
+
? `${promptArg} — specifically: what is the ${field}?`
|
|
423
|
+
: `What is the ${field}?`;
|
|
424
|
+
const qa = quickAnswer({ question, content, maxPassages: 1, url: u });
|
|
425
|
+
extracted[field] = qa.answer || '';
|
|
426
|
+
total += qa.confidence;
|
|
427
|
+
}
|
|
428
|
+
if ('source' in schema)
|
|
429
|
+
extracted['source'] = u;
|
|
430
|
+
confidence = Object.keys(schema).length > 0 ? total / Object.keys(schema).length : 0;
|
|
431
|
+
}
|
|
432
|
+
else if (promptArg) {
|
|
433
|
+
const qa = quickAnswer({ question: promptArg, content, maxPassages: 3, url: u });
|
|
434
|
+
confidence = qa.confidence;
|
|
435
|
+
}
|
|
436
|
+
agentResults.push({
|
|
437
|
+
url: u,
|
|
438
|
+
title,
|
|
439
|
+
extracted,
|
|
440
|
+
content: content.slice(0, 500) + (content.length > 500 ? '…' : ''),
|
|
441
|
+
confidence,
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
catch { /* skip failed URLs */ }
|
|
445
|
+
}));
|
|
446
|
+
return textResult(safeStringify({
|
|
447
|
+
success: true,
|
|
448
|
+
data: { results: agentResults, totalSources: agentResults.length },
|
|
449
|
+
}));
|
|
450
|
+
};
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* handleMeta — the 'webpeel' meta tool.
|
|
3
|
+
* Parses plain-language intent and routes to the appropriate handler.
|
|
4
|
+
*/
|
|
5
|
+
import { parseIntent } from '../smart-router.js';
|
|
6
|
+
import { handleRead } from './read.js';
|
|
7
|
+
import { handleSee } from './see.js';
|
|
8
|
+
import { handleFind } from './find.js';
|
|
9
|
+
import { handleExtract } from './extract.js';
|
|
10
|
+
import { handleMonitor } from './monitor.js';
|
|
11
|
+
import { handleAct } from './act.js';
|
|
12
|
+
export const handleMeta = async (args, ctx) => {
|
|
13
|
+
const task = args['task'];
|
|
14
|
+
if (!task || typeof task !== 'string')
|
|
15
|
+
throw new Error('task is required');
|
|
16
|
+
const parsed = parseIntent(task);
|
|
17
|
+
const routedArgs = { ...parsed.params };
|
|
18
|
+
if (parsed.url)
|
|
19
|
+
routedArgs['url'] = parsed.url;
|
|
20
|
+
if (parsed.query)
|
|
21
|
+
routedArgs['query'] = parsed.query;
|
|
22
|
+
// Forward LLM BYOK params so extract/read handlers can use them
|
|
23
|
+
if (args['llmProvider'])
|
|
24
|
+
routedArgs['llmProvider'] = args['llmProvider'];
|
|
25
|
+
if (args['llmApiKey'])
|
|
26
|
+
routedArgs['llmApiKey'] = args['llmApiKey'];
|
|
27
|
+
if (args['llmModel'])
|
|
28
|
+
routedArgs['llmModel'] = args['llmModel'];
|
|
29
|
+
if (args['llmBaseUrl'])
|
|
30
|
+
routedArgs['llmBaseUrl'] = args['llmBaseUrl'];
|
|
31
|
+
switch (parsed.intent) {
|
|
32
|
+
case 'read': return handleRead(routedArgs, ctx);
|
|
33
|
+
case 'see': return handleSee(routedArgs, ctx);
|
|
34
|
+
case 'find': return handleFind(routedArgs, ctx);
|
|
35
|
+
case 'extract': return handleExtract(routedArgs, ctx);
|
|
36
|
+
case 'monitor': return handleMonitor(routedArgs, ctx);
|
|
37
|
+
case 'act': return handleAct(routedArgs, ctx);
|
|
38
|
+
default: return handleRead(routedArgs, ctx);
|
|
39
|
+
}
|
|
40
|
+
};
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* handleMonitor — watch a URL for changes, with optional webhook.
|
|
3
|
+
*/
|
|
4
|
+
import { peel } from '../../index.js';
|
|
5
|
+
import { textResult, safeStringify, timeout } from './types.js';
|
|
6
|
+
export const handleMonitor = async (args, _ctx) => {
|
|
7
|
+
const url = args['url'];
|
|
8
|
+
if (!url || typeof url !== 'string')
|
|
9
|
+
throw new Error('url is required');
|
|
10
|
+
if (url.length > 2048)
|
|
11
|
+
throw new Error('URL too long (max 2048 characters)');
|
|
12
|
+
const webhook = args['webhook'];
|
|
13
|
+
const selector = args['selector'];
|
|
14
|
+
const render = args['render'] || false;
|
|
15
|
+
if (webhook) {
|
|
16
|
+
// Webhook-based persistent monitoring requires hosted API
|
|
17
|
+
return textResult(safeStringify({
|
|
18
|
+
message: 'Persistent webhook monitoring requires the hosted API (api.webpeel.dev). ' +
|
|
19
|
+
'Use webpeel_monitor without webhook= for one-time change detection.',
|
|
20
|
+
url,
|
|
21
|
+
webhook,
|
|
22
|
+
}));
|
|
23
|
+
}
|
|
24
|
+
// One-time change snapshot (change_track logic)
|
|
25
|
+
const options = {
|
|
26
|
+
render: render || false,
|
|
27
|
+
...(selector ? { selector } : {}),
|
|
28
|
+
};
|
|
29
|
+
const result = await Promise.race([
|
|
30
|
+
peel(url, options),
|
|
31
|
+
timeout(60000, 'Monitor'),
|
|
32
|
+
]);
|
|
33
|
+
return textResult(safeStringify({
|
|
34
|
+
url: result.url,
|
|
35
|
+
title: result.title,
|
|
36
|
+
fingerprint: result.fingerprint,
|
|
37
|
+
tokens: result.tokens,
|
|
38
|
+
contentType: result.contentType,
|
|
39
|
+
lastChecked: new Date().toISOString(),
|
|
40
|
+
}));
|
|
41
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* handleObserve — give agents structured "eyes" on a web page.
|
|
3
|
+
*
|
|
4
|
+
* Returns interactive elements (links, buttons, inputs, forms) so agents
|
|
5
|
+
* can discover what's actionable before deciding what to do.
|
|
6
|
+
*/
|
|
7
|
+
import { type McpHandler } from './types.js';
|
|
8
|
+
export declare const handleObserve: McpHandler;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* handleObserve — give agents structured "eyes" on a web page.
|
|
3
|
+
*
|
|
4
|
+
* Returns interactive elements (links, buttons, inputs, forms) so agents
|
|
5
|
+
* can discover what's actionable before deciding what to do.
|
|
6
|
+
*/
|
|
7
|
+
import { textResult, safeStringify, timeout } from './types.js';
|
|
8
|
+
export const handleObserve = async (args, _ctx) => {
|
|
9
|
+
const url = args['url'];
|
|
10
|
+
if (!url || typeof url !== 'string')
|
|
11
|
+
throw new Error('url is required');
|
|
12
|
+
if (url.length > 2048)
|
|
13
|
+
throw new Error('URL too long (max 2048 characters)');
|
|
14
|
+
const selector = args['selector'];
|
|
15
|
+
const screenshot = Boolean(args['screenshot']);
|
|
16
|
+
const maxElements = typeof args['max_elements'] === 'number' ? args['max_elements'] : 50;
|
|
17
|
+
const stealth = Boolean(args['stealth']);
|
|
18
|
+
// Resolve viewport
|
|
19
|
+
let viewport = 'desktop';
|
|
20
|
+
const vpArg = args['viewport'];
|
|
21
|
+
if (vpArg === 'mobile' || vpArg === 'tablet')
|
|
22
|
+
viewport = vpArg;
|
|
23
|
+
const { observe } = await import('../../core/observe.js');
|
|
24
|
+
const result = await Promise.race([
|
|
25
|
+
observe({
|
|
26
|
+
url,
|
|
27
|
+
selector,
|
|
28
|
+
viewport,
|
|
29
|
+
screenshot,
|
|
30
|
+
maxElements,
|
|
31
|
+
timeout: 60000,
|
|
32
|
+
stealth,
|
|
33
|
+
}),
|
|
34
|
+
timeout(90000, 'Observe'),
|
|
35
|
+
]);
|
|
36
|
+
return textResult(safeStringify(result));
|
|
37
|
+
};
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* handleRead — fetch any URL as clean markdown.
|
|
3
|
+
* Auto-detects YouTube URLs and extracts transcripts.
|
|
4
|
+
*/
|
|
5
|
+
import { peel } from '../../index.js';
|
|
6
|
+
import { textResult, safeStringify, timeout } from './types.js';
|
|
7
|
+
function isYouTubeUrl(url) {
|
|
8
|
+
return /(?:youtube\.com|youtu\.be)/i.test(url);
|
|
9
|
+
}
|
|
10
|
+
export const handleRead = async (args, _ctx) => {
|
|
11
|
+
const url = args['url'];
|
|
12
|
+
if (!url || typeof url !== 'string')
|
|
13
|
+
throw new Error('url is required');
|
|
14
|
+
if (url.length > 2048)
|
|
15
|
+
throw new Error('URL too long (max 2048 characters)');
|
|
16
|
+
const format = args['format'] || 'markdown';
|
|
17
|
+
const render = args['render'] || false;
|
|
18
|
+
const question = args['question'];
|
|
19
|
+
const summary = args['summary'] || false;
|
|
20
|
+
const budgetArg = args['budget'];
|
|
21
|
+
const readable = args['readable'] || false;
|
|
22
|
+
// YouTube auto-detection
|
|
23
|
+
if (isYouTubeUrl(url)) {
|
|
24
|
+
const { getYouTubeTranscript } = await import('../../core/youtube.js');
|
|
25
|
+
const language = args['language'] || 'en';
|
|
26
|
+
const transcript = await Promise.race([
|
|
27
|
+
getYouTubeTranscript(url, { language }),
|
|
28
|
+
timeout(60000, 'YouTube transcript'),
|
|
29
|
+
]);
|
|
30
|
+
return textResult(safeStringify(transcript));
|
|
31
|
+
}
|
|
32
|
+
// Build summary extraction options if requested
|
|
33
|
+
const extractOpts = summary
|
|
34
|
+
? { prompt: 'Summarize this webpage in 2-3 concise sentences.' }
|
|
35
|
+
: undefined;
|
|
36
|
+
const options = {
|
|
37
|
+
render,
|
|
38
|
+
format: format,
|
|
39
|
+
question,
|
|
40
|
+
budget: budgetArg ?? 4000,
|
|
41
|
+
readable,
|
|
42
|
+
...(extractOpts ? { extract: extractOpts } : {}),
|
|
43
|
+
};
|
|
44
|
+
const result = await Promise.race([
|
|
45
|
+
peel(url, options),
|
|
46
|
+
timeout(60000, 'MCP read'),
|
|
47
|
+
]);
|
|
48
|
+
// Auth wall detection — return a helpful message with instructions
|
|
49
|
+
if (result.authRequired) {
|
|
50
|
+
let authHost = url;
|
|
51
|
+
try {
|
|
52
|
+
authHost = new URL(url).hostname.replace('www.', '');
|
|
53
|
+
}
|
|
54
|
+
catch { /* ignore */ }
|
|
55
|
+
return textResult(`🔐 Authentication Required\n\n` +
|
|
56
|
+
`This page is behind a login wall and cannot be accessed without authentication.\n\n` +
|
|
57
|
+
`To access this content:\n` +
|
|
58
|
+
`1. Ask the user to run: webpeel profile create ${authHost}\n` +
|
|
59
|
+
`2. They'll log in via a browser that opens\n` +
|
|
60
|
+
`3. Then fetch with: webpeel_read with url="${url}" and profile="${authHost}"\n\n` +
|
|
61
|
+
`Partial content (if any):\n${result.content?.slice(0, 500) || '(none)'}`);
|
|
62
|
+
}
|
|
63
|
+
const out = {
|
|
64
|
+
url: result.url || url,
|
|
65
|
+
title: result.title || '',
|
|
66
|
+
tokens: result.tokens || 0,
|
|
67
|
+
content: result.content,
|
|
68
|
+
};
|
|
69
|
+
if (result.metadata)
|
|
70
|
+
out['metadata'] = result.metadata;
|
|
71
|
+
if (result.quickAnswer)
|
|
72
|
+
out['quickAnswer'] = result.quickAnswer;
|
|
73
|
+
if (result.extracted)
|
|
74
|
+
out['extracted'] = result.extracted;
|
|
75
|
+
if (result.images)
|
|
76
|
+
out['images'] = result.images;
|
|
77
|
+
return textResult(safeStringify(out));
|
|
78
|
+
};
|