@iflow-mcp/jakeliume-webpeel 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +313 -0
- package/dist/cache.d.ts +30 -0
- package/dist/cache.js +139 -0
- package/dist/cli/commands/auth.d.ts +5 -0
- package/dist/cli/commands/auth.js +411 -0
- package/dist/cli/commands/doctor.d.ts +37 -0
- package/dist/cli/commands/doctor.js +371 -0
- package/dist/cli/commands/fetch.d.ts +6 -0
- package/dist/cli/commands/fetch.js +1345 -0
- package/dist/cli/commands/guide.d.ts +2 -0
- package/dist/cli/commands/guide.js +183 -0
- package/dist/cli/commands/interact.d.ts +5 -0
- package/dist/cli/commands/interact.js +840 -0
- package/dist/cli/commands/jobs.d.ts +5 -0
- package/dist/cli/commands/jobs.js +997 -0
- package/dist/cli/commands/monitor.d.ts +12 -0
- package/dist/cli/commands/monitor.js +197 -0
- package/dist/cli/commands/observe.d.ts +12 -0
- package/dist/cli/commands/observe.js +158 -0
- package/dist/cli/commands/screenshot.d.ts +5 -0
- package/dist/cli/commands/screenshot.js +282 -0
- package/dist/cli/commands/search.d.ts +5 -0
- package/dist/cli/commands/search.js +1021 -0
- package/dist/cli/commands/setup.d.ts +13 -0
- package/dist/cli/commands/setup.js +244 -0
- package/dist/cli/commands/skill.d.ts +15 -0
- package/dist/cli/commands/skill.js +195 -0
- package/dist/cli/utils.d.ts +84 -0
- package/dist/cli/utils.js +806 -0
- package/dist/cli-auth.d.ts +75 -0
- package/dist/cli-auth.js +369 -0
- package/dist/cli.d.ts +17 -0
- package/dist/cli.js +99 -0
- package/dist/core/actions.d.ts +69 -0
- package/dist/core/actions.js +495 -0
- package/dist/core/agent.d.ts +98 -0
- package/dist/core/agent.js +558 -0
- package/dist/core/answer.d.ts +42 -0
- package/dist/core/answer.js +395 -0
- package/dist/core/application-tracker.d.ts +84 -0
- package/dist/core/application-tracker.js +184 -0
- package/dist/core/apply.d.ts +162 -0
- package/dist/core/apply.js +816 -0
- package/dist/core/auth-detection.d.ts +35 -0
- package/dist/core/auth-detection.js +358 -0
- package/dist/core/auto-extract.d.ts +82 -0
- package/dist/core/auto-extract.js +604 -0
- package/dist/core/auto-interact.d.ts +23 -0
- package/dist/core/auto-interact.js +246 -0
- package/dist/core/bm25-filter.d.ts +66 -0
- package/dist/core/bm25-filter.js +288 -0
- package/dist/core/branding.d.ts +54 -0
- package/dist/core/branding.js +234 -0
- package/dist/core/browser-fetch.d.ts +323 -0
- package/dist/core/browser-fetch.js +1600 -0
- package/dist/core/browser-pool.d.ts +91 -0
- package/dist/core/browser-pool.js +550 -0
- package/dist/core/budget.d.ts +42 -0
- package/dist/core/budget.js +324 -0
- package/dist/core/business-intel.d.ts +47 -0
- package/dist/core/business-intel.js +279 -0
- package/dist/core/cache.d.ts +13 -0
- package/dist/core/cache.js +121 -0
- package/dist/core/cf-worker-proxy.d.ts +32 -0
- package/dist/core/cf-worker-proxy.js +87 -0
- package/dist/core/challenge-detection.d.ts +26 -0
- package/dist/core/challenge-detection.js +468 -0
- package/dist/core/change-tracking.d.ts +75 -0
- package/dist/core/change-tracking.js +276 -0
- package/dist/core/chunker.d.ts +46 -0
- package/dist/core/chunker.js +249 -0
- package/dist/core/chunking.d.ts +42 -0
- package/dist/core/chunking.js +181 -0
- package/dist/core/circuit-breaker.d.ts +44 -0
- package/dist/core/circuit-breaker.js +85 -0
- package/dist/core/content-pruner.d.ts +47 -0
- package/dist/core/content-pruner.js +425 -0
- package/dist/core/cookie-cache.d.ts +60 -0
- package/dist/core/cookie-cache.js +163 -0
- package/dist/core/crawl-checkpoint.d.ts +54 -0
- package/dist/core/crawl-checkpoint.js +104 -0
- package/dist/core/crawler.d.ts +84 -0
- package/dist/core/crawler.js +349 -0
- package/dist/core/cross-verify.d.ts +27 -0
- package/dist/core/cross-verify.js +93 -0
- package/dist/core/deep-fetch.d.ts +74 -0
- package/dist/core/deep-fetch.js +405 -0
- package/dist/core/deep-research.d.ts +141 -0
- package/dist/core/deep-research.js +972 -0
- package/dist/core/design-analysis.d.ts +70 -0
- package/dist/core/design-analysis.js +490 -0
- package/dist/core/design-compare.d.ts +38 -0
- package/dist/core/design-compare.js +264 -0
- package/dist/core/diff.d.ts +61 -0
- package/dist/core/diff.js +289 -0
- package/dist/core/dns-cache.d.ts +20 -0
- package/dist/core/dns-cache.js +198 -0
- package/dist/core/documents.d.ts +23 -0
- package/dist/core/documents.js +123 -0
- package/dist/core/domain-memory.d.ts +66 -0
- package/dist/core/domain-memory.js +163 -0
- package/dist/core/domain-verify.d.ts +40 -0
- package/dist/core/domain-verify.js +379 -0
- package/dist/core/engine-ranker.d.ts +112 -0
- package/dist/core/engine-ranker.js +395 -0
- package/dist/core/extract-inline.d.ts +38 -0
- package/dist/core/extract-inline.js +215 -0
- package/dist/core/extract-listings.d.ts +38 -0
- package/dist/core/extract-listings.js +461 -0
- package/dist/core/extract.d.ts +9 -0
- package/dist/core/extract.js +139 -0
- package/dist/core/fetch-cache.d.ts +57 -0
- package/dist/core/fetch-cache.js +95 -0
- package/dist/core/fetcher.d.ts +13 -0
- package/dist/core/fetcher.js +12 -0
- package/dist/core/google-cache.d.ts +29 -0
- package/dist/core/google-cache.js +180 -0
- package/dist/core/google-serp-parser.d.ts +82 -0
- package/dist/core/google-serp-parser.js +287 -0
- package/dist/core/hotel-search.d.ts +122 -0
- package/dist/core/hotel-search.js +382 -0
- package/dist/core/http-fetch.d.ts +72 -0
- package/dist/core/http-fetch.js +820 -0
- package/dist/core/human.d.ts +175 -0
- package/dist/core/human.js +680 -0
- package/dist/core/image-caption.d.ts +44 -0
- package/dist/core/image-caption.js +271 -0
- package/dist/core/jobs.d.ts +75 -0
- package/dist/core/jobs.js +634 -0
- package/dist/core/json-ld.d.ts +15 -0
- package/dist/core/json-ld.js +617 -0
- package/dist/core/language-detect.d.ts +18 -0
- package/dist/core/language-detect.js +135 -0
- package/dist/core/links.d.ts +10 -0
- package/dist/core/links.js +44 -0
- package/dist/core/llm-extract.d.ts +71 -0
- package/dist/core/llm-extract.js +507 -0
- package/dist/core/llm-provider.d.ts +100 -0
- package/dist/core/llm-provider.js +702 -0
- package/dist/core/local-search.d.ts +60 -0
- package/dist/core/local-search.js +308 -0
- package/dist/core/logger.d.ts +28 -0
- package/dist/core/logger.js +104 -0
- package/dist/core/map.d.ts +33 -0
- package/dist/core/map.js +127 -0
- package/dist/core/markdown.d.ts +92 -0
- package/dist/core/markdown.js +809 -0
- package/dist/core/metadata.d.ts +34 -0
- package/dist/core/metadata.js +422 -0
- package/dist/core/observe.d.ts +113 -0
- package/dist/core/observe.js +395 -0
- package/dist/core/ocr.d.ts +12 -0
- package/dist/core/ocr.js +33 -0
- package/dist/core/paginate.d.ts +31 -0
- package/dist/core/paginate.js +106 -0
- package/dist/core/pdf.d.ts +8 -0
- package/dist/core/pdf.js +25 -0
- package/dist/core/peel-tls.d.ts +25 -0
- package/dist/core/peel-tls.js +220 -0
- package/dist/core/pipeline.d.ts +132 -0
- package/dist/core/pipeline.js +1666 -0
- package/dist/core/profiles.d.ts +61 -0
- package/dist/core/profiles.js +350 -0
- package/dist/core/prompt-guard.d.ts +30 -0
- package/dist/core/prompt-guard.js +119 -0
- package/dist/core/proxy-config.d.ts +90 -0
- package/dist/core/proxy-config.js +172 -0
- package/dist/core/quick-answer.d.ts +53 -0
- package/dist/core/quick-answer.js +833 -0
- package/dist/core/rate-governor.d.ts +80 -0
- package/dist/core/rate-governor.js +238 -0
- package/dist/core/readability.d.ts +57 -0
- package/dist/core/readability.js +533 -0
- package/dist/core/research.d.ts +66 -0
- package/dist/core/research.js +270 -0
- package/dist/core/retry.d.ts +60 -0
- package/dist/core/retry.js +119 -0
- package/dist/core/safe-browsing.d.ts +30 -0
- package/dist/core/safe-browsing.js +206 -0
- package/dist/core/schema-extraction.d.ts +66 -0
- package/dist/core/schema-extraction.js +352 -0
- package/dist/core/schema-postprocess.d.ts +32 -0
- package/dist/core/schema-postprocess.js +469 -0
- package/dist/core/schema-templates.d.ts +19 -0
- package/dist/core/schema-templates.js +143 -0
- package/dist/core/screenshot.d.ts +224 -0
- package/dist/core/screenshot.js +207 -0
- package/dist/core/search-engines.d.ts +25 -0
- package/dist/core/search-engines.js +182 -0
- package/dist/core/search-provider.d.ts +243 -0
- package/dist/core/search-provider.js +1629 -0
- package/dist/core/searxng-provider.d.ts +35 -0
- package/dist/core/searxng-provider.js +105 -0
- package/dist/core/selective-evidence.d.ts +151 -0
- package/dist/core/selective-evidence.js +389 -0
- package/dist/core/site-search.d.ts +44 -0
- package/dist/core/site-search.js +252 -0
- package/dist/core/sitemap.d.ts +23 -0
- package/dist/core/sitemap.js +105 -0
- package/dist/core/source-credibility.d.ts +29 -0
- package/dist/core/source-credibility.js +584 -0
- package/dist/core/source-scoring.d.ts +166 -0
- package/dist/core/source-scoring.js +396 -0
- package/dist/core/stemmer.d.ts +38 -0
- package/dist/core/stemmer.js +509 -0
- package/dist/core/strategies.d.ts +104 -0
- package/dist/core/strategies.js +1044 -0
- package/dist/core/strategy-hooks.d.ts +145 -0
- package/dist/core/strategy-hooks.js +74 -0
- package/dist/core/structured-extract.d.ts +43 -0
- package/dist/core/structured-extract.js +550 -0
- package/dist/core/summarize.d.ts +17 -0
- package/dist/core/summarize.js +78 -0
- package/dist/core/synonyms.d.ts +42 -0
- package/dist/core/synonyms.js +184 -0
- package/dist/core/system-monitor.d.ts +61 -0
- package/dist/core/system-monitor.js +133 -0
- package/dist/core/table-format.d.ts +30 -0
- package/dist/core/table-format.js +146 -0
- package/dist/core/threat-feeds.d.ts +23 -0
- package/dist/core/threat-feeds.js +104 -0
- package/dist/core/timing.d.ts +21 -0
- package/dist/core/timing.js +33 -0
- package/dist/core/transcript-export.d.ts +47 -0
- package/dist/core/transcript-export.js +107 -0
- package/dist/core/user-agents.d.ts +82 -0
- package/dist/core/user-agents.js +239 -0
- package/dist/core/vertical-search.d.ts +54 -0
- package/dist/core/vertical-search.js +158 -0
- package/dist/core/watch-manager.d.ts +175 -0
- package/dist/core/watch-manager.js +416 -0
- package/dist/core/watch.d.ts +101 -0
- package/dist/core/watch.js +389 -0
- package/dist/core/youtube.d.ts +130 -0
- package/dist/core/youtube.js +1175 -0
- package/dist/ee/challenge-re-export.d.ts +1 -0
- package/dist/ee/challenge-re-export.js +1 -0
- package/dist/ee/challenge-solver.d.ts +72 -0
- package/dist/ee/challenge-solver.js +720 -0
- package/dist/ee/domain-extractors.d.ts +8 -0
- package/dist/ee/domain-extractors.js +8 -0
- package/dist/ee/domain-intel.d.ts +16 -0
- package/dist/ee/domain-intel.js +133 -0
- package/dist/ee/extractors/allrecipes.d.ts +2 -0
- package/dist/ee/extractors/allrecipes.js +120 -0
- package/dist/ee/extractors/amazon.d.ts +2 -0
- package/dist/ee/extractors/amazon.js +78 -0
- package/dist/ee/extractors/arxiv.d.ts +2 -0
- package/dist/ee/extractors/arxiv.js +137 -0
- package/dist/ee/extractors/bestbuy.d.ts +2 -0
- package/dist/ee/extractors/bestbuy.js +78 -0
- package/dist/ee/extractors/carscom.d.ts +2 -0
- package/dist/ee/extractors/carscom.js +121 -0
- package/dist/ee/extractors/coingecko.d.ts +2 -0
- package/dist/ee/extractors/coingecko.js +134 -0
- package/dist/ee/extractors/craigslist.d.ts +2 -0
- package/dist/ee/extractors/craigslist.js +92 -0
- package/dist/ee/extractors/devto.d.ts +2 -0
- package/dist/ee/extractors/devto.js +135 -0
- package/dist/ee/extractors/ebay.d.ts +2 -0
- package/dist/ee/extractors/ebay.js +90 -0
- package/dist/ee/extractors/espn.d.ts +2 -0
- package/dist/ee/extractors/espn.js +260 -0
- package/dist/ee/extractors/etsy.d.ts +2 -0
- package/dist/ee/extractors/etsy.js +52 -0
- package/dist/ee/extractors/facebook.d.ts +2 -0
- package/dist/ee/extractors/facebook.js +46 -0
- package/dist/ee/extractors/github.d.ts +2 -0
- package/dist/ee/extractors/github.js +196 -0
- package/dist/ee/extractors/google-flights.d.ts +2 -0
- package/dist/ee/extractors/google-flights.js +176 -0
- package/dist/ee/extractors/hackernews.d.ts +2 -0
- package/dist/ee/extractors/hackernews.js +147 -0
- package/dist/ee/extractors/imdb.d.ts +2 -0
- package/dist/ee/extractors/imdb.js +172 -0
- package/dist/ee/extractors/index.d.ts +26 -0
- package/dist/ee/extractors/index.js +247 -0
- package/dist/ee/extractors/instagram.d.ts +2 -0
- package/dist/ee/extractors/instagram.js +102 -0
- package/dist/ee/extractors/kalshi.d.ts +2 -0
- package/dist/ee/extractors/kalshi.js +121 -0
- package/dist/ee/extractors/kayak-cars.d.ts +2 -0
- package/dist/ee/extractors/kayak-cars.js +270 -0
- package/dist/ee/extractors/linkedin.d.ts +2 -0
- package/dist/ee/extractors/linkedin.js +113 -0
- package/dist/ee/extractors/medium.d.ts +2 -0
- package/dist/ee/extractors/medium.js +130 -0
- package/dist/ee/extractors/news.d.ts +4 -0
- package/dist/ee/extractors/news.js +173 -0
- package/dist/ee/extractors/npm.d.ts +2 -0
- package/dist/ee/extractors/npm.js +86 -0
- package/dist/ee/extractors/pdf.d.ts +2 -0
- package/dist/ee/extractors/pdf.js +108 -0
- package/dist/ee/extractors/pinterest.d.ts +2 -0
- package/dist/ee/extractors/pinterest.js +34 -0
- package/dist/ee/extractors/polymarket.d.ts +2 -0
- package/dist/ee/extractors/polymarket.js +358 -0
- package/dist/ee/extractors/producthunt.d.ts +2 -0
- package/dist/ee/extractors/producthunt.js +88 -0
- package/dist/ee/extractors/pubmed.d.ts +2 -0
- package/dist/ee/extractors/pubmed.js +162 -0
- package/dist/ee/extractors/pypi.d.ts +2 -0
- package/dist/ee/extractors/pypi.js +80 -0
- package/dist/ee/extractors/reddit.d.ts +2 -0
- package/dist/ee/extractors/reddit.js +438 -0
- package/dist/ee/extractors/redfin.d.ts +2 -0
- package/dist/ee/extractors/redfin.js +156 -0
- package/dist/ee/extractors/semanticscholar.d.ts +2 -0
- package/dist/ee/extractors/semanticscholar.js +131 -0
- package/dist/ee/extractors/shared.d.ts +12 -0
- package/dist/ee/extractors/shared.js +76 -0
- package/dist/ee/extractors/soundcloud.d.ts +2 -0
- package/dist/ee/extractors/soundcloud.js +34 -0
- package/dist/ee/extractors/sportsbetting.d.ts +2 -0
- package/dist/ee/extractors/sportsbetting.js +37 -0
- package/dist/ee/extractors/spotify.d.ts +2 -0
- package/dist/ee/extractors/spotify.js +34 -0
- package/dist/ee/extractors/stackoverflow.d.ts +2 -0
- package/dist/ee/extractors/stackoverflow.js +61 -0
- package/dist/ee/extractors/substack.d.ts +2 -0
- package/dist/ee/extractors/substack.js +115 -0
- package/dist/ee/extractors/substackroot.d.ts +2 -0
- package/dist/ee/extractors/substackroot.js +46 -0
- package/dist/ee/extractors/tiktok.d.ts +2 -0
- package/dist/ee/extractors/tiktok.js +29 -0
- package/dist/ee/extractors/tradingview.d.ts +2 -0
- package/dist/ee/extractors/tradingview.js +182 -0
- package/dist/ee/extractors/twitch.d.ts +2 -0
- package/dist/ee/extractors/twitch.js +36 -0
- package/dist/ee/extractors/twitter.d.ts +2 -0
- package/dist/ee/extractors/twitter.js +327 -0
- package/dist/ee/extractors/types.d.ts +14 -0
- package/dist/ee/extractors/types.js +1 -0
- package/dist/ee/extractors/walmart.d.ts +2 -0
- package/dist/ee/extractors/walmart.js +50 -0
- package/dist/ee/extractors/weather.d.ts +2 -0
- package/dist/ee/extractors/weather.js +133 -0
- package/dist/ee/extractors/wikipedia.d.ts +4 -0
- package/dist/ee/extractors/wikipedia.js +235 -0
- package/dist/ee/extractors/yelp.d.ts +2 -0
- package/dist/ee/extractors/yelp.js +216 -0
- package/dist/ee/extractors/youtube.d.ts +2 -0
- package/dist/ee/extractors/youtube.js +189 -0
- package/dist/ee/extractors/zillow.d.ts +54 -0
- package/dist/ee/extractors/zillow.js +247 -0
- package/dist/ee/extractors-re-export.d.ts +1 -0
- package/dist/ee/extractors-re-export.js +1 -0
- package/dist/ee/premium-hooks.d.ts +20 -0
- package/dist/ee/premium-hooks.js +50 -0
- package/dist/ee/spa-detection.d.ts +2 -0
- package/dist/ee/spa-detection.js +2 -0
- package/dist/ee/stability.d.ts +4 -0
- package/dist/ee/stability.js +29 -0
- package/dist/ee/swr-cache.d.ts +14 -0
- package/dist/ee/swr-cache.js +34 -0
- package/dist/index.d.ts +143 -0
- package/dist/index.js +291 -0
- package/dist/integrations/index.d.ts +2 -0
- package/dist/integrations/index.js +2 -0
- package/dist/integrations/langchain.d.ts +64 -0
- package/dist/integrations/langchain.js +115 -0
- package/dist/integrations/llamaindex.d.ts +50 -0
- package/dist/integrations/llamaindex.js +91 -0
- package/dist/mcp/handlers/act.d.ts +5 -0
- package/dist/mcp/handlers/act.js +34 -0
- package/dist/mcp/handlers/definitions.d.ts +6 -0
- package/dist/mcp/handlers/definitions.js +395 -0
- package/dist/mcp/handlers/extract.d.ts +7 -0
- package/dist/mcp/handlers/extract.js +135 -0
- package/dist/mcp/handlers/fetch.d.ts +6 -0
- package/dist/mcp/handlers/fetch.js +98 -0
- package/dist/mcp/handlers/find.d.ts +5 -0
- package/dist/mcp/handlers/find.js +137 -0
- package/dist/mcp/handlers/index.d.ts +13 -0
- package/dist/mcp/handlers/index.js +63 -0
- package/dist/mcp/handlers/legacy.d.ts +25 -0
- package/dist/mcp/handlers/legacy.js +450 -0
- package/dist/mcp/handlers/meta.d.ts +6 -0
- package/dist/mcp/handlers/meta.js +40 -0
- package/dist/mcp/handlers/monitor.d.ts +5 -0
- package/dist/mcp/handlers/monitor.js +41 -0
- package/dist/mcp/handlers/observe.d.ts +8 -0
- package/dist/mcp/handlers/observe.js +37 -0
- package/dist/mcp/handlers/read.d.ts +6 -0
- package/dist/mcp/handlers/read.js +78 -0
- package/dist/mcp/handlers/see.d.ts +5 -0
- package/dist/mcp/handlers/see.js +75 -0
- package/dist/mcp/handlers/types.d.ts +29 -0
- package/dist/mcp/handlers/types.js +28 -0
- package/dist/mcp/server.d.ts +7 -0
- package/dist/mcp/server.js +108 -0
- package/dist/mcp/smart-router.d.ts +23 -0
- package/dist/mcp/smart-router.js +178 -0
- package/dist/server/app.d.ts +14 -0
- package/dist/server/app.js +632 -0
- package/dist/server/auth-store.d.ts +28 -0
- package/dist/server/auth-store.js +88 -0
- package/dist/server/bull-queues.d.ts +60 -0
- package/dist/server/bull-queues.js +90 -0
- package/dist/server/email-service.d.ts +55 -0
- package/dist/server/email-service.js +291 -0
- package/dist/server/job-queue.d.ts +100 -0
- package/dist/server/job-queue.js +145 -0
- package/dist/server/logger.d.ts +10 -0
- package/dist/server/logger.js +37 -0
- package/dist/server/middleware/audit-log.d.ts +14 -0
- package/dist/server/middleware/audit-log.js +73 -0
- package/dist/server/middleware/auth.d.ts +35 -0
- package/dist/server/middleware/auth.js +225 -0
- package/dist/server/middleware/rate-limit.d.ts +50 -0
- package/dist/server/middleware/rate-limit.js +270 -0
- package/dist/server/middleware/scope-guard.d.ts +25 -0
- package/dist/server/middleware/scope-guard.js +45 -0
- package/dist/server/middleware/url-validator.d.ts +15 -0
- package/dist/server/middleware/url-validator.js +201 -0
- package/dist/server/openapi.yaml +6418 -0
- package/dist/server/pg-auth-store.d.ts +146 -0
- package/dist/server/pg-auth-store.js +576 -0
- package/dist/server/pg-job-queue.d.ts +59 -0
- package/dist/server/pg-job-queue.js +375 -0
- package/dist/server/routes/activity.d.ts +6 -0
- package/dist/server/routes/activity.js +79 -0
- package/dist/server/routes/admin-active.d.ts +7 -0
- package/dist/server/routes/admin-active.js +120 -0
- package/dist/server/routes/admin-stats.d.ts +7 -0
- package/dist/server/routes/admin-stats.js +176 -0
- package/dist/server/routes/agent.d.ts +24 -0
- package/dist/server/routes/agent.js +480 -0
- package/dist/server/routes/answer.d.ts +5 -0
- package/dist/server/routes/answer.js +125 -0
- package/dist/server/routes/ask.d.ts +28 -0
- package/dist/server/routes/ask.js +295 -0
- package/dist/server/routes/batch.d.ts +6 -0
- package/dist/server/routes/batch.js +493 -0
- package/dist/server/routes/cache-warm.d.ts +25 -0
- package/dist/server/routes/cache-warm.js +212 -0
- package/dist/server/routes/cli-usage.d.ts +6 -0
- package/dist/server/routes/cli-usage.js +127 -0
- package/dist/server/routes/compat.d.ts +23 -0
- package/dist/server/routes/compat.js +652 -0
- package/dist/server/routes/crawl.d.ts +13 -0
- package/dist/server/routes/crawl.js +287 -0
- package/dist/server/routes/deep-fetch.d.ts +8 -0
- package/dist/server/routes/deep-fetch.js +57 -0
- package/dist/server/routes/deep-research.d.ts +11 -0
- package/dist/server/routes/deep-research.js +232 -0
- package/dist/server/routes/demo.d.ts +24 -0
- package/dist/server/routes/demo.js +517 -0
- package/dist/server/routes/do.d.ts +8 -0
- package/dist/server/routes/do.js +72 -0
- package/dist/server/routes/extract.d.ts +14 -0
- package/dist/server/routes/extract.js +325 -0
- package/dist/server/routes/feed.d.ts +15 -0
- package/dist/server/routes/feed.js +311 -0
- package/dist/server/routes/fetch-queue.d.ts +13 -0
- package/dist/server/routes/fetch-queue.js +357 -0
- package/dist/server/routes/fetch.d.ts +7 -0
- package/dist/server/routes/fetch.js +1274 -0
- package/dist/server/routes/go.d.ts +14 -0
- package/dist/server/routes/go.js +81 -0
- package/dist/server/routes/health.d.ts +11 -0
- package/dist/server/routes/health.js +141 -0
- package/dist/server/routes/jobs.d.ts +7 -0
- package/dist/server/routes/jobs.js +574 -0
- package/dist/server/routes/map.d.ts +11 -0
- package/dist/server/routes/map.js +116 -0
- package/dist/server/routes/mcp.d.ts +14 -0
- package/dist/server/routes/mcp.js +197 -0
- package/dist/server/routes/metrics.d.ts +37 -0
- package/dist/server/routes/metrics.js +149 -0
- package/dist/server/routes/oauth.d.ts +9 -0
- package/dist/server/routes/oauth.js +396 -0
- package/dist/server/routes/playground.d.ts +17 -0
- package/dist/server/routes/playground.js +283 -0
- package/dist/server/routes/reader.d.ts +18 -0
- package/dist/server/routes/reader.js +192 -0
- package/dist/server/routes/research.d.ts +14 -0
- package/dist/server/routes/research.js +482 -0
- package/dist/server/routes/screenshot.d.ts +22 -0
- package/dist/server/routes/screenshot.js +820 -0
- package/dist/server/routes/search.d.ts +6 -0
- package/dist/server/routes/search.js +874 -0
- package/dist/server/routes/session.d.ts +17 -0
- package/dist/server/routes/session.js +548 -0
- package/dist/server/routes/share.d.ts +18 -0
- package/dist/server/routes/share.js +462 -0
- package/dist/server/routes/smart-search/handlers/cars.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/cars.js +102 -0
- package/dist/server/routes/smart-search/handlers/flights.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/flights.js +72 -0
- package/dist/server/routes/smart-search/handlers/general.d.ts +13 -0
- package/dist/server/routes/smart-search/handlers/general.js +717 -0
- package/dist/server/routes/smart-search/handlers/hotels.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/hotels.js +88 -0
- package/dist/server/routes/smart-search/handlers/products.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/products.js +1309 -0
- package/dist/server/routes/smart-search/handlers/rental.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/rental.js +154 -0
- package/dist/server/routes/smart-search/handlers/restaurants.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/restaurants.js +225 -0
- package/dist/server/routes/smart-search/handlers/transit-verdict.d.ts +41 -0
- package/dist/server/routes/smart-search/handlers/transit-verdict.js +224 -0
- package/dist/server/routes/smart-search/index.d.ts +19 -0
- package/dist/server/routes/smart-search/index.js +546 -0
- package/dist/server/routes/smart-search/intent.d.ts +3 -0
- package/dist/server/routes/smart-search/intent.js +264 -0
- package/dist/server/routes/smart-search/llm.d.ts +16 -0
- package/dist/server/routes/smart-search/llm.js +70 -0
- package/dist/server/routes/smart-search/sources/reddit.d.ts +18 -0
- package/dist/server/routes/smart-search/sources/reddit.js +34 -0
- package/dist/server/routes/smart-search/sources/yelp.d.ts +25 -0
- package/dist/server/routes/smart-search/sources/yelp.js +171 -0
- package/dist/server/routes/smart-search/sources/youtube.d.ts +8 -0
- package/dist/server/routes/smart-search/sources/youtube.js +9 -0
- package/dist/server/routes/smart-search/types.d.ts +81 -0
- package/dist/server/routes/smart-search/types.js +1 -0
- package/dist/server/routes/smart-search/utils.d.ts +20 -0
- package/dist/server/routes/smart-search/utils.js +146 -0
- package/dist/server/routes/stats.d.ts +6 -0
- package/dist/server/routes/stats.js +71 -0
- package/dist/server/routes/stripe.d.ts +15 -0
- package/dist/server/routes/stripe.js +296 -0
- package/dist/server/routes/transcript-export.d.ts +10 -0
- package/dist/server/routes/transcript-export.js +178 -0
- package/dist/server/routes/usage.d.ts +9 -0
- package/dist/server/routes/usage.js +279 -0
- package/dist/server/routes/users.d.ts +8 -0
- package/dist/server/routes/users.js +1867 -0
- package/dist/server/routes/watch.d.ts +15 -0
- package/dist/server/routes/watch.js +309 -0
- package/dist/server/routes/webhooks.d.ts +26 -0
- package/dist/server/routes/webhooks.js +170 -0
- package/dist/server/routes/youtube.d.ts +6 -0
- package/dist/server/routes/youtube.js +130 -0
- package/dist/server/sentry.d.ts +14 -0
- package/dist/server/sentry.js +104 -0
- package/dist/server/types.d.ts +15 -0
- package/dist/server/types.js +7 -0
- package/dist/server/utils/response.d.ts +44 -0
- package/dist/server/utils/response.js +69 -0
- package/dist/server/utils/sse.d.ts +22 -0
- package/dist/server/utils/sse.js +38 -0
- package/dist/types.d.ts +552 -0
- package/dist/types.js +39 -0
- package/llms.txt +105 -0
- package/package.json +189 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Monitor command: content change detection for URLs
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* webpeel monitor <url> - Fetch & snapshot (or diff if prev exists)
|
|
6
|
+
* webpeel monitor <url> --interval 300 - Watch mode: re-check every 5 minutes
|
|
7
|
+
* webpeel monitor <url> --json - JSON output for automation
|
|
8
|
+
* webpeel monitor <url> --render - Use browser rendering
|
|
9
|
+
* webpeel monitor <url> --selector <css> - Monitor specific section only
|
|
10
|
+
*/
|
|
11
|
+
import type { Command } from 'commander';
|
|
12
|
+
export declare function registerMonitorCommands(program: Command): void;
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Monitor command: content change detection for URLs
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* webpeel monitor <url> - Fetch & snapshot (or diff if prev exists)
|
|
6
|
+
* webpeel monitor <url> --interval 300 - Watch mode: re-check every 5 minutes
|
|
7
|
+
* webpeel monitor <url> --json - JSON output for automation
|
|
8
|
+
* webpeel monitor <url> --render - Use browser rendering
|
|
9
|
+
* webpeel monitor <url> --selector <css> - Monitor specific section only
|
|
10
|
+
*/
|
|
11
|
+
import { createHash } from 'crypto';
|
|
12
|
+
import ora from 'ora';
|
|
13
|
+
import { peel, cleanup } from '../../index.js';
|
|
14
|
+
import { trackChange, getSnapshot } from '../../core/change-tracking.js';
|
|
15
|
+
// ─── helpers ─────────────────────────────────────────────────────────────────
|
|
16
|
+
function fingerprint(content) {
|
|
17
|
+
return createHash('sha256').update(content).digest('hex');
|
|
18
|
+
}
|
|
19
|
+
function formatTimestamp(iso) {
|
|
20
|
+
return new Date(iso).toLocaleString();
|
|
21
|
+
}
|
|
22
|
+
/** Run a single monitor check. Returns exit code (0 = ok, 1 = error). */
|
|
23
|
+
async function runMonitorCheck(url, options) {
|
|
24
|
+
const spinner = options.silent || options.json ? null : ora(`Checking ${url}...`).start();
|
|
25
|
+
try {
|
|
26
|
+
const peelOptions = {
|
|
27
|
+
render: options.render || false,
|
|
28
|
+
selector: options.selector,
|
|
29
|
+
format: 'markdown',
|
|
30
|
+
timeout: options.timeout,
|
|
31
|
+
raw: false,
|
|
32
|
+
};
|
|
33
|
+
const result = await peel(url, peelOptions);
|
|
34
|
+
const content = result.content ?? '';
|
|
35
|
+
const fp = fingerprint(content);
|
|
36
|
+
if (spinner)
|
|
37
|
+
spinner.succeed(`Fetched in ${result.elapsed ?? 0}ms`);
|
|
38
|
+
const changeResult = await trackChange(url, content, fp);
|
|
39
|
+
if (options.json) {
|
|
40
|
+
const out = {
|
|
41
|
+
success: true,
|
|
42
|
+
url,
|
|
43
|
+
changeStatus: changeResult.changeStatus,
|
|
44
|
+
previousScrapeAt: changeResult.previousScrapeAt,
|
|
45
|
+
checkedAt: new Date().toISOString(),
|
|
46
|
+
};
|
|
47
|
+
if (changeResult.diff) {
|
|
48
|
+
out.diff = {
|
|
49
|
+
additions: changeResult.diff.additions,
|
|
50
|
+
deletions: changeResult.diff.deletions,
|
|
51
|
+
text: changeResult.diff.text,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
process.stdout.write(JSON.stringify(out, null, 2) + '\n');
|
|
55
|
+
return { exitCode: 0, changed: changeResult.changeStatus === 'changed' };
|
|
56
|
+
}
|
|
57
|
+
// Human-readable output
|
|
58
|
+
switch (changeResult.changeStatus) {
|
|
59
|
+
case 'new': {
|
|
60
|
+
console.log(`📸 Baseline snapshot saved for ${url}`);
|
|
61
|
+
break;
|
|
62
|
+
}
|
|
63
|
+
case 'same': {
|
|
64
|
+
const since = changeResult.previousScrapeAt
|
|
65
|
+
? formatTimestamp(changeResult.previousScrapeAt)
|
|
66
|
+
: 'last check';
|
|
67
|
+
console.log(`✅ No changes detected since ${since}`);
|
|
68
|
+
break;
|
|
69
|
+
}
|
|
70
|
+
case 'changed': {
|
|
71
|
+
const since = changeResult.previousScrapeAt
|
|
72
|
+
? formatTimestamp(changeResult.previousScrapeAt)
|
|
73
|
+
: 'last check';
|
|
74
|
+
console.log(`\n🔔 Changes detected since ${since}:`);
|
|
75
|
+
console.log('');
|
|
76
|
+
if (changeResult.diff) {
|
|
77
|
+
const { additions, deletions, text } = changeResult.diff;
|
|
78
|
+
// Print unified diff with color
|
|
79
|
+
if (text) {
|
|
80
|
+
for (const line of text.split('\n')) {
|
|
81
|
+
if (line.startsWith('+')) {
|
|
82
|
+
process.stdout.write(`\x1b[32m${line}\x1b[0m\n`);
|
|
83
|
+
}
|
|
84
|
+
else if (line.startsWith('-')) {
|
|
85
|
+
process.stdout.write(`\x1b[31m${line}\x1b[0m\n`);
|
|
86
|
+
}
|
|
87
|
+
else if (line.startsWith('@')) {
|
|
88
|
+
process.stdout.write(`\x1b[36m${line}\x1b[0m\n`);
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
process.stdout.write(`${line}\n`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
console.log('');
|
|
96
|
+
console.log(`📊 Summary: \x1b[32m+${additions} lines added\x1b[0m, \x1b[31m-${deletions} lines removed\x1b[0m`);
|
|
97
|
+
}
|
|
98
|
+
break;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return { exitCode: 0, changed: changeResult.changeStatus === 'changed' };
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
if (spinner)
|
|
105
|
+
spinner.fail('Monitor check failed');
|
|
106
|
+
const msg = error instanceof Error ? error.message : 'Unknown error';
|
|
107
|
+
if (options.json) {
|
|
108
|
+
process.stdout.write(JSON.stringify({ success: false, url, error: msg }) + '\n');
|
|
109
|
+
}
|
|
110
|
+
else {
|
|
111
|
+
console.error(`\nError: ${msg}`);
|
|
112
|
+
}
|
|
113
|
+
return { exitCode: 1, changed: false };
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
// ─── registerMonitorCommands ──────────────────────────────────────────────────
|
|
117
|
+
export function registerMonitorCommands(program) {
|
|
118
|
+
program
|
|
119
|
+
.command('monitor <url>')
|
|
120
|
+
.description('Monitor a URL for content changes (saves snapshots, shows diffs)')
|
|
121
|
+
.option('-r, --render', 'Use headless browser (for JS-heavy sites)')
|
|
122
|
+
.option('--selector <css>', 'CSS selector — only monitor this section of the page')
|
|
123
|
+
.option('-i, --interval <seconds>', 'Watch mode: re-check every N seconds', parseInt)
|
|
124
|
+
.option('--json', 'Output as JSON (for automation/scripting)')
|
|
125
|
+
.option('-s, --silent', 'Silent mode (no spinner)')
|
|
126
|
+
.option('-t, --timeout <ms>', 'Request timeout (ms)', (v) => parseInt(v, 10), 30000)
|
|
127
|
+
.addHelpText('after', `
|
|
128
|
+
Examples:
|
|
129
|
+
webpeel monitor https://example.com/pricing # First run: save baseline
|
|
130
|
+
webpeel monitor https://example.com/pricing # Second run: show diff
|
|
131
|
+
webpeel monitor https://example.com/pricing --interval 300 # Watch every 5 min
|
|
132
|
+
webpeel monitor https://example.com/pricing --render # JS-rendered page
|
|
133
|
+
webpeel monitor https://example.com/pricing --json # JSON output
|
|
134
|
+
webpeel monitor https://example.com/pricing --selector .price # Monitor prices only
|
|
135
|
+
`)
|
|
136
|
+
.action(async (url, options) => {
|
|
137
|
+
// Validate URL
|
|
138
|
+
try {
|
|
139
|
+
const parsed = new URL(url);
|
|
140
|
+
if (!['http:', 'https:'].includes(parsed.protocol)) {
|
|
141
|
+
console.error('Error: Only HTTP and HTTPS protocols are allowed');
|
|
142
|
+
process.exit(1);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
console.error(`Error: Invalid URL format: ${url}`);
|
|
147
|
+
process.exit(1);
|
|
148
|
+
}
|
|
149
|
+
const checkOptions = {
|
|
150
|
+
json: options.json || false,
|
|
151
|
+
render: options.render || false,
|
|
152
|
+
selector: options.selector,
|
|
153
|
+
silent: options.silent || false,
|
|
154
|
+
timeout: options.timeout,
|
|
155
|
+
};
|
|
156
|
+
const intervalSec = options.interval;
|
|
157
|
+
if (!intervalSec) {
|
|
158
|
+
// Single run
|
|
159
|
+
const { exitCode } = await runMonitorCheck(url, checkOptions);
|
|
160
|
+
await cleanup();
|
|
161
|
+
process.exit(exitCode);
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
// Watch mode
|
|
165
|
+
if (intervalSec < 5) {
|
|
166
|
+
console.error('Error: --interval must be at least 5 seconds');
|
|
167
|
+
process.exit(1);
|
|
168
|
+
}
|
|
169
|
+
if (!options.json && !options.silent) {
|
|
170
|
+
const prev = await getSnapshot(url);
|
|
171
|
+
const baselineNote = prev
|
|
172
|
+
? `(baseline exists from ${formatTimestamp(new Date(prev.timestamp).toISOString())})`
|
|
173
|
+
: '(no baseline yet — first run will save it)';
|
|
174
|
+
console.log(`👁 Watching ${url} every ${intervalSec}s ${baselineNote}`);
|
|
175
|
+
console.log('Press Ctrl+C to stop.\n');
|
|
176
|
+
}
|
|
177
|
+
// Initial check immediately
|
|
178
|
+
await runMonitorCheck(url, checkOptions);
|
|
179
|
+
// Then loop
|
|
180
|
+
const loop = setInterval(async () => {
|
|
181
|
+
if (!options.json && !options.silent) {
|
|
182
|
+
console.log(`\n⏰ ${new Date().toLocaleTimeString()} — checking...`);
|
|
183
|
+
}
|
|
184
|
+
await runMonitorCheck(url, checkOptions);
|
|
185
|
+
}, intervalSec * 1000);
|
|
186
|
+
// Graceful shutdown
|
|
187
|
+
process.on('SIGINT', async () => {
|
|
188
|
+
clearInterval(loop);
|
|
189
|
+
if (!options.json && !options.silent) {
|
|
190
|
+
console.log('\n👋 Stopped monitoring.');
|
|
191
|
+
}
|
|
192
|
+
await cleanup();
|
|
193
|
+
process.exit(0);
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
});
|
|
197
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Observe command: structured page observation for AI agents
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* webpeel observe <url> - List all interactive elements
|
|
6
|
+
* webpeel observe <url> --json - JSON output for agents
|
|
7
|
+
* webpeel observe <url> --screenshot - Include screenshot
|
|
8
|
+
* webpeel observe <url> --selector main - Scope to a section
|
|
9
|
+
* webpeel observe <url> --viewport mobile - Mobile viewport
|
|
10
|
+
*/
|
|
11
|
+
import type { Command } from 'commander';
|
|
12
|
+
export declare function registerObserveCommand(program: Command): void;
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Observe command: structured page observation for AI agents
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* webpeel observe <url> - List all interactive elements
|
|
6
|
+
* webpeel observe <url> --json - JSON output for agents
|
|
7
|
+
* webpeel observe <url> --screenshot - Include screenshot
|
|
8
|
+
* webpeel observe <url> --selector main - Scope to a section
|
|
9
|
+
* webpeel observe <url> --viewport mobile - Mobile viewport
|
|
10
|
+
*/
|
|
11
|
+
import ora from 'ora';
|
|
12
|
+
import { writeFileSync } from 'fs';
|
|
13
|
+
import { cleanup } from '../../index.js';
|
|
14
|
+
import { writeStdout } from '../utils.js';
|
|
15
|
+
export function registerObserveCommand(program) {
|
|
16
|
+
program
|
|
17
|
+
.command('observe <url>')
|
|
18
|
+
.alias('obs')
|
|
19
|
+
.description('Observe a page — list interactive elements (links, buttons, inputs, forms) for agent use')
|
|
20
|
+
.option('--json', 'Output as JSON (default for piped output)')
|
|
21
|
+
.option('-s, --silent', 'Silent mode (no spinner)')
|
|
22
|
+
.option('--selector <css>', 'CSS selector to scope observation')
|
|
23
|
+
.option('--viewport <v>', 'Viewport: desktop (default), mobile, tablet, or WxH', 'desktop')
|
|
24
|
+
.option('--screenshot', 'Include a screenshot')
|
|
25
|
+
.option('--screenshot-full-page', 'Full-page screenshot')
|
|
26
|
+
.option('--screenshot-output <path>', 'Save screenshot to file instead of base64 in JSON')
|
|
27
|
+
.option('--max-elements <n>', 'Max elements per category (default: 50)', (v) => parseInt(v, 10), 50)
|
|
28
|
+
.option('-t, --timeout <ms>', 'Request timeout in ms', (v) => parseInt(v, 10), 30000)
|
|
29
|
+
.option('--stealth', 'Stealth mode for bot-protected sites')
|
|
30
|
+
.addHelpText('after', `
|
|
31
|
+
Examples:
|
|
32
|
+
webpeel observe https://news.ycombinator.com # See all interactive elements
|
|
33
|
+
webpeel observe https://github.com --json # JSON for agent consumption
|
|
34
|
+
webpeel observe https://amazon.com/dp/... --selector main # Scope to main content
|
|
35
|
+
webpeel observe https://example.com --viewport mobile # Mobile viewport
|
|
36
|
+
webpeel observe https://example.com --screenshot -o snap.png # With screenshot
|
|
37
|
+
`)
|
|
38
|
+
.action(async (url, options) => {
|
|
39
|
+
// Validate URL
|
|
40
|
+
try {
|
|
41
|
+
const parsed = new URL(url);
|
|
42
|
+
if (!['http:', 'https:'].includes(parsed.protocol)) {
|
|
43
|
+
console.error('Error: Only HTTP and HTTPS protocols are allowed');
|
|
44
|
+
process.exit(1);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
console.error(`Error: Invalid URL format: ${url}`);
|
|
49
|
+
process.exit(1);
|
|
50
|
+
}
|
|
51
|
+
const spinner = options.silent ? null : ora('Observing page...').start();
|
|
52
|
+
try {
|
|
53
|
+
const { observe } = await import('../../core/observe.js');
|
|
54
|
+
// Parse viewport
|
|
55
|
+
let viewport = 'desktop';
|
|
56
|
+
const vpArg = options.viewport;
|
|
57
|
+
if (vpArg === 'mobile' || vpArg === 'tablet') {
|
|
58
|
+
viewport = vpArg;
|
|
59
|
+
}
|
|
60
|
+
else if (vpArg && vpArg.includes('x')) {
|
|
61
|
+
const [w, h] = vpArg.split('x').map(Number);
|
|
62
|
+
if (w && h)
|
|
63
|
+
viewport = { width: w, height: h };
|
|
64
|
+
}
|
|
65
|
+
const result = await observe({
|
|
66
|
+
url,
|
|
67
|
+
selector: options.selector,
|
|
68
|
+
viewport,
|
|
69
|
+
screenshot: options.screenshot || false,
|
|
70
|
+
screenshotFullPage: options.screenshotFullPage || false,
|
|
71
|
+
maxElements: options.maxElements,
|
|
72
|
+
timeout: options.timeout,
|
|
73
|
+
stealth: options.stealth || false,
|
|
74
|
+
});
|
|
75
|
+
if (spinner) {
|
|
76
|
+
spinner.succeed(`Observed ${result.totalElements} elements in ${result.elapsed}ms`);
|
|
77
|
+
}
|
|
78
|
+
// Save screenshot to file if requested
|
|
79
|
+
if (result.screenshot && options.screenshotOutput) {
|
|
80
|
+
const buf = Buffer.from(result.screenshot, 'base64');
|
|
81
|
+
writeFileSync(options.screenshotOutput, buf);
|
|
82
|
+
if (!options.silent) {
|
|
83
|
+
console.error(`Screenshot saved to: ${options.screenshotOutput} (${(buf.length / 1024).toFixed(1)} KB)`);
|
|
84
|
+
}
|
|
85
|
+
// Don't include screenshot in JSON output when saved to file
|
|
86
|
+
delete result.screenshot;
|
|
87
|
+
}
|
|
88
|
+
if (options.json) {
|
|
89
|
+
await writeStdout(JSON.stringify(result, null, 2) + '\n');
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
// Human-readable output
|
|
93
|
+
console.log(`\n🔍 ${result.title}`);
|
|
94
|
+
console.log(` ${result.url}`);
|
|
95
|
+
console.log(` Viewport: ${result.viewport.width}×${result.viewport.height} | Page: ${result.scroll.width}×${result.scroll.height}\n`);
|
|
96
|
+
const { elements } = result;
|
|
97
|
+
if (elements.links.length > 0) {
|
|
98
|
+
console.log(`📎 Links (${elements.links.length}):`);
|
|
99
|
+
for (const el of elements.links.slice(0, 15)) {
|
|
100
|
+
const vp = el.inViewport ? '👁' : ' ';
|
|
101
|
+
const href = el.attributes.href ? ` → ${el.attributes.href.slice(0, 60)}` : '';
|
|
102
|
+
console.log(` ${vp} [${el.ref}] ${el.text || '(no text)'}${href}`);
|
|
103
|
+
}
|
|
104
|
+
if (elements.links.length > 15) {
|
|
105
|
+
console.log(` ... and ${elements.links.length - 15} more`);
|
|
106
|
+
}
|
|
107
|
+
console.log('');
|
|
108
|
+
}
|
|
109
|
+
if (elements.buttons.length > 0) {
|
|
110
|
+
console.log(`🔘 Buttons (${elements.buttons.length}):`);
|
|
111
|
+
for (const el of elements.buttons) {
|
|
112
|
+
const vp = el.inViewport ? '👁' : ' ';
|
|
113
|
+
console.log(` ${vp} [${el.ref}] "${el.text || '(no text)'}" — selector: ${el.selector}`);
|
|
114
|
+
}
|
|
115
|
+
console.log('');
|
|
116
|
+
}
|
|
117
|
+
if (elements.inputs.length > 0) {
|
|
118
|
+
console.log(`✏️ Inputs (${elements.inputs.length}):`);
|
|
119
|
+
for (const el of elements.inputs) {
|
|
120
|
+
const vp = el.inViewport ? '👁' : ' ';
|
|
121
|
+
const type = el.attributes.type || 'text';
|
|
122
|
+
const label = el.text || el.attributes.placeholder || el.attributes.name || 'unnamed';
|
|
123
|
+
console.log(` ${vp} [${el.ref}] ${label} (${type}) — selector: ${el.selector}`);
|
|
124
|
+
}
|
|
125
|
+
console.log('');
|
|
126
|
+
}
|
|
127
|
+
if (elements.selects.length > 0) {
|
|
128
|
+
console.log(`📋 Selects (${elements.selects.length}):`);
|
|
129
|
+
for (const el of elements.selects) {
|
|
130
|
+
const vp = el.inViewport ? '👁' : ' ';
|
|
131
|
+
const opts = el.attributes.options || '';
|
|
132
|
+
console.log(` ${vp} [${el.ref}] ${el.text || el.attributes.name || 'dropdown'}${opts ? ` [${opts}]` : ''}`);
|
|
133
|
+
}
|
|
134
|
+
console.log('');
|
|
135
|
+
}
|
|
136
|
+
if (elements.forms.length > 0) {
|
|
137
|
+
console.log(`📝 Forms (${elements.forms.length}):`);
|
|
138
|
+
for (const el of elements.forms) {
|
|
139
|
+
const fields = el.attributes.fields || '?';
|
|
140
|
+
const action = el.attributes.action || '';
|
|
141
|
+
console.log(` [${el.ref}] ${el.text || 'form'} — ${fields} fields${action ? ` → ${action.slice(0, 60)}` : ''}`);
|
|
142
|
+
}
|
|
143
|
+
console.log('');
|
|
144
|
+
}
|
|
145
|
+
console.log(`📊 Summary: ${result.summary}`);
|
|
146
|
+
}
|
|
147
|
+
await cleanup();
|
|
148
|
+
process.exit(0);
|
|
149
|
+
}
|
|
150
|
+
catch (error) {
|
|
151
|
+
if (spinner)
|
|
152
|
+
spinner.fail('Observation failed');
|
|
153
|
+
console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
154
|
+
await cleanup();
|
|
155
|
+
process.exit(1);
|
|
156
|
+
}
|
|
157
|
+
});
|
|
158
|
+
}
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Screenshot commands: screenshot, brand, design-compare
|
|
3
|
+
*/
|
|
4
|
+
import ora from 'ora';
|
|
5
|
+
import { writeFileSync } from 'fs';
|
|
6
|
+
import { cleanup } from '../../index.js';
|
|
7
|
+
import { checkUsage, showUsageFooter } from '../../cli-auth.js';
|
|
8
|
+
import { parseActions, extractColors, extractFonts } from '../utils.js';
|
|
9
|
+
import { peel } from '../../index.js';
|
|
10
|
+
export function registerScreenshotCommands(program) {
|
|
11
|
+
// ── screenshot command ────────────────────────────────────────────────────
|
|
12
|
+
program
|
|
13
|
+
.command('screenshot <url>')
|
|
14
|
+
.alias('snap')
|
|
15
|
+
.description('Take a screenshot of a URL and save as PNG/JPEG')
|
|
16
|
+
.option('--full-page', 'Capture full page (not just viewport)')
|
|
17
|
+
.option('--width <px>', 'Viewport width in pixels (default: 1280)', parseInt)
|
|
18
|
+
.option('--height <px>', 'Viewport height in pixels (default: 720)', parseInt)
|
|
19
|
+
.option('--format <fmt>', 'Image format: png (default) or jpeg', 'png')
|
|
20
|
+
.option('--quality <n>', 'JPEG quality 1-100 (ignored for PNG)', parseInt)
|
|
21
|
+
.option('-w, --wait <ms>', 'Wait time after page load (ms)', parseInt)
|
|
22
|
+
.option('-t, --timeout <ms>', 'Request timeout (ms)', (v) => parseInt(v, 10), 30000)
|
|
23
|
+
.option('--stealth', 'Use stealth mode to bypass bot detection')
|
|
24
|
+
.option('--action <actions...>', 'Page actions before screenshot (e.g., "click:.btn" "wait:2000")')
|
|
25
|
+
.option('--scroll-through', 'Auto-scroll page before screenshot (triggers lazy content + scroll animations)')
|
|
26
|
+
.option('-o, --output <path>', 'Output file path (default: screenshot.png)')
|
|
27
|
+
.option('-s, --silent', 'Silent mode (no spinner)')
|
|
28
|
+
.option('--json', 'Output base64 JSON instead of binary file')
|
|
29
|
+
.action(async (url, options) => {
|
|
30
|
+
// Validate URL
|
|
31
|
+
try {
|
|
32
|
+
const parsed = new URL(url);
|
|
33
|
+
if (!['http:', 'https:'].includes(parsed.protocol)) {
|
|
34
|
+
console.error('Error: Only HTTP and HTTPS protocols are allowed');
|
|
35
|
+
process.exit(1);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
console.error(`Error: Invalid URL format: ${url}`);
|
|
40
|
+
process.exit(1);
|
|
41
|
+
}
|
|
42
|
+
// Check usage quota
|
|
43
|
+
const usageCheck = await checkUsage();
|
|
44
|
+
if (!usageCheck.allowed) {
|
|
45
|
+
console.error(usageCheck.message);
|
|
46
|
+
process.exit(1);
|
|
47
|
+
}
|
|
48
|
+
const spinner = options.silent ? null : ora('Taking screenshot...').start();
|
|
49
|
+
try {
|
|
50
|
+
// Validate format
|
|
51
|
+
const format = options.format?.toLowerCase();
|
|
52
|
+
if (format && !['png', 'jpeg', 'jpg'].includes(format)) {
|
|
53
|
+
console.error('Error: --format must be png, jpeg, or jpg');
|
|
54
|
+
process.exit(1);
|
|
55
|
+
}
|
|
56
|
+
// Parse actions
|
|
57
|
+
let actions;
|
|
58
|
+
if (options.action && options.action.length > 0) {
|
|
59
|
+
try {
|
|
60
|
+
actions = parseActions(options.action);
|
|
61
|
+
}
|
|
62
|
+
catch (e) {
|
|
63
|
+
console.error(`Error: ${e.message}`);
|
|
64
|
+
process.exit(1);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
const { takeScreenshot } = await import('../../core/screenshot.js');
|
|
68
|
+
const result = await takeScreenshot(url, {
|
|
69
|
+
fullPage: options.fullPage || false,
|
|
70
|
+
width: options.width,
|
|
71
|
+
height: options.height,
|
|
72
|
+
format: format || 'png',
|
|
73
|
+
quality: options.quality,
|
|
74
|
+
waitFor: options.wait,
|
|
75
|
+
timeout: options.timeout,
|
|
76
|
+
stealth: options.stealth || false,
|
|
77
|
+
actions,
|
|
78
|
+
scrollThrough: options.scrollThrough || false,
|
|
79
|
+
});
|
|
80
|
+
if (spinner) {
|
|
81
|
+
spinner.succeed(`Screenshot taken (${result.format})`);
|
|
82
|
+
}
|
|
83
|
+
// Show usage footer for free/anonymous users
|
|
84
|
+
if (usageCheck.usageInfo && !options.silent) {
|
|
85
|
+
showUsageFooter(usageCheck.usageInfo, usageCheck.isAnonymous || false, true);
|
|
86
|
+
}
|
|
87
|
+
if (options.json) {
|
|
88
|
+
// Output JSON with base64
|
|
89
|
+
const jsonStr = JSON.stringify({
|
|
90
|
+
url: result.url,
|
|
91
|
+
format: result.format,
|
|
92
|
+
contentType: result.contentType,
|
|
93
|
+
screenshot: result.screenshot,
|
|
94
|
+
}, null, 2);
|
|
95
|
+
await new Promise((resolve, reject) => {
|
|
96
|
+
process.stdout.write(jsonStr + '\n', (err) => {
|
|
97
|
+
if (err)
|
|
98
|
+
reject(err);
|
|
99
|
+
else
|
|
100
|
+
resolve();
|
|
101
|
+
});
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
else {
|
|
105
|
+
// Save to file
|
|
106
|
+
const ext = result.format === 'jpeg' ? 'jpg' : 'png';
|
|
107
|
+
const outputPath = options.output || `screenshot.${ext}`;
|
|
108
|
+
const buffer = Buffer.from(result.screenshot, 'base64');
|
|
109
|
+
writeFileSync(outputPath, buffer);
|
|
110
|
+
if (!options.silent) {
|
|
111
|
+
console.error(`Screenshot saved to: ${outputPath} (${(buffer.length / 1024).toFixed(1)} KB)`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
await cleanup();
|
|
115
|
+
process.exit(0);
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
if (spinner) {
|
|
119
|
+
spinner.fail('Screenshot failed');
|
|
120
|
+
}
|
|
121
|
+
if (error instanceof Error) {
|
|
122
|
+
const msg = error.message;
|
|
123
|
+
// Detect missing browser binary and give an actionable error
|
|
124
|
+
if (msg.includes("Executable doesn't exist") || msg.includes('browserType.launch') || msg.includes('Chromium is not installed')) {
|
|
125
|
+
console.error('\n\x1b[31m❌ Browser not installed.\x1b[0m');
|
|
126
|
+
console.error('\x1b[36m Run: npx playwright install chromium\x1b[0m');
|
|
127
|
+
console.error('\x1b[36m Then retry your screenshot command.\x1b[0m');
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
console.error(`\nError: ${msg}`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
console.error('\nError: Unknown error occurred');
|
|
135
|
+
}
|
|
136
|
+
await cleanup();
|
|
137
|
+
process.exit(1);
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
// ── brand command ─────────────────────────────────────────────────────────
|
|
141
|
+
program
|
|
142
|
+
.command('brand <url>')
|
|
143
|
+
.description('Extract branding and design system from a URL')
|
|
144
|
+
.option('-s, --silent', 'Silent mode (no spinner)')
|
|
145
|
+
.option('--json', 'Output as JSON (default)')
|
|
146
|
+
.action(async (url, options) => {
|
|
147
|
+
const spinner = options.silent ? null : ora('Extracting branding...').start();
|
|
148
|
+
try {
|
|
149
|
+
const result = await peel(url, {
|
|
150
|
+
extract: {
|
|
151
|
+
selectors: {
|
|
152
|
+
primaryColor: 'meta[name="theme-color"]',
|
|
153
|
+
title: 'title',
|
|
154
|
+
logo: 'img[class*="logo"], img[alt*="logo"]',
|
|
155
|
+
},
|
|
156
|
+
},
|
|
157
|
+
});
|
|
158
|
+
if (spinner) {
|
|
159
|
+
spinner.succeed(`Extracted branding in ${result.elapsed}ms`);
|
|
160
|
+
}
|
|
161
|
+
// Extract branding data from metadata and page
|
|
162
|
+
const branding = {
|
|
163
|
+
url: result.url,
|
|
164
|
+
title: result.title,
|
|
165
|
+
colors: extractColors(result.content),
|
|
166
|
+
fonts: extractFonts(result.content),
|
|
167
|
+
extracted: result.extracted,
|
|
168
|
+
metadata: result.metadata,
|
|
169
|
+
};
|
|
170
|
+
console.log(JSON.stringify(branding, null, 2));
|
|
171
|
+
await cleanup();
|
|
172
|
+
process.exit(0);
|
|
173
|
+
}
|
|
174
|
+
catch (error) {
|
|
175
|
+
if (spinner)
|
|
176
|
+
spinner.fail('Branding extraction failed');
|
|
177
|
+
console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
178
|
+
await cleanup();
|
|
179
|
+
process.exit(1);
|
|
180
|
+
}
|
|
181
|
+
});
|
|
182
|
+
// ── design-compare command ────────────────────────────────────────────────
|
|
183
|
+
program
|
|
184
|
+
.command('design-compare <url>')
|
|
185
|
+
.description('Compare the design of a subject URL against a reference URL')
|
|
186
|
+
.option('--ref <url>', 'Reference URL to compare against (required)')
|
|
187
|
+
.option('--width <px>', 'Viewport width in pixels (default: 1440)', parseInt)
|
|
188
|
+
.option('--height <px>', 'Viewport height in pixels (default: 900)', parseInt)
|
|
189
|
+
.option('-o, --output <path>', 'Save comparison report to a JSON file')
|
|
190
|
+
.option('-s, --silent', 'Silent mode (no spinner)')
|
|
191
|
+
.option('--json', 'Output comparison as JSON to stdout')
|
|
192
|
+
.action(async (url, options) => {
|
|
193
|
+
// Validate subject URL
|
|
194
|
+
try {
|
|
195
|
+
const parsed = new URL(url);
|
|
196
|
+
if (!['http:', 'https:'].includes(parsed.protocol)) {
|
|
197
|
+
console.error('Error: Only HTTP and HTTPS protocols are allowed');
|
|
198
|
+
process.exit(1);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
catch {
|
|
202
|
+
console.error(`Error: Invalid URL format: ${url}`);
|
|
203
|
+
process.exit(1);
|
|
204
|
+
}
|
|
205
|
+
// Validate --ref
|
|
206
|
+
if (!options.ref) {
|
|
207
|
+
console.error('Error: --ref <url> is required');
|
|
208
|
+
process.exit(1);
|
|
209
|
+
}
|
|
210
|
+
try {
|
|
211
|
+
const parsedRef = new URL(options.ref);
|
|
212
|
+
if (!['http:', 'https:'].includes(parsedRef.protocol)) {
|
|
213
|
+
console.error('Error: --ref must be an HTTP or HTTPS URL');
|
|
214
|
+
process.exit(1);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
catch {
|
|
218
|
+
console.error(`Error: Invalid --ref URL format: ${options.ref}`);
|
|
219
|
+
process.exit(1);
|
|
220
|
+
}
|
|
221
|
+
const spinner = options.silent ? null : ora(`Comparing designs: ${url} vs ${options.ref}...`).start();
|
|
222
|
+
try {
|
|
223
|
+
const { takeDesignComparison } = await import('../../core/screenshot.js');
|
|
224
|
+
const result = await takeDesignComparison(url, options.ref, {
|
|
225
|
+
width: options.width,
|
|
226
|
+
height: options.height,
|
|
227
|
+
});
|
|
228
|
+
if (spinner)
|
|
229
|
+
spinner.succeed('Design comparison complete');
|
|
230
|
+
const { comparison } = result;
|
|
231
|
+
const output = {
|
|
232
|
+
subjectUrl: result.subjectUrl,
|
|
233
|
+
referenceUrl: result.referenceUrl,
|
|
234
|
+
score: comparison.score,
|
|
235
|
+
summary: comparison.summary,
|
|
236
|
+
gaps: comparison.gaps,
|
|
237
|
+
subjectAnalysis: comparison.subjectAnalysis,
|
|
238
|
+
referenceAnalysis: comparison.referenceAnalysis,
|
|
239
|
+
};
|
|
240
|
+
if (options.output) {
|
|
241
|
+
writeFileSync(options.output, JSON.stringify(output, null, 2));
|
|
242
|
+
if (!options.silent)
|
|
243
|
+
console.error(`Report saved to: ${options.output}`);
|
|
244
|
+
}
|
|
245
|
+
if (options.json || !options.output) {
|
|
246
|
+
const jsonStr = JSON.stringify(output, null, 2);
|
|
247
|
+
await new Promise((resolve, reject) => {
|
|
248
|
+
process.stdout.write(jsonStr + '\n', (err) => {
|
|
249
|
+
if (err)
|
|
250
|
+
reject(err);
|
|
251
|
+
else
|
|
252
|
+
resolve();
|
|
253
|
+
});
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
else if (!options.silent) {
|
|
257
|
+
// Human-readable summary
|
|
258
|
+
console.log(`\n🎨 Design Comparison`);
|
|
259
|
+
console.log(`Subject: ${result.subjectUrl}`);
|
|
260
|
+
console.log(`Reference: ${result.referenceUrl}`);
|
|
261
|
+
console.log(`Score: ${comparison.score}/10`);
|
|
262
|
+
console.log(`\n${comparison.summary}`);
|
|
263
|
+
if (comparison.gaps.length > 0) {
|
|
264
|
+
console.log(`\nGaps (${comparison.gaps.length}):`);
|
|
265
|
+
for (const gap of comparison.gaps) {
|
|
266
|
+
const sev = gap.severity === 'high' ? '🔴' : gap.severity === 'medium' ? '🟡' : '🟢';
|
|
267
|
+
console.log(` ${sev} ${gap.property}: ${gap.description}`);
|
|
268
|
+
console.log(` Subject: ${gap.subject}`);
|
|
269
|
+
console.log(` Reference: ${gap.reference}`);
|
|
270
|
+
console.log(` Suggestion: ${gap.suggestion}`);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
catch (error) {
|
|
276
|
+
if (spinner)
|
|
277
|
+
spinner.fail('Design comparison failed');
|
|
278
|
+
console.error(`Error: ${error.message}`);
|
|
279
|
+
process.exit(1);
|
|
280
|
+
}
|
|
281
|
+
});
|
|
282
|
+
}
|