@iflow-mcp/jakeliume-webpeel 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +313 -0
- package/dist/cache.d.ts +30 -0
- package/dist/cache.js +139 -0
- package/dist/cli/commands/auth.d.ts +5 -0
- package/dist/cli/commands/auth.js +411 -0
- package/dist/cli/commands/doctor.d.ts +37 -0
- package/dist/cli/commands/doctor.js +371 -0
- package/dist/cli/commands/fetch.d.ts +6 -0
- package/dist/cli/commands/fetch.js +1345 -0
- package/dist/cli/commands/guide.d.ts +2 -0
- package/dist/cli/commands/guide.js +183 -0
- package/dist/cli/commands/interact.d.ts +5 -0
- package/dist/cli/commands/interact.js +840 -0
- package/dist/cli/commands/jobs.d.ts +5 -0
- package/dist/cli/commands/jobs.js +997 -0
- package/dist/cli/commands/monitor.d.ts +12 -0
- package/dist/cli/commands/monitor.js +197 -0
- package/dist/cli/commands/observe.d.ts +12 -0
- package/dist/cli/commands/observe.js +158 -0
- package/dist/cli/commands/screenshot.d.ts +5 -0
- package/dist/cli/commands/screenshot.js +282 -0
- package/dist/cli/commands/search.d.ts +5 -0
- package/dist/cli/commands/search.js +1021 -0
- package/dist/cli/commands/setup.d.ts +13 -0
- package/dist/cli/commands/setup.js +244 -0
- package/dist/cli/commands/skill.d.ts +15 -0
- package/dist/cli/commands/skill.js +195 -0
- package/dist/cli/utils.d.ts +84 -0
- package/dist/cli/utils.js +806 -0
- package/dist/cli-auth.d.ts +75 -0
- package/dist/cli-auth.js +369 -0
- package/dist/cli.d.ts +17 -0
- package/dist/cli.js +99 -0
- package/dist/core/actions.d.ts +69 -0
- package/dist/core/actions.js +495 -0
- package/dist/core/agent.d.ts +98 -0
- package/dist/core/agent.js +558 -0
- package/dist/core/answer.d.ts +42 -0
- package/dist/core/answer.js +395 -0
- package/dist/core/application-tracker.d.ts +84 -0
- package/dist/core/application-tracker.js +184 -0
- package/dist/core/apply.d.ts +162 -0
- package/dist/core/apply.js +816 -0
- package/dist/core/auth-detection.d.ts +35 -0
- package/dist/core/auth-detection.js +358 -0
- package/dist/core/auto-extract.d.ts +82 -0
- package/dist/core/auto-extract.js +604 -0
- package/dist/core/auto-interact.d.ts +23 -0
- package/dist/core/auto-interact.js +246 -0
- package/dist/core/bm25-filter.d.ts +66 -0
- package/dist/core/bm25-filter.js +288 -0
- package/dist/core/branding.d.ts +54 -0
- package/dist/core/branding.js +234 -0
- package/dist/core/browser-fetch.d.ts +323 -0
- package/dist/core/browser-fetch.js +1600 -0
- package/dist/core/browser-pool.d.ts +91 -0
- package/dist/core/browser-pool.js +550 -0
- package/dist/core/budget.d.ts +42 -0
- package/dist/core/budget.js +324 -0
- package/dist/core/business-intel.d.ts +47 -0
- package/dist/core/business-intel.js +279 -0
- package/dist/core/cache.d.ts +13 -0
- package/dist/core/cache.js +121 -0
- package/dist/core/cf-worker-proxy.d.ts +32 -0
- package/dist/core/cf-worker-proxy.js +87 -0
- package/dist/core/challenge-detection.d.ts +26 -0
- package/dist/core/challenge-detection.js +468 -0
- package/dist/core/change-tracking.d.ts +75 -0
- package/dist/core/change-tracking.js +276 -0
- package/dist/core/chunker.d.ts +46 -0
- package/dist/core/chunker.js +249 -0
- package/dist/core/chunking.d.ts +42 -0
- package/dist/core/chunking.js +181 -0
- package/dist/core/circuit-breaker.d.ts +44 -0
- package/dist/core/circuit-breaker.js +85 -0
- package/dist/core/content-pruner.d.ts +47 -0
- package/dist/core/content-pruner.js +425 -0
- package/dist/core/cookie-cache.d.ts +60 -0
- package/dist/core/cookie-cache.js +163 -0
- package/dist/core/crawl-checkpoint.d.ts +54 -0
- package/dist/core/crawl-checkpoint.js +104 -0
- package/dist/core/crawler.d.ts +84 -0
- package/dist/core/crawler.js +349 -0
- package/dist/core/cross-verify.d.ts +27 -0
- package/dist/core/cross-verify.js +93 -0
- package/dist/core/deep-fetch.d.ts +74 -0
- package/dist/core/deep-fetch.js +405 -0
- package/dist/core/deep-research.d.ts +141 -0
- package/dist/core/deep-research.js +972 -0
- package/dist/core/design-analysis.d.ts +70 -0
- package/dist/core/design-analysis.js +490 -0
- package/dist/core/design-compare.d.ts +38 -0
- package/dist/core/design-compare.js +264 -0
- package/dist/core/diff.d.ts +61 -0
- package/dist/core/diff.js +289 -0
- package/dist/core/dns-cache.d.ts +20 -0
- package/dist/core/dns-cache.js +198 -0
- package/dist/core/documents.d.ts +23 -0
- package/dist/core/documents.js +123 -0
- package/dist/core/domain-memory.d.ts +66 -0
- package/dist/core/domain-memory.js +163 -0
- package/dist/core/domain-verify.d.ts +40 -0
- package/dist/core/domain-verify.js +379 -0
- package/dist/core/engine-ranker.d.ts +112 -0
- package/dist/core/engine-ranker.js +395 -0
- package/dist/core/extract-inline.d.ts +38 -0
- package/dist/core/extract-inline.js +215 -0
- package/dist/core/extract-listings.d.ts +38 -0
- package/dist/core/extract-listings.js +461 -0
- package/dist/core/extract.d.ts +9 -0
- package/dist/core/extract.js +139 -0
- package/dist/core/fetch-cache.d.ts +57 -0
- package/dist/core/fetch-cache.js +95 -0
- package/dist/core/fetcher.d.ts +13 -0
- package/dist/core/fetcher.js +12 -0
- package/dist/core/google-cache.d.ts +29 -0
- package/dist/core/google-cache.js +180 -0
- package/dist/core/google-serp-parser.d.ts +82 -0
- package/dist/core/google-serp-parser.js +287 -0
- package/dist/core/hotel-search.d.ts +122 -0
- package/dist/core/hotel-search.js +382 -0
- package/dist/core/http-fetch.d.ts +72 -0
- package/dist/core/http-fetch.js +820 -0
- package/dist/core/human.d.ts +175 -0
- package/dist/core/human.js +680 -0
- package/dist/core/image-caption.d.ts +44 -0
- package/dist/core/image-caption.js +271 -0
- package/dist/core/jobs.d.ts +75 -0
- package/dist/core/jobs.js +634 -0
- package/dist/core/json-ld.d.ts +15 -0
- package/dist/core/json-ld.js +617 -0
- package/dist/core/language-detect.d.ts +18 -0
- package/dist/core/language-detect.js +135 -0
- package/dist/core/links.d.ts +10 -0
- package/dist/core/links.js +44 -0
- package/dist/core/llm-extract.d.ts +71 -0
- package/dist/core/llm-extract.js +507 -0
- package/dist/core/llm-provider.d.ts +100 -0
- package/dist/core/llm-provider.js +702 -0
- package/dist/core/local-search.d.ts +60 -0
- package/dist/core/local-search.js +308 -0
- package/dist/core/logger.d.ts +28 -0
- package/dist/core/logger.js +104 -0
- package/dist/core/map.d.ts +33 -0
- package/dist/core/map.js +127 -0
- package/dist/core/markdown.d.ts +92 -0
- package/dist/core/markdown.js +809 -0
- package/dist/core/metadata.d.ts +34 -0
- package/dist/core/metadata.js +422 -0
- package/dist/core/observe.d.ts +113 -0
- package/dist/core/observe.js +395 -0
- package/dist/core/ocr.d.ts +12 -0
- package/dist/core/ocr.js +33 -0
- package/dist/core/paginate.d.ts +31 -0
- package/dist/core/paginate.js +106 -0
- package/dist/core/pdf.d.ts +8 -0
- package/dist/core/pdf.js +25 -0
- package/dist/core/peel-tls.d.ts +25 -0
- package/dist/core/peel-tls.js +220 -0
- package/dist/core/pipeline.d.ts +132 -0
- package/dist/core/pipeline.js +1666 -0
- package/dist/core/profiles.d.ts +61 -0
- package/dist/core/profiles.js +350 -0
- package/dist/core/prompt-guard.d.ts +30 -0
- package/dist/core/prompt-guard.js +119 -0
- package/dist/core/proxy-config.d.ts +90 -0
- package/dist/core/proxy-config.js +172 -0
- package/dist/core/quick-answer.d.ts +53 -0
- package/dist/core/quick-answer.js +833 -0
- package/dist/core/rate-governor.d.ts +80 -0
- package/dist/core/rate-governor.js +238 -0
- package/dist/core/readability.d.ts +57 -0
- package/dist/core/readability.js +533 -0
- package/dist/core/research.d.ts +66 -0
- package/dist/core/research.js +270 -0
- package/dist/core/retry.d.ts +60 -0
- package/dist/core/retry.js +119 -0
- package/dist/core/safe-browsing.d.ts +30 -0
- package/dist/core/safe-browsing.js +206 -0
- package/dist/core/schema-extraction.d.ts +66 -0
- package/dist/core/schema-extraction.js +352 -0
- package/dist/core/schema-postprocess.d.ts +32 -0
- package/dist/core/schema-postprocess.js +469 -0
- package/dist/core/schema-templates.d.ts +19 -0
- package/dist/core/schema-templates.js +143 -0
- package/dist/core/screenshot.d.ts +224 -0
- package/dist/core/screenshot.js +207 -0
- package/dist/core/search-engines.d.ts +25 -0
- package/dist/core/search-engines.js +182 -0
- package/dist/core/search-provider.d.ts +243 -0
- package/dist/core/search-provider.js +1629 -0
- package/dist/core/searxng-provider.d.ts +35 -0
- package/dist/core/searxng-provider.js +105 -0
- package/dist/core/selective-evidence.d.ts +151 -0
- package/dist/core/selective-evidence.js +389 -0
- package/dist/core/site-search.d.ts +44 -0
- package/dist/core/site-search.js +252 -0
- package/dist/core/sitemap.d.ts +23 -0
- package/dist/core/sitemap.js +105 -0
- package/dist/core/source-credibility.d.ts +29 -0
- package/dist/core/source-credibility.js +584 -0
- package/dist/core/source-scoring.d.ts +166 -0
- package/dist/core/source-scoring.js +396 -0
- package/dist/core/stemmer.d.ts +38 -0
- package/dist/core/stemmer.js +509 -0
- package/dist/core/strategies.d.ts +104 -0
- package/dist/core/strategies.js +1044 -0
- package/dist/core/strategy-hooks.d.ts +145 -0
- package/dist/core/strategy-hooks.js +74 -0
- package/dist/core/structured-extract.d.ts +43 -0
- package/dist/core/structured-extract.js +550 -0
- package/dist/core/summarize.d.ts +17 -0
- package/dist/core/summarize.js +78 -0
- package/dist/core/synonyms.d.ts +42 -0
- package/dist/core/synonyms.js +184 -0
- package/dist/core/system-monitor.d.ts +61 -0
- package/dist/core/system-monitor.js +133 -0
- package/dist/core/table-format.d.ts +30 -0
- package/dist/core/table-format.js +146 -0
- package/dist/core/threat-feeds.d.ts +23 -0
- package/dist/core/threat-feeds.js +104 -0
- package/dist/core/timing.d.ts +21 -0
- package/dist/core/timing.js +33 -0
- package/dist/core/transcript-export.d.ts +47 -0
- package/dist/core/transcript-export.js +107 -0
- package/dist/core/user-agents.d.ts +82 -0
- package/dist/core/user-agents.js +239 -0
- package/dist/core/vertical-search.d.ts +54 -0
- package/dist/core/vertical-search.js +158 -0
- package/dist/core/watch-manager.d.ts +175 -0
- package/dist/core/watch-manager.js +416 -0
- package/dist/core/watch.d.ts +101 -0
- package/dist/core/watch.js +389 -0
- package/dist/core/youtube.d.ts +130 -0
- package/dist/core/youtube.js +1175 -0
- package/dist/ee/challenge-re-export.d.ts +1 -0
- package/dist/ee/challenge-re-export.js +1 -0
- package/dist/ee/challenge-solver.d.ts +72 -0
- package/dist/ee/challenge-solver.js +720 -0
- package/dist/ee/domain-extractors.d.ts +8 -0
- package/dist/ee/domain-extractors.js +8 -0
- package/dist/ee/domain-intel.d.ts +16 -0
- package/dist/ee/domain-intel.js +133 -0
- package/dist/ee/extractors/allrecipes.d.ts +2 -0
- package/dist/ee/extractors/allrecipes.js +120 -0
- package/dist/ee/extractors/amazon.d.ts +2 -0
- package/dist/ee/extractors/amazon.js +78 -0
- package/dist/ee/extractors/arxiv.d.ts +2 -0
- package/dist/ee/extractors/arxiv.js +137 -0
- package/dist/ee/extractors/bestbuy.d.ts +2 -0
- package/dist/ee/extractors/bestbuy.js +78 -0
- package/dist/ee/extractors/carscom.d.ts +2 -0
- package/dist/ee/extractors/carscom.js +121 -0
- package/dist/ee/extractors/coingecko.d.ts +2 -0
- package/dist/ee/extractors/coingecko.js +134 -0
- package/dist/ee/extractors/craigslist.d.ts +2 -0
- package/dist/ee/extractors/craigslist.js +92 -0
- package/dist/ee/extractors/devto.d.ts +2 -0
- package/dist/ee/extractors/devto.js +135 -0
- package/dist/ee/extractors/ebay.d.ts +2 -0
- package/dist/ee/extractors/ebay.js +90 -0
- package/dist/ee/extractors/espn.d.ts +2 -0
- package/dist/ee/extractors/espn.js +260 -0
- package/dist/ee/extractors/etsy.d.ts +2 -0
- package/dist/ee/extractors/etsy.js +52 -0
- package/dist/ee/extractors/facebook.d.ts +2 -0
- package/dist/ee/extractors/facebook.js +46 -0
- package/dist/ee/extractors/github.d.ts +2 -0
- package/dist/ee/extractors/github.js +196 -0
- package/dist/ee/extractors/google-flights.d.ts +2 -0
- package/dist/ee/extractors/google-flights.js +176 -0
- package/dist/ee/extractors/hackernews.d.ts +2 -0
- package/dist/ee/extractors/hackernews.js +147 -0
- package/dist/ee/extractors/imdb.d.ts +2 -0
- package/dist/ee/extractors/imdb.js +172 -0
- package/dist/ee/extractors/index.d.ts +26 -0
- package/dist/ee/extractors/index.js +247 -0
- package/dist/ee/extractors/instagram.d.ts +2 -0
- package/dist/ee/extractors/instagram.js +102 -0
- package/dist/ee/extractors/kalshi.d.ts +2 -0
- package/dist/ee/extractors/kalshi.js +121 -0
- package/dist/ee/extractors/kayak-cars.d.ts +2 -0
- package/dist/ee/extractors/kayak-cars.js +270 -0
- package/dist/ee/extractors/linkedin.d.ts +2 -0
- package/dist/ee/extractors/linkedin.js +113 -0
- package/dist/ee/extractors/medium.d.ts +2 -0
- package/dist/ee/extractors/medium.js +130 -0
- package/dist/ee/extractors/news.d.ts +4 -0
- package/dist/ee/extractors/news.js +173 -0
- package/dist/ee/extractors/npm.d.ts +2 -0
- package/dist/ee/extractors/npm.js +86 -0
- package/dist/ee/extractors/pdf.d.ts +2 -0
- package/dist/ee/extractors/pdf.js +108 -0
- package/dist/ee/extractors/pinterest.d.ts +2 -0
- package/dist/ee/extractors/pinterest.js +34 -0
- package/dist/ee/extractors/polymarket.d.ts +2 -0
- package/dist/ee/extractors/polymarket.js +358 -0
- package/dist/ee/extractors/producthunt.d.ts +2 -0
- package/dist/ee/extractors/producthunt.js +88 -0
- package/dist/ee/extractors/pubmed.d.ts +2 -0
- package/dist/ee/extractors/pubmed.js +162 -0
- package/dist/ee/extractors/pypi.d.ts +2 -0
- package/dist/ee/extractors/pypi.js +80 -0
- package/dist/ee/extractors/reddit.d.ts +2 -0
- package/dist/ee/extractors/reddit.js +438 -0
- package/dist/ee/extractors/redfin.d.ts +2 -0
- package/dist/ee/extractors/redfin.js +156 -0
- package/dist/ee/extractors/semanticscholar.d.ts +2 -0
- package/dist/ee/extractors/semanticscholar.js +131 -0
- package/dist/ee/extractors/shared.d.ts +12 -0
- package/dist/ee/extractors/shared.js +76 -0
- package/dist/ee/extractors/soundcloud.d.ts +2 -0
- package/dist/ee/extractors/soundcloud.js +34 -0
- package/dist/ee/extractors/sportsbetting.d.ts +2 -0
- package/dist/ee/extractors/sportsbetting.js +37 -0
- package/dist/ee/extractors/spotify.d.ts +2 -0
- package/dist/ee/extractors/spotify.js +34 -0
- package/dist/ee/extractors/stackoverflow.d.ts +2 -0
- package/dist/ee/extractors/stackoverflow.js +61 -0
- package/dist/ee/extractors/substack.d.ts +2 -0
- package/dist/ee/extractors/substack.js +115 -0
- package/dist/ee/extractors/substackroot.d.ts +2 -0
- package/dist/ee/extractors/substackroot.js +46 -0
- package/dist/ee/extractors/tiktok.d.ts +2 -0
- package/dist/ee/extractors/tiktok.js +29 -0
- package/dist/ee/extractors/tradingview.d.ts +2 -0
- package/dist/ee/extractors/tradingview.js +182 -0
- package/dist/ee/extractors/twitch.d.ts +2 -0
- package/dist/ee/extractors/twitch.js +36 -0
- package/dist/ee/extractors/twitter.d.ts +2 -0
- package/dist/ee/extractors/twitter.js +327 -0
- package/dist/ee/extractors/types.d.ts +14 -0
- package/dist/ee/extractors/types.js +1 -0
- package/dist/ee/extractors/walmart.d.ts +2 -0
- package/dist/ee/extractors/walmart.js +50 -0
- package/dist/ee/extractors/weather.d.ts +2 -0
- package/dist/ee/extractors/weather.js +133 -0
- package/dist/ee/extractors/wikipedia.d.ts +4 -0
- package/dist/ee/extractors/wikipedia.js +235 -0
- package/dist/ee/extractors/yelp.d.ts +2 -0
- package/dist/ee/extractors/yelp.js +216 -0
- package/dist/ee/extractors/youtube.d.ts +2 -0
- package/dist/ee/extractors/youtube.js +189 -0
- package/dist/ee/extractors/zillow.d.ts +54 -0
- package/dist/ee/extractors/zillow.js +247 -0
- package/dist/ee/extractors-re-export.d.ts +1 -0
- package/dist/ee/extractors-re-export.js +1 -0
- package/dist/ee/premium-hooks.d.ts +20 -0
- package/dist/ee/premium-hooks.js +50 -0
- package/dist/ee/spa-detection.d.ts +2 -0
- package/dist/ee/spa-detection.js +2 -0
- package/dist/ee/stability.d.ts +4 -0
- package/dist/ee/stability.js +29 -0
- package/dist/ee/swr-cache.d.ts +14 -0
- package/dist/ee/swr-cache.js +34 -0
- package/dist/index.d.ts +143 -0
- package/dist/index.js +291 -0
- package/dist/integrations/index.d.ts +2 -0
- package/dist/integrations/index.js +2 -0
- package/dist/integrations/langchain.d.ts +64 -0
- package/dist/integrations/langchain.js +115 -0
- package/dist/integrations/llamaindex.d.ts +50 -0
- package/dist/integrations/llamaindex.js +91 -0
- package/dist/mcp/handlers/act.d.ts +5 -0
- package/dist/mcp/handlers/act.js +34 -0
- package/dist/mcp/handlers/definitions.d.ts +6 -0
- package/dist/mcp/handlers/definitions.js +395 -0
- package/dist/mcp/handlers/extract.d.ts +7 -0
- package/dist/mcp/handlers/extract.js +135 -0
- package/dist/mcp/handlers/fetch.d.ts +6 -0
- package/dist/mcp/handlers/fetch.js +98 -0
- package/dist/mcp/handlers/find.d.ts +5 -0
- package/dist/mcp/handlers/find.js +137 -0
- package/dist/mcp/handlers/index.d.ts +13 -0
- package/dist/mcp/handlers/index.js +63 -0
- package/dist/mcp/handlers/legacy.d.ts +25 -0
- package/dist/mcp/handlers/legacy.js +450 -0
- package/dist/mcp/handlers/meta.d.ts +6 -0
- package/dist/mcp/handlers/meta.js +40 -0
- package/dist/mcp/handlers/monitor.d.ts +5 -0
- package/dist/mcp/handlers/monitor.js +41 -0
- package/dist/mcp/handlers/observe.d.ts +8 -0
- package/dist/mcp/handlers/observe.js +37 -0
- package/dist/mcp/handlers/read.d.ts +6 -0
- package/dist/mcp/handlers/read.js +78 -0
- package/dist/mcp/handlers/see.d.ts +5 -0
- package/dist/mcp/handlers/see.js +75 -0
- package/dist/mcp/handlers/types.d.ts +29 -0
- package/dist/mcp/handlers/types.js +28 -0
- package/dist/mcp/server.d.ts +7 -0
- package/dist/mcp/server.js +108 -0
- package/dist/mcp/smart-router.d.ts +23 -0
- package/dist/mcp/smart-router.js +178 -0
- package/dist/server/app.d.ts +14 -0
- package/dist/server/app.js +632 -0
- package/dist/server/auth-store.d.ts +28 -0
- package/dist/server/auth-store.js +88 -0
- package/dist/server/bull-queues.d.ts +60 -0
- package/dist/server/bull-queues.js +90 -0
- package/dist/server/email-service.d.ts +55 -0
- package/dist/server/email-service.js +291 -0
- package/dist/server/job-queue.d.ts +100 -0
- package/dist/server/job-queue.js +145 -0
- package/dist/server/logger.d.ts +10 -0
- package/dist/server/logger.js +37 -0
- package/dist/server/middleware/audit-log.d.ts +14 -0
- package/dist/server/middleware/audit-log.js +73 -0
- package/dist/server/middleware/auth.d.ts +35 -0
- package/dist/server/middleware/auth.js +225 -0
- package/dist/server/middleware/rate-limit.d.ts +50 -0
- package/dist/server/middleware/rate-limit.js +270 -0
- package/dist/server/middleware/scope-guard.d.ts +25 -0
- package/dist/server/middleware/scope-guard.js +45 -0
- package/dist/server/middleware/url-validator.d.ts +15 -0
- package/dist/server/middleware/url-validator.js +201 -0
- package/dist/server/openapi.yaml +6418 -0
- package/dist/server/pg-auth-store.d.ts +146 -0
- package/dist/server/pg-auth-store.js +576 -0
- package/dist/server/pg-job-queue.d.ts +59 -0
- package/dist/server/pg-job-queue.js +375 -0
- package/dist/server/routes/activity.d.ts +6 -0
- package/dist/server/routes/activity.js +79 -0
- package/dist/server/routes/admin-active.d.ts +7 -0
- package/dist/server/routes/admin-active.js +120 -0
- package/dist/server/routes/admin-stats.d.ts +7 -0
- package/dist/server/routes/admin-stats.js +176 -0
- package/dist/server/routes/agent.d.ts +24 -0
- package/dist/server/routes/agent.js +480 -0
- package/dist/server/routes/answer.d.ts +5 -0
- package/dist/server/routes/answer.js +125 -0
- package/dist/server/routes/ask.d.ts +28 -0
- package/dist/server/routes/ask.js +295 -0
- package/dist/server/routes/batch.d.ts +6 -0
- package/dist/server/routes/batch.js +493 -0
- package/dist/server/routes/cache-warm.d.ts +25 -0
- package/dist/server/routes/cache-warm.js +212 -0
- package/dist/server/routes/cli-usage.d.ts +6 -0
- package/dist/server/routes/cli-usage.js +127 -0
- package/dist/server/routes/compat.d.ts +23 -0
- package/dist/server/routes/compat.js +652 -0
- package/dist/server/routes/crawl.d.ts +13 -0
- package/dist/server/routes/crawl.js +287 -0
- package/dist/server/routes/deep-fetch.d.ts +8 -0
- package/dist/server/routes/deep-fetch.js +57 -0
- package/dist/server/routes/deep-research.d.ts +11 -0
- package/dist/server/routes/deep-research.js +232 -0
- package/dist/server/routes/demo.d.ts +24 -0
- package/dist/server/routes/demo.js +517 -0
- package/dist/server/routes/do.d.ts +8 -0
- package/dist/server/routes/do.js +72 -0
- package/dist/server/routes/extract.d.ts +14 -0
- package/dist/server/routes/extract.js +325 -0
- package/dist/server/routes/feed.d.ts +15 -0
- package/dist/server/routes/feed.js +311 -0
- package/dist/server/routes/fetch-queue.d.ts +13 -0
- package/dist/server/routes/fetch-queue.js +357 -0
- package/dist/server/routes/fetch.d.ts +7 -0
- package/dist/server/routes/fetch.js +1274 -0
- package/dist/server/routes/go.d.ts +14 -0
- package/dist/server/routes/go.js +81 -0
- package/dist/server/routes/health.d.ts +11 -0
- package/dist/server/routes/health.js +141 -0
- package/dist/server/routes/jobs.d.ts +7 -0
- package/dist/server/routes/jobs.js +574 -0
- package/dist/server/routes/map.d.ts +11 -0
- package/dist/server/routes/map.js +116 -0
- package/dist/server/routes/mcp.d.ts +14 -0
- package/dist/server/routes/mcp.js +197 -0
- package/dist/server/routes/metrics.d.ts +37 -0
- package/dist/server/routes/metrics.js +149 -0
- package/dist/server/routes/oauth.d.ts +9 -0
- package/dist/server/routes/oauth.js +396 -0
- package/dist/server/routes/playground.d.ts +17 -0
- package/dist/server/routes/playground.js +283 -0
- package/dist/server/routes/reader.d.ts +18 -0
- package/dist/server/routes/reader.js +192 -0
- package/dist/server/routes/research.d.ts +14 -0
- package/dist/server/routes/research.js +482 -0
- package/dist/server/routes/screenshot.d.ts +22 -0
- package/dist/server/routes/screenshot.js +820 -0
- package/dist/server/routes/search.d.ts +6 -0
- package/dist/server/routes/search.js +874 -0
- package/dist/server/routes/session.d.ts +17 -0
- package/dist/server/routes/session.js +548 -0
- package/dist/server/routes/share.d.ts +18 -0
- package/dist/server/routes/share.js +462 -0
- package/dist/server/routes/smart-search/handlers/cars.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/cars.js +102 -0
- package/dist/server/routes/smart-search/handlers/flights.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/flights.js +72 -0
- package/dist/server/routes/smart-search/handlers/general.d.ts +13 -0
- package/dist/server/routes/smart-search/handlers/general.js +717 -0
- package/dist/server/routes/smart-search/handlers/hotels.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/hotels.js +88 -0
- package/dist/server/routes/smart-search/handlers/products.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/products.js +1309 -0
- package/dist/server/routes/smart-search/handlers/rental.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/rental.js +154 -0
- package/dist/server/routes/smart-search/handlers/restaurants.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/restaurants.js +225 -0
- package/dist/server/routes/smart-search/handlers/transit-verdict.d.ts +41 -0
- package/dist/server/routes/smart-search/handlers/transit-verdict.js +224 -0
- package/dist/server/routes/smart-search/index.d.ts +19 -0
- package/dist/server/routes/smart-search/index.js +546 -0
- package/dist/server/routes/smart-search/intent.d.ts +3 -0
- package/dist/server/routes/smart-search/intent.js +264 -0
- package/dist/server/routes/smart-search/llm.d.ts +16 -0
- package/dist/server/routes/smart-search/llm.js +70 -0
- package/dist/server/routes/smart-search/sources/reddit.d.ts +18 -0
- package/dist/server/routes/smart-search/sources/reddit.js +34 -0
- package/dist/server/routes/smart-search/sources/yelp.d.ts +25 -0
- package/dist/server/routes/smart-search/sources/yelp.js +171 -0
- package/dist/server/routes/smart-search/sources/youtube.d.ts +8 -0
- package/dist/server/routes/smart-search/sources/youtube.js +9 -0
- package/dist/server/routes/smart-search/types.d.ts +81 -0
- package/dist/server/routes/smart-search/types.js +1 -0
- package/dist/server/routes/smart-search/utils.d.ts +20 -0
- package/dist/server/routes/smart-search/utils.js +146 -0
- package/dist/server/routes/stats.d.ts +6 -0
- package/dist/server/routes/stats.js +71 -0
- package/dist/server/routes/stripe.d.ts +15 -0
- package/dist/server/routes/stripe.js +296 -0
- package/dist/server/routes/transcript-export.d.ts +10 -0
- package/dist/server/routes/transcript-export.js +178 -0
- package/dist/server/routes/usage.d.ts +9 -0
- package/dist/server/routes/usage.js +279 -0
- package/dist/server/routes/users.d.ts +8 -0
- package/dist/server/routes/users.js +1867 -0
- package/dist/server/routes/watch.d.ts +15 -0
- package/dist/server/routes/watch.js +309 -0
- package/dist/server/routes/webhooks.d.ts +26 -0
- package/dist/server/routes/webhooks.js +170 -0
- package/dist/server/routes/youtube.d.ts +6 -0
- package/dist/server/routes/youtube.js +130 -0
- package/dist/server/sentry.d.ts +14 -0
- package/dist/server/sentry.js +104 -0
- package/dist/server/types.d.ts +15 -0
- package/dist/server/types.js +7 -0
- package/dist/server/utils/response.d.ts +44 -0
- package/dist/server/utils/response.js +69 -0
- package/dist/server/utils/sse.d.ts +22 -0
- package/dist/server/utils/sse.js +38 -0
- package/dist/types.d.ts +552 -0
- package/dist/types.js +39 -0
- package/llms.txt +105 -0
- package/package.json +189 -0
|
@@ -0,0 +1,576 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PostgreSQL-backed auth store for production deployments
|
|
3
|
+
* Uses SHA-256 hashing for API keys and tracks WEEKLY usage with burst limits
|
|
4
|
+
*/
|
|
5
|
+
import pg from 'pg';
|
|
6
|
+
import crypto from 'crypto';
|
|
7
|
+
const { Pool } = pg;
|
|
8
|
+
// Extra usage cost constants
|
|
9
|
+
const EXTRA_USAGE_RATES = {
|
|
10
|
+
basic: 0.002, // $0.002 per basic fetch
|
|
11
|
+
stealth: 0.01, // $0.01 per stealth fetch
|
|
12
|
+
captcha: 0.02, // $0.02 per CAPTCHA solve
|
|
13
|
+
search: 0.001, // $0.001 per search
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* PostgreSQL auth store for production
|
|
17
|
+
*/
|
|
18
|
+
export class PostgresAuthStore {
|
|
19
|
+
pool;
|
|
20
|
+
constructor(connectionString) {
|
|
21
|
+
const dbUrl = connectionString || process.env.DATABASE_URL;
|
|
22
|
+
if (!dbUrl) {
|
|
23
|
+
throw new Error('DATABASE_URL environment variable is required for PostgresAuthStore');
|
|
24
|
+
}
|
|
25
|
+
this.pool = new Pool({
|
|
26
|
+
connectionString: dbUrl,
|
|
27
|
+
// TLS: enabled when DATABASE_URL contains sslmode=require.
|
|
28
|
+
// Secure by default (rejectUnauthorized: true); set PG_REJECT_UNAUTHORIZED=false
|
|
29
|
+
// only for managed DBs (Render/Neon/Supabase) that use self-signed certs.
|
|
30
|
+
ssl: process.env.DATABASE_URL?.includes('sslmode=require')
|
|
31
|
+
? { rejectUnauthorized: process.env.PG_REJECT_UNAUTHORIZED !== 'false' }
|
|
32
|
+
: undefined,
|
|
33
|
+
max: 20,
|
|
34
|
+
idleTimeoutMillis: 30000,
|
|
35
|
+
connectionTimeoutMillis: 10000,
|
|
36
|
+
});
|
|
37
|
+
// Run idempotent schema migrations on startup
|
|
38
|
+
this.ensureSchema().catch(err => console.error('[pg-auth-store] Schema migration failed:', err));
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Run idempotent schema migrations.
|
|
42
|
+
* Safe to call on every startup — all statements use IF NOT EXISTS / IF EXISTS.
|
|
43
|
+
*/
|
|
44
|
+
async ensureSchema() {
|
|
45
|
+
// ── Auto-fix schema mismatches on startup ──────────────────────────────
|
|
46
|
+
// These run as idempotent ALTER/CREATE IF NOT EXISTS so they're safe to
|
|
47
|
+
// run every boot. Prevents "column X does not exist" crashes after migrations.
|
|
48
|
+
const fixes = [
|
|
49
|
+
// api_keys
|
|
50
|
+
`ALTER TABLE api_keys ADD COLUMN IF NOT EXISTS scope VARCHAR(20) NOT NULL DEFAULT 'full'`,
|
|
51
|
+
// weekly_usage
|
|
52
|
+
`ALTER TABLE weekly_usage ADD COLUMN IF NOT EXISTS rollover_credits INTEGER DEFAULT 0`,
|
|
53
|
+
// burst_usage
|
|
54
|
+
`DO $$ BEGIN IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='burst_usage' AND column_name='hour') THEN ALTER TABLE burst_usage RENAME COLUMN hour TO hour_bucket; END IF; END $$`,
|
|
55
|
+
`ALTER TABLE burst_usage ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT now()`,
|
|
56
|
+
// usage_logs
|
|
57
|
+
`ALTER TABLE usage_logs ADD COLUMN IF NOT EXISTS processing_time_ms INTEGER`,
|
|
58
|
+
// oauth_accounts — ensure provider_id exists (not provider_account_id)
|
|
59
|
+
`DO $$ BEGIN IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='oauth_accounts' AND column_name='provider_account_id') THEN ALTER TABLE oauth_accounts RENAME COLUMN provider_account_id TO provider_id; END IF; END $$`,
|
|
60
|
+
// refresh_tokens — token_hash should be nullable (code uses id as JWT identifier)
|
|
61
|
+
`ALTER TABLE refresh_tokens ALTER COLUMN token_hash DROP NOT NULL`,
|
|
62
|
+
// users — ensure name/avatar columns exist
|
|
63
|
+
`ALTER TABLE users ADD COLUMN IF NOT EXISTS name TEXT`,
|
|
64
|
+
`ALTER TABLE users ADD COLUMN IF NOT EXISTS avatar_url TEXT`,
|
|
65
|
+
// shared_reads
|
|
66
|
+
`CREATE TABLE IF NOT EXISTS shared_reads (
|
|
67
|
+
id VARCHAR(12) PRIMARY KEY, url TEXT NOT NULL, title TEXT, content TEXT NOT NULL,
|
|
68
|
+
tokens INTEGER, created_by TEXT, created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
69
|
+
expires_at TIMESTAMPTZ DEFAULT NOW() + INTERVAL '30 days', view_count INTEGER DEFAULT 0
|
|
70
|
+
)`,
|
|
71
|
+
`CREATE INDEX IF NOT EXISTS idx_shared_reads_url ON shared_reads(url)`,
|
|
72
|
+
`CREATE INDEX IF NOT EXISTS idx_shared_reads_created_by ON shared_reads(created_by)`,
|
|
73
|
+
// password_reset_tokens
|
|
74
|
+
`CREATE TABLE IF NOT EXISTS password_reset_tokens (
|
|
75
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
76
|
+
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
|
77
|
+
token_hash TEXT NOT NULL,
|
|
78
|
+
expires_at TIMESTAMPTZ NOT NULL,
|
|
79
|
+
used BOOLEAN DEFAULT false,
|
|
80
|
+
created_at TIMESTAMPTZ DEFAULT now()
|
|
81
|
+
)`,
|
|
82
|
+
`CREATE INDEX IF NOT EXISTS idx_prt_token_hash ON password_reset_tokens(token_hash)`,
|
|
83
|
+
`CREATE INDEX IF NOT EXISTS idx_prt_user_id ON password_reset_tokens(user_id)`,
|
|
84
|
+
// usage_logs + api_keys + users indexes for admin analytics queries
|
|
85
|
+
`CREATE INDEX IF NOT EXISTS idx_usage_logs_user_id ON usage_logs(user_id)`,
|
|
86
|
+
`CREATE INDEX IF NOT EXISTS idx_usage_logs_created_at ON usage_logs(created_at)`,
|
|
87
|
+
`CREATE INDEX IF NOT EXISTS idx_usage_logs_endpoint ON usage_logs(endpoint)`,
|
|
88
|
+
`CREATE INDEX IF NOT EXISTS idx_api_keys_last_used ON api_keys(last_used_at)`,
|
|
89
|
+
`CREATE INDEX IF NOT EXISTS idx_users_created_at ON users(created_at)`,
|
|
90
|
+
];
|
|
91
|
+
for (const sql of fixes) {
|
|
92
|
+
try {
|
|
93
|
+
await this.pool.query(sql);
|
|
94
|
+
}
|
|
95
|
+
catch (err) {
|
|
96
|
+
// Log but don't crash — individual fixes may fail on older schemas
|
|
97
|
+
console.warn('[webpeel] Schema fix warning:', err.message?.slice(0, 100));
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Hash API key with SHA-256
|
|
103
|
+
* SECURITY: Never store raw API keys
|
|
104
|
+
*/
|
|
105
|
+
hashKey(key) {
|
|
106
|
+
return crypto.createHash('sha256').update(key).digest('hex');
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Get current ISO week in YYYY-WXX format (e.g., "2026-W07")
|
|
110
|
+
*/
|
|
111
|
+
getCurrentWeek() {
|
|
112
|
+
const now = new Date();
|
|
113
|
+
const year = now.getUTCFullYear();
|
|
114
|
+
const jan4 = new Date(Date.UTC(year, 0, 4));
|
|
115
|
+
const weekNum = Math.ceil(((now.getTime() - jan4.getTime()) / 86400000 + jan4.getUTCDay() + 1) / 7);
|
|
116
|
+
return `${year}-W${String(weekNum).padStart(2, '0')}`;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Get previous ISO week in YYYY-WXX format
|
|
120
|
+
*/
|
|
121
|
+
getPreviousWeek() {
|
|
122
|
+
const now = new Date();
|
|
123
|
+
const lastWeek = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000);
|
|
124
|
+
const year = lastWeek.getUTCFullYear();
|
|
125
|
+
const jan4 = new Date(Date.UTC(year, 0, 4));
|
|
126
|
+
const weekNum = Math.ceil(((lastWeek.getTime() - jan4.getTime()) / 86400000 + jan4.getUTCDay() + 1) / 7);
|
|
127
|
+
return `${year}-W${String(weekNum).padStart(2, '0')}`;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Get next Monday 00:00 UTC (week reset time)
|
|
131
|
+
*/
|
|
132
|
+
getWeekResetTime() {
|
|
133
|
+
const now = new Date();
|
|
134
|
+
const dayOfWeek = now.getUTCDay();
|
|
135
|
+
const daysUntilMonday = dayOfWeek === 0 ? 1 : 8 - dayOfWeek;
|
|
136
|
+
const nextMonday = new Date(now);
|
|
137
|
+
nextMonday.setUTCDate(now.getUTCDate() + daysUntilMonday);
|
|
138
|
+
nextMonday.setUTCHours(0, 0, 0, 0);
|
|
139
|
+
return nextMonday;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Get current hour bucket in YYYY-MM-DDTHH format (UTC)
|
|
143
|
+
*/
|
|
144
|
+
getCurrentHour() {
|
|
145
|
+
const now = new Date();
|
|
146
|
+
return now.toISOString().substring(0, 13); // "2026-02-12T20"
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Get human-readable time until next hour
|
|
150
|
+
*/
|
|
151
|
+
getTimeUntilNextHour() {
|
|
152
|
+
const now = new Date();
|
|
153
|
+
const minutesRemaining = 59 - now.getUTCMinutes();
|
|
154
|
+
if (minutesRemaining === 0) {
|
|
155
|
+
return '< 1 min';
|
|
156
|
+
}
|
|
157
|
+
return `${minutesRemaining} min`;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Validate API key and return user info
|
|
161
|
+
* SECURITY: Uses SHA-256 hash comparison, updates last_used_at
|
|
162
|
+
*/
|
|
163
|
+
async validateKey(key) {
|
|
164
|
+
if (!key || typeof key !== 'string') {
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
const keyHash = this.hashKey(key);
|
|
168
|
+
try {
|
|
169
|
+
const result = await this.pool.query(`SELECT
|
|
170
|
+
ak.id,
|
|
171
|
+
ak.user_id,
|
|
172
|
+
ak.key_prefix,
|
|
173
|
+
ak.name,
|
|
174
|
+
ak.scope,
|
|
175
|
+
u.tier,
|
|
176
|
+
u.rate_limit,
|
|
177
|
+
u.weekly_limit,
|
|
178
|
+
u.burst_limit,
|
|
179
|
+
u.email
|
|
180
|
+
FROM api_keys ak
|
|
181
|
+
JOIN users u ON ak.user_id = u.id
|
|
182
|
+
WHERE ak.key_hash = $1 AND ak.is_active = true
|
|
183
|
+
AND (ak.expires_at IS NULL OR ak.expires_at > NOW())`, [keyHash]);
|
|
184
|
+
if (result.rows.length === 0) {
|
|
185
|
+
return null;
|
|
186
|
+
}
|
|
187
|
+
const row = result.rows[0];
|
|
188
|
+
// Update last_used_at (fire and forget, don't wait)
|
|
189
|
+
this.pool.query('UPDATE api_keys SET last_used_at = now() WHERE id = $1', [row.id]).catch(err => console.error('Failed to update last_used_at:', err));
|
|
190
|
+
return {
|
|
191
|
+
key,
|
|
192
|
+
tier: row.tier,
|
|
193
|
+
rateLimit: row.rate_limit,
|
|
194
|
+
accountId: row.user_id,
|
|
195
|
+
createdAt: new Date(),
|
|
196
|
+
scope: row.scope || 'full',
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
catch (error) {
|
|
200
|
+
console.error('Failed to validate API key:', error);
|
|
201
|
+
return null;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Check if a key exists but is expired (used to return specific 401 error)
|
|
206
|
+
*/
|
|
207
|
+
async isKeyExpired(key) {
|
|
208
|
+
if (!key || typeof key !== 'string')
|
|
209
|
+
return false;
|
|
210
|
+
const keyHash = this.hashKey(key);
|
|
211
|
+
try {
|
|
212
|
+
const result = await this.pool.query(`SELECT 1 FROM api_keys WHERE key_hash = $1 AND is_active = true AND expires_at IS NOT NULL AND expires_at <= NOW()`, [keyHash]);
|
|
213
|
+
return result.rows.length > 0;
|
|
214
|
+
}
|
|
215
|
+
catch {
|
|
216
|
+
return false;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Track weekly usage for an API key
|
|
221
|
+
* SECURITY: Uses UPSERT to prevent race conditions
|
|
222
|
+
*/
|
|
223
|
+
async trackUsage(key, fetchType) {
|
|
224
|
+
const keyHash = this.hashKey(key);
|
|
225
|
+
const week = this.getCurrentWeek();
|
|
226
|
+
try {
|
|
227
|
+
// Get API key ID and user ID
|
|
228
|
+
const keyResult = await this.pool.query('SELECT id, user_id FROM api_keys WHERE key_hash = $1', [keyHash]);
|
|
229
|
+
if (keyResult.rows.length === 0) {
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
const { id: apiKeyId, user_id: userId } = keyResult.rows[0];
|
|
233
|
+
// Determine which counter to increment
|
|
234
|
+
const columnMap = {
|
|
235
|
+
basic: 'basic_count',
|
|
236
|
+
stealth: 'stealth_count',
|
|
237
|
+
captcha: 'captcha_count',
|
|
238
|
+
search: 'search_count',
|
|
239
|
+
};
|
|
240
|
+
const column = columnMap[fetchType];
|
|
241
|
+
// UPSERT usage record (total_count is GENERATED, don't touch it)
|
|
242
|
+
await this.pool.query(`INSERT INTO weekly_usage (user_id, api_key_id, week, ${column})
|
|
243
|
+
VALUES ($1, $2, $3, 1)
|
|
244
|
+
ON CONFLICT (api_key_id, week)
|
|
245
|
+
DO UPDATE SET
|
|
246
|
+
${column} = weekly_usage.${column} + 1,
|
|
247
|
+
updated_at = now()`, [userId, apiKeyId, week]);
|
|
248
|
+
}
|
|
249
|
+
catch (error) {
|
|
250
|
+
console.error('Failed to track usage:', error);
|
|
251
|
+
throw error;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Track burst usage (hourly limit)
|
|
256
|
+
*/
|
|
257
|
+
async trackBurstUsage(key) {
|
|
258
|
+
const keyHash = this.hashKey(key);
|
|
259
|
+
const hourBucket = this.getCurrentHour();
|
|
260
|
+
try {
|
|
261
|
+
const keyResult = await this.pool.query('SELECT id FROM api_keys WHERE key_hash = $1', [keyHash]);
|
|
262
|
+
if (keyResult.rows.length === 0) {
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
const apiKeyId = keyResult.rows[0].id;
|
|
266
|
+
// UPSERT burst usage
|
|
267
|
+
await this.pool.query(`INSERT INTO burst_usage (api_key_id, hour_bucket, count)
|
|
268
|
+
VALUES ($1, $2, 1)
|
|
269
|
+
ON CONFLICT (api_key_id, hour_bucket)
|
|
270
|
+
DO UPDATE SET
|
|
271
|
+
count = burst_usage.count + 1,
|
|
272
|
+
updated_at = now()`, [apiKeyId, hourBucket]);
|
|
273
|
+
}
|
|
274
|
+
catch (error) {
|
|
275
|
+
console.error('Failed to track burst usage:', error);
|
|
276
|
+
throw error;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* Check burst limit (hourly)
|
|
281
|
+
*/
|
|
282
|
+
async checkBurstLimit(key) {
|
|
283
|
+
const keyHash = this.hashKey(key);
|
|
284
|
+
const hourBucket = this.getCurrentHour();
|
|
285
|
+
try {
|
|
286
|
+
const result = await this.pool.query(`SELECT
|
|
287
|
+
u.burst_limit,
|
|
288
|
+
COALESCE(bu.count, 0) as count
|
|
289
|
+
FROM api_keys ak
|
|
290
|
+
JOIN users u ON ak.user_id = u.id
|
|
291
|
+
LEFT JOIN burst_usage bu ON bu.api_key_id = ak.id AND bu.hour_bucket = $2
|
|
292
|
+
WHERE ak.key_hash = $1`, [keyHash, hourBucket]);
|
|
293
|
+
if (result.rows.length === 0) {
|
|
294
|
+
return {
|
|
295
|
+
allowed: false,
|
|
296
|
+
burst: {
|
|
297
|
+
hourBucket,
|
|
298
|
+
count: 0,
|
|
299
|
+
limit: 0,
|
|
300
|
+
remaining: 0,
|
|
301
|
+
resetsIn: this.getTimeUntilNextHour(),
|
|
302
|
+
},
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
const row = result.rows[0];
|
|
306
|
+
const allowed = row.count < row.burst_limit;
|
|
307
|
+
return {
|
|
308
|
+
allowed,
|
|
309
|
+
burst: {
|
|
310
|
+
hourBucket,
|
|
311
|
+
count: row.count,
|
|
312
|
+
limit: row.burst_limit,
|
|
313
|
+
remaining: Math.max(0, row.burst_limit - row.count),
|
|
314
|
+
resetsIn: this.getTimeUntilNextHour(),
|
|
315
|
+
},
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
catch (error) {
|
|
319
|
+
console.error('Failed to check burst limit:', error);
|
|
320
|
+
return {
|
|
321
|
+
allowed: false,
|
|
322
|
+
burst: {
|
|
323
|
+
hourBucket,
|
|
324
|
+
count: 0,
|
|
325
|
+
limit: 0,
|
|
326
|
+
remaining: 0,
|
|
327
|
+
resetsIn: this.getTimeUntilNextHour(),
|
|
328
|
+
},
|
|
329
|
+
};
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* Get weekly usage info for an API key with rollover calculation
|
|
334
|
+
*/
|
|
335
|
+
async getUsage(key) {
|
|
336
|
+
const keyHash = this.hashKey(key);
|
|
337
|
+
const currentWeek = this.getCurrentWeek();
|
|
338
|
+
const previousWeek = this.getPreviousWeek();
|
|
339
|
+
try {
|
|
340
|
+
const result = await this.pool.query(`SELECT
|
|
341
|
+
u.weekly_limit,
|
|
342
|
+
COALESCE(curr.basic_count, 0) as basic_count,
|
|
343
|
+
COALESCE(curr.stealth_count, 0) as stealth_count,
|
|
344
|
+
COALESCE(curr.captcha_count, 0) as captcha_count,
|
|
345
|
+
COALESCE(curr.search_count, 0) as search_count,
|
|
346
|
+
COALESCE(curr.total_count, 0) as current_used,
|
|
347
|
+
COALESCE(prev.total_count, 0) as prev_used,
|
|
348
|
+
COALESCE(curr.rollover_credits, 0) as rollover_credits
|
|
349
|
+
FROM api_keys ak
|
|
350
|
+
JOIN users u ON ak.user_id = u.id
|
|
351
|
+
LEFT JOIN weekly_usage curr ON curr.api_key_id = ak.id AND curr.week = $2
|
|
352
|
+
LEFT JOIN weekly_usage prev ON prev.api_key_id = ak.id AND prev.week = $3
|
|
353
|
+
WHERE ak.key_hash = $1`, [keyHash, currentWeek, previousWeek]);
|
|
354
|
+
if (result.rows.length === 0) {
|
|
355
|
+
return null;
|
|
356
|
+
}
|
|
357
|
+
const row = result.rows[0];
|
|
358
|
+
const weeklyLimit = row.weekly_limit;
|
|
359
|
+
const currentUsed = row.current_used;
|
|
360
|
+
const prevUsed = row.prev_used;
|
|
361
|
+
const rolloverCredits = row.rollover_credits;
|
|
362
|
+
// Calculate rollover: MIN(unused_last_week, weekly_limit)
|
|
363
|
+
const prevUnused = Math.max(0, weeklyLimit - prevUsed);
|
|
364
|
+
const calculatedRollover = Math.min(prevUnused, weeklyLimit);
|
|
365
|
+
// Update rollover if it's the first access this week
|
|
366
|
+
if (rolloverCredits === 0 && calculatedRollover > 0) {
|
|
367
|
+
await this.pool.query(`INSERT INTO weekly_usage (user_id, api_key_id, week, rollover_credits, updated_at)
|
|
368
|
+
SELECT user_id, id, $2, $3, now()
|
|
369
|
+
FROM api_keys WHERE key_hash = $1
|
|
370
|
+
ON CONFLICT (api_key_id, week)
|
|
371
|
+
DO UPDATE SET rollover_credits = $3`, [keyHash, currentWeek, calculatedRollover]);
|
|
372
|
+
}
|
|
373
|
+
const effectiveRollover = rolloverCredits > 0 ? rolloverCredits : calculatedRollover;
|
|
374
|
+
const totalAvailable = weeklyLimit + effectiveRollover;
|
|
375
|
+
const remaining = Math.max(0, totalAvailable - currentUsed);
|
|
376
|
+
const percentUsed = totalAvailable > 0 ? Math.round((currentUsed / totalAvailable) * 100) : 0;
|
|
377
|
+
return {
|
|
378
|
+
week: currentWeek,
|
|
379
|
+
basicCount: row.basic_count,
|
|
380
|
+
stealthCount: row.stealth_count,
|
|
381
|
+
captchaCount: row.captcha_count,
|
|
382
|
+
searchCount: row.search_count,
|
|
383
|
+
totalUsed: currentUsed,
|
|
384
|
+
weeklyLimit,
|
|
385
|
+
rolloverCredits: effectiveRollover,
|
|
386
|
+
totalAvailable,
|
|
387
|
+
remaining,
|
|
388
|
+
percentUsed,
|
|
389
|
+
resetsAt: this.getWeekResetTime().toISOString(),
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
catch (error) {
|
|
393
|
+
console.error('Failed to get usage:', error);
|
|
394
|
+
return null;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
/**
|
|
398
|
+
* Check if API key has exceeded weekly limit
|
|
399
|
+
*/
|
|
400
|
+
async checkLimit(key) {
|
|
401
|
+
const usage = await this.getUsage(key);
|
|
402
|
+
if (!usage) {
|
|
403
|
+
return { allowed: false };
|
|
404
|
+
}
|
|
405
|
+
const allowed = usage.remaining > 0;
|
|
406
|
+
return { allowed, usage };
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* Get extra usage info for a user
|
|
410
|
+
*/
|
|
411
|
+
async getExtraUsageInfo(key) {
|
|
412
|
+
const keyHash = this.hashKey(key);
|
|
413
|
+
try {
|
|
414
|
+
const result = await this.pool.query(`SELECT
|
|
415
|
+
u.extra_usage_enabled,
|
|
416
|
+
u.extra_usage_balance,
|
|
417
|
+
u.extra_usage_spent,
|
|
418
|
+
u.extra_usage_spending_limit,
|
|
419
|
+
u.auto_reload_enabled,
|
|
420
|
+
u.extra_usage_period_start
|
|
421
|
+
FROM api_keys ak
|
|
422
|
+
JOIN users u ON ak.user_id = u.id
|
|
423
|
+
WHERE ak.key_hash = $1`, [keyHash]);
|
|
424
|
+
if (result.rows.length === 0) {
|
|
425
|
+
return null;
|
|
426
|
+
}
|
|
427
|
+
const row = result.rows[0];
|
|
428
|
+
// Calculate next month reset (1st of next month, 00:00 UTC)
|
|
429
|
+
const now = new Date();
|
|
430
|
+
const nextMonth = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1, 0, 0, 0, 0));
|
|
431
|
+
const percentUsed = row.extra_usage_spending_limit > 0
|
|
432
|
+
? Math.round((parseFloat(row.extra_usage_spent) / parseFloat(row.extra_usage_spending_limit)) * 100)
|
|
433
|
+
: 0;
|
|
434
|
+
return {
|
|
435
|
+
enabled: row.extra_usage_enabled,
|
|
436
|
+
balance: parseFloat(row.extra_usage_balance),
|
|
437
|
+
spent: parseFloat(row.extra_usage_spent),
|
|
438
|
+
spendingLimit: parseFloat(row.extra_usage_spending_limit),
|
|
439
|
+
autoReload: row.auto_reload_enabled,
|
|
440
|
+
percentUsed,
|
|
441
|
+
resetsAt: nextMonth.toISOString(),
|
|
442
|
+
};
|
|
443
|
+
}
|
|
444
|
+
catch (error) {
|
|
445
|
+
console.error('Failed to get extra usage info:', error);
|
|
446
|
+
return null;
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
/**
|
|
450
|
+
* Check if extra usage can be used
|
|
451
|
+
*/
|
|
452
|
+
async canUseExtraUsage(key) {
|
|
453
|
+
const info = await this.getExtraUsageInfo(key);
|
|
454
|
+
if (!info || !info.enabled) {
|
|
455
|
+
return false;
|
|
456
|
+
}
|
|
457
|
+
// Check if under spending limit and has balance
|
|
458
|
+
return info.balance > 0 && info.spent < info.spendingLimit;
|
|
459
|
+
}
|
|
460
|
+
/**
|
|
461
|
+
* Track extra usage and deduct from balance
|
|
462
|
+
*/
|
|
463
|
+
async trackExtraUsage(key, fetchType, url, processingTimeMs, statusCode) {
|
|
464
|
+
const keyHash = this.hashKey(key);
|
|
465
|
+
const cost = EXTRA_USAGE_RATES[fetchType];
|
|
466
|
+
try {
|
|
467
|
+
// Get API key and user info
|
|
468
|
+
const keyResult = await this.pool.query(`SELECT
|
|
469
|
+
ak.id as api_key_id,
|
|
470
|
+
ak.user_id,
|
|
471
|
+
u.extra_usage_balance,
|
|
472
|
+
u.extra_usage_spent
|
|
473
|
+
FROM api_keys ak
|
|
474
|
+
JOIN users u ON ak.user_id = u.id
|
|
475
|
+
WHERE ak.key_hash = $1`, [keyHash]);
|
|
476
|
+
if (keyResult.rows.length === 0) {
|
|
477
|
+
return { success: false, cost: 0, newBalance: 0 };
|
|
478
|
+
}
|
|
479
|
+
const { api_key_id, user_id, extra_usage_balance } = keyResult.rows[0];
|
|
480
|
+
const currentBalance = parseFloat(extra_usage_balance);
|
|
481
|
+
if (currentBalance < cost) {
|
|
482
|
+
return { success: false, cost, newBalance: currentBalance };
|
|
483
|
+
}
|
|
484
|
+
// Start transaction
|
|
485
|
+
const client = await this.pool.connect();
|
|
486
|
+
try {
|
|
487
|
+
await client.query('BEGIN');
|
|
488
|
+
// Deduct from balance and add to spent
|
|
489
|
+
const updateResult = await client.query(`UPDATE users
|
|
490
|
+
SET
|
|
491
|
+
extra_usage_balance = extra_usage_balance - $1,
|
|
492
|
+
extra_usage_spent = extra_usage_spent + $1,
|
|
493
|
+
updated_at = now()
|
|
494
|
+
WHERE id = $2
|
|
495
|
+
RETURNING extra_usage_balance`, [cost, user_id]);
|
|
496
|
+
const newBalance = parseFloat(updateResult.rows[0].extra_usage_balance);
|
|
497
|
+
// Log to extra_usage_logs
|
|
498
|
+
await client.query(`INSERT INTO extra_usage_logs
|
|
499
|
+
(user_id, api_key_id, fetch_type, url, cost, processing_time_ms, status_code)
|
|
500
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7)`, [user_id, api_key_id, fetchType, url, cost, processingTimeMs, statusCode]);
|
|
501
|
+
await client.query('COMMIT');
|
|
502
|
+
return { success: true, cost, newBalance };
|
|
503
|
+
}
|
|
504
|
+
catch (error) {
|
|
505
|
+
await client.query('ROLLBACK');
|
|
506
|
+
throw error;
|
|
507
|
+
}
|
|
508
|
+
finally {
|
|
509
|
+
client.release();
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
catch (error) {
|
|
513
|
+
console.error('Failed to track extra usage:', error);
|
|
514
|
+
return { success: false, cost, newBalance: 0 };
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
/**
|
|
518
|
+
* Generate a cryptographically secure API key
|
|
519
|
+
* Format: wp_live_ + 32 random hex chars (total 40 chars)
|
|
520
|
+
*/
|
|
521
|
+
static generateApiKey() {
|
|
522
|
+
const randomBytes = crypto.randomBytes(16).toString('hex');
|
|
523
|
+
return `wp_live_${randomBytes}`;
|
|
524
|
+
}
|
|
525
|
+
/**
|
|
526
|
+
* Get key prefix (first 12 characters for display)
|
|
527
|
+
*/
|
|
528
|
+
static getKeyPrefix(key) {
|
|
529
|
+
return key.substring(0, 12);
|
|
530
|
+
}
|
|
531
|
+
/**
|
|
532
|
+
* GDPR: Delete all data associated with a user account (atomic transaction).
|
|
533
|
+
* Removes: api_keys, usage, weekly_usage, burst_usage, extra_usage_logs,
|
|
534
|
+
* usage_logs, watches, oauth_accounts, refresh_tokens, shared_reads,
|
|
535
|
+
* and finally the user row itself.
|
|
536
|
+
*/
|
|
537
|
+
async deleteAccount(userId) {
|
|
538
|
+
const client = await this.pool.connect();
|
|
539
|
+
try {
|
|
540
|
+
await client.query('BEGIN');
|
|
541
|
+
// Delete per-key usage data first (FK → api_keys)
|
|
542
|
+
await client.query(`DELETE FROM weekly_usage WHERE api_key_id IN (SELECT id FROM api_keys WHERE user_id = $1)`, [userId]);
|
|
543
|
+
await client.query(`DELETE FROM burst_usage WHERE api_key_id IN (SELECT id FROM api_keys WHERE user_id = $1)`, [userId]);
|
|
544
|
+
await client.query(`DELETE FROM extra_usage_logs WHERE user_id = $1`, [userId]);
|
|
545
|
+
await client.query(`DELETE FROM usage_logs WHERE user_id = $1`, [userId]);
|
|
546
|
+
// usage table (legacy monthly usage)
|
|
547
|
+
await client.query(`DELETE FROM usage WHERE user_id = $1`, [userId]);
|
|
548
|
+
// watches (account_id column, no FK constraint to users in migration 007)
|
|
549
|
+
await client.query(`DELETE FROM watches WHERE account_id = $1`, [userId]);
|
|
550
|
+
// shared_reads created by this user (created_by is TEXT, not FK)
|
|
551
|
+
await client.query(`DELETE FROM shared_reads WHERE created_by = $1`, [userId]);
|
|
552
|
+
// oauth_accounts and refresh_tokens have ON DELETE CASCADE via users FK
|
|
553
|
+
// but we delete them explicitly to be safe and for auditability
|
|
554
|
+
await client.query(`DELETE FROM oauth_accounts WHERE user_id = $1`, [userId]);
|
|
555
|
+
await client.query(`DELETE FROM refresh_tokens WHERE user_id = $1`, [userId]);
|
|
556
|
+
// api_keys — has ON DELETE CASCADE but deleted explicitly here
|
|
557
|
+
await client.query(`DELETE FROM api_keys WHERE user_id = $1`, [userId]);
|
|
558
|
+
// Finally delete the user row
|
|
559
|
+
await client.query(`DELETE FROM users WHERE id = $1`, [userId]);
|
|
560
|
+
await client.query('COMMIT');
|
|
561
|
+
}
|
|
562
|
+
catch (error) {
|
|
563
|
+
await client.query('ROLLBACK');
|
|
564
|
+
throw error;
|
|
565
|
+
}
|
|
566
|
+
finally {
|
|
567
|
+
client.release();
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
/**
|
|
571
|
+
* Close the database pool
|
|
572
|
+
*/
|
|
573
|
+
async close() {
|
|
574
|
+
await this.pool.end();
|
|
575
|
+
}
|
|
576
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PostgreSQL-backed job queue for production deployments
|
|
3
|
+
* Uses same Pool pattern as pg-auth-store.ts
|
|
4
|
+
*/
|
|
5
|
+
import type { Job, WebhookConfig } from './job-queue.js';
|
|
6
|
+
export declare class PostgresJobQueue {
|
|
7
|
+
private pool;
|
|
8
|
+
private cleanupInterval;
|
|
9
|
+
constructor(connectionString?: string);
|
|
10
|
+
/**
|
|
11
|
+
* Create jobs table if it doesn't exist
|
|
12
|
+
*/
|
|
13
|
+
private initTable;
|
|
14
|
+
/**
|
|
15
|
+
* Create a new job
|
|
16
|
+
*/
|
|
17
|
+
createJob(type: Job['type'], webhook?: WebhookConfig, ownerId?: string): Promise<Job>;
|
|
18
|
+
/**
|
|
19
|
+
* Get a job by ID
|
|
20
|
+
*/
|
|
21
|
+
getJob(id: string): Promise<Job | null>;
|
|
22
|
+
/**
|
|
23
|
+
* Update a job
|
|
24
|
+
*/
|
|
25
|
+
updateJob(id: string, update: Partial<Job>): Promise<void>;
|
|
26
|
+
/**
|
|
27
|
+
* Cancel a job
|
|
28
|
+
*/
|
|
29
|
+
cancelJob(id: string): Promise<boolean>;
|
|
30
|
+
/**
|
|
31
|
+
* List jobs with optional filters
|
|
32
|
+
*/
|
|
33
|
+
listJobs(options?: {
|
|
34
|
+
type?: string;
|
|
35
|
+
status?: string;
|
|
36
|
+
limit?: number;
|
|
37
|
+
ownerId?: string;
|
|
38
|
+
}): Promise<Job[]>;
|
|
39
|
+
/**
|
|
40
|
+
* Remove expired jobs (called periodically)
|
|
41
|
+
*/
|
|
42
|
+
private cleanExpired;
|
|
43
|
+
/**
|
|
44
|
+
* Remove old completed/failed jobs (>7 days)
|
|
45
|
+
*/
|
|
46
|
+
private cleanupOldJobs;
|
|
47
|
+
/**
|
|
48
|
+
* Map database row to Job object
|
|
49
|
+
*/
|
|
50
|
+
private mapRowToJob;
|
|
51
|
+
/**
|
|
52
|
+
* Clean up interval on shutdown
|
|
53
|
+
*/
|
|
54
|
+
destroy(): void;
|
|
55
|
+
/**
|
|
56
|
+
* Close the database pool
|
|
57
|
+
*/
|
|
58
|
+
close(): Promise<void>;
|
|
59
|
+
}
|