@iflow-mcp/jakeliume-webpeel 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +313 -0
- package/dist/cache.d.ts +30 -0
- package/dist/cache.js +139 -0
- package/dist/cli/commands/auth.d.ts +5 -0
- package/dist/cli/commands/auth.js +411 -0
- package/dist/cli/commands/doctor.d.ts +37 -0
- package/dist/cli/commands/doctor.js +371 -0
- package/dist/cli/commands/fetch.d.ts +6 -0
- package/dist/cli/commands/fetch.js +1345 -0
- package/dist/cli/commands/guide.d.ts +2 -0
- package/dist/cli/commands/guide.js +183 -0
- package/dist/cli/commands/interact.d.ts +5 -0
- package/dist/cli/commands/interact.js +840 -0
- package/dist/cli/commands/jobs.d.ts +5 -0
- package/dist/cli/commands/jobs.js +997 -0
- package/dist/cli/commands/monitor.d.ts +12 -0
- package/dist/cli/commands/monitor.js +197 -0
- package/dist/cli/commands/observe.d.ts +12 -0
- package/dist/cli/commands/observe.js +158 -0
- package/dist/cli/commands/screenshot.d.ts +5 -0
- package/dist/cli/commands/screenshot.js +282 -0
- package/dist/cli/commands/search.d.ts +5 -0
- package/dist/cli/commands/search.js +1021 -0
- package/dist/cli/commands/setup.d.ts +13 -0
- package/dist/cli/commands/setup.js +244 -0
- package/dist/cli/commands/skill.d.ts +15 -0
- package/dist/cli/commands/skill.js +195 -0
- package/dist/cli/utils.d.ts +84 -0
- package/dist/cli/utils.js +806 -0
- package/dist/cli-auth.d.ts +75 -0
- package/dist/cli-auth.js +369 -0
- package/dist/cli.d.ts +17 -0
- package/dist/cli.js +99 -0
- package/dist/core/actions.d.ts +69 -0
- package/dist/core/actions.js +495 -0
- package/dist/core/agent.d.ts +98 -0
- package/dist/core/agent.js +558 -0
- package/dist/core/answer.d.ts +42 -0
- package/dist/core/answer.js +395 -0
- package/dist/core/application-tracker.d.ts +84 -0
- package/dist/core/application-tracker.js +184 -0
- package/dist/core/apply.d.ts +162 -0
- package/dist/core/apply.js +816 -0
- package/dist/core/auth-detection.d.ts +35 -0
- package/dist/core/auth-detection.js +358 -0
- package/dist/core/auto-extract.d.ts +82 -0
- package/dist/core/auto-extract.js +604 -0
- package/dist/core/auto-interact.d.ts +23 -0
- package/dist/core/auto-interact.js +246 -0
- package/dist/core/bm25-filter.d.ts +66 -0
- package/dist/core/bm25-filter.js +288 -0
- package/dist/core/branding.d.ts +54 -0
- package/dist/core/branding.js +234 -0
- package/dist/core/browser-fetch.d.ts +323 -0
- package/dist/core/browser-fetch.js +1600 -0
- package/dist/core/browser-pool.d.ts +91 -0
- package/dist/core/browser-pool.js +550 -0
- package/dist/core/budget.d.ts +42 -0
- package/dist/core/budget.js +324 -0
- package/dist/core/business-intel.d.ts +47 -0
- package/dist/core/business-intel.js +279 -0
- package/dist/core/cache.d.ts +13 -0
- package/dist/core/cache.js +121 -0
- package/dist/core/cf-worker-proxy.d.ts +32 -0
- package/dist/core/cf-worker-proxy.js +87 -0
- package/dist/core/challenge-detection.d.ts +26 -0
- package/dist/core/challenge-detection.js +468 -0
- package/dist/core/change-tracking.d.ts +75 -0
- package/dist/core/change-tracking.js +276 -0
- package/dist/core/chunker.d.ts +46 -0
- package/dist/core/chunker.js +249 -0
- package/dist/core/chunking.d.ts +42 -0
- package/dist/core/chunking.js +181 -0
- package/dist/core/circuit-breaker.d.ts +44 -0
- package/dist/core/circuit-breaker.js +85 -0
- package/dist/core/content-pruner.d.ts +47 -0
- package/dist/core/content-pruner.js +425 -0
- package/dist/core/cookie-cache.d.ts +60 -0
- package/dist/core/cookie-cache.js +163 -0
- package/dist/core/crawl-checkpoint.d.ts +54 -0
- package/dist/core/crawl-checkpoint.js +104 -0
- package/dist/core/crawler.d.ts +84 -0
- package/dist/core/crawler.js +349 -0
- package/dist/core/cross-verify.d.ts +27 -0
- package/dist/core/cross-verify.js +93 -0
- package/dist/core/deep-fetch.d.ts +74 -0
- package/dist/core/deep-fetch.js +405 -0
- package/dist/core/deep-research.d.ts +141 -0
- package/dist/core/deep-research.js +972 -0
- package/dist/core/design-analysis.d.ts +70 -0
- package/dist/core/design-analysis.js +490 -0
- package/dist/core/design-compare.d.ts +38 -0
- package/dist/core/design-compare.js +264 -0
- package/dist/core/diff.d.ts +61 -0
- package/dist/core/diff.js +289 -0
- package/dist/core/dns-cache.d.ts +20 -0
- package/dist/core/dns-cache.js +198 -0
- package/dist/core/documents.d.ts +23 -0
- package/dist/core/documents.js +123 -0
- package/dist/core/domain-memory.d.ts +66 -0
- package/dist/core/domain-memory.js +163 -0
- package/dist/core/domain-verify.d.ts +40 -0
- package/dist/core/domain-verify.js +379 -0
- package/dist/core/engine-ranker.d.ts +112 -0
- package/dist/core/engine-ranker.js +395 -0
- package/dist/core/extract-inline.d.ts +38 -0
- package/dist/core/extract-inline.js +215 -0
- package/dist/core/extract-listings.d.ts +38 -0
- package/dist/core/extract-listings.js +461 -0
- package/dist/core/extract.d.ts +9 -0
- package/dist/core/extract.js +139 -0
- package/dist/core/fetch-cache.d.ts +57 -0
- package/dist/core/fetch-cache.js +95 -0
- package/dist/core/fetcher.d.ts +13 -0
- package/dist/core/fetcher.js +12 -0
- package/dist/core/google-cache.d.ts +29 -0
- package/dist/core/google-cache.js +180 -0
- package/dist/core/google-serp-parser.d.ts +82 -0
- package/dist/core/google-serp-parser.js +287 -0
- package/dist/core/hotel-search.d.ts +122 -0
- package/dist/core/hotel-search.js +382 -0
- package/dist/core/http-fetch.d.ts +72 -0
- package/dist/core/http-fetch.js +820 -0
- package/dist/core/human.d.ts +175 -0
- package/dist/core/human.js +680 -0
- package/dist/core/image-caption.d.ts +44 -0
- package/dist/core/image-caption.js +271 -0
- package/dist/core/jobs.d.ts +75 -0
- package/dist/core/jobs.js +634 -0
- package/dist/core/json-ld.d.ts +15 -0
- package/dist/core/json-ld.js +617 -0
- package/dist/core/language-detect.d.ts +18 -0
- package/dist/core/language-detect.js +135 -0
- package/dist/core/links.d.ts +10 -0
- package/dist/core/links.js +44 -0
- package/dist/core/llm-extract.d.ts +71 -0
- package/dist/core/llm-extract.js +507 -0
- package/dist/core/llm-provider.d.ts +100 -0
- package/dist/core/llm-provider.js +702 -0
- package/dist/core/local-search.d.ts +60 -0
- package/dist/core/local-search.js +308 -0
- package/dist/core/logger.d.ts +28 -0
- package/dist/core/logger.js +104 -0
- package/dist/core/map.d.ts +33 -0
- package/dist/core/map.js +127 -0
- package/dist/core/markdown.d.ts +92 -0
- package/dist/core/markdown.js +809 -0
- package/dist/core/metadata.d.ts +34 -0
- package/dist/core/metadata.js +422 -0
- package/dist/core/observe.d.ts +113 -0
- package/dist/core/observe.js +395 -0
- package/dist/core/ocr.d.ts +12 -0
- package/dist/core/ocr.js +33 -0
- package/dist/core/paginate.d.ts +31 -0
- package/dist/core/paginate.js +106 -0
- package/dist/core/pdf.d.ts +8 -0
- package/dist/core/pdf.js +25 -0
- package/dist/core/peel-tls.d.ts +25 -0
- package/dist/core/peel-tls.js +220 -0
- package/dist/core/pipeline.d.ts +132 -0
- package/dist/core/pipeline.js +1666 -0
- package/dist/core/profiles.d.ts +61 -0
- package/dist/core/profiles.js +350 -0
- package/dist/core/prompt-guard.d.ts +30 -0
- package/dist/core/prompt-guard.js +119 -0
- package/dist/core/proxy-config.d.ts +90 -0
- package/dist/core/proxy-config.js +172 -0
- package/dist/core/quick-answer.d.ts +53 -0
- package/dist/core/quick-answer.js +833 -0
- package/dist/core/rate-governor.d.ts +80 -0
- package/dist/core/rate-governor.js +238 -0
- package/dist/core/readability.d.ts +57 -0
- package/dist/core/readability.js +533 -0
- package/dist/core/research.d.ts +66 -0
- package/dist/core/research.js +270 -0
- package/dist/core/retry.d.ts +60 -0
- package/dist/core/retry.js +119 -0
- package/dist/core/safe-browsing.d.ts +30 -0
- package/dist/core/safe-browsing.js +206 -0
- package/dist/core/schema-extraction.d.ts +66 -0
- package/dist/core/schema-extraction.js +352 -0
- package/dist/core/schema-postprocess.d.ts +32 -0
- package/dist/core/schema-postprocess.js +469 -0
- package/dist/core/schema-templates.d.ts +19 -0
- package/dist/core/schema-templates.js +143 -0
- package/dist/core/screenshot.d.ts +224 -0
- package/dist/core/screenshot.js +207 -0
- package/dist/core/search-engines.d.ts +25 -0
- package/dist/core/search-engines.js +182 -0
- package/dist/core/search-provider.d.ts +243 -0
- package/dist/core/search-provider.js +1629 -0
- package/dist/core/searxng-provider.d.ts +35 -0
- package/dist/core/searxng-provider.js +105 -0
- package/dist/core/selective-evidence.d.ts +151 -0
- package/dist/core/selective-evidence.js +389 -0
- package/dist/core/site-search.d.ts +44 -0
- package/dist/core/site-search.js +252 -0
- package/dist/core/sitemap.d.ts +23 -0
- package/dist/core/sitemap.js +105 -0
- package/dist/core/source-credibility.d.ts +29 -0
- package/dist/core/source-credibility.js +584 -0
- package/dist/core/source-scoring.d.ts +166 -0
- package/dist/core/source-scoring.js +396 -0
- package/dist/core/stemmer.d.ts +38 -0
- package/dist/core/stemmer.js +509 -0
- package/dist/core/strategies.d.ts +104 -0
- package/dist/core/strategies.js +1044 -0
- package/dist/core/strategy-hooks.d.ts +145 -0
- package/dist/core/strategy-hooks.js +74 -0
- package/dist/core/structured-extract.d.ts +43 -0
- package/dist/core/structured-extract.js +550 -0
- package/dist/core/summarize.d.ts +17 -0
- package/dist/core/summarize.js +78 -0
- package/dist/core/synonyms.d.ts +42 -0
- package/dist/core/synonyms.js +184 -0
- package/dist/core/system-monitor.d.ts +61 -0
- package/dist/core/system-monitor.js +133 -0
- package/dist/core/table-format.d.ts +30 -0
- package/dist/core/table-format.js +146 -0
- package/dist/core/threat-feeds.d.ts +23 -0
- package/dist/core/threat-feeds.js +104 -0
- package/dist/core/timing.d.ts +21 -0
- package/dist/core/timing.js +33 -0
- package/dist/core/transcript-export.d.ts +47 -0
- package/dist/core/transcript-export.js +107 -0
- package/dist/core/user-agents.d.ts +82 -0
- package/dist/core/user-agents.js +239 -0
- package/dist/core/vertical-search.d.ts +54 -0
- package/dist/core/vertical-search.js +158 -0
- package/dist/core/watch-manager.d.ts +175 -0
- package/dist/core/watch-manager.js +416 -0
- package/dist/core/watch.d.ts +101 -0
- package/dist/core/watch.js +389 -0
- package/dist/core/youtube.d.ts +130 -0
- package/dist/core/youtube.js +1175 -0
- package/dist/ee/challenge-re-export.d.ts +1 -0
- package/dist/ee/challenge-re-export.js +1 -0
- package/dist/ee/challenge-solver.d.ts +72 -0
- package/dist/ee/challenge-solver.js +720 -0
- package/dist/ee/domain-extractors.d.ts +8 -0
- package/dist/ee/domain-extractors.js +8 -0
- package/dist/ee/domain-intel.d.ts +16 -0
- package/dist/ee/domain-intel.js +133 -0
- package/dist/ee/extractors/allrecipes.d.ts +2 -0
- package/dist/ee/extractors/allrecipes.js +120 -0
- package/dist/ee/extractors/amazon.d.ts +2 -0
- package/dist/ee/extractors/amazon.js +78 -0
- package/dist/ee/extractors/arxiv.d.ts +2 -0
- package/dist/ee/extractors/arxiv.js +137 -0
- package/dist/ee/extractors/bestbuy.d.ts +2 -0
- package/dist/ee/extractors/bestbuy.js +78 -0
- package/dist/ee/extractors/carscom.d.ts +2 -0
- package/dist/ee/extractors/carscom.js +121 -0
- package/dist/ee/extractors/coingecko.d.ts +2 -0
- package/dist/ee/extractors/coingecko.js +134 -0
- package/dist/ee/extractors/craigslist.d.ts +2 -0
- package/dist/ee/extractors/craigslist.js +92 -0
- package/dist/ee/extractors/devto.d.ts +2 -0
- package/dist/ee/extractors/devto.js +135 -0
- package/dist/ee/extractors/ebay.d.ts +2 -0
- package/dist/ee/extractors/ebay.js +90 -0
- package/dist/ee/extractors/espn.d.ts +2 -0
- package/dist/ee/extractors/espn.js +260 -0
- package/dist/ee/extractors/etsy.d.ts +2 -0
- package/dist/ee/extractors/etsy.js +52 -0
- package/dist/ee/extractors/facebook.d.ts +2 -0
- package/dist/ee/extractors/facebook.js +46 -0
- package/dist/ee/extractors/github.d.ts +2 -0
- package/dist/ee/extractors/github.js +196 -0
- package/dist/ee/extractors/google-flights.d.ts +2 -0
- package/dist/ee/extractors/google-flights.js +176 -0
- package/dist/ee/extractors/hackernews.d.ts +2 -0
- package/dist/ee/extractors/hackernews.js +147 -0
- package/dist/ee/extractors/imdb.d.ts +2 -0
- package/dist/ee/extractors/imdb.js +172 -0
- package/dist/ee/extractors/index.d.ts +26 -0
- package/dist/ee/extractors/index.js +247 -0
- package/dist/ee/extractors/instagram.d.ts +2 -0
- package/dist/ee/extractors/instagram.js +102 -0
- package/dist/ee/extractors/kalshi.d.ts +2 -0
- package/dist/ee/extractors/kalshi.js +121 -0
- package/dist/ee/extractors/kayak-cars.d.ts +2 -0
- package/dist/ee/extractors/kayak-cars.js +270 -0
- package/dist/ee/extractors/linkedin.d.ts +2 -0
- package/dist/ee/extractors/linkedin.js +113 -0
- package/dist/ee/extractors/medium.d.ts +2 -0
- package/dist/ee/extractors/medium.js +130 -0
- package/dist/ee/extractors/news.d.ts +4 -0
- package/dist/ee/extractors/news.js +173 -0
- package/dist/ee/extractors/npm.d.ts +2 -0
- package/dist/ee/extractors/npm.js +86 -0
- package/dist/ee/extractors/pdf.d.ts +2 -0
- package/dist/ee/extractors/pdf.js +108 -0
- package/dist/ee/extractors/pinterest.d.ts +2 -0
- package/dist/ee/extractors/pinterest.js +34 -0
- package/dist/ee/extractors/polymarket.d.ts +2 -0
- package/dist/ee/extractors/polymarket.js +358 -0
- package/dist/ee/extractors/producthunt.d.ts +2 -0
- package/dist/ee/extractors/producthunt.js +88 -0
- package/dist/ee/extractors/pubmed.d.ts +2 -0
- package/dist/ee/extractors/pubmed.js +162 -0
- package/dist/ee/extractors/pypi.d.ts +2 -0
- package/dist/ee/extractors/pypi.js +80 -0
- package/dist/ee/extractors/reddit.d.ts +2 -0
- package/dist/ee/extractors/reddit.js +438 -0
- package/dist/ee/extractors/redfin.d.ts +2 -0
- package/dist/ee/extractors/redfin.js +156 -0
- package/dist/ee/extractors/semanticscholar.d.ts +2 -0
- package/dist/ee/extractors/semanticscholar.js +131 -0
- package/dist/ee/extractors/shared.d.ts +12 -0
- package/dist/ee/extractors/shared.js +76 -0
- package/dist/ee/extractors/soundcloud.d.ts +2 -0
- package/dist/ee/extractors/soundcloud.js +34 -0
- package/dist/ee/extractors/sportsbetting.d.ts +2 -0
- package/dist/ee/extractors/sportsbetting.js +37 -0
- package/dist/ee/extractors/spotify.d.ts +2 -0
- package/dist/ee/extractors/spotify.js +34 -0
- package/dist/ee/extractors/stackoverflow.d.ts +2 -0
- package/dist/ee/extractors/stackoverflow.js +61 -0
- package/dist/ee/extractors/substack.d.ts +2 -0
- package/dist/ee/extractors/substack.js +115 -0
- package/dist/ee/extractors/substackroot.d.ts +2 -0
- package/dist/ee/extractors/substackroot.js +46 -0
- package/dist/ee/extractors/tiktok.d.ts +2 -0
- package/dist/ee/extractors/tiktok.js +29 -0
- package/dist/ee/extractors/tradingview.d.ts +2 -0
- package/dist/ee/extractors/tradingview.js +182 -0
- package/dist/ee/extractors/twitch.d.ts +2 -0
- package/dist/ee/extractors/twitch.js +36 -0
- package/dist/ee/extractors/twitter.d.ts +2 -0
- package/dist/ee/extractors/twitter.js +327 -0
- package/dist/ee/extractors/types.d.ts +14 -0
- package/dist/ee/extractors/types.js +1 -0
- package/dist/ee/extractors/walmart.d.ts +2 -0
- package/dist/ee/extractors/walmart.js +50 -0
- package/dist/ee/extractors/weather.d.ts +2 -0
- package/dist/ee/extractors/weather.js +133 -0
- package/dist/ee/extractors/wikipedia.d.ts +4 -0
- package/dist/ee/extractors/wikipedia.js +235 -0
- package/dist/ee/extractors/yelp.d.ts +2 -0
- package/dist/ee/extractors/yelp.js +216 -0
- package/dist/ee/extractors/youtube.d.ts +2 -0
- package/dist/ee/extractors/youtube.js +189 -0
- package/dist/ee/extractors/zillow.d.ts +54 -0
- package/dist/ee/extractors/zillow.js +247 -0
- package/dist/ee/extractors-re-export.d.ts +1 -0
- package/dist/ee/extractors-re-export.js +1 -0
- package/dist/ee/premium-hooks.d.ts +20 -0
- package/dist/ee/premium-hooks.js +50 -0
- package/dist/ee/spa-detection.d.ts +2 -0
- package/dist/ee/spa-detection.js +2 -0
- package/dist/ee/stability.d.ts +4 -0
- package/dist/ee/stability.js +29 -0
- package/dist/ee/swr-cache.d.ts +14 -0
- package/dist/ee/swr-cache.js +34 -0
- package/dist/index.d.ts +143 -0
- package/dist/index.js +291 -0
- package/dist/integrations/index.d.ts +2 -0
- package/dist/integrations/index.js +2 -0
- package/dist/integrations/langchain.d.ts +64 -0
- package/dist/integrations/langchain.js +115 -0
- package/dist/integrations/llamaindex.d.ts +50 -0
- package/dist/integrations/llamaindex.js +91 -0
- package/dist/mcp/handlers/act.d.ts +5 -0
- package/dist/mcp/handlers/act.js +34 -0
- package/dist/mcp/handlers/definitions.d.ts +6 -0
- package/dist/mcp/handlers/definitions.js +395 -0
- package/dist/mcp/handlers/extract.d.ts +7 -0
- package/dist/mcp/handlers/extract.js +135 -0
- package/dist/mcp/handlers/fetch.d.ts +6 -0
- package/dist/mcp/handlers/fetch.js +98 -0
- package/dist/mcp/handlers/find.d.ts +5 -0
- package/dist/mcp/handlers/find.js +137 -0
- package/dist/mcp/handlers/index.d.ts +13 -0
- package/dist/mcp/handlers/index.js +63 -0
- package/dist/mcp/handlers/legacy.d.ts +25 -0
- package/dist/mcp/handlers/legacy.js +450 -0
- package/dist/mcp/handlers/meta.d.ts +6 -0
- package/dist/mcp/handlers/meta.js +40 -0
- package/dist/mcp/handlers/monitor.d.ts +5 -0
- package/dist/mcp/handlers/monitor.js +41 -0
- package/dist/mcp/handlers/observe.d.ts +8 -0
- package/dist/mcp/handlers/observe.js +37 -0
- package/dist/mcp/handlers/read.d.ts +6 -0
- package/dist/mcp/handlers/read.js +78 -0
- package/dist/mcp/handlers/see.d.ts +5 -0
- package/dist/mcp/handlers/see.js +75 -0
- package/dist/mcp/handlers/types.d.ts +29 -0
- package/dist/mcp/handlers/types.js +28 -0
- package/dist/mcp/server.d.ts +7 -0
- package/dist/mcp/server.js +108 -0
- package/dist/mcp/smart-router.d.ts +23 -0
- package/dist/mcp/smart-router.js +178 -0
- package/dist/server/app.d.ts +14 -0
- package/dist/server/app.js +632 -0
- package/dist/server/auth-store.d.ts +28 -0
- package/dist/server/auth-store.js +88 -0
- package/dist/server/bull-queues.d.ts +60 -0
- package/dist/server/bull-queues.js +90 -0
- package/dist/server/email-service.d.ts +55 -0
- package/dist/server/email-service.js +291 -0
- package/dist/server/job-queue.d.ts +100 -0
- package/dist/server/job-queue.js +145 -0
- package/dist/server/logger.d.ts +10 -0
- package/dist/server/logger.js +37 -0
- package/dist/server/middleware/audit-log.d.ts +14 -0
- package/dist/server/middleware/audit-log.js +73 -0
- package/dist/server/middleware/auth.d.ts +35 -0
- package/dist/server/middleware/auth.js +225 -0
- package/dist/server/middleware/rate-limit.d.ts +50 -0
- package/dist/server/middleware/rate-limit.js +270 -0
- package/dist/server/middleware/scope-guard.d.ts +25 -0
- package/dist/server/middleware/scope-guard.js +45 -0
- package/dist/server/middleware/url-validator.d.ts +15 -0
- package/dist/server/middleware/url-validator.js +201 -0
- package/dist/server/openapi.yaml +6418 -0
- package/dist/server/pg-auth-store.d.ts +146 -0
- package/dist/server/pg-auth-store.js +576 -0
- package/dist/server/pg-job-queue.d.ts +59 -0
- package/dist/server/pg-job-queue.js +375 -0
- package/dist/server/routes/activity.d.ts +6 -0
- package/dist/server/routes/activity.js +79 -0
- package/dist/server/routes/admin-active.d.ts +7 -0
- package/dist/server/routes/admin-active.js +120 -0
- package/dist/server/routes/admin-stats.d.ts +7 -0
- package/dist/server/routes/admin-stats.js +176 -0
- package/dist/server/routes/agent.d.ts +24 -0
- package/dist/server/routes/agent.js +480 -0
- package/dist/server/routes/answer.d.ts +5 -0
- package/dist/server/routes/answer.js +125 -0
- package/dist/server/routes/ask.d.ts +28 -0
- package/dist/server/routes/ask.js +295 -0
- package/dist/server/routes/batch.d.ts +6 -0
- package/dist/server/routes/batch.js +493 -0
- package/dist/server/routes/cache-warm.d.ts +25 -0
- package/dist/server/routes/cache-warm.js +212 -0
- package/dist/server/routes/cli-usage.d.ts +6 -0
- package/dist/server/routes/cli-usage.js +127 -0
- package/dist/server/routes/compat.d.ts +23 -0
- package/dist/server/routes/compat.js +652 -0
- package/dist/server/routes/crawl.d.ts +13 -0
- package/dist/server/routes/crawl.js +287 -0
- package/dist/server/routes/deep-fetch.d.ts +8 -0
- package/dist/server/routes/deep-fetch.js +57 -0
- package/dist/server/routes/deep-research.d.ts +11 -0
- package/dist/server/routes/deep-research.js +232 -0
- package/dist/server/routes/demo.d.ts +24 -0
- package/dist/server/routes/demo.js +517 -0
- package/dist/server/routes/do.d.ts +8 -0
- package/dist/server/routes/do.js +72 -0
- package/dist/server/routes/extract.d.ts +14 -0
- package/dist/server/routes/extract.js +325 -0
- package/dist/server/routes/feed.d.ts +15 -0
- package/dist/server/routes/feed.js +311 -0
- package/dist/server/routes/fetch-queue.d.ts +13 -0
- package/dist/server/routes/fetch-queue.js +357 -0
- package/dist/server/routes/fetch.d.ts +7 -0
- package/dist/server/routes/fetch.js +1274 -0
- package/dist/server/routes/go.d.ts +14 -0
- package/dist/server/routes/go.js +81 -0
- package/dist/server/routes/health.d.ts +11 -0
- package/dist/server/routes/health.js +141 -0
- package/dist/server/routes/jobs.d.ts +7 -0
- package/dist/server/routes/jobs.js +574 -0
- package/dist/server/routes/map.d.ts +11 -0
- package/dist/server/routes/map.js +116 -0
- package/dist/server/routes/mcp.d.ts +14 -0
- package/dist/server/routes/mcp.js +197 -0
- package/dist/server/routes/metrics.d.ts +37 -0
- package/dist/server/routes/metrics.js +149 -0
- package/dist/server/routes/oauth.d.ts +9 -0
- package/dist/server/routes/oauth.js +396 -0
- package/dist/server/routes/playground.d.ts +17 -0
- package/dist/server/routes/playground.js +283 -0
- package/dist/server/routes/reader.d.ts +18 -0
- package/dist/server/routes/reader.js +192 -0
- package/dist/server/routes/research.d.ts +14 -0
- package/dist/server/routes/research.js +482 -0
- package/dist/server/routes/screenshot.d.ts +22 -0
- package/dist/server/routes/screenshot.js +820 -0
- package/dist/server/routes/search.d.ts +6 -0
- package/dist/server/routes/search.js +874 -0
- package/dist/server/routes/session.d.ts +17 -0
- package/dist/server/routes/session.js +548 -0
- package/dist/server/routes/share.d.ts +18 -0
- package/dist/server/routes/share.js +462 -0
- package/dist/server/routes/smart-search/handlers/cars.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/cars.js +102 -0
- package/dist/server/routes/smart-search/handlers/flights.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/flights.js +72 -0
- package/dist/server/routes/smart-search/handlers/general.d.ts +13 -0
- package/dist/server/routes/smart-search/handlers/general.js +717 -0
- package/dist/server/routes/smart-search/handlers/hotels.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/hotels.js +88 -0
- package/dist/server/routes/smart-search/handlers/products.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/products.js +1309 -0
- package/dist/server/routes/smart-search/handlers/rental.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/rental.js +154 -0
- package/dist/server/routes/smart-search/handlers/restaurants.d.ts +2 -0
- package/dist/server/routes/smart-search/handlers/restaurants.js +225 -0
- package/dist/server/routes/smart-search/handlers/transit-verdict.d.ts +41 -0
- package/dist/server/routes/smart-search/handlers/transit-verdict.js +224 -0
- package/dist/server/routes/smart-search/index.d.ts +19 -0
- package/dist/server/routes/smart-search/index.js +546 -0
- package/dist/server/routes/smart-search/intent.d.ts +3 -0
- package/dist/server/routes/smart-search/intent.js +264 -0
- package/dist/server/routes/smart-search/llm.d.ts +16 -0
- package/dist/server/routes/smart-search/llm.js +70 -0
- package/dist/server/routes/smart-search/sources/reddit.d.ts +18 -0
- package/dist/server/routes/smart-search/sources/reddit.js +34 -0
- package/dist/server/routes/smart-search/sources/yelp.d.ts +25 -0
- package/dist/server/routes/smart-search/sources/yelp.js +171 -0
- package/dist/server/routes/smart-search/sources/youtube.d.ts +8 -0
- package/dist/server/routes/smart-search/sources/youtube.js +9 -0
- package/dist/server/routes/smart-search/types.d.ts +81 -0
- package/dist/server/routes/smart-search/types.js +1 -0
- package/dist/server/routes/smart-search/utils.d.ts +20 -0
- package/dist/server/routes/smart-search/utils.js +146 -0
- package/dist/server/routes/stats.d.ts +6 -0
- package/dist/server/routes/stats.js +71 -0
- package/dist/server/routes/stripe.d.ts +15 -0
- package/dist/server/routes/stripe.js +296 -0
- package/dist/server/routes/transcript-export.d.ts +10 -0
- package/dist/server/routes/transcript-export.js +178 -0
- package/dist/server/routes/usage.d.ts +9 -0
- package/dist/server/routes/usage.js +279 -0
- package/dist/server/routes/users.d.ts +8 -0
- package/dist/server/routes/users.js +1867 -0
- package/dist/server/routes/watch.d.ts +15 -0
- package/dist/server/routes/watch.js +309 -0
- package/dist/server/routes/webhooks.d.ts +26 -0
- package/dist/server/routes/webhooks.js +170 -0
- package/dist/server/routes/youtube.d.ts +6 -0
- package/dist/server/routes/youtube.js +130 -0
- package/dist/server/sentry.d.ts +14 -0
- package/dist/server/sentry.js +104 -0
- package/dist/server/types.d.ts +15 -0
- package/dist/server/types.js +7 -0
- package/dist/server/utils/response.d.ts +44 -0
- package/dist/server/utils/response.js +69 -0
- package/dist/server/utils/sse.d.ts +22 -0
- package/dist/server/utils/sse.js +38 -0
- package/dist/types.d.ts +552 -0
- package/dist/types.js +39 -0
- package/llms.txt +105 -0
- package/package.json +189 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Smart Search endpoint — intent detection + travel/commerce routing
|
|
3
|
+
* POST /v1/search/smart
|
|
4
|
+
*
|
|
5
|
+
* Detects user intent from natural language and routes to the best source:
|
|
6
|
+
* - cars → Cars.com with browser rendering + structured extraction
|
|
7
|
+
* - flights → Google Flights with browser rendering + flight extractor
|
|
8
|
+
* - hotels → Google Hotels with browser rendering
|
|
9
|
+
* - rental → Kayak with browser rendering + rental extractor
|
|
10
|
+
* - restaurants → Yelp Fusion API extractor
|
|
11
|
+
* - products → Amazon search with structured extraction
|
|
12
|
+
* - general → SearXNG with smart enrichment (peel() for top 3)
|
|
13
|
+
*/
|
|
14
|
+
import { Router } from 'express';
|
|
15
|
+
import '../../types.js';
|
|
16
|
+
import { AuthStore } from '../../auth-store.js';
|
|
17
|
+
export type { SearchIntent, SmartSearchResult, TransactionalVerdict, VerdictOption } from './types.js';
|
|
18
|
+
export { detectSearchIntent } from './intent.js';
|
|
19
|
+
export declare function createSmartSearchRouter(authStore: AuthStore): Router;
|
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Smart Search endpoint — intent detection + travel/commerce routing
|
|
3
|
+
* POST /v1/search/smart
|
|
4
|
+
*
|
|
5
|
+
* Detects user intent from natural language and routes to the best source:
|
|
6
|
+
* - cars → Cars.com with browser rendering + structured extraction
|
|
7
|
+
* - flights → Google Flights with browser rendering + flight extractor
|
|
8
|
+
* - hotels → Google Hotels with browser rendering
|
|
9
|
+
* - rental → Kayak with browser rendering + rental extractor
|
|
10
|
+
* - restaurants → Yelp Fusion API extractor
|
|
11
|
+
* - products → Amazon search with structured extraction
|
|
12
|
+
* - general → SearXNG with smart enrichment (peel() for top 3)
|
|
13
|
+
*/
|
|
14
|
+
import { Router } from 'express';
|
|
15
|
+
import '../../types.js'; // Augments Express.Request with requestId, auth
|
|
16
|
+
// @ts-ignore — ioredis CJS/ESM interop
|
|
17
|
+
import IoRedisModule from 'ioredis';
|
|
18
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
19
|
+
const IoRedis = IoRedisModule.default ?? IoRedisModule;
|
|
20
|
+
export { detectSearchIntent } from './intent.js';
|
|
21
|
+
import { detectSearchIntent, classifyIntentWithLLM } from './intent.js';
|
|
22
|
+
import { callLLMQuick, sanitizeSearchQuery, PROMPT_INJECTION_DEFENSE } from './llm.js';
|
|
23
|
+
import { handleCarSearch } from './handlers/cars.js';
|
|
24
|
+
import { handleFlightSearch } from './handlers/flights.js';
|
|
25
|
+
import { handleHotelSearch } from './handlers/hotels.js';
|
|
26
|
+
import { handleRentalSearch } from './handlers/rental.js';
|
|
27
|
+
import { handleRestaurantSearch } from './handlers/restaurants.js';
|
|
28
|
+
import { handleProductSearch } from './handlers/products.js';
|
|
29
|
+
import { handleGeneralSearch } from './handlers/general.js';
|
|
30
|
+
import { fetchYelpResults } from './sources/yelp.js';
|
|
31
|
+
import { fetchRedditResults } from './sources/reddit.js';
|
|
32
|
+
import { fetchYouTubeResults } from './sources/youtube.js';
|
|
33
|
+
// ─── Redis client (lazy singleton for smart-search caching) ───────────────
|
|
34
|
+
function buildSmartRedis() {
|
|
35
|
+
const url = process.env.REDIS_URL || 'redis://redis:6379';
|
|
36
|
+
const password = process.env.REDIS_PASSWORD || undefined;
|
|
37
|
+
try {
|
|
38
|
+
const parsed = new URL(url);
|
|
39
|
+
return new IoRedis({
|
|
40
|
+
host: parsed.hostname,
|
|
41
|
+
port: parseInt(parsed.port || '6379', 10),
|
|
42
|
+
password,
|
|
43
|
+
db: parseInt(parsed.pathname?.slice(1) || '0', 10) || 0,
|
|
44
|
+
lazyConnect: true,
|
|
45
|
+
maxRetriesPerRequest: 1,
|
|
46
|
+
enableOfflineQueue: false,
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return new IoRedis({ host: 'redis', port: 6379, password, lazyConnect: true, maxRetriesPerRequest: 1, enableOfflineQueue: false });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
let _smartRedis = null;
|
|
54
|
+
function getSmartRedis() {
|
|
55
|
+
if (!_smartRedis)
|
|
56
|
+
_smartRedis = buildSmartRedis();
|
|
57
|
+
return _smartRedis;
|
|
58
|
+
}
|
|
59
|
+
// TTL by intent type (seconds)
|
|
60
|
+
const CACHE_TTL = {
|
|
61
|
+
restaurants: 1800, // 30 min
|
|
62
|
+
cars: 900, // 15 min
|
|
63
|
+
products: 900, // 15 min
|
|
64
|
+
flights: 600, // 10 min
|
|
65
|
+
hotels: 600, // 10 min
|
|
66
|
+
rental: 1800, // 30 min
|
|
67
|
+
general: 3600, // 60 min
|
|
68
|
+
};
|
|
69
|
+
// ─── Loading message by intent type ────────────────────────────────────────
|
|
70
|
+
function getLoadingMessage(type) {
|
|
71
|
+
const msgs = {
|
|
72
|
+
cars: 'Searching cars on Cars.com…',
|
|
73
|
+
flights: 'Searching for flights...',
|
|
74
|
+
hotels: 'Searching for hotels...',
|
|
75
|
+
rental: 'Searching for rental cars...',
|
|
76
|
+
restaurants: 'Finding restaurants on Yelp…',
|
|
77
|
+
products: 'Searching Amazon for products…',
|
|
78
|
+
general: '🔍 Searching and analyzing results...',
|
|
79
|
+
};
|
|
80
|
+
return msgs[type] || 'Searching…';
|
|
81
|
+
}
|
|
82
|
+
// ─── Router ────────────────────────────────────────────────────────────────
|
|
83
|
+
// Log LLM provider at startup (uses unified config from core)
|
|
84
|
+
{
|
|
85
|
+
// Dynamic import to avoid top-level await issues
|
|
86
|
+
import('../../../core/llm-provider.js').then(({ getQuickLLMConfig }) => {
|
|
87
|
+
const cfg = getQuickLLMConfig();
|
|
88
|
+
const _llmProvider = cfg?.provider ?? 'none';
|
|
89
|
+
const _llmModel = cfg?.model ?? 'n/a';
|
|
90
|
+
const _baseUrl = cfg?.baseUrl;
|
|
91
|
+
const suffix = _baseUrl ? ` via ${_baseUrl}` : '';
|
|
92
|
+
console.log(`[smart-search] LLM provider: ${_llmProvider} (${_llmModel}${suffix})`);
|
|
93
|
+
}).catch(() => {
|
|
94
|
+
console.log('[smart-search] LLM provider: none (config load failed)');
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
export function createSmartSearchRouter(authStore) {
|
|
98
|
+
const router = Router();
|
|
99
|
+
router.post('/v1/search/smart', async (req, res) => {
|
|
100
|
+
try {
|
|
101
|
+
// Authentication: API key OR anonymous (rate-limited by IP)
|
|
102
|
+
const authId = req.auth?.keyInfo?.accountId || req.user?.userId;
|
|
103
|
+
const isAnonymous = !authId;
|
|
104
|
+
if (isAnonymous) {
|
|
105
|
+
// Rate limit anonymous users: 5 searches per day per IP
|
|
106
|
+
const clientIp = req.headers['x-forwarded-for']?.split(',')[0]?.trim()
|
|
107
|
+
|| req.headers['cf-connecting-ip']
|
|
108
|
+
|| req.socket.remoteAddress
|
|
109
|
+
|| 'unknown';
|
|
110
|
+
const anonKey = `anon:smart:${clientIp}`;
|
|
111
|
+
try {
|
|
112
|
+
const redis = getSmartRedis();
|
|
113
|
+
const count = await redis.incr(anonKey);
|
|
114
|
+
if (count === 1) {
|
|
115
|
+
// Set 24-hour expiry on first request
|
|
116
|
+
await redis.expire(anonKey, 86400);
|
|
117
|
+
}
|
|
118
|
+
if (count > 5) {
|
|
119
|
+
res.status(429).json({
|
|
120
|
+
success: false,
|
|
121
|
+
error: {
|
|
122
|
+
type: 'anonymous_limit_exceeded',
|
|
123
|
+
message: 'Free search limit reached (5/day). Sign up for 500 fetches/week free.',
|
|
124
|
+
signupUrl: 'https://app.webpeel.dev/signup',
|
|
125
|
+
},
|
|
126
|
+
requestId: req.requestId,
|
|
127
|
+
});
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
// Redis failed — allow the request (graceful degradation)
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
const { q, location, zip, language: reqLanguage } = req.body;
|
|
136
|
+
if (!q || typeof q !== 'string' || !q.trim()) {
|
|
137
|
+
res.status(400).json({
|
|
138
|
+
success: false,
|
|
139
|
+
error: {
|
|
140
|
+
type: 'invalid_request',
|
|
141
|
+
message: 'Missing or invalid "q" field in request body',
|
|
142
|
+
hint: 'POST /v1/search/smart with JSON body: { "q": "your search query" }',
|
|
143
|
+
docs: 'https://webpeel.dev/docs/api-reference#smart-search',
|
|
144
|
+
},
|
|
145
|
+
requestId: req.requestId,
|
|
146
|
+
});
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
const query = q.trim();
|
|
150
|
+
const normalizedLocation = typeof location === 'string' ? location.trim() : '';
|
|
151
|
+
const intent = detectSearchIntent(query);
|
|
152
|
+
// If regex returned 'general' as fallback (not from an explicit pattern match),
|
|
153
|
+
// try LLM classification to catch typos, other languages, creative phrasing.
|
|
154
|
+
// Skip LLM override if regex matched a specific pattern (comparison, local, service queries)
|
|
155
|
+
// — those were INTENTIONALLY set to 'general'.
|
|
156
|
+
const queryLower = query.toLowerCase();
|
|
157
|
+
const isExplicitGeneral = (/\b(compare|vs\.?|versus|which is better|difference between)\b/.test(queryLower) ||
|
|
158
|
+
(/\b(near me|near\s+\w+|open now|open today|open on|what time|is .* open|hours|closest|nearest)\b/.test(queryLower) && /\b(buy|where|store|shop|near|close to|around)\b/.test(queryLower)) ||
|
|
159
|
+
(/\b(plumber|electrician|mechanic|dentist|doctor|lawyer|therapist|vet|salon|barber|gym|daycare)\b/.test(queryLower) && /\b(near|in|around|open|best|cheap|emergency)\b/.test(queryLower)) ||
|
|
160
|
+
(/\b(cruise|vacation|resort|trip|travel|getaway|tour|safari|honeymoon|disneyland|disney|universal|six flags|theme park)\b/.test(queryLower) && /\b(cheap|cheapest|price|ticket|book|deal|package)\b/.test(queryLower)) ||
|
|
161
|
+
(/\b(bus|buses|coach|greyhound|flixbus|megabus|busbud|wanderu|train|trains|amtrak|ferry|ferries)\b/.test(queryLower) && /\b(ticket|tickets|book|booking|cheap|cheapest|price|fare|fares|route|take|trip|from|to)\b/.test(queryLower)));
|
|
162
|
+
if (intent.type === 'general' && !isExplicitGeneral && process.env.OLLAMA_URL) {
|
|
163
|
+
try {
|
|
164
|
+
const llmType = await classifyIntentWithLLM(query);
|
|
165
|
+
if (llmType !== 'general') {
|
|
166
|
+
console.log(`[smart-search] LLM reclassified "${query}" from general → ${llmType}`);
|
|
167
|
+
intent.type = llmType;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
catch (err) {
|
|
171
|
+
// Graceful degradation — regex result stands
|
|
172
|
+
console.warn('[smart-search] LLM intent classification failed:', err.message);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
// Override zip from request body if provided
|
|
176
|
+
if (zip && intent.params) {
|
|
177
|
+
intent.params.zip = zip;
|
|
178
|
+
}
|
|
179
|
+
// Also try to extract location context from request body when provided
|
|
180
|
+
if (normalizedLocation && intent.type === 'restaurants') {
|
|
181
|
+
intent.params = { ...(intent.params || {}), location: normalizedLocation };
|
|
182
|
+
intent.location = normalizedLocation;
|
|
183
|
+
}
|
|
184
|
+
if (normalizedLocation && intent.params?.localIntent === 'true') {
|
|
185
|
+
intent.params.location = normalizedLocation;
|
|
186
|
+
intent.params.localLocation = normalizedLocation;
|
|
187
|
+
intent.params.localNeedsUserLocation = 'false';
|
|
188
|
+
}
|
|
189
|
+
// ── Cache check (before streaming — HIT skips SSE entirely) ─────────
|
|
190
|
+
const SMART_CACHE_VERSION = 'v7'; // bump when intent routing or location wiring changes
|
|
191
|
+
const cacheLocation = normalizedLocation || intent.params?.location || intent.params?.localLocation || intent.params?.zip || '';
|
|
192
|
+
const cacheKey = `smart:${SMART_CACHE_VERSION}:${intent.type}:${query.toLowerCase().trim().replace(/\s+/g, ' ')}:${cacheLocation.toLowerCase().replace(/\s+/g, ' ') || 'none'}`;
|
|
193
|
+
try {
|
|
194
|
+
const redis = getSmartRedis();
|
|
195
|
+
const cached = await redis.get(cacheKey);
|
|
196
|
+
if (cached) {
|
|
197
|
+
const parsed = JSON.parse(cached);
|
|
198
|
+
console.log(`[smart-search] Cache HIT: ${cacheKey} (${parsed.fetchTimeMs}ms original)`);
|
|
199
|
+
// Ensure safety field exists on cached responses
|
|
200
|
+
if (!parsed.safety) {
|
|
201
|
+
parsed.safety = {
|
|
202
|
+
verified: true,
|
|
203
|
+
promptInjectionsBlocked: 0,
|
|
204
|
+
maliciousPatternsStripped: 0,
|
|
205
|
+
sourcesChecked: parsed.sources?.length || 0,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
// Attach suggestedDomains from intent
|
|
209
|
+
if (intent.suggestedDomains?.length) {
|
|
210
|
+
parsed.suggestedDomains = intent.suggestedDomains;
|
|
211
|
+
}
|
|
212
|
+
res.setHeader('X-Intent-Type', intent.type);
|
|
213
|
+
res.setHeader('X-Source', parsed.source);
|
|
214
|
+
res.setHeader('X-Processing-Time', '0');
|
|
215
|
+
res.setHeader('X-Cache', 'HIT');
|
|
216
|
+
res.setHeader('X-Cache-Key', cacheKey);
|
|
217
|
+
res.setHeader('Cache-Control', 'no-store');
|
|
218
|
+
res.json({ success: true, data: parsed });
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
catch (err) {
|
|
223
|
+
console.warn('[smart-search] Redis cache error (non-fatal):', err.message);
|
|
224
|
+
}
|
|
225
|
+
// ── SSE Streaming path ────────────────────────────────────────────────
|
|
226
|
+
const streamRequested = req.body?.stream === true || req.body?.stream === 'true';
|
|
227
|
+
if (streamRequested) {
|
|
228
|
+
res.setHeader('Content-Type', 'text/event-stream');
|
|
229
|
+
res.setHeader('Cache-Control', 'no-cache');
|
|
230
|
+
res.setHeader('Connection', 'keep-alive');
|
|
231
|
+
res.setHeader('X-Accel-Buffering', 'no');
|
|
232
|
+
const sendEvent = (event, data) => {
|
|
233
|
+
res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
234
|
+
if (typeof res.flush === 'function')
|
|
235
|
+
res.flush();
|
|
236
|
+
};
|
|
237
|
+
sendEvent('intent', {
|
|
238
|
+
type: intent.type,
|
|
239
|
+
query,
|
|
240
|
+
loadingMessage: getLoadingMessage(intent.type),
|
|
241
|
+
});
|
|
242
|
+
try {
|
|
243
|
+
const t0Stream = Date.now();
|
|
244
|
+
if (intent.type === 'restaurants') {
|
|
245
|
+
// Restaurant: stream each source as it arrives
|
|
246
|
+
const loc = intent.params.location || 'New York, NY';
|
|
247
|
+
const kw = intent.query
|
|
248
|
+
.replace(/\b(best|top|good|cheap|affordable|near me|near|around|in|find|search|looking for)\b/gi, '')
|
|
249
|
+
.replace(/\s+/g, ' ')
|
|
250
|
+
.trim();
|
|
251
|
+
let yelpData = null;
|
|
252
|
+
sendEvent('progress', { step: 'searching_yelp', message: 'Searching Yelp for restaurants...' });
|
|
253
|
+
try {
|
|
254
|
+
yelpData = await Promise.race([
|
|
255
|
+
fetchYelpResults(kw, loc),
|
|
256
|
+
new Promise((_, rej) => setTimeout(() => rej(new Error('yelp timeout')), 10000)),
|
|
257
|
+
]);
|
|
258
|
+
sendEvent('progress', { step: 'yelp_done', message: `Found ${yelpData?.businesses?.length || 0} restaurants on Yelp` });
|
|
259
|
+
if (yelpData?.businesses?.length > 0) {
|
|
260
|
+
yelpData.businesses.sort((a, b) => {
|
|
261
|
+
const scoreA = (a.rating || 0) * Math.log2((a.reviewCount || 0) + 1);
|
|
262
|
+
const scoreB = (b.rating || 0) * Math.log2((b.reviewCount || 0) + 1);
|
|
263
|
+
return scoreB - scoreA;
|
|
264
|
+
});
|
|
265
|
+
yelpData.businesses = yelpData.businesses.filter((b) => !b.isClosed);
|
|
266
|
+
if (process.env.GOOGLE_PLACES_API_KEY) {
|
|
267
|
+
sendEvent('progress', { step: 'checking_google', message: 'Verifying hours on Google Maps...' });
|
|
268
|
+
}
|
|
269
|
+
sendEvent('source', { source: 'yelp', businesses: yelpData.businesses.slice(0, 10) });
|
|
270
|
+
if (process.env.GOOGLE_PLACES_API_KEY) {
|
|
271
|
+
sendEvent('progress', { step: 'google_done', message: 'Hours verified for top 3 restaurants' });
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
catch {
|
|
276
|
+
sendEvent('progress', { step: 'yelp_done', message: 'Found 0 restaurants on Yelp' });
|
|
277
|
+
}
|
|
278
|
+
sendEvent('progress', { step: 'fetching_reviews', message: 'Finding Reddit discussions and YouTube reviews...' });
|
|
279
|
+
const [redditSettled, youtubeSettled] = await Promise.allSettled([
|
|
280
|
+
Promise.race([
|
|
281
|
+
fetchRedditResults(kw, loc),
|
|
282
|
+
new Promise((_, rej) => setTimeout(() => rej(new Error('reddit timeout')), 8000)),
|
|
283
|
+
]),
|
|
284
|
+
Promise.race([
|
|
285
|
+
fetchYouTubeResults(kw, loc),
|
|
286
|
+
new Promise((_, rej) => setTimeout(() => rej(new Error('youtube timeout')), 5000)),
|
|
287
|
+
]),
|
|
288
|
+
]);
|
|
289
|
+
const redditData = redditSettled.status === 'fulfilled' ? redditSettled.value : null;
|
|
290
|
+
const youtubeData = youtubeSettled.status === 'fulfilled' ? youtubeSettled.value : null;
|
|
291
|
+
if (redditData) {
|
|
292
|
+
sendEvent('source', { source: 'reddit', thread: redditData.thread, otherThreads: redditData.otherThreads });
|
|
293
|
+
}
|
|
294
|
+
if (youtubeData && youtubeData.videos?.length) {
|
|
295
|
+
sendEvent('source', { source: 'youtube', videos: youtubeData.videos });
|
|
296
|
+
}
|
|
297
|
+
let answer;
|
|
298
|
+
const ollamaUrl = process.env.OLLAMA_URL;
|
|
299
|
+
if (ollamaUrl && yelpData?.businesses?.length > 0) {
|
|
300
|
+
sendEvent('progress', { step: 'generating_ai', message: 'Generating AI recommendation...' });
|
|
301
|
+
try {
|
|
302
|
+
const yelpLines = yelpData.businesses.slice(0, 3).map((b, i) => {
|
|
303
|
+
const openStatus = b.isClosed ? 'PERMANENTLY CLOSED' : (b.isOpenNow ? 'OPEN NOW' : 'Closed right now');
|
|
304
|
+
const txns = b.transactions?.length > 0 ? `Available: ${b.transactions.join(', ')}` : '';
|
|
305
|
+
const googleInfo = b.googleRating ? ` | Google: ⭐${b.googleRating} (${b.googleReviewCount} reviews)` : '';
|
|
306
|
+
return `[${i + 1}] ${b.name} ⭐${b.rating} (${b.reviewCount?.toLocaleString()} reviews) ${b.price || ''} — ${b.address}
|
|
307
|
+
${openStatus} | Today: ${b.todayHours || 'hours not available'} | ${txns} | Categories: ${b.categories || ''}${googleInfo}
|
|
308
|
+
URL: ${b.url || ''}`;
|
|
309
|
+
}).join('\n');
|
|
310
|
+
const yelpCitations = yelpData.businesses.slice(0, 3).map((b, i) => `[${i + 1}] ${b.url || 'yelp.com'}`).join('\n');
|
|
311
|
+
const redditHint = redditData && redditData.otherThreads?.slice(0, 2).map((t) => t.title).join('; ') || '';
|
|
312
|
+
const systemPrompt = `${PROMPT_INJECTION_DEFENSE}Recommend top 3 restaurants. For each: name with inline citation [1][2][3], why it's good, open/closed status, hours.
|
|
313
|
+
Cite sources inline using [1], [2], [3] notation matching the numbered sources. At the end, list Sources with their URLs.
|
|
314
|
+
Be specific. Max 200 words.
|
|
315
|
+
`;
|
|
316
|
+
const userMessage = `Query: ${sanitizeSearchQuery(intent.query)}\n\nTop restaurants:\n${yelpLines}${redditHint ? '\n\nReddit mentions: ' + redditHint : ''}\n\nSources:\n${yelpCitations}`;
|
|
317
|
+
const text = await callLLMQuick(`${systemPrompt}\n\n${userMessage}`, { maxTokens: 250, timeoutMs: 5000, temperature: 0.3 });
|
|
318
|
+
if (text)
|
|
319
|
+
answer = text;
|
|
320
|
+
}
|
|
321
|
+
catch { /* LLM failure — no answer */ }
|
|
322
|
+
}
|
|
323
|
+
if (answer) {
|
|
324
|
+
sendEvent('answer', { answer });
|
|
325
|
+
}
|
|
326
|
+
sendEvent('done', { fetchTimeMs: Date.now() - t0Stream, answer: answer || undefined });
|
|
327
|
+
// Cache the streaming result for restaurants
|
|
328
|
+
try {
|
|
329
|
+
const redis = getSmartRedis();
|
|
330
|
+
const ttl = CACHE_TTL[intent.type] || 600;
|
|
331
|
+
const yelpUrl = yelpData?.url || `https://www.yelp.com/search?find_desc=${encodeURIComponent(kw)}&find_loc=${encodeURIComponent(loc)}`;
|
|
332
|
+
const contentParts = [];
|
|
333
|
+
if (yelpData?.businesses?.length > 0) {
|
|
334
|
+
contentParts.push(`## Yelp (${yelpData.businesses.length} restaurants)`);
|
|
335
|
+
yelpData.businesses.slice(0, 10).forEach((b, i) => {
|
|
336
|
+
const openStatus = b.isClosed ? ' · ⛔ Permanently Closed' : (b.isOpenNow ? ' · 🟢 Open Now' : ' · 🔴 Closed');
|
|
337
|
+
contentParts.push(`${i + 1}. **${b.name}** ⭐${b.rating} (${(b.reviewCount || 0).toLocaleString()} reviews)${b.price ? ' · ' + b.price : ''}${openStatus}${b.address ? ' — ' + b.address : ''}`);
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
if (redditData) {
|
|
341
|
+
contentParts.push('');
|
|
342
|
+
contentParts.push('## Reddit Recommendations');
|
|
343
|
+
if (redditData.thread)
|
|
344
|
+
contentParts.push(`**${redditData.thread.title}**`);
|
|
345
|
+
}
|
|
346
|
+
if (youtubeData && youtubeData.videos?.length) {
|
|
347
|
+
contentParts.push('');
|
|
348
|
+
contentParts.push('## YouTube Reviews');
|
|
349
|
+
youtubeData.videos.forEach((v) => contentParts.push(`🎬 [${v.title}](${v.url})`));
|
|
350
|
+
}
|
|
351
|
+
const cachedSources = [];
|
|
352
|
+
if (yelpData)
|
|
353
|
+
cachedSources.push({ title: 'Yelp', url: yelpUrl, domain: 'yelp.com' });
|
|
354
|
+
if (redditData?.thread)
|
|
355
|
+
cachedSources.push({ title: redditData.thread.title, url: redditData.thread.url, domain: 'reddit.com' });
|
|
356
|
+
if (youtubeData?.videos?.[0])
|
|
357
|
+
cachedSources.push({ title: youtubeData.videos[0].title, url: youtubeData.videos[0].url, domain: 'youtube.com' });
|
|
358
|
+
const cacheResult = {
|
|
359
|
+
type: 'restaurants',
|
|
360
|
+
source: 'Yelp + Reddit + YouTube',
|
|
361
|
+
sourceUrl: yelpUrl,
|
|
362
|
+
content: contentParts.join('\n'),
|
|
363
|
+
title: `${kw} in ${loc}`,
|
|
364
|
+
domainData: yelpData?.domainData,
|
|
365
|
+
structured: yelpData?.domainData?.structured,
|
|
366
|
+
tokens: contentParts.join('\n').split(/\s+/).length,
|
|
367
|
+
fetchTimeMs: Date.now() - t0Stream,
|
|
368
|
+
...(answer !== undefined ? { answer } : {}),
|
|
369
|
+
...(cachedSources.length > 0 ? { sources: cachedSources } : {}),
|
|
370
|
+
safety: {
|
|
371
|
+
verified: true,
|
|
372
|
+
promptInjectionsBlocked: 0,
|
|
373
|
+
maliciousPatternsStripped: 0,
|
|
374
|
+
sourcesChecked: cachedSources.length,
|
|
375
|
+
},
|
|
376
|
+
...(intent.suggestedDomains?.length ? { suggestedDomains: intent.suggestedDomains } : {}),
|
|
377
|
+
};
|
|
378
|
+
await redis.setex(cacheKey, ttl, JSON.stringify(cacheResult));
|
|
379
|
+
console.log(`[smart-search] SSE Cache WRITE: ${cacheKey} (TTL: ${ttl}s)`);
|
|
380
|
+
}
|
|
381
|
+
catch { /* non-fatal */ }
|
|
382
|
+
res.end();
|
|
383
|
+
}
|
|
384
|
+
else {
|
|
385
|
+
// All other intent types: run the existing handler, emit full result
|
|
386
|
+
const typeLabels = {
|
|
387
|
+
cars: 'Searching Cars.com for vehicles...',
|
|
388
|
+
flights: 'Finding flights and prices...',
|
|
389
|
+
hotels: 'Searching for hotels and rates...',
|
|
390
|
+
rental: 'Searching rental car prices...',
|
|
391
|
+
products: 'Searching for products and prices...',
|
|
392
|
+
general: 'Searching the web...',
|
|
393
|
+
};
|
|
394
|
+
sendEvent('progress', { step: 'searching', message: typeLabels[intent.type] || 'Searching...' });
|
|
395
|
+
let streamResult;
|
|
396
|
+
switch (intent.type) {
|
|
397
|
+
case 'cars':
|
|
398
|
+
streamResult = await handleCarSearch(intent);
|
|
399
|
+
break;
|
|
400
|
+
case 'flights':
|
|
401
|
+
streamResult = await handleFlightSearch(intent);
|
|
402
|
+
break;
|
|
403
|
+
case 'hotels':
|
|
404
|
+
streamResult = await handleHotelSearch(intent);
|
|
405
|
+
break;
|
|
406
|
+
case 'rental':
|
|
407
|
+
streamResult = await handleRentalSearch(intent);
|
|
408
|
+
break;
|
|
409
|
+
case 'products':
|
|
410
|
+
streamResult = await handleProductSearch(intent);
|
|
411
|
+
break;
|
|
412
|
+
default:
|
|
413
|
+
streamResult = await handleGeneralSearch(query);
|
|
414
|
+
}
|
|
415
|
+
const resultCount = streamResult.structured?.listings?.length ?? streamResult.results?.length ?? null;
|
|
416
|
+
sendEvent('progress', { step: 'complete', message: `Found ${resultCount !== null ? resultCount : 'results'}` });
|
|
417
|
+
if (streamResult.answer) {
|
|
418
|
+
sendEvent('progress', { step: 'ai_done', message: 'AI summary generated' });
|
|
419
|
+
}
|
|
420
|
+
if (!streamResult.loadingMessage) {
|
|
421
|
+
streamResult.loadingMessage = getLoadingMessage(intent.type);
|
|
422
|
+
}
|
|
423
|
+
// Attach safety summary for streaming non-restaurant results
|
|
424
|
+
streamResult.safety = {
|
|
425
|
+
verified: true,
|
|
426
|
+
promptInjectionsBlocked: 0,
|
|
427
|
+
maliciousPatternsStripped: 0,
|
|
428
|
+
sourcesChecked: streamResult.sources?.length || 0,
|
|
429
|
+
};
|
|
430
|
+
// Attach suggestedDomains from intent
|
|
431
|
+
if (intent.suggestedDomains?.length) {
|
|
432
|
+
streamResult.suggestedDomains = intent.suggestedDomains;
|
|
433
|
+
}
|
|
434
|
+
sendEvent('result', streamResult);
|
|
435
|
+
sendEvent('done', { fetchTimeMs: streamResult.fetchTimeMs });
|
|
436
|
+
// Cache the streaming result
|
|
437
|
+
try {
|
|
438
|
+
const redis = getSmartRedis();
|
|
439
|
+
const ttl = CACHE_TTL[intent.type] || 600;
|
|
440
|
+
await redis.setex(cacheKey, ttl, JSON.stringify(streamResult));
|
|
441
|
+
console.log(`[smart-search] SSE Cache WRITE: ${cacheKey} (TTL: ${ttl}s)`);
|
|
442
|
+
}
|
|
443
|
+
catch { /* non-fatal */ }
|
|
444
|
+
res.end();
|
|
445
|
+
}
|
|
446
|
+
// Track usage for streaming path too
|
|
447
|
+
const pgStoreStream = authStore;
|
|
448
|
+
if (req.auth?.keyInfo?.key && typeof pgStoreStream.trackUsage === 'function') {
|
|
449
|
+
if (typeof pgStoreStream.trackBurstUsage === 'function') {
|
|
450
|
+
await pgStoreStream.trackBurstUsage(req.auth.keyInfo.key);
|
|
451
|
+
}
|
|
452
|
+
if (!req.auth?.softLimited) {
|
|
453
|
+
await pgStoreStream.trackUsage(req.auth.keyInfo.key, 'search');
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
catch (err) {
|
|
458
|
+
sendEvent('error', { message: err.message });
|
|
459
|
+
res.end();
|
|
460
|
+
}
|
|
461
|
+
return; // Don't fall through to non-streaming response
|
|
462
|
+
}
|
|
463
|
+
let smartResult;
|
|
464
|
+
switch (intent.type) {
|
|
465
|
+
case 'cars':
|
|
466
|
+
smartResult = await handleCarSearch(intent);
|
|
467
|
+
break;
|
|
468
|
+
case 'flights':
|
|
469
|
+
smartResult = await handleFlightSearch(intent);
|
|
470
|
+
break;
|
|
471
|
+
case 'hotels':
|
|
472
|
+
smartResult = await handleHotelSearch(intent);
|
|
473
|
+
break;
|
|
474
|
+
case 'rental':
|
|
475
|
+
smartResult = await handleRentalSearch(intent);
|
|
476
|
+
break;
|
|
477
|
+
case 'restaurants':
|
|
478
|
+
smartResult = await handleRestaurantSearch(intent, reqLanguage);
|
|
479
|
+
break;
|
|
480
|
+
case 'products':
|
|
481
|
+
smartResult = await handleProductSearch(intent);
|
|
482
|
+
break;
|
|
483
|
+
default:
|
|
484
|
+
smartResult = await handleGeneralSearch(query);
|
|
485
|
+
}
|
|
486
|
+
if (!smartResult.loadingMessage) {
|
|
487
|
+
smartResult.loadingMessage = getLoadingMessage(intent.type);
|
|
488
|
+
}
|
|
489
|
+
// ── Attach safety summary ─────────────────────────────────────────────
|
|
490
|
+
smartResult.safety = {
|
|
491
|
+
verified: true, // smart search already sanitizes all LLM inputs
|
|
492
|
+
promptInjectionsBlocked: 0, // stripped before they reach the LLM
|
|
493
|
+
maliciousPatternsStripped: 0,
|
|
494
|
+
sourcesChecked: smartResult.sources?.length || 0,
|
|
495
|
+
};
|
|
496
|
+
// Attach suggestedDomains from intent
|
|
497
|
+
if (intent.suggestedDomains?.length) {
|
|
498
|
+
smartResult.suggestedDomains = intent.suggestedDomains;
|
|
499
|
+
}
|
|
500
|
+
// ── Cache write ───────────────────────────────────────────────────────
|
|
501
|
+
try {
|
|
502
|
+
const redis = getSmartRedis();
|
|
503
|
+
const ttl = CACHE_TTL[smartResult.type] || 600;
|
|
504
|
+
await redis.setex(cacheKey, ttl, JSON.stringify(smartResult));
|
|
505
|
+
res.setHeader('X-Cache', 'MISS');
|
|
506
|
+
res.setHeader('X-Cache-Key', cacheKey);
|
|
507
|
+
console.log(`[smart-search] Cache WRITE: ${cacheKey} (TTL: ${ttl}s)`);
|
|
508
|
+
}
|
|
509
|
+
catch (err) {
|
|
510
|
+
console.warn('[smart-search] Redis cache write error (non-fatal):', err.message);
|
|
511
|
+
}
|
|
512
|
+
// Track usage
|
|
513
|
+
const pgStore = authStore;
|
|
514
|
+
if (req.auth?.keyInfo?.key && typeof pgStore.trackUsage === 'function') {
|
|
515
|
+
if (typeof pgStore.trackBurstUsage === 'function') {
|
|
516
|
+
await pgStore.trackBurstUsage(req.auth.keyInfo.key);
|
|
517
|
+
}
|
|
518
|
+
if (!req.auth?.softLimited) {
|
|
519
|
+
await pgStore.trackUsage(req.auth.keyInfo.key, 'search');
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
res.setHeader('X-Intent-Type', intent.type);
|
|
523
|
+
res.setHeader('X-Source', smartResult.source);
|
|
524
|
+
res.setHeader('X-Processing-Time', smartResult.fetchTimeMs.toString());
|
|
525
|
+
res.setHeader('Cache-Control', 'no-store');
|
|
526
|
+
res.json({
|
|
527
|
+
success: true,
|
|
528
|
+
data: smartResult,
|
|
529
|
+
});
|
|
530
|
+
}
|
|
531
|
+
catch (error) {
|
|
532
|
+
const err = error;
|
|
533
|
+
console.error('Smart search error:', err.message, err.stack);
|
|
534
|
+
res.status(500).json({
|
|
535
|
+
success: false,
|
|
536
|
+
error: {
|
|
537
|
+
type: 'smart_search_failed',
|
|
538
|
+
message: err.message || 'Smart search failed. Please try again.',
|
|
539
|
+
docs: 'https://webpeel.dev/docs/api-reference#smart-search',
|
|
540
|
+
},
|
|
541
|
+
requestId: req.requestId,
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
});
|
|
545
|
+
return router;
|
|
546
|
+
}
|