@iflow-mcp/jakeliume-webpeel 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (547) hide show
  1. package/LICENSE +15 -0
  2. package/README.md +313 -0
  3. package/dist/cache.d.ts +30 -0
  4. package/dist/cache.js +139 -0
  5. package/dist/cli/commands/auth.d.ts +5 -0
  6. package/dist/cli/commands/auth.js +411 -0
  7. package/dist/cli/commands/doctor.d.ts +37 -0
  8. package/dist/cli/commands/doctor.js +371 -0
  9. package/dist/cli/commands/fetch.d.ts +6 -0
  10. package/dist/cli/commands/fetch.js +1345 -0
  11. package/dist/cli/commands/guide.d.ts +2 -0
  12. package/dist/cli/commands/guide.js +183 -0
  13. package/dist/cli/commands/interact.d.ts +5 -0
  14. package/dist/cli/commands/interact.js +840 -0
  15. package/dist/cli/commands/jobs.d.ts +5 -0
  16. package/dist/cli/commands/jobs.js +997 -0
  17. package/dist/cli/commands/monitor.d.ts +12 -0
  18. package/dist/cli/commands/monitor.js +197 -0
  19. package/dist/cli/commands/observe.d.ts +12 -0
  20. package/dist/cli/commands/observe.js +158 -0
  21. package/dist/cli/commands/screenshot.d.ts +5 -0
  22. package/dist/cli/commands/screenshot.js +282 -0
  23. package/dist/cli/commands/search.d.ts +5 -0
  24. package/dist/cli/commands/search.js +1021 -0
  25. package/dist/cli/commands/setup.d.ts +13 -0
  26. package/dist/cli/commands/setup.js +244 -0
  27. package/dist/cli/commands/skill.d.ts +15 -0
  28. package/dist/cli/commands/skill.js +195 -0
  29. package/dist/cli/utils.d.ts +84 -0
  30. package/dist/cli/utils.js +806 -0
  31. package/dist/cli-auth.d.ts +75 -0
  32. package/dist/cli-auth.js +369 -0
  33. package/dist/cli.d.ts +17 -0
  34. package/dist/cli.js +99 -0
  35. package/dist/core/actions.d.ts +69 -0
  36. package/dist/core/actions.js +495 -0
  37. package/dist/core/agent.d.ts +98 -0
  38. package/dist/core/agent.js +558 -0
  39. package/dist/core/answer.d.ts +42 -0
  40. package/dist/core/answer.js +395 -0
  41. package/dist/core/application-tracker.d.ts +84 -0
  42. package/dist/core/application-tracker.js +184 -0
  43. package/dist/core/apply.d.ts +162 -0
  44. package/dist/core/apply.js +816 -0
  45. package/dist/core/auth-detection.d.ts +35 -0
  46. package/dist/core/auth-detection.js +358 -0
  47. package/dist/core/auto-extract.d.ts +82 -0
  48. package/dist/core/auto-extract.js +604 -0
  49. package/dist/core/auto-interact.d.ts +23 -0
  50. package/dist/core/auto-interact.js +246 -0
  51. package/dist/core/bm25-filter.d.ts +66 -0
  52. package/dist/core/bm25-filter.js +288 -0
  53. package/dist/core/branding.d.ts +54 -0
  54. package/dist/core/branding.js +234 -0
  55. package/dist/core/browser-fetch.d.ts +323 -0
  56. package/dist/core/browser-fetch.js +1600 -0
  57. package/dist/core/browser-pool.d.ts +91 -0
  58. package/dist/core/browser-pool.js +550 -0
  59. package/dist/core/budget.d.ts +42 -0
  60. package/dist/core/budget.js +324 -0
  61. package/dist/core/business-intel.d.ts +47 -0
  62. package/dist/core/business-intel.js +279 -0
  63. package/dist/core/cache.d.ts +13 -0
  64. package/dist/core/cache.js +121 -0
  65. package/dist/core/cf-worker-proxy.d.ts +32 -0
  66. package/dist/core/cf-worker-proxy.js +87 -0
  67. package/dist/core/challenge-detection.d.ts +26 -0
  68. package/dist/core/challenge-detection.js +468 -0
  69. package/dist/core/change-tracking.d.ts +75 -0
  70. package/dist/core/change-tracking.js +276 -0
  71. package/dist/core/chunker.d.ts +46 -0
  72. package/dist/core/chunker.js +249 -0
  73. package/dist/core/chunking.d.ts +42 -0
  74. package/dist/core/chunking.js +181 -0
  75. package/dist/core/circuit-breaker.d.ts +44 -0
  76. package/dist/core/circuit-breaker.js +85 -0
  77. package/dist/core/content-pruner.d.ts +47 -0
  78. package/dist/core/content-pruner.js +425 -0
  79. package/dist/core/cookie-cache.d.ts +60 -0
  80. package/dist/core/cookie-cache.js +163 -0
  81. package/dist/core/crawl-checkpoint.d.ts +54 -0
  82. package/dist/core/crawl-checkpoint.js +104 -0
  83. package/dist/core/crawler.d.ts +84 -0
  84. package/dist/core/crawler.js +349 -0
  85. package/dist/core/cross-verify.d.ts +27 -0
  86. package/dist/core/cross-verify.js +93 -0
  87. package/dist/core/deep-fetch.d.ts +74 -0
  88. package/dist/core/deep-fetch.js +405 -0
  89. package/dist/core/deep-research.d.ts +141 -0
  90. package/dist/core/deep-research.js +972 -0
  91. package/dist/core/design-analysis.d.ts +70 -0
  92. package/dist/core/design-analysis.js +490 -0
  93. package/dist/core/design-compare.d.ts +38 -0
  94. package/dist/core/design-compare.js +264 -0
  95. package/dist/core/diff.d.ts +61 -0
  96. package/dist/core/diff.js +289 -0
  97. package/dist/core/dns-cache.d.ts +20 -0
  98. package/dist/core/dns-cache.js +198 -0
  99. package/dist/core/documents.d.ts +23 -0
  100. package/dist/core/documents.js +123 -0
  101. package/dist/core/domain-memory.d.ts +66 -0
  102. package/dist/core/domain-memory.js +163 -0
  103. package/dist/core/domain-verify.d.ts +40 -0
  104. package/dist/core/domain-verify.js +379 -0
  105. package/dist/core/engine-ranker.d.ts +112 -0
  106. package/dist/core/engine-ranker.js +395 -0
  107. package/dist/core/extract-inline.d.ts +38 -0
  108. package/dist/core/extract-inline.js +215 -0
  109. package/dist/core/extract-listings.d.ts +38 -0
  110. package/dist/core/extract-listings.js +461 -0
  111. package/dist/core/extract.d.ts +9 -0
  112. package/dist/core/extract.js +139 -0
  113. package/dist/core/fetch-cache.d.ts +57 -0
  114. package/dist/core/fetch-cache.js +95 -0
  115. package/dist/core/fetcher.d.ts +13 -0
  116. package/dist/core/fetcher.js +12 -0
  117. package/dist/core/google-cache.d.ts +29 -0
  118. package/dist/core/google-cache.js +180 -0
  119. package/dist/core/google-serp-parser.d.ts +82 -0
  120. package/dist/core/google-serp-parser.js +287 -0
  121. package/dist/core/hotel-search.d.ts +122 -0
  122. package/dist/core/hotel-search.js +382 -0
  123. package/dist/core/http-fetch.d.ts +72 -0
  124. package/dist/core/http-fetch.js +820 -0
  125. package/dist/core/human.d.ts +175 -0
  126. package/dist/core/human.js +680 -0
  127. package/dist/core/image-caption.d.ts +44 -0
  128. package/dist/core/image-caption.js +271 -0
  129. package/dist/core/jobs.d.ts +75 -0
  130. package/dist/core/jobs.js +634 -0
  131. package/dist/core/json-ld.d.ts +15 -0
  132. package/dist/core/json-ld.js +617 -0
  133. package/dist/core/language-detect.d.ts +18 -0
  134. package/dist/core/language-detect.js +135 -0
  135. package/dist/core/links.d.ts +10 -0
  136. package/dist/core/links.js +44 -0
  137. package/dist/core/llm-extract.d.ts +71 -0
  138. package/dist/core/llm-extract.js +507 -0
  139. package/dist/core/llm-provider.d.ts +100 -0
  140. package/dist/core/llm-provider.js +702 -0
  141. package/dist/core/local-search.d.ts +60 -0
  142. package/dist/core/local-search.js +308 -0
  143. package/dist/core/logger.d.ts +28 -0
  144. package/dist/core/logger.js +104 -0
  145. package/dist/core/map.d.ts +33 -0
  146. package/dist/core/map.js +127 -0
  147. package/dist/core/markdown.d.ts +92 -0
  148. package/dist/core/markdown.js +809 -0
  149. package/dist/core/metadata.d.ts +34 -0
  150. package/dist/core/metadata.js +422 -0
  151. package/dist/core/observe.d.ts +113 -0
  152. package/dist/core/observe.js +395 -0
  153. package/dist/core/ocr.d.ts +12 -0
  154. package/dist/core/ocr.js +33 -0
  155. package/dist/core/paginate.d.ts +31 -0
  156. package/dist/core/paginate.js +106 -0
  157. package/dist/core/pdf.d.ts +8 -0
  158. package/dist/core/pdf.js +25 -0
  159. package/dist/core/peel-tls.d.ts +25 -0
  160. package/dist/core/peel-tls.js +220 -0
  161. package/dist/core/pipeline.d.ts +132 -0
  162. package/dist/core/pipeline.js +1666 -0
  163. package/dist/core/profiles.d.ts +61 -0
  164. package/dist/core/profiles.js +350 -0
  165. package/dist/core/prompt-guard.d.ts +30 -0
  166. package/dist/core/prompt-guard.js +119 -0
  167. package/dist/core/proxy-config.d.ts +90 -0
  168. package/dist/core/proxy-config.js +172 -0
  169. package/dist/core/quick-answer.d.ts +53 -0
  170. package/dist/core/quick-answer.js +833 -0
  171. package/dist/core/rate-governor.d.ts +80 -0
  172. package/dist/core/rate-governor.js +238 -0
  173. package/dist/core/readability.d.ts +57 -0
  174. package/dist/core/readability.js +533 -0
  175. package/dist/core/research.d.ts +66 -0
  176. package/dist/core/research.js +270 -0
  177. package/dist/core/retry.d.ts +60 -0
  178. package/dist/core/retry.js +119 -0
  179. package/dist/core/safe-browsing.d.ts +30 -0
  180. package/dist/core/safe-browsing.js +206 -0
  181. package/dist/core/schema-extraction.d.ts +66 -0
  182. package/dist/core/schema-extraction.js +352 -0
  183. package/dist/core/schema-postprocess.d.ts +32 -0
  184. package/dist/core/schema-postprocess.js +469 -0
  185. package/dist/core/schema-templates.d.ts +19 -0
  186. package/dist/core/schema-templates.js +143 -0
  187. package/dist/core/screenshot.d.ts +224 -0
  188. package/dist/core/screenshot.js +207 -0
  189. package/dist/core/search-engines.d.ts +25 -0
  190. package/dist/core/search-engines.js +182 -0
  191. package/dist/core/search-provider.d.ts +243 -0
  192. package/dist/core/search-provider.js +1629 -0
  193. package/dist/core/searxng-provider.d.ts +35 -0
  194. package/dist/core/searxng-provider.js +105 -0
  195. package/dist/core/selective-evidence.d.ts +151 -0
  196. package/dist/core/selective-evidence.js +389 -0
  197. package/dist/core/site-search.d.ts +44 -0
  198. package/dist/core/site-search.js +252 -0
  199. package/dist/core/sitemap.d.ts +23 -0
  200. package/dist/core/sitemap.js +105 -0
  201. package/dist/core/source-credibility.d.ts +29 -0
  202. package/dist/core/source-credibility.js +584 -0
  203. package/dist/core/source-scoring.d.ts +166 -0
  204. package/dist/core/source-scoring.js +396 -0
  205. package/dist/core/stemmer.d.ts +38 -0
  206. package/dist/core/stemmer.js +509 -0
  207. package/dist/core/strategies.d.ts +104 -0
  208. package/dist/core/strategies.js +1044 -0
  209. package/dist/core/strategy-hooks.d.ts +145 -0
  210. package/dist/core/strategy-hooks.js +74 -0
  211. package/dist/core/structured-extract.d.ts +43 -0
  212. package/dist/core/structured-extract.js +550 -0
  213. package/dist/core/summarize.d.ts +17 -0
  214. package/dist/core/summarize.js +78 -0
  215. package/dist/core/synonyms.d.ts +42 -0
  216. package/dist/core/synonyms.js +184 -0
  217. package/dist/core/system-monitor.d.ts +61 -0
  218. package/dist/core/system-monitor.js +133 -0
  219. package/dist/core/table-format.d.ts +30 -0
  220. package/dist/core/table-format.js +146 -0
  221. package/dist/core/threat-feeds.d.ts +23 -0
  222. package/dist/core/threat-feeds.js +104 -0
  223. package/dist/core/timing.d.ts +21 -0
  224. package/dist/core/timing.js +33 -0
  225. package/dist/core/transcript-export.d.ts +47 -0
  226. package/dist/core/transcript-export.js +107 -0
  227. package/dist/core/user-agents.d.ts +82 -0
  228. package/dist/core/user-agents.js +239 -0
  229. package/dist/core/vertical-search.d.ts +54 -0
  230. package/dist/core/vertical-search.js +158 -0
  231. package/dist/core/watch-manager.d.ts +175 -0
  232. package/dist/core/watch-manager.js +416 -0
  233. package/dist/core/watch.d.ts +101 -0
  234. package/dist/core/watch.js +389 -0
  235. package/dist/core/youtube.d.ts +130 -0
  236. package/dist/core/youtube.js +1175 -0
  237. package/dist/ee/challenge-re-export.d.ts +1 -0
  238. package/dist/ee/challenge-re-export.js +1 -0
  239. package/dist/ee/challenge-solver.d.ts +72 -0
  240. package/dist/ee/challenge-solver.js +720 -0
  241. package/dist/ee/domain-extractors.d.ts +8 -0
  242. package/dist/ee/domain-extractors.js +8 -0
  243. package/dist/ee/domain-intel.d.ts +16 -0
  244. package/dist/ee/domain-intel.js +133 -0
  245. package/dist/ee/extractors/allrecipes.d.ts +2 -0
  246. package/dist/ee/extractors/allrecipes.js +120 -0
  247. package/dist/ee/extractors/amazon.d.ts +2 -0
  248. package/dist/ee/extractors/amazon.js +78 -0
  249. package/dist/ee/extractors/arxiv.d.ts +2 -0
  250. package/dist/ee/extractors/arxiv.js +137 -0
  251. package/dist/ee/extractors/bestbuy.d.ts +2 -0
  252. package/dist/ee/extractors/bestbuy.js +78 -0
  253. package/dist/ee/extractors/carscom.d.ts +2 -0
  254. package/dist/ee/extractors/carscom.js +121 -0
  255. package/dist/ee/extractors/coingecko.d.ts +2 -0
  256. package/dist/ee/extractors/coingecko.js +134 -0
  257. package/dist/ee/extractors/craigslist.d.ts +2 -0
  258. package/dist/ee/extractors/craigslist.js +92 -0
  259. package/dist/ee/extractors/devto.d.ts +2 -0
  260. package/dist/ee/extractors/devto.js +135 -0
  261. package/dist/ee/extractors/ebay.d.ts +2 -0
  262. package/dist/ee/extractors/ebay.js +90 -0
  263. package/dist/ee/extractors/espn.d.ts +2 -0
  264. package/dist/ee/extractors/espn.js +260 -0
  265. package/dist/ee/extractors/etsy.d.ts +2 -0
  266. package/dist/ee/extractors/etsy.js +52 -0
  267. package/dist/ee/extractors/facebook.d.ts +2 -0
  268. package/dist/ee/extractors/facebook.js +46 -0
  269. package/dist/ee/extractors/github.d.ts +2 -0
  270. package/dist/ee/extractors/github.js +196 -0
  271. package/dist/ee/extractors/google-flights.d.ts +2 -0
  272. package/dist/ee/extractors/google-flights.js +176 -0
  273. package/dist/ee/extractors/hackernews.d.ts +2 -0
  274. package/dist/ee/extractors/hackernews.js +147 -0
  275. package/dist/ee/extractors/imdb.d.ts +2 -0
  276. package/dist/ee/extractors/imdb.js +172 -0
  277. package/dist/ee/extractors/index.d.ts +26 -0
  278. package/dist/ee/extractors/index.js +247 -0
  279. package/dist/ee/extractors/instagram.d.ts +2 -0
  280. package/dist/ee/extractors/instagram.js +102 -0
  281. package/dist/ee/extractors/kalshi.d.ts +2 -0
  282. package/dist/ee/extractors/kalshi.js +121 -0
  283. package/dist/ee/extractors/kayak-cars.d.ts +2 -0
  284. package/dist/ee/extractors/kayak-cars.js +270 -0
  285. package/dist/ee/extractors/linkedin.d.ts +2 -0
  286. package/dist/ee/extractors/linkedin.js +113 -0
  287. package/dist/ee/extractors/medium.d.ts +2 -0
  288. package/dist/ee/extractors/medium.js +130 -0
  289. package/dist/ee/extractors/news.d.ts +4 -0
  290. package/dist/ee/extractors/news.js +173 -0
  291. package/dist/ee/extractors/npm.d.ts +2 -0
  292. package/dist/ee/extractors/npm.js +86 -0
  293. package/dist/ee/extractors/pdf.d.ts +2 -0
  294. package/dist/ee/extractors/pdf.js +108 -0
  295. package/dist/ee/extractors/pinterest.d.ts +2 -0
  296. package/dist/ee/extractors/pinterest.js +34 -0
  297. package/dist/ee/extractors/polymarket.d.ts +2 -0
  298. package/dist/ee/extractors/polymarket.js +358 -0
  299. package/dist/ee/extractors/producthunt.d.ts +2 -0
  300. package/dist/ee/extractors/producthunt.js +88 -0
  301. package/dist/ee/extractors/pubmed.d.ts +2 -0
  302. package/dist/ee/extractors/pubmed.js +162 -0
  303. package/dist/ee/extractors/pypi.d.ts +2 -0
  304. package/dist/ee/extractors/pypi.js +80 -0
  305. package/dist/ee/extractors/reddit.d.ts +2 -0
  306. package/dist/ee/extractors/reddit.js +438 -0
  307. package/dist/ee/extractors/redfin.d.ts +2 -0
  308. package/dist/ee/extractors/redfin.js +156 -0
  309. package/dist/ee/extractors/semanticscholar.d.ts +2 -0
  310. package/dist/ee/extractors/semanticscholar.js +131 -0
  311. package/dist/ee/extractors/shared.d.ts +12 -0
  312. package/dist/ee/extractors/shared.js +76 -0
  313. package/dist/ee/extractors/soundcloud.d.ts +2 -0
  314. package/dist/ee/extractors/soundcloud.js +34 -0
  315. package/dist/ee/extractors/sportsbetting.d.ts +2 -0
  316. package/dist/ee/extractors/sportsbetting.js +37 -0
  317. package/dist/ee/extractors/spotify.d.ts +2 -0
  318. package/dist/ee/extractors/spotify.js +34 -0
  319. package/dist/ee/extractors/stackoverflow.d.ts +2 -0
  320. package/dist/ee/extractors/stackoverflow.js +61 -0
  321. package/dist/ee/extractors/substack.d.ts +2 -0
  322. package/dist/ee/extractors/substack.js +115 -0
  323. package/dist/ee/extractors/substackroot.d.ts +2 -0
  324. package/dist/ee/extractors/substackroot.js +46 -0
  325. package/dist/ee/extractors/tiktok.d.ts +2 -0
  326. package/dist/ee/extractors/tiktok.js +29 -0
  327. package/dist/ee/extractors/tradingview.d.ts +2 -0
  328. package/dist/ee/extractors/tradingview.js +182 -0
  329. package/dist/ee/extractors/twitch.d.ts +2 -0
  330. package/dist/ee/extractors/twitch.js +36 -0
  331. package/dist/ee/extractors/twitter.d.ts +2 -0
  332. package/dist/ee/extractors/twitter.js +327 -0
  333. package/dist/ee/extractors/types.d.ts +14 -0
  334. package/dist/ee/extractors/types.js +1 -0
  335. package/dist/ee/extractors/walmart.d.ts +2 -0
  336. package/dist/ee/extractors/walmart.js +50 -0
  337. package/dist/ee/extractors/weather.d.ts +2 -0
  338. package/dist/ee/extractors/weather.js +133 -0
  339. package/dist/ee/extractors/wikipedia.d.ts +4 -0
  340. package/dist/ee/extractors/wikipedia.js +235 -0
  341. package/dist/ee/extractors/yelp.d.ts +2 -0
  342. package/dist/ee/extractors/yelp.js +216 -0
  343. package/dist/ee/extractors/youtube.d.ts +2 -0
  344. package/dist/ee/extractors/youtube.js +189 -0
  345. package/dist/ee/extractors/zillow.d.ts +54 -0
  346. package/dist/ee/extractors/zillow.js +247 -0
  347. package/dist/ee/extractors-re-export.d.ts +1 -0
  348. package/dist/ee/extractors-re-export.js +1 -0
  349. package/dist/ee/premium-hooks.d.ts +20 -0
  350. package/dist/ee/premium-hooks.js +50 -0
  351. package/dist/ee/spa-detection.d.ts +2 -0
  352. package/dist/ee/spa-detection.js +2 -0
  353. package/dist/ee/stability.d.ts +4 -0
  354. package/dist/ee/stability.js +29 -0
  355. package/dist/ee/swr-cache.d.ts +14 -0
  356. package/dist/ee/swr-cache.js +34 -0
  357. package/dist/index.d.ts +143 -0
  358. package/dist/index.js +291 -0
  359. package/dist/integrations/index.d.ts +2 -0
  360. package/dist/integrations/index.js +2 -0
  361. package/dist/integrations/langchain.d.ts +64 -0
  362. package/dist/integrations/langchain.js +115 -0
  363. package/dist/integrations/llamaindex.d.ts +50 -0
  364. package/dist/integrations/llamaindex.js +91 -0
  365. package/dist/mcp/handlers/act.d.ts +5 -0
  366. package/dist/mcp/handlers/act.js +34 -0
  367. package/dist/mcp/handlers/definitions.d.ts +6 -0
  368. package/dist/mcp/handlers/definitions.js +395 -0
  369. package/dist/mcp/handlers/extract.d.ts +7 -0
  370. package/dist/mcp/handlers/extract.js +135 -0
  371. package/dist/mcp/handlers/fetch.d.ts +6 -0
  372. package/dist/mcp/handlers/fetch.js +98 -0
  373. package/dist/mcp/handlers/find.d.ts +5 -0
  374. package/dist/mcp/handlers/find.js +137 -0
  375. package/dist/mcp/handlers/index.d.ts +13 -0
  376. package/dist/mcp/handlers/index.js +63 -0
  377. package/dist/mcp/handlers/legacy.d.ts +25 -0
  378. package/dist/mcp/handlers/legacy.js +450 -0
  379. package/dist/mcp/handlers/meta.d.ts +6 -0
  380. package/dist/mcp/handlers/meta.js +40 -0
  381. package/dist/mcp/handlers/monitor.d.ts +5 -0
  382. package/dist/mcp/handlers/monitor.js +41 -0
  383. package/dist/mcp/handlers/observe.d.ts +8 -0
  384. package/dist/mcp/handlers/observe.js +37 -0
  385. package/dist/mcp/handlers/read.d.ts +6 -0
  386. package/dist/mcp/handlers/read.js +78 -0
  387. package/dist/mcp/handlers/see.d.ts +5 -0
  388. package/dist/mcp/handlers/see.js +75 -0
  389. package/dist/mcp/handlers/types.d.ts +29 -0
  390. package/dist/mcp/handlers/types.js +28 -0
  391. package/dist/mcp/server.d.ts +7 -0
  392. package/dist/mcp/server.js +108 -0
  393. package/dist/mcp/smart-router.d.ts +23 -0
  394. package/dist/mcp/smart-router.js +178 -0
  395. package/dist/server/app.d.ts +14 -0
  396. package/dist/server/app.js +632 -0
  397. package/dist/server/auth-store.d.ts +28 -0
  398. package/dist/server/auth-store.js +88 -0
  399. package/dist/server/bull-queues.d.ts +60 -0
  400. package/dist/server/bull-queues.js +90 -0
  401. package/dist/server/email-service.d.ts +55 -0
  402. package/dist/server/email-service.js +291 -0
  403. package/dist/server/job-queue.d.ts +100 -0
  404. package/dist/server/job-queue.js +145 -0
  405. package/dist/server/logger.d.ts +10 -0
  406. package/dist/server/logger.js +37 -0
  407. package/dist/server/middleware/audit-log.d.ts +14 -0
  408. package/dist/server/middleware/audit-log.js +73 -0
  409. package/dist/server/middleware/auth.d.ts +35 -0
  410. package/dist/server/middleware/auth.js +225 -0
  411. package/dist/server/middleware/rate-limit.d.ts +50 -0
  412. package/dist/server/middleware/rate-limit.js +270 -0
  413. package/dist/server/middleware/scope-guard.d.ts +25 -0
  414. package/dist/server/middleware/scope-guard.js +45 -0
  415. package/dist/server/middleware/url-validator.d.ts +15 -0
  416. package/dist/server/middleware/url-validator.js +201 -0
  417. package/dist/server/openapi.yaml +6418 -0
  418. package/dist/server/pg-auth-store.d.ts +146 -0
  419. package/dist/server/pg-auth-store.js +576 -0
  420. package/dist/server/pg-job-queue.d.ts +59 -0
  421. package/dist/server/pg-job-queue.js +375 -0
  422. package/dist/server/routes/activity.d.ts +6 -0
  423. package/dist/server/routes/activity.js +79 -0
  424. package/dist/server/routes/admin-active.d.ts +7 -0
  425. package/dist/server/routes/admin-active.js +120 -0
  426. package/dist/server/routes/admin-stats.d.ts +7 -0
  427. package/dist/server/routes/admin-stats.js +176 -0
  428. package/dist/server/routes/agent.d.ts +24 -0
  429. package/dist/server/routes/agent.js +480 -0
  430. package/dist/server/routes/answer.d.ts +5 -0
  431. package/dist/server/routes/answer.js +125 -0
  432. package/dist/server/routes/ask.d.ts +28 -0
  433. package/dist/server/routes/ask.js +295 -0
  434. package/dist/server/routes/batch.d.ts +6 -0
  435. package/dist/server/routes/batch.js +493 -0
  436. package/dist/server/routes/cache-warm.d.ts +25 -0
  437. package/dist/server/routes/cache-warm.js +212 -0
  438. package/dist/server/routes/cli-usage.d.ts +6 -0
  439. package/dist/server/routes/cli-usage.js +127 -0
  440. package/dist/server/routes/compat.d.ts +23 -0
  441. package/dist/server/routes/compat.js +652 -0
  442. package/dist/server/routes/crawl.d.ts +13 -0
  443. package/dist/server/routes/crawl.js +287 -0
  444. package/dist/server/routes/deep-fetch.d.ts +8 -0
  445. package/dist/server/routes/deep-fetch.js +57 -0
  446. package/dist/server/routes/deep-research.d.ts +11 -0
  447. package/dist/server/routes/deep-research.js +232 -0
  448. package/dist/server/routes/demo.d.ts +24 -0
  449. package/dist/server/routes/demo.js +517 -0
  450. package/dist/server/routes/do.d.ts +8 -0
  451. package/dist/server/routes/do.js +72 -0
  452. package/dist/server/routes/extract.d.ts +14 -0
  453. package/dist/server/routes/extract.js +325 -0
  454. package/dist/server/routes/feed.d.ts +15 -0
  455. package/dist/server/routes/feed.js +311 -0
  456. package/dist/server/routes/fetch-queue.d.ts +13 -0
  457. package/dist/server/routes/fetch-queue.js +357 -0
  458. package/dist/server/routes/fetch.d.ts +7 -0
  459. package/dist/server/routes/fetch.js +1274 -0
  460. package/dist/server/routes/go.d.ts +14 -0
  461. package/dist/server/routes/go.js +81 -0
  462. package/dist/server/routes/health.d.ts +11 -0
  463. package/dist/server/routes/health.js +141 -0
  464. package/dist/server/routes/jobs.d.ts +7 -0
  465. package/dist/server/routes/jobs.js +574 -0
  466. package/dist/server/routes/map.d.ts +11 -0
  467. package/dist/server/routes/map.js +116 -0
  468. package/dist/server/routes/mcp.d.ts +14 -0
  469. package/dist/server/routes/mcp.js +197 -0
  470. package/dist/server/routes/metrics.d.ts +37 -0
  471. package/dist/server/routes/metrics.js +149 -0
  472. package/dist/server/routes/oauth.d.ts +9 -0
  473. package/dist/server/routes/oauth.js +396 -0
  474. package/dist/server/routes/playground.d.ts +17 -0
  475. package/dist/server/routes/playground.js +283 -0
  476. package/dist/server/routes/reader.d.ts +18 -0
  477. package/dist/server/routes/reader.js +192 -0
  478. package/dist/server/routes/research.d.ts +14 -0
  479. package/dist/server/routes/research.js +482 -0
  480. package/dist/server/routes/screenshot.d.ts +22 -0
  481. package/dist/server/routes/screenshot.js +820 -0
  482. package/dist/server/routes/search.d.ts +6 -0
  483. package/dist/server/routes/search.js +874 -0
  484. package/dist/server/routes/session.d.ts +17 -0
  485. package/dist/server/routes/session.js +548 -0
  486. package/dist/server/routes/share.d.ts +18 -0
  487. package/dist/server/routes/share.js +462 -0
  488. package/dist/server/routes/smart-search/handlers/cars.d.ts +2 -0
  489. package/dist/server/routes/smart-search/handlers/cars.js +102 -0
  490. package/dist/server/routes/smart-search/handlers/flights.d.ts +2 -0
  491. package/dist/server/routes/smart-search/handlers/flights.js +72 -0
  492. package/dist/server/routes/smart-search/handlers/general.d.ts +13 -0
  493. package/dist/server/routes/smart-search/handlers/general.js +717 -0
  494. package/dist/server/routes/smart-search/handlers/hotels.d.ts +2 -0
  495. package/dist/server/routes/smart-search/handlers/hotels.js +88 -0
  496. package/dist/server/routes/smart-search/handlers/products.d.ts +2 -0
  497. package/dist/server/routes/smart-search/handlers/products.js +1309 -0
  498. package/dist/server/routes/smart-search/handlers/rental.d.ts +2 -0
  499. package/dist/server/routes/smart-search/handlers/rental.js +154 -0
  500. package/dist/server/routes/smart-search/handlers/restaurants.d.ts +2 -0
  501. package/dist/server/routes/smart-search/handlers/restaurants.js +225 -0
  502. package/dist/server/routes/smart-search/handlers/transit-verdict.d.ts +41 -0
  503. package/dist/server/routes/smart-search/handlers/transit-verdict.js +224 -0
  504. package/dist/server/routes/smart-search/index.d.ts +19 -0
  505. package/dist/server/routes/smart-search/index.js +546 -0
  506. package/dist/server/routes/smart-search/intent.d.ts +3 -0
  507. package/dist/server/routes/smart-search/intent.js +264 -0
  508. package/dist/server/routes/smart-search/llm.d.ts +16 -0
  509. package/dist/server/routes/smart-search/llm.js +70 -0
  510. package/dist/server/routes/smart-search/sources/reddit.d.ts +18 -0
  511. package/dist/server/routes/smart-search/sources/reddit.js +34 -0
  512. package/dist/server/routes/smart-search/sources/yelp.d.ts +25 -0
  513. package/dist/server/routes/smart-search/sources/yelp.js +171 -0
  514. package/dist/server/routes/smart-search/sources/youtube.d.ts +8 -0
  515. package/dist/server/routes/smart-search/sources/youtube.js +9 -0
  516. package/dist/server/routes/smart-search/types.d.ts +81 -0
  517. package/dist/server/routes/smart-search/types.js +1 -0
  518. package/dist/server/routes/smart-search/utils.d.ts +20 -0
  519. package/dist/server/routes/smart-search/utils.js +146 -0
  520. package/dist/server/routes/stats.d.ts +6 -0
  521. package/dist/server/routes/stats.js +71 -0
  522. package/dist/server/routes/stripe.d.ts +15 -0
  523. package/dist/server/routes/stripe.js +296 -0
  524. package/dist/server/routes/transcript-export.d.ts +10 -0
  525. package/dist/server/routes/transcript-export.js +178 -0
  526. package/dist/server/routes/usage.d.ts +9 -0
  527. package/dist/server/routes/usage.js +279 -0
  528. package/dist/server/routes/users.d.ts +8 -0
  529. package/dist/server/routes/users.js +1867 -0
  530. package/dist/server/routes/watch.d.ts +15 -0
  531. package/dist/server/routes/watch.js +309 -0
  532. package/dist/server/routes/webhooks.d.ts +26 -0
  533. package/dist/server/routes/webhooks.js +170 -0
  534. package/dist/server/routes/youtube.d.ts +6 -0
  535. package/dist/server/routes/youtube.js +130 -0
  536. package/dist/server/sentry.d.ts +14 -0
  537. package/dist/server/sentry.js +104 -0
  538. package/dist/server/types.d.ts +15 -0
  539. package/dist/server/types.js +7 -0
  540. package/dist/server/utils/response.d.ts +44 -0
  541. package/dist/server/utils/response.js +69 -0
  542. package/dist/server/utils/sse.d.ts +22 -0
  543. package/dist/server/utils/sse.js +38 -0
  544. package/dist/types.d.ts +552 -0
  545. package/dist/types.js +39 -0
  546. package/llms.txt +105 -0
  547. package/package.json +189 -0
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Smart Search endpoint — intent detection + travel/commerce routing
3
+ * POST /v1/search/smart
4
+ *
5
+ * Detects user intent from natural language and routes to the best source:
6
+ * - cars → Cars.com with browser rendering + structured extraction
7
+ * - flights → Google Flights with browser rendering + flight extractor
8
+ * - hotels → Google Hotels with browser rendering
9
+ * - rental → Kayak with browser rendering + rental extractor
10
+ * - restaurants → Yelp Fusion API extractor
11
+ * - products → Amazon search with structured extraction
12
+ * - general → SearXNG with smart enrichment (peel() for top 3)
13
+ */
14
+ import { Router } from 'express';
15
+ import '../../types.js';
16
+ import { AuthStore } from '../../auth-store.js';
17
+ export type { SearchIntent, SmartSearchResult, TransactionalVerdict, VerdictOption } from './types.js';
18
+ export { detectSearchIntent } from './intent.js';
19
+ export declare function createSmartSearchRouter(authStore: AuthStore): Router;
@@ -0,0 +1,546 @@
1
+ /**
2
+ * Smart Search endpoint — intent detection + travel/commerce routing
3
+ * POST /v1/search/smart
4
+ *
5
+ * Detects user intent from natural language and routes to the best source:
6
+ * - cars → Cars.com with browser rendering + structured extraction
7
+ * - flights → Google Flights with browser rendering + flight extractor
8
+ * - hotels → Google Hotels with browser rendering
9
+ * - rental → Kayak with browser rendering + rental extractor
10
+ * - restaurants → Yelp Fusion API extractor
11
+ * - products → Amazon search with structured extraction
12
+ * - general → SearXNG with smart enrichment (peel() for top 3)
13
+ */
14
+ import { Router } from 'express';
15
+ import '../../types.js'; // Augments Express.Request with requestId, auth
16
+ // @ts-ignore — ioredis CJS/ESM interop
17
+ import IoRedisModule from 'ioredis';
18
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
19
+ const IoRedis = IoRedisModule.default ?? IoRedisModule;
20
+ export { detectSearchIntent } from './intent.js';
21
+ import { detectSearchIntent, classifyIntentWithLLM } from './intent.js';
22
+ import { callLLMQuick, sanitizeSearchQuery, PROMPT_INJECTION_DEFENSE } from './llm.js';
23
+ import { handleCarSearch } from './handlers/cars.js';
24
+ import { handleFlightSearch } from './handlers/flights.js';
25
+ import { handleHotelSearch } from './handlers/hotels.js';
26
+ import { handleRentalSearch } from './handlers/rental.js';
27
+ import { handleRestaurantSearch } from './handlers/restaurants.js';
28
+ import { handleProductSearch } from './handlers/products.js';
29
+ import { handleGeneralSearch } from './handlers/general.js';
30
+ import { fetchYelpResults } from './sources/yelp.js';
31
+ import { fetchRedditResults } from './sources/reddit.js';
32
+ import { fetchYouTubeResults } from './sources/youtube.js';
33
+ // ─── Redis client (lazy singleton for smart-search caching) ───────────────
34
+ function buildSmartRedis() {
35
+ const url = process.env.REDIS_URL || 'redis://redis:6379';
36
+ const password = process.env.REDIS_PASSWORD || undefined;
37
+ try {
38
+ const parsed = new URL(url);
39
+ return new IoRedis({
40
+ host: parsed.hostname,
41
+ port: parseInt(parsed.port || '6379', 10),
42
+ password,
43
+ db: parseInt(parsed.pathname?.slice(1) || '0', 10) || 0,
44
+ lazyConnect: true,
45
+ maxRetriesPerRequest: 1,
46
+ enableOfflineQueue: false,
47
+ });
48
+ }
49
+ catch {
50
+ return new IoRedis({ host: 'redis', port: 6379, password, lazyConnect: true, maxRetriesPerRequest: 1, enableOfflineQueue: false });
51
+ }
52
+ }
53
+ let _smartRedis = null;
54
+ function getSmartRedis() {
55
+ if (!_smartRedis)
56
+ _smartRedis = buildSmartRedis();
57
+ return _smartRedis;
58
+ }
59
+ // TTL by intent type (seconds)
60
+ const CACHE_TTL = {
61
+ restaurants: 1800, // 30 min
62
+ cars: 900, // 15 min
63
+ products: 900, // 15 min
64
+ flights: 600, // 10 min
65
+ hotels: 600, // 10 min
66
+ rental: 1800, // 30 min
67
+ general: 3600, // 60 min
68
+ };
69
+ // ─── Loading message by intent type ────────────────────────────────────────
70
+ function getLoadingMessage(type) {
71
+ const msgs = {
72
+ cars: 'Searching cars on Cars.com…',
73
+ flights: 'Searching for flights...',
74
+ hotels: 'Searching for hotels...',
75
+ rental: 'Searching for rental cars...',
76
+ restaurants: 'Finding restaurants on Yelp…',
77
+ products: 'Searching Amazon for products…',
78
+ general: '🔍 Searching and analyzing results...',
79
+ };
80
+ return msgs[type] || 'Searching…';
81
+ }
82
+ // ─── Router ────────────────────────────────────────────────────────────────
83
+ // Log LLM provider at startup (uses unified config from core)
84
+ {
85
+ // Dynamic import to avoid top-level await issues
86
+ import('../../../core/llm-provider.js').then(({ getQuickLLMConfig }) => {
87
+ const cfg = getQuickLLMConfig();
88
+ const _llmProvider = cfg?.provider ?? 'none';
89
+ const _llmModel = cfg?.model ?? 'n/a';
90
+ const _baseUrl = cfg?.baseUrl;
91
+ const suffix = _baseUrl ? ` via ${_baseUrl}` : '';
92
+ console.log(`[smart-search] LLM provider: ${_llmProvider} (${_llmModel}${suffix})`);
93
+ }).catch(() => {
94
+ console.log('[smart-search] LLM provider: none (config load failed)');
95
+ });
96
+ }
97
+ export function createSmartSearchRouter(authStore) {
98
+ const router = Router();
99
+ router.post('/v1/search/smart', async (req, res) => {
100
+ try {
101
+ // Authentication: API key OR anonymous (rate-limited by IP)
102
+ const authId = req.auth?.keyInfo?.accountId || req.user?.userId;
103
+ const isAnonymous = !authId;
104
+ if (isAnonymous) {
105
+ // Rate limit anonymous users: 5 searches per day per IP
106
+ const clientIp = req.headers['x-forwarded-for']?.split(',')[0]?.trim()
107
+ || req.headers['cf-connecting-ip']
108
+ || req.socket.remoteAddress
109
+ || 'unknown';
110
+ const anonKey = `anon:smart:${clientIp}`;
111
+ try {
112
+ const redis = getSmartRedis();
113
+ const count = await redis.incr(anonKey);
114
+ if (count === 1) {
115
+ // Set 24-hour expiry on first request
116
+ await redis.expire(anonKey, 86400);
117
+ }
118
+ if (count > 5) {
119
+ res.status(429).json({
120
+ success: false,
121
+ error: {
122
+ type: 'anonymous_limit_exceeded',
123
+ message: 'Free search limit reached (5/day). Sign up for 500 fetches/week free.',
124
+ signupUrl: 'https://app.webpeel.dev/signup',
125
+ },
126
+ requestId: req.requestId,
127
+ });
128
+ return;
129
+ }
130
+ }
131
+ catch {
132
+ // Redis failed — allow the request (graceful degradation)
133
+ }
134
+ }
135
+ const { q, location, zip, language: reqLanguage } = req.body;
136
+ if (!q || typeof q !== 'string' || !q.trim()) {
137
+ res.status(400).json({
138
+ success: false,
139
+ error: {
140
+ type: 'invalid_request',
141
+ message: 'Missing or invalid "q" field in request body',
142
+ hint: 'POST /v1/search/smart with JSON body: { "q": "your search query" }',
143
+ docs: 'https://webpeel.dev/docs/api-reference#smart-search',
144
+ },
145
+ requestId: req.requestId,
146
+ });
147
+ return;
148
+ }
149
+ const query = q.trim();
150
+ const normalizedLocation = typeof location === 'string' ? location.trim() : '';
151
+ const intent = detectSearchIntent(query);
152
+ // If regex returned 'general' as fallback (not from an explicit pattern match),
153
+ // try LLM classification to catch typos, other languages, creative phrasing.
154
+ // Skip LLM override if regex matched a specific pattern (comparison, local, service queries)
155
+ // — those were INTENTIONALLY set to 'general'.
156
+ const queryLower = query.toLowerCase();
157
+ const isExplicitGeneral = (/\b(compare|vs\.?|versus|which is better|difference between)\b/.test(queryLower) ||
158
+ (/\b(near me|near\s+\w+|open now|open today|open on|what time|is .* open|hours|closest|nearest)\b/.test(queryLower) && /\b(buy|where|store|shop|near|close to|around)\b/.test(queryLower)) ||
159
+ (/\b(plumber|electrician|mechanic|dentist|doctor|lawyer|therapist|vet|salon|barber|gym|daycare)\b/.test(queryLower) && /\b(near|in|around|open|best|cheap|emergency)\b/.test(queryLower)) ||
160
+ (/\b(cruise|vacation|resort|trip|travel|getaway|tour|safari|honeymoon|disneyland|disney|universal|six flags|theme park)\b/.test(queryLower) && /\b(cheap|cheapest|price|ticket|book|deal|package)\b/.test(queryLower)) ||
161
+ (/\b(bus|buses|coach|greyhound|flixbus|megabus|busbud|wanderu|train|trains|amtrak|ferry|ferries)\b/.test(queryLower) && /\b(ticket|tickets|book|booking|cheap|cheapest|price|fare|fares|route|take|trip|from|to)\b/.test(queryLower)));
162
+ if (intent.type === 'general' && !isExplicitGeneral && process.env.OLLAMA_URL) {
163
+ try {
164
+ const llmType = await classifyIntentWithLLM(query);
165
+ if (llmType !== 'general') {
166
+ console.log(`[smart-search] LLM reclassified "${query}" from general → ${llmType}`);
167
+ intent.type = llmType;
168
+ }
169
+ }
170
+ catch (err) {
171
+ // Graceful degradation — regex result stands
172
+ console.warn('[smart-search] LLM intent classification failed:', err.message);
173
+ }
174
+ }
175
+ // Override zip from request body if provided
176
+ if (zip && intent.params) {
177
+ intent.params.zip = zip;
178
+ }
179
+ // Also try to extract location context from request body when provided
180
+ if (normalizedLocation && intent.type === 'restaurants') {
181
+ intent.params = { ...(intent.params || {}), location: normalizedLocation };
182
+ intent.location = normalizedLocation;
183
+ }
184
+ if (normalizedLocation && intent.params?.localIntent === 'true') {
185
+ intent.params.location = normalizedLocation;
186
+ intent.params.localLocation = normalizedLocation;
187
+ intent.params.localNeedsUserLocation = 'false';
188
+ }
189
+ // ── Cache check (before streaming — HIT skips SSE entirely) ─────────
190
+ const SMART_CACHE_VERSION = 'v7'; // bump when intent routing or location wiring changes
191
+ const cacheLocation = normalizedLocation || intent.params?.location || intent.params?.localLocation || intent.params?.zip || '';
192
+ const cacheKey = `smart:${SMART_CACHE_VERSION}:${intent.type}:${query.toLowerCase().trim().replace(/\s+/g, ' ')}:${cacheLocation.toLowerCase().replace(/\s+/g, ' ') || 'none'}`;
193
+ try {
194
+ const redis = getSmartRedis();
195
+ const cached = await redis.get(cacheKey);
196
+ if (cached) {
197
+ const parsed = JSON.parse(cached);
198
+ console.log(`[smart-search] Cache HIT: ${cacheKey} (${parsed.fetchTimeMs}ms original)`);
199
+ // Ensure safety field exists on cached responses
200
+ if (!parsed.safety) {
201
+ parsed.safety = {
202
+ verified: true,
203
+ promptInjectionsBlocked: 0,
204
+ maliciousPatternsStripped: 0,
205
+ sourcesChecked: parsed.sources?.length || 0,
206
+ };
207
+ }
208
+ // Attach suggestedDomains from intent
209
+ if (intent.suggestedDomains?.length) {
210
+ parsed.suggestedDomains = intent.suggestedDomains;
211
+ }
212
+ res.setHeader('X-Intent-Type', intent.type);
213
+ res.setHeader('X-Source', parsed.source);
214
+ res.setHeader('X-Processing-Time', '0');
215
+ res.setHeader('X-Cache', 'HIT');
216
+ res.setHeader('X-Cache-Key', cacheKey);
217
+ res.setHeader('Cache-Control', 'no-store');
218
+ res.json({ success: true, data: parsed });
219
+ return;
220
+ }
221
+ }
222
+ catch (err) {
223
+ console.warn('[smart-search] Redis cache error (non-fatal):', err.message);
224
+ }
225
+ // ── SSE Streaming path ────────────────────────────────────────────────
226
+ const streamRequested = req.body?.stream === true || req.body?.stream === 'true';
227
+ if (streamRequested) {
228
+ res.setHeader('Content-Type', 'text/event-stream');
229
+ res.setHeader('Cache-Control', 'no-cache');
230
+ res.setHeader('Connection', 'keep-alive');
231
+ res.setHeader('X-Accel-Buffering', 'no');
232
+ const sendEvent = (event, data) => {
233
+ res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
234
+ if (typeof res.flush === 'function')
235
+ res.flush();
236
+ };
237
+ sendEvent('intent', {
238
+ type: intent.type,
239
+ query,
240
+ loadingMessage: getLoadingMessage(intent.type),
241
+ });
242
+ try {
243
+ const t0Stream = Date.now();
244
+ if (intent.type === 'restaurants') {
245
+ // Restaurant: stream each source as it arrives
246
+ const loc = intent.params.location || 'New York, NY';
247
+ const kw = intent.query
248
+ .replace(/\b(best|top|good|cheap|affordable|near me|near|around|in|find|search|looking for)\b/gi, '')
249
+ .replace(/\s+/g, ' ')
250
+ .trim();
251
+ let yelpData = null;
252
+ sendEvent('progress', { step: 'searching_yelp', message: 'Searching Yelp for restaurants...' });
253
+ try {
254
+ yelpData = await Promise.race([
255
+ fetchYelpResults(kw, loc),
256
+ new Promise((_, rej) => setTimeout(() => rej(new Error('yelp timeout')), 10000)),
257
+ ]);
258
+ sendEvent('progress', { step: 'yelp_done', message: `Found ${yelpData?.businesses?.length || 0} restaurants on Yelp` });
259
+ if (yelpData?.businesses?.length > 0) {
260
+ yelpData.businesses.sort((a, b) => {
261
+ const scoreA = (a.rating || 0) * Math.log2((a.reviewCount || 0) + 1);
262
+ const scoreB = (b.rating || 0) * Math.log2((b.reviewCount || 0) + 1);
263
+ return scoreB - scoreA;
264
+ });
265
+ yelpData.businesses = yelpData.businesses.filter((b) => !b.isClosed);
266
+ if (process.env.GOOGLE_PLACES_API_KEY) {
267
+ sendEvent('progress', { step: 'checking_google', message: 'Verifying hours on Google Maps...' });
268
+ }
269
+ sendEvent('source', { source: 'yelp', businesses: yelpData.businesses.slice(0, 10) });
270
+ if (process.env.GOOGLE_PLACES_API_KEY) {
271
+ sendEvent('progress', { step: 'google_done', message: 'Hours verified for top 3 restaurants' });
272
+ }
273
+ }
274
+ }
275
+ catch {
276
+ sendEvent('progress', { step: 'yelp_done', message: 'Found 0 restaurants on Yelp' });
277
+ }
278
+ sendEvent('progress', { step: 'fetching_reviews', message: 'Finding Reddit discussions and YouTube reviews...' });
279
+ const [redditSettled, youtubeSettled] = await Promise.allSettled([
280
+ Promise.race([
281
+ fetchRedditResults(kw, loc),
282
+ new Promise((_, rej) => setTimeout(() => rej(new Error('reddit timeout')), 8000)),
283
+ ]),
284
+ Promise.race([
285
+ fetchYouTubeResults(kw, loc),
286
+ new Promise((_, rej) => setTimeout(() => rej(new Error('youtube timeout')), 5000)),
287
+ ]),
288
+ ]);
289
+ const redditData = redditSettled.status === 'fulfilled' ? redditSettled.value : null;
290
+ const youtubeData = youtubeSettled.status === 'fulfilled' ? youtubeSettled.value : null;
291
+ if (redditData) {
292
+ sendEvent('source', { source: 'reddit', thread: redditData.thread, otherThreads: redditData.otherThreads });
293
+ }
294
+ if (youtubeData && youtubeData.videos?.length) {
295
+ sendEvent('source', { source: 'youtube', videos: youtubeData.videos });
296
+ }
297
+ let answer;
298
+ const ollamaUrl = process.env.OLLAMA_URL;
299
+ if (ollamaUrl && yelpData?.businesses?.length > 0) {
300
+ sendEvent('progress', { step: 'generating_ai', message: 'Generating AI recommendation...' });
301
+ try {
302
+ const yelpLines = yelpData.businesses.slice(0, 3).map((b, i) => {
303
+ const openStatus = b.isClosed ? 'PERMANENTLY CLOSED' : (b.isOpenNow ? 'OPEN NOW' : 'Closed right now');
304
+ const txns = b.transactions?.length > 0 ? `Available: ${b.transactions.join(', ')}` : '';
305
+ const googleInfo = b.googleRating ? ` | Google: ⭐${b.googleRating} (${b.googleReviewCount} reviews)` : '';
306
+ return `[${i + 1}] ${b.name} ⭐${b.rating} (${b.reviewCount?.toLocaleString()} reviews) ${b.price || ''} — ${b.address}
307
+ ${openStatus} | Today: ${b.todayHours || 'hours not available'} | ${txns} | Categories: ${b.categories || ''}${googleInfo}
308
+ URL: ${b.url || ''}`;
309
+ }).join('\n');
310
+ const yelpCitations = yelpData.businesses.slice(0, 3).map((b, i) => `[${i + 1}] ${b.url || 'yelp.com'}`).join('\n');
311
+ const redditHint = redditData && redditData.otherThreads?.slice(0, 2).map((t) => t.title).join('; ') || '';
312
+ const systemPrompt = `${PROMPT_INJECTION_DEFENSE}Recommend top 3 restaurants. For each: name with inline citation [1][2][3], why it's good, open/closed status, hours.
313
+ Cite sources inline using [1], [2], [3] notation matching the numbered sources. At the end, list Sources with their URLs.
314
+ Be specific. Max 200 words.
315
+ `;
316
+ const userMessage = `Query: ${sanitizeSearchQuery(intent.query)}\n\nTop restaurants:\n${yelpLines}${redditHint ? '\n\nReddit mentions: ' + redditHint : ''}\n\nSources:\n${yelpCitations}`;
317
+ const text = await callLLMQuick(`${systemPrompt}\n\n${userMessage}`, { maxTokens: 250, timeoutMs: 5000, temperature: 0.3 });
318
+ if (text)
319
+ answer = text;
320
+ }
321
+ catch { /* LLM failure — no answer */ }
322
+ }
323
+ if (answer) {
324
+ sendEvent('answer', { answer });
325
+ }
326
+ sendEvent('done', { fetchTimeMs: Date.now() - t0Stream, answer: answer || undefined });
327
+ // Cache the streaming result for restaurants
328
+ try {
329
+ const redis = getSmartRedis();
330
+ const ttl = CACHE_TTL[intent.type] || 600;
331
+ const yelpUrl = yelpData?.url || `https://www.yelp.com/search?find_desc=${encodeURIComponent(kw)}&find_loc=${encodeURIComponent(loc)}`;
332
+ const contentParts = [];
333
+ if (yelpData?.businesses?.length > 0) {
334
+ contentParts.push(`## Yelp (${yelpData.businesses.length} restaurants)`);
335
+ yelpData.businesses.slice(0, 10).forEach((b, i) => {
336
+ const openStatus = b.isClosed ? ' · ⛔ Permanently Closed' : (b.isOpenNow ? ' · 🟢 Open Now' : ' · 🔴 Closed');
337
+ contentParts.push(`${i + 1}. **${b.name}** ⭐${b.rating} (${(b.reviewCount || 0).toLocaleString()} reviews)${b.price ? ' · ' + b.price : ''}${openStatus}${b.address ? ' — ' + b.address : ''}`);
338
+ });
339
+ }
340
+ if (redditData) {
341
+ contentParts.push('');
342
+ contentParts.push('## Reddit Recommendations');
343
+ if (redditData.thread)
344
+ contentParts.push(`**${redditData.thread.title}**`);
345
+ }
346
+ if (youtubeData && youtubeData.videos?.length) {
347
+ contentParts.push('');
348
+ contentParts.push('## YouTube Reviews');
349
+ youtubeData.videos.forEach((v) => contentParts.push(`🎬 [${v.title}](${v.url})`));
350
+ }
351
+ const cachedSources = [];
352
+ if (yelpData)
353
+ cachedSources.push({ title: 'Yelp', url: yelpUrl, domain: 'yelp.com' });
354
+ if (redditData?.thread)
355
+ cachedSources.push({ title: redditData.thread.title, url: redditData.thread.url, domain: 'reddit.com' });
356
+ if (youtubeData?.videos?.[0])
357
+ cachedSources.push({ title: youtubeData.videos[0].title, url: youtubeData.videos[0].url, domain: 'youtube.com' });
358
+ const cacheResult = {
359
+ type: 'restaurants',
360
+ source: 'Yelp + Reddit + YouTube',
361
+ sourceUrl: yelpUrl,
362
+ content: contentParts.join('\n'),
363
+ title: `${kw} in ${loc}`,
364
+ domainData: yelpData?.domainData,
365
+ structured: yelpData?.domainData?.structured,
366
+ tokens: contentParts.join('\n').split(/\s+/).length,
367
+ fetchTimeMs: Date.now() - t0Stream,
368
+ ...(answer !== undefined ? { answer } : {}),
369
+ ...(cachedSources.length > 0 ? { sources: cachedSources } : {}),
370
+ safety: {
371
+ verified: true,
372
+ promptInjectionsBlocked: 0,
373
+ maliciousPatternsStripped: 0,
374
+ sourcesChecked: cachedSources.length,
375
+ },
376
+ ...(intent.suggestedDomains?.length ? { suggestedDomains: intent.suggestedDomains } : {}),
377
+ };
378
+ await redis.setex(cacheKey, ttl, JSON.stringify(cacheResult));
379
+ console.log(`[smart-search] SSE Cache WRITE: ${cacheKey} (TTL: ${ttl}s)`);
380
+ }
381
+ catch { /* non-fatal */ }
382
+ res.end();
383
+ }
384
+ else {
385
+ // All other intent types: run the existing handler, emit full result
386
+ const typeLabels = {
387
+ cars: 'Searching Cars.com for vehicles...',
388
+ flights: 'Finding flights and prices...',
389
+ hotels: 'Searching for hotels and rates...',
390
+ rental: 'Searching rental car prices...',
391
+ products: 'Searching for products and prices...',
392
+ general: 'Searching the web...',
393
+ };
394
+ sendEvent('progress', { step: 'searching', message: typeLabels[intent.type] || 'Searching...' });
395
+ let streamResult;
396
+ switch (intent.type) {
397
+ case 'cars':
398
+ streamResult = await handleCarSearch(intent);
399
+ break;
400
+ case 'flights':
401
+ streamResult = await handleFlightSearch(intent);
402
+ break;
403
+ case 'hotels':
404
+ streamResult = await handleHotelSearch(intent);
405
+ break;
406
+ case 'rental':
407
+ streamResult = await handleRentalSearch(intent);
408
+ break;
409
+ case 'products':
410
+ streamResult = await handleProductSearch(intent);
411
+ break;
412
+ default:
413
+ streamResult = await handleGeneralSearch(query);
414
+ }
415
+ const resultCount = streamResult.structured?.listings?.length ?? streamResult.results?.length ?? null;
416
+ sendEvent('progress', { step: 'complete', message: `Found ${resultCount !== null ? resultCount : 'results'}` });
417
+ if (streamResult.answer) {
418
+ sendEvent('progress', { step: 'ai_done', message: 'AI summary generated' });
419
+ }
420
+ if (!streamResult.loadingMessage) {
421
+ streamResult.loadingMessage = getLoadingMessage(intent.type);
422
+ }
423
+ // Attach safety summary for streaming non-restaurant results
424
+ streamResult.safety = {
425
+ verified: true,
426
+ promptInjectionsBlocked: 0,
427
+ maliciousPatternsStripped: 0,
428
+ sourcesChecked: streamResult.sources?.length || 0,
429
+ };
430
+ // Attach suggestedDomains from intent
431
+ if (intent.suggestedDomains?.length) {
432
+ streamResult.suggestedDomains = intent.suggestedDomains;
433
+ }
434
+ sendEvent('result', streamResult);
435
+ sendEvent('done', { fetchTimeMs: streamResult.fetchTimeMs });
436
+ // Cache the streaming result
437
+ try {
438
+ const redis = getSmartRedis();
439
+ const ttl = CACHE_TTL[intent.type] || 600;
440
+ await redis.setex(cacheKey, ttl, JSON.stringify(streamResult));
441
+ console.log(`[smart-search] SSE Cache WRITE: ${cacheKey} (TTL: ${ttl}s)`);
442
+ }
443
+ catch { /* non-fatal */ }
444
+ res.end();
445
+ }
446
+ // Track usage for streaming path too
447
+ const pgStoreStream = authStore;
448
+ if (req.auth?.keyInfo?.key && typeof pgStoreStream.trackUsage === 'function') {
449
+ if (typeof pgStoreStream.trackBurstUsage === 'function') {
450
+ await pgStoreStream.trackBurstUsage(req.auth.keyInfo.key);
451
+ }
452
+ if (!req.auth?.softLimited) {
453
+ await pgStoreStream.trackUsage(req.auth.keyInfo.key, 'search');
454
+ }
455
+ }
456
+ }
457
+ catch (err) {
458
+ sendEvent('error', { message: err.message });
459
+ res.end();
460
+ }
461
+ return; // Don't fall through to non-streaming response
462
+ }
463
+ let smartResult;
464
+ switch (intent.type) {
465
+ case 'cars':
466
+ smartResult = await handleCarSearch(intent);
467
+ break;
468
+ case 'flights':
469
+ smartResult = await handleFlightSearch(intent);
470
+ break;
471
+ case 'hotels':
472
+ smartResult = await handleHotelSearch(intent);
473
+ break;
474
+ case 'rental':
475
+ smartResult = await handleRentalSearch(intent);
476
+ break;
477
+ case 'restaurants':
478
+ smartResult = await handleRestaurantSearch(intent, reqLanguage);
479
+ break;
480
+ case 'products':
481
+ smartResult = await handleProductSearch(intent);
482
+ break;
483
+ default:
484
+ smartResult = await handleGeneralSearch(query);
485
+ }
486
+ if (!smartResult.loadingMessage) {
487
+ smartResult.loadingMessage = getLoadingMessage(intent.type);
488
+ }
489
+ // ── Attach safety summary ─────────────────────────────────────────────
490
+ smartResult.safety = {
491
+ verified: true, // smart search already sanitizes all LLM inputs
492
+ promptInjectionsBlocked: 0, // stripped before they reach the LLM
493
+ maliciousPatternsStripped: 0,
494
+ sourcesChecked: smartResult.sources?.length || 0,
495
+ };
496
+ // Attach suggestedDomains from intent
497
+ if (intent.suggestedDomains?.length) {
498
+ smartResult.suggestedDomains = intent.suggestedDomains;
499
+ }
500
+ // ── Cache write ───────────────────────────────────────────────────────
501
+ try {
502
+ const redis = getSmartRedis();
503
+ const ttl = CACHE_TTL[smartResult.type] || 600;
504
+ await redis.setex(cacheKey, ttl, JSON.stringify(smartResult));
505
+ res.setHeader('X-Cache', 'MISS');
506
+ res.setHeader('X-Cache-Key', cacheKey);
507
+ console.log(`[smart-search] Cache WRITE: ${cacheKey} (TTL: ${ttl}s)`);
508
+ }
509
+ catch (err) {
510
+ console.warn('[smart-search] Redis cache write error (non-fatal):', err.message);
511
+ }
512
+ // Track usage
513
+ const pgStore = authStore;
514
+ if (req.auth?.keyInfo?.key && typeof pgStore.trackUsage === 'function') {
515
+ if (typeof pgStore.trackBurstUsage === 'function') {
516
+ await pgStore.trackBurstUsage(req.auth.keyInfo.key);
517
+ }
518
+ if (!req.auth?.softLimited) {
519
+ await pgStore.trackUsage(req.auth.keyInfo.key, 'search');
520
+ }
521
+ }
522
+ res.setHeader('X-Intent-Type', intent.type);
523
+ res.setHeader('X-Source', smartResult.source);
524
+ res.setHeader('X-Processing-Time', smartResult.fetchTimeMs.toString());
525
+ res.setHeader('Cache-Control', 'no-store');
526
+ res.json({
527
+ success: true,
528
+ data: smartResult,
529
+ });
530
+ }
531
+ catch (error) {
532
+ const err = error;
533
+ console.error('Smart search error:', err.message, err.stack);
534
+ res.status(500).json({
535
+ success: false,
536
+ error: {
537
+ type: 'smart_search_failed',
538
+ message: err.message || 'Smart search failed. Please try again.',
539
+ docs: 'https://webpeel.dev/docs/api-reference#smart-search',
540
+ },
541
+ requestId: req.requestId,
542
+ });
543
+ }
544
+ });
545
+ return router;
546
+ }
@@ -0,0 +1,3 @@
1
+ import type { SearchIntent } from './types.js';
2
+ export declare function detectSearchIntent(query: string): SearchIntent;
3
+ export declare function classifyIntentWithLLM(query: string): Promise<SearchIntent['type']>;