@iflow-mcp/jakeliume-webpeel 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (547) hide show
  1. package/LICENSE +15 -0
  2. package/README.md +313 -0
  3. package/dist/cache.d.ts +30 -0
  4. package/dist/cache.js +139 -0
  5. package/dist/cli/commands/auth.d.ts +5 -0
  6. package/dist/cli/commands/auth.js +411 -0
  7. package/dist/cli/commands/doctor.d.ts +37 -0
  8. package/dist/cli/commands/doctor.js +371 -0
  9. package/dist/cli/commands/fetch.d.ts +6 -0
  10. package/dist/cli/commands/fetch.js +1345 -0
  11. package/dist/cli/commands/guide.d.ts +2 -0
  12. package/dist/cli/commands/guide.js +183 -0
  13. package/dist/cli/commands/interact.d.ts +5 -0
  14. package/dist/cli/commands/interact.js +840 -0
  15. package/dist/cli/commands/jobs.d.ts +5 -0
  16. package/dist/cli/commands/jobs.js +997 -0
  17. package/dist/cli/commands/monitor.d.ts +12 -0
  18. package/dist/cli/commands/monitor.js +197 -0
  19. package/dist/cli/commands/observe.d.ts +12 -0
  20. package/dist/cli/commands/observe.js +158 -0
  21. package/dist/cli/commands/screenshot.d.ts +5 -0
  22. package/dist/cli/commands/screenshot.js +282 -0
  23. package/dist/cli/commands/search.d.ts +5 -0
  24. package/dist/cli/commands/search.js +1021 -0
  25. package/dist/cli/commands/setup.d.ts +13 -0
  26. package/dist/cli/commands/setup.js +244 -0
  27. package/dist/cli/commands/skill.d.ts +15 -0
  28. package/dist/cli/commands/skill.js +195 -0
  29. package/dist/cli/utils.d.ts +84 -0
  30. package/dist/cli/utils.js +806 -0
  31. package/dist/cli-auth.d.ts +75 -0
  32. package/dist/cli-auth.js +369 -0
  33. package/dist/cli.d.ts +17 -0
  34. package/dist/cli.js +99 -0
  35. package/dist/core/actions.d.ts +69 -0
  36. package/dist/core/actions.js +495 -0
  37. package/dist/core/agent.d.ts +98 -0
  38. package/dist/core/agent.js +558 -0
  39. package/dist/core/answer.d.ts +42 -0
  40. package/dist/core/answer.js +395 -0
  41. package/dist/core/application-tracker.d.ts +84 -0
  42. package/dist/core/application-tracker.js +184 -0
  43. package/dist/core/apply.d.ts +162 -0
  44. package/dist/core/apply.js +816 -0
  45. package/dist/core/auth-detection.d.ts +35 -0
  46. package/dist/core/auth-detection.js +358 -0
  47. package/dist/core/auto-extract.d.ts +82 -0
  48. package/dist/core/auto-extract.js +604 -0
  49. package/dist/core/auto-interact.d.ts +23 -0
  50. package/dist/core/auto-interact.js +246 -0
  51. package/dist/core/bm25-filter.d.ts +66 -0
  52. package/dist/core/bm25-filter.js +288 -0
  53. package/dist/core/branding.d.ts +54 -0
  54. package/dist/core/branding.js +234 -0
  55. package/dist/core/browser-fetch.d.ts +323 -0
  56. package/dist/core/browser-fetch.js +1600 -0
  57. package/dist/core/browser-pool.d.ts +91 -0
  58. package/dist/core/browser-pool.js +550 -0
  59. package/dist/core/budget.d.ts +42 -0
  60. package/dist/core/budget.js +324 -0
  61. package/dist/core/business-intel.d.ts +47 -0
  62. package/dist/core/business-intel.js +279 -0
  63. package/dist/core/cache.d.ts +13 -0
  64. package/dist/core/cache.js +121 -0
  65. package/dist/core/cf-worker-proxy.d.ts +32 -0
  66. package/dist/core/cf-worker-proxy.js +87 -0
  67. package/dist/core/challenge-detection.d.ts +26 -0
  68. package/dist/core/challenge-detection.js +468 -0
  69. package/dist/core/change-tracking.d.ts +75 -0
  70. package/dist/core/change-tracking.js +276 -0
  71. package/dist/core/chunker.d.ts +46 -0
  72. package/dist/core/chunker.js +249 -0
  73. package/dist/core/chunking.d.ts +42 -0
  74. package/dist/core/chunking.js +181 -0
  75. package/dist/core/circuit-breaker.d.ts +44 -0
  76. package/dist/core/circuit-breaker.js +85 -0
  77. package/dist/core/content-pruner.d.ts +47 -0
  78. package/dist/core/content-pruner.js +425 -0
  79. package/dist/core/cookie-cache.d.ts +60 -0
  80. package/dist/core/cookie-cache.js +163 -0
  81. package/dist/core/crawl-checkpoint.d.ts +54 -0
  82. package/dist/core/crawl-checkpoint.js +104 -0
  83. package/dist/core/crawler.d.ts +84 -0
  84. package/dist/core/crawler.js +349 -0
  85. package/dist/core/cross-verify.d.ts +27 -0
  86. package/dist/core/cross-verify.js +93 -0
  87. package/dist/core/deep-fetch.d.ts +74 -0
  88. package/dist/core/deep-fetch.js +405 -0
  89. package/dist/core/deep-research.d.ts +141 -0
  90. package/dist/core/deep-research.js +972 -0
  91. package/dist/core/design-analysis.d.ts +70 -0
  92. package/dist/core/design-analysis.js +490 -0
  93. package/dist/core/design-compare.d.ts +38 -0
  94. package/dist/core/design-compare.js +264 -0
  95. package/dist/core/diff.d.ts +61 -0
  96. package/dist/core/diff.js +289 -0
  97. package/dist/core/dns-cache.d.ts +20 -0
  98. package/dist/core/dns-cache.js +198 -0
  99. package/dist/core/documents.d.ts +23 -0
  100. package/dist/core/documents.js +123 -0
  101. package/dist/core/domain-memory.d.ts +66 -0
  102. package/dist/core/domain-memory.js +163 -0
  103. package/dist/core/domain-verify.d.ts +40 -0
  104. package/dist/core/domain-verify.js +379 -0
  105. package/dist/core/engine-ranker.d.ts +112 -0
  106. package/dist/core/engine-ranker.js +395 -0
  107. package/dist/core/extract-inline.d.ts +38 -0
  108. package/dist/core/extract-inline.js +215 -0
  109. package/dist/core/extract-listings.d.ts +38 -0
  110. package/dist/core/extract-listings.js +461 -0
  111. package/dist/core/extract.d.ts +9 -0
  112. package/dist/core/extract.js +139 -0
  113. package/dist/core/fetch-cache.d.ts +57 -0
  114. package/dist/core/fetch-cache.js +95 -0
  115. package/dist/core/fetcher.d.ts +13 -0
  116. package/dist/core/fetcher.js +12 -0
  117. package/dist/core/google-cache.d.ts +29 -0
  118. package/dist/core/google-cache.js +180 -0
  119. package/dist/core/google-serp-parser.d.ts +82 -0
  120. package/dist/core/google-serp-parser.js +287 -0
  121. package/dist/core/hotel-search.d.ts +122 -0
  122. package/dist/core/hotel-search.js +382 -0
  123. package/dist/core/http-fetch.d.ts +72 -0
  124. package/dist/core/http-fetch.js +820 -0
  125. package/dist/core/human.d.ts +175 -0
  126. package/dist/core/human.js +680 -0
  127. package/dist/core/image-caption.d.ts +44 -0
  128. package/dist/core/image-caption.js +271 -0
  129. package/dist/core/jobs.d.ts +75 -0
  130. package/dist/core/jobs.js +634 -0
  131. package/dist/core/json-ld.d.ts +15 -0
  132. package/dist/core/json-ld.js +617 -0
  133. package/dist/core/language-detect.d.ts +18 -0
  134. package/dist/core/language-detect.js +135 -0
  135. package/dist/core/links.d.ts +10 -0
  136. package/dist/core/links.js +44 -0
  137. package/dist/core/llm-extract.d.ts +71 -0
  138. package/dist/core/llm-extract.js +507 -0
  139. package/dist/core/llm-provider.d.ts +100 -0
  140. package/dist/core/llm-provider.js +702 -0
  141. package/dist/core/local-search.d.ts +60 -0
  142. package/dist/core/local-search.js +308 -0
  143. package/dist/core/logger.d.ts +28 -0
  144. package/dist/core/logger.js +104 -0
  145. package/dist/core/map.d.ts +33 -0
  146. package/dist/core/map.js +127 -0
  147. package/dist/core/markdown.d.ts +92 -0
  148. package/dist/core/markdown.js +809 -0
  149. package/dist/core/metadata.d.ts +34 -0
  150. package/dist/core/metadata.js +422 -0
  151. package/dist/core/observe.d.ts +113 -0
  152. package/dist/core/observe.js +395 -0
  153. package/dist/core/ocr.d.ts +12 -0
  154. package/dist/core/ocr.js +33 -0
  155. package/dist/core/paginate.d.ts +31 -0
  156. package/dist/core/paginate.js +106 -0
  157. package/dist/core/pdf.d.ts +8 -0
  158. package/dist/core/pdf.js +25 -0
  159. package/dist/core/peel-tls.d.ts +25 -0
  160. package/dist/core/peel-tls.js +220 -0
  161. package/dist/core/pipeline.d.ts +132 -0
  162. package/dist/core/pipeline.js +1666 -0
  163. package/dist/core/profiles.d.ts +61 -0
  164. package/dist/core/profiles.js +350 -0
  165. package/dist/core/prompt-guard.d.ts +30 -0
  166. package/dist/core/prompt-guard.js +119 -0
  167. package/dist/core/proxy-config.d.ts +90 -0
  168. package/dist/core/proxy-config.js +172 -0
  169. package/dist/core/quick-answer.d.ts +53 -0
  170. package/dist/core/quick-answer.js +833 -0
  171. package/dist/core/rate-governor.d.ts +80 -0
  172. package/dist/core/rate-governor.js +238 -0
  173. package/dist/core/readability.d.ts +57 -0
  174. package/dist/core/readability.js +533 -0
  175. package/dist/core/research.d.ts +66 -0
  176. package/dist/core/research.js +270 -0
  177. package/dist/core/retry.d.ts +60 -0
  178. package/dist/core/retry.js +119 -0
  179. package/dist/core/safe-browsing.d.ts +30 -0
  180. package/dist/core/safe-browsing.js +206 -0
  181. package/dist/core/schema-extraction.d.ts +66 -0
  182. package/dist/core/schema-extraction.js +352 -0
  183. package/dist/core/schema-postprocess.d.ts +32 -0
  184. package/dist/core/schema-postprocess.js +469 -0
  185. package/dist/core/schema-templates.d.ts +19 -0
  186. package/dist/core/schema-templates.js +143 -0
  187. package/dist/core/screenshot.d.ts +224 -0
  188. package/dist/core/screenshot.js +207 -0
  189. package/dist/core/search-engines.d.ts +25 -0
  190. package/dist/core/search-engines.js +182 -0
  191. package/dist/core/search-provider.d.ts +243 -0
  192. package/dist/core/search-provider.js +1629 -0
  193. package/dist/core/searxng-provider.d.ts +35 -0
  194. package/dist/core/searxng-provider.js +105 -0
  195. package/dist/core/selective-evidence.d.ts +151 -0
  196. package/dist/core/selective-evidence.js +389 -0
  197. package/dist/core/site-search.d.ts +44 -0
  198. package/dist/core/site-search.js +252 -0
  199. package/dist/core/sitemap.d.ts +23 -0
  200. package/dist/core/sitemap.js +105 -0
  201. package/dist/core/source-credibility.d.ts +29 -0
  202. package/dist/core/source-credibility.js +584 -0
  203. package/dist/core/source-scoring.d.ts +166 -0
  204. package/dist/core/source-scoring.js +396 -0
  205. package/dist/core/stemmer.d.ts +38 -0
  206. package/dist/core/stemmer.js +509 -0
  207. package/dist/core/strategies.d.ts +104 -0
  208. package/dist/core/strategies.js +1044 -0
  209. package/dist/core/strategy-hooks.d.ts +145 -0
  210. package/dist/core/strategy-hooks.js +74 -0
  211. package/dist/core/structured-extract.d.ts +43 -0
  212. package/dist/core/structured-extract.js +550 -0
  213. package/dist/core/summarize.d.ts +17 -0
  214. package/dist/core/summarize.js +78 -0
  215. package/dist/core/synonyms.d.ts +42 -0
  216. package/dist/core/synonyms.js +184 -0
  217. package/dist/core/system-monitor.d.ts +61 -0
  218. package/dist/core/system-monitor.js +133 -0
  219. package/dist/core/table-format.d.ts +30 -0
  220. package/dist/core/table-format.js +146 -0
  221. package/dist/core/threat-feeds.d.ts +23 -0
  222. package/dist/core/threat-feeds.js +104 -0
  223. package/dist/core/timing.d.ts +21 -0
  224. package/dist/core/timing.js +33 -0
  225. package/dist/core/transcript-export.d.ts +47 -0
  226. package/dist/core/transcript-export.js +107 -0
  227. package/dist/core/user-agents.d.ts +82 -0
  228. package/dist/core/user-agents.js +239 -0
  229. package/dist/core/vertical-search.d.ts +54 -0
  230. package/dist/core/vertical-search.js +158 -0
  231. package/dist/core/watch-manager.d.ts +175 -0
  232. package/dist/core/watch-manager.js +416 -0
  233. package/dist/core/watch.d.ts +101 -0
  234. package/dist/core/watch.js +389 -0
  235. package/dist/core/youtube.d.ts +130 -0
  236. package/dist/core/youtube.js +1175 -0
  237. package/dist/ee/challenge-re-export.d.ts +1 -0
  238. package/dist/ee/challenge-re-export.js +1 -0
  239. package/dist/ee/challenge-solver.d.ts +72 -0
  240. package/dist/ee/challenge-solver.js +720 -0
  241. package/dist/ee/domain-extractors.d.ts +8 -0
  242. package/dist/ee/domain-extractors.js +8 -0
  243. package/dist/ee/domain-intel.d.ts +16 -0
  244. package/dist/ee/domain-intel.js +133 -0
  245. package/dist/ee/extractors/allrecipes.d.ts +2 -0
  246. package/dist/ee/extractors/allrecipes.js +120 -0
  247. package/dist/ee/extractors/amazon.d.ts +2 -0
  248. package/dist/ee/extractors/amazon.js +78 -0
  249. package/dist/ee/extractors/arxiv.d.ts +2 -0
  250. package/dist/ee/extractors/arxiv.js +137 -0
  251. package/dist/ee/extractors/bestbuy.d.ts +2 -0
  252. package/dist/ee/extractors/bestbuy.js +78 -0
  253. package/dist/ee/extractors/carscom.d.ts +2 -0
  254. package/dist/ee/extractors/carscom.js +121 -0
  255. package/dist/ee/extractors/coingecko.d.ts +2 -0
  256. package/dist/ee/extractors/coingecko.js +134 -0
  257. package/dist/ee/extractors/craigslist.d.ts +2 -0
  258. package/dist/ee/extractors/craigslist.js +92 -0
  259. package/dist/ee/extractors/devto.d.ts +2 -0
  260. package/dist/ee/extractors/devto.js +135 -0
  261. package/dist/ee/extractors/ebay.d.ts +2 -0
  262. package/dist/ee/extractors/ebay.js +90 -0
  263. package/dist/ee/extractors/espn.d.ts +2 -0
  264. package/dist/ee/extractors/espn.js +260 -0
  265. package/dist/ee/extractors/etsy.d.ts +2 -0
  266. package/dist/ee/extractors/etsy.js +52 -0
  267. package/dist/ee/extractors/facebook.d.ts +2 -0
  268. package/dist/ee/extractors/facebook.js +46 -0
  269. package/dist/ee/extractors/github.d.ts +2 -0
  270. package/dist/ee/extractors/github.js +196 -0
  271. package/dist/ee/extractors/google-flights.d.ts +2 -0
  272. package/dist/ee/extractors/google-flights.js +176 -0
  273. package/dist/ee/extractors/hackernews.d.ts +2 -0
  274. package/dist/ee/extractors/hackernews.js +147 -0
  275. package/dist/ee/extractors/imdb.d.ts +2 -0
  276. package/dist/ee/extractors/imdb.js +172 -0
  277. package/dist/ee/extractors/index.d.ts +26 -0
  278. package/dist/ee/extractors/index.js +247 -0
  279. package/dist/ee/extractors/instagram.d.ts +2 -0
  280. package/dist/ee/extractors/instagram.js +102 -0
  281. package/dist/ee/extractors/kalshi.d.ts +2 -0
  282. package/dist/ee/extractors/kalshi.js +121 -0
  283. package/dist/ee/extractors/kayak-cars.d.ts +2 -0
  284. package/dist/ee/extractors/kayak-cars.js +270 -0
  285. package/dist/ee/extractors/linkedin.d.ts +2 -0
  286. package/dist/ee/extractors/linkedin.js +113 -0
  287. package/dist/ee/extractors/medium.d.ts +2 -0
  288. package/dist/ee/extractors/medium.js +130 -0
  289. package/dist/ee/extractors/news.d.ts +4 -0
  290. package/dist/ee/extractors/news.js +173 -0
  291. package/dist/ee/extractors/npm.d.ts +2 -0
  292. package/dist/ee/extractors/npm.js +86 -0
  293. package/dist/ee/extractors/pdf.d.ts +2 -0
  294. package/dist/ee/extractors/pdf.js +108 -0
  295. package/dist/ee/extractors/pinterest.d.ts +2 -0
  296. package/dist/ee/extractors/pinterest.js +34 -0
  297. package/dist/ee/extractors/polymarket.d.ts +2 -0
  298. package/dist/ee/extractors/polymarket.js +358 -0
  299. package/dist/ee/extractors/producthunt.d.ts +2 -0
  300. package/dist/ee/extractors/producthunt.js +88 -0
  301. package/dist/ee/extractors/pubmed.d.ts +2 -0
  302. package/dist/ee/extractors/pubmed.js +162 -0
  303. package/dist/ee/extractors/pypi.d.ts +2 -0
  304. package/dist/ee/extractors/pypi.js +80 -0
  305. package/dist/ee/extractors/reddit.d.ts +2 -0
  306. package/dist/ee/extractors/reddit.js +438 -0
  307. package/dist/ee/extractors/redfin.d.ts +2 -0
  308. package/dist/ee/extractors/redfin.js +156 -0
  309. package/dist/ee/extractors/semanticscholar.d.ts +2 -0
  310. package/dist/ee/extractors/semanticscholar.js +131 -0
  311. package/dist/ee/extractors/shared.d.ts +12 -0
  312. package/dist/ee/extractors/shared.js +76 -0
  313. package/dist/ee/extractors/soundcloud.d.ts +2 -0
  314. package/dist/ee/extractors/soundcloud.js +34 -0
  315. package/dist/ee/extractors/sportsbetting.d.ts +2 -0
  316. package/dist/ee/extractors/sportsbetting.js +37 -0
  317. package/dist/ee/extractors/spotify.d.ts +2 -0
  318. package/dist/ee/extractors/spotify.js +34 -0
  319. package/dist/ee/extractors/stackoverflow.d.ts +2 -0
  320. package/dist/ee/extractors/stackoverflow.js +61 -0
  321. package/dist/ee/extractors/substack.d.ts +2 -0
  322. package/dist/ee/extractors/substack.js +115 -0
  323. package/dist/ee/extractors/substackroot.d.ts +2 -0
  324. package/dist/ee/extractors/substackroot.js +46 -0
  325. package/dist/ee/extractors/tiktok.d.ts +2 -0
  326. package/dist/ee/extractors/tiktok.js +29 -0
  327. package/dist/ee/extractors/tradingview.d.ts +2 -0
  328. package/dist/ee/extractors/tradingview.js +182 -0
  329. package/dist/ee/extractors/twitch.d.ts +2 -0
  330. package/dist/ee/extractors/twitch.js +36 -0
  331. package/dist/ee/extractors/twitter.d.ts +2 -0
  332. package/dist/ee/extractors/twitter.js +327 -0
  333. package/dist/ee/extractors/types.d.ts +14 -0
  334. package/dist/ee/extractors/types.js +1 -0
  335. package/dist/ee/extractors/walmart.d.ts +2 -0
  336. package/dist/ee/extractors/walmart.js +50 -0
  337. package/dist/ee/extractors/weather.d.ts +2 -0
  338. package/dist/ee/extractors/weather.js +133 -0
  339. package/dist/ee/extractors/wikipedia.d.ts +4 -0
  340. package/dist/ee/extractors/wikipedia.js +235 -0
  341. package/dist/ee/extractors/yelp.d.ts +2 -0
  342. package/dist/ee/extractors/yelp.js +216 -0
  343. package/dist/ee/extractors/youtube.d.ts +2 -0
  344. package/dist/ee/extractors/youtube.js +189 -0
  345. package/dist/ee/extractors/zillow.d.ts +54 -0
  346. package/dist/ee/extractors/zillow.js +247 -0
  347. package/dist/ee/extractors-re-export.d.ts +1 -0
  348. package/dist/ee/extractors-re-export.js +1 -0
  349. package/dist/ee/premium-hooks.d.ts +20 -0
  350. package/dist/ee/premium-hooks.js +50 -0
  351. package/dist/ee/spa-detection.d.ts +2 -0
  352. package/dist/ee/spa-detection.js +2 -0
  353. package/dist/ee/stability.d.ts +4 -0
  354. package/dist/ee/stability.js +29 -0
  355. package/dist/ee/swr-cache.d.ts +14 -0
  356. package/dist/ee/swr-cache.js +34 -0
  357. package/dist/index.d.ts +143 -0
  358. package/dist/index.js +291 -0
  359. package/dist/integrations/index.d.ts +2 -0
  360. package/dist/integrations/index.js +2 -0
  361. package/dist/integrations/langchain.d.ts +64 -0
  362. package/dist/integrations/langchain.js +115 -0
  363. package/dist/integrations/llamaindex.d.ts +50 -0
  364. package/dist/integrations/llamaindex.js +91 -0
  365. package/dist/mcp/handlers/act.d.ts +5 -0
  366. package/dist/mcp/handlers/act.js +34 -0
  367. package/dist/mcp/handlers/definitions.d.ts +6 -0
  368. package/dist/mcp/handlers/definitions.js +395 -0
  369. package/dist/mcp/handlers/extract.d.ts +7 -0
  370. package/dist/mcp/handlers/extract.js +135 -0
  371. package/dist/mcp/handlers/fetch.d.ts +6 -0
  372. package/dist/mcp/handlers/fetch.js +98 -0
  373. package/dist/mcp/handlers/find.d.ts +5 -0
  374. package/dist/mcp/handlers/find.js +137 -0
  375. package/dist/mcp/handlers/index.d.ts +13 -0
  376. package/dist/mcp/handlers/index.js +63 -0
  377. package/dist/mcp/handlers/legacy.d.ts +25 -0
  378. package/dist/mcp/handlers/legacy.js +450 -0
  379. package/dist/mcp/handlers/meta.d.ts +6 -0
  380. package/dist/mcp/handlers/meta.js +40 -0
  381. package/dist/mcp/handlers/monitor.d.ts +5 -0
  382. package/dist/mcp/handlers/monitor.js +41 -0
  383. package/dist/mcp/handlers/observe.d.ts +8 -0
  384. package/dist/mcp/handlers/observe.js +37 -0
  385. package/dist/mcp/handlers/read.d.ts +6 -0
  386. package/dist/mcp/handlers/read.js +78 -0
  387. package/dist/mcp/handlers/see.d.ts +5 -0
  388. package/dist/mcp/handlers/see.js +75 -0
  389. package/dist/mcp/handlers/types.d.ts +29 -0
  390. package/dist/mcp/handlers/types.js +28 -0
  391. package/dist/mcp/server.d.ts +7 -0
  392. package/dist/mcp/server.js +108 -0
  393. package/dist/mcp/smart-router.d.ts +23 -0
  394. package/dist/mcp/smart-router.js +178 -0
  395. package/dist/server/app.d.ts +14 -0
  396. package/dist/server/app.js +632 -0
  397. package/dist/server/auth-store.d.ts +28 -0
  398. package/dist/server/auth-store.js +88 -0
  399. package/dist/server/bull-queues.d.ts +60 -0
  400. package/dist/server/bull-queues.js +90 -0
  401. package/dist/server/email-service.d.ts +55 -0
  402. package/dist/server/email-service.js +291 -0
  403. package/dist/server/job-queue.d.ts +100 -0
  404. package/dist/server/job-queue.js +145 -0
  405. package/dist/server/logger.d.ts +10 -0
  406. package/dist/server/logger.js +37 -0
  407. package/dist/server/middleware/audit-log.d.ts +14 -0
  408. package/dist/server/middleware/audit-log.js +73 -0
  409. package/dist/server/middleware/auth.d.ts +35 -0
  410. package/dist/server/middleware/auth.js +225 -0
  411. package/dist/server/middleware/rate-limit.d.ts +50 -0
  412. package/dist/server/middleware/rate-limit.js +270 -0
  413. package/dist/server/middleware/scope-guard.d.ts +25 -0
  414. package/dist/server/middleware/scope-guard.js +45 -0
  415. package/dist/server/middleware/url-validator.d.ts +15 -0
  416. package/dist/server/middleware/url-validator.js +201 -0
  417. package/dist/server/openapi.yaml +6418 -0
  418. package/dist/server/pg-auth-store.d.ts +146 -0
  419. package/dist/server/pg-auth-store.js +576 -0
  420. package/dist/server/pg-job-queue.d.ts +59 -0
  421. package/dist/server/pg-job-queue.js +375 -0
  422. package/dist/server/routes/activity.d.ts +6 -0
  423. package/dist/server/routes/activity.js +79 -0
  424. package/dist/server/routes/admin-active.d.ts +7 -0
  425. package/dist/server/routes/admin-active.js +120 -0
  426. package/dist/server/routes/admin-stats.d.ts +7 -0
  427. package/dist/server/routes/admin-stats.js +176 -0
  428. package/dist/server/routes/agent.d.ts +24 -0
  429. package/dist/server/routes/agent.js +480 -0
  430. package/dist/server/routes/answer.d.ts +5 -0
  431. package/dist/server/routes/answer.js +125 -0
  432. package/dist/server/routes/ask.d.ts +28 -0
  433. package/dist/server/routes/ask.js +295 -0
  434. package/dist/server/routes/batch.d.ts +6 -0
  435. package/dist/server/routes/batch.js +493 -0
  436. package/dist/server/routes/cache-warm.d.ts +25 -0
  437. package/dist/server/routes/cache-warm.js +212 -0
  438. package/dist/server/routes/cli-usage.d.ts +6 -0
  439. package/dist/server/routes/cli-usage.js +127 -0
  440. package/dist/server/routes/compat.d.ts +23 -0
  441. package/dist/server/routes/compat.js +652 -0
  442. package/dist/server/routes/crawl.d.ts +13 -0
  443. package/dist/server/routes/crawl.js +287 -0
  444. package/dist/server/routes/deep-fetch.d.ts +8 -0
  445. package/dist/server/routes/deep-fetch.js +57 -0
  446. package/dist/server/routes/deep-research.d.ts +11 -0
  447. package/dist/server/routes/deep-research.js +232 -0
  448. package/dist/server/routes/demo.d.ts +24 -0
  449. package/dist/server/routes/demo.js +517 -0
  450. package/dist/server/routes/do.d.ts +8 -0
  451. package/dist/server/routes/do.js +72 -0
  452. package/dist/server/routes/extract.d.ts +14 -0
  453. package/dist/server/routes/extract.js +325 -0
  454. package/dist/server/routes/feed.d.ts +15 -0
  455. package/dist/server/routes/feed.js +311 -0
  456. package/dist/server/routes/fetch-queue.d.ts +13 -0
  457. package/dist/server/routes/fetch-queue.js +357 -0
  458. package/dist/server/routes/fetch.d.ts +7 -0
  459. package/dist/server/routes/fetch.js +1274 -0
  460. package/dist/server/routes/go.d.ts +14 -0
  461. package/dist/server/routes/go.js +81 -0
  462. package/dist/server/routes/health.d.ts +11 -0
  463. package/dist/server/routes/health.js +141 -0
  464. package/dist/server/routes/jobs.d.ts +7 -0
  465. package/dist/server/routes/jobs.js +574 -0
  466. package/dist/server/routes/map.d.ts +11 -0
  467. package/dist/server/routes/map.js +116 -0
  468. package/dist/server/routes/mcp.d.ts +14 -0
  469. package/dist/server/routes/mcp.js +197 -0
  470. package/dist/server/routes/metrics.d.ts +37 -0
  471. package/dist/server/routes/metrics.js +149 -0
  472. package/dist/server/routes/oauth.d.ts +9 -0
  473. package/dist/server/routes/oauth.js +396 -0
  474. package/dist/server/routes/playground.d.ts +17 -0
  475. package/dist/server/routes/playground.js +283 -0
  476. package/dist/server/routes/reader.d.ts +18 -0
  477. package/dist/server/routes/reader.js +192 -0
  478. package/dist/server/routes/research.d.ts +14 -0
  479. package/dist/server/routes/research.js +482 -0
  480. package/dist/server/routes/screenshot.d.ts +22 -0
  481. package/dist/server/routes/screenshot.js +820 -0
  482. package/dist/server/routes/search.d.ts +6 -0
  483. package/dist/server/routes/search.js +874 -0
  484. package/dist/server/routes/session.d.ts +17 -0
  485. package/dist/server/routes/session.js +548 -0
  486. package/dist/server/routes/share.d.ts +18 -0
  487. package/dist/server/routes/share.js +462 -0
  488. package/dist/server/routes/smart-search/handlers/cars.d.ts +2 -0
  489. package/dist/server/routes/smart-search/handlers/cars.js +102 -0
  490. package/dist/server/routes/smart-search/handlers/flights.d.ts +2 -0
  491. package/dist/server/routes/smart-search/handlers/flights.js +72 -0
  492. package/dist/server/routes/smart-search/handlers/general.d.ts +13 -0
  493. package/dist/server/routes/smart-search/handlers/general.js +717 -0
  494. package/dist/server/routes/smart-search/handlers/hotels.d.ts +2 -0
  495. package/dist/server/routes/smart-search/handlers/hotels.js +88 -0
  496. package/dist/server/routes/smart-search/handlers/products.d.ts +2 -0
  497. package/dist/server/routes/smart-search/handlers/products.js +1309 -0
  498. package/dist/server/routes/smart-search/handlers/rental.d.ts +2 -0
  499. package/dist/server/routes/smart-search/handlers/rental.js +154 -0
  500. package/dist/server/routes/smart-search/handlers/restaurants.d.ts +2 -0
  501. package/dist/server/routes/smart-search/handlers/restaurants.js +225 -0
  502. package/dist/server/routes/smart-search/handlers/transit-verdict.d.ts +41 -0
  503. package/dist/server/routes/smart-search/handlers/transit-verdict.js +224 -0
  504. package/dist/server/routes/smart-search/index.d.ts +19 -0
  505. package/dist/server/routes/smart-search/index.js +546 -0
  506. package/dist/server/routes/smart-search/intent.d.ts +3 -0
  507. package/dist/server/routes/smart-search/intent.js +264 -0
  508. package/dist/server/routes/smart-search/llm.d.ts +16 -0
  509. package/dist/server/routes/smart-search/llm.js +70 -0
  510. package/dist/server/routes/smart-search/sources/reddit.d.ts +18 -0
  511. package/dist/server/routes/smart-search/sources/reddit.js +34 -0
  512. package/dist/server/routes/smart-search/sources/yelp.d.ts +25 -0
  513. package/dist/server/routes/smart-search/sources/yelp.js +171 -0
  514. package/dist/server/routes/smart-search/sources/youtube.d.ts +8 -0
  515. package/dist/server/routes/smart-search/sources/youtube.js +9 -0
  516. package/dist/server/routes/smart-search/types.d.ts +81 -0
  517. package/dist/server/routes/smart-search/types.js +1 -0
  518. package/dist/server/routes/smart-search/utils.d.ts +20 -0
  519. package/dist/server/routes/smart-search/utils.js +146 -0
  520. package/dist/server/routes/stats.d.ts +6 -0
  521. package/dist/server/routes/stats.js +71 -0
  522. package/dist/server/routes/stripe.d.ts +15 -0
  523. package/dist/server/routes/stripe.js +296 -0
  524. package/dist/server/routes/transcript-export.d.ts +10 -0
  525. package/dist/server/routes/transcript-export.js +178 -0
  526. package/dist/server/routes/usage.d.ts +9 -0
  527. package/dist/server/routes/usage.js +279 -0
  528. package/dist/server/routes/users.d.ts +8 -0
  529. package/dist/server/routes/users.js +1867 -0
  530. package/dist/server/routes/watch.d.ts +15 -0
  531. package/dist/server/routes/watch.js +309 -0
  532. package/dist/server/routes/webhooks.d.ts +26 -0
  533. package/dist/server/routes/webhooks.js +170 -0
  534. package/dist/server/routes/youtube.d.ts +6 -0
  535. package/dist/server/routes/youtube.js +130 -0
  536. package/dist/server/sentry.d.ts +14 -0
  537. package/dist/server/sentry.js +104 -0
  538. package/dist/server/types.d.ts +15 -0
  539. package/dist/server/types.js +7 -0
  540. package/dist/server/utils/response.d.ts +44 -0
  541. package/dist/server/utils/response.js +69 -0
  542. package/dist/server/utils/sse.d.ts +22 -0
  543. package/dist/server/utils/sse.js +38 -0
  544. package/dist/types.d.ts +552 -0
  545. package/dist/types.js +39 -0
  546. package/llms.txt +105 -0
  547. package/package.json +189 -0
package/dist/server/routes/share.js ADDED
@@ -0,0 +1,462 @@
1
+ /**
2
+ * Shareable read links โ€” short public URLs for fetched content
3
+ *
4
+ * POST /v1/share โ€” create a short link (auth required, 50/day limit)
5
+ * GET /s/:id โ€” serve shared content (public, no auth)
6
+ *
7
+ * IDs are 9-char base64url strings (crypto.randomBytes(6).toString('base64url').slice(0, 9))
8
+ * Shares expire after 30 days. view_count is incremented on every public read.
9
+ */
10
+ import { Router } from 'express';
11
+ import crypto from 'crypto';
12
+ import { createLogger } from '../logger.js';
13
+ import { peel } from '../../index.js';
14
+ import { validateUrlForSSRF, SSRFError } from '../middleware/url-validator.js';
15
+ const log = createLogger('share');
16
+ // โ”€โ”€โ”€ Helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
17
+ /** Generate a cryptographically secure 9-char base64url ID.
18
+ * randomBytes(7) โ†’ base64url gives 10 chars (7*4/3=9.33โ†’10), slice to 9.
19
+ * Note: randomBytes(6) โ†’ base64url gives only 8 chars (6/3*4=8), so we need 7+ bytes.
20
+ */
21
+ export function generateShareId() {
22
+ return crypto.randomBytes(7).toString('base64url').slice(0, 9);
23
+ }
24
+ /** Base URL for share links */
25
+ function getBaseUrl() {
26
+ return process.env.API_BASE_URL || 'https://api.webpeel.dev';
27
+ }
28
+ /** Simple markdown โ†’ HTML renderer (no external deps) */
29
+ function markdownToHtml(md) {
30
+ let html = md
31
+ // Escape raw HTML in content to prevent XSS
32
+ .replace(/&/g, '&')
33
+ .replace(/</g, '&lt;')
34
+ .replace(/>/g, '&gt;')
35
+ // Code blocks (``` ... ```)
36
+ .replace(/```[\w]*\n([\s\S]*?)```/g, (_m, code) => `<pre><code>${code.trim()}</code></pre>`)
37
+ // Inline code
38
+ .replace(/`([^`]+)`/g, '<code>$1</code>')
39
+ // Bold + italic
40
+ .replace(/\*\*\*([^*]+)\*\*\*/g, '<strong><em>$1</em></strong>')
41
+ // Bold
42
+ .replace(/\*\*([^*]+)\*\*/g, '<strong>$1</strong>')
43
+ // Italic
44
+ .replace(/\*([^*]+)\*/g, '<em>$1</em>')
45
+ // Headings
46
+ .replace(/^### (.+)$/gm, '<h3>$1</h3>')
47
+ .replace(/^## (.+)$/gm, '<h2>$1</h2>')
48
+ .replace(/^# (.+)$/gm, '<h1>$1</h1>')
49
+ // Horizontal rule
50
+ .replace(/^---$/gm, '<hr>')
51
+ // Blockquote
52
+ .replace(/^> (.+)$/gm, '<blockquote>$1</blockquote>')
53
+ // Unordered list items
54
+ .replace(/^[\*\-] (.+)$/gm, '<li>$1</li>')
55
+ // Ordered list items
56
+ .replace(/^\d+\. (.+)$/gm, '<li>$1</li>')
57
+ // Links
58
+ .replace(/\[([^\]]+)\]\(([^)]+)\)/g, '<a href="$2" rel="noopener noreferrer">$1</a>')
59
+ // Images
60
+ .replace(/!\[([^\]]*)\]\(([^)]+)\)/g, '<img src="$2" alt="$1">')
61
+ // Double newlines โ†’ paragraph breaks
62
+ .replace(/\n\n+/g, '\n</p><p>\n')
63
+ // Remaining single newlines โ†’ <br>
64
+ .replace(/\n/g, '<br>\n');
65
+ // Wrap consecutive <li> items in <ul>
66
+ html = html.replace(/(<li>.*?<\/li>\n?)+/gs, (m) => `<ul>\n${m}</ul>\n`);
67
+ return `<p>\n${html}\n</p>`;
68
+ }
69
+ /** Build the full HTML page for a shared read */
70
+ function buildHtmlPage(share) {
71
+ const title = share.title ? `${share.title} โ€” WebPeel` : 'Shared Read โ€” WebPeel';
72
+ const description = share.content.slice(0, 200).replace(/\n/g, ' ').replace(/"/g, '&quot;') + 'โ€ฆ';
73
+ const canonicalUrl = `${getBaseUrl()}/s/${share.id}`;
74
+ const originalUrl = share.url
75
+ .replace(/&/g, '&amp;')
76
+ .replace(/</g, '&lt;')
77
+ .replace(/>/g, '&gt;');
78
+ const bodyHtml = markdownToHtml(share.content);
79
+ return `<!DOCTYPE html>
80
+ <html lang="en">
81
+ <head>
82
+ <meta charset="UTF-8">
83
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
84
+ <title>${title.replace(/</g, '&lt;')}</title>
85
+ <meta name="description" content="${description}">
86
+ <link rel="canonical" href="${canonicalUrl}">
87
+
88
+ <!-- Open Graph -->
89
+ <meta property="og:title" content="${(share.title || 'Shared Read').replace(/</g, '&lt;')}">
90
+ <meta property="og:description" content="${description}">
91
+ <meta property="og:url" content="${canonicalUrl}">
92
+ <meta property="og:type" content="article">
93
+ <meta property="og:site_name" content="WebPeel">
94
+
95
+ <!-- Twitter Card -->
96
+ <meta name="twitter:card" content="summary">
97
+ <meta name="twitter:title" content="${(share.title || 'Shared Read').replace(/</g, '&lt;')}">
98
+ <meta name="twitter:description" content="${description}">
99
+ <meta name="twitter:site" content="@webpeel">
100
+
101
+ <style>
102
+ *, *::before, *::after { box-sizing: border-box; }
103
+ :root {
104
+ --bg: #0f0f11;
105
+ --surface: #1a1a1f;
106
+ --border: #2a2a35;
107
+ --text: #e4e4e7;
108
+ --muted: #71717a;
109
+ --accent: #818cf8;
110
+ --link: #6366f1;
111
+ --code-bg: #1e1e28;
112
+ --max-w: 760px;
113
+ }
114
+ html { background: var(--bg); }
115
+ body {
116
+ margin: 0;
117
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
118
+ font-size: 16px;
119
+ line-height: 1.75;
120
+ color: var(--text);
121
+ background: var(--bg);
122
+ padding: 0 16px;
123
+ }
124
+
125
+ /* Top bar */
126
+ .topbar {
127
+ display: flex;
128
+ align-items: center;
129
+ justify-content: space-between;
130
+ max-width: var(--max-w);
131
+ margin: 0 auto;
132
+ padding: 20px 0 16px;
133
+ border-bottom: 1px solid var(--border);
134
+ gap: 12px;
135
+ flex-wrap: wrap;
136
+ }
137
+ .logo { display: flex; align-items: center; gap: 8px; text-decoration: none; color: var(--text); }
138
+ .logo-mark {
139
+ width: 28px; height: 28px;
140
+ background: var(--accent);
141
+ border-radius: 7px;
142
+ display: flex; align-items: center; justify-content: center;
143
+ font-size: 14px; font-weight: 700; color: #fff; letter-spacing: -0.5px;
144
+ }
145
+ .logo-name { font-weight: 600; font-size: 15px; }
146
+ .source-link {
147
+ font-size: 12px; color: var(--muted);
148
+ text-decoration: none; max-width: 300px;
149
+ overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
150
+ }
151
+ .source-link:hover { color: var(--accent); }
152
+
153
+ /* Main content */
154
+ main {
155
+ max-width: var(--max-w);
156
+ margin: 32px auto;
157
+ }
158
+ h1 { font-size: 1.75rem; font-weight: 700; line-height: 1.25; margin: 0 0 24px; color: var(--text); }
159
+ h2 { font-size: 1.35rem; font-weight: 600; margin: 28px 0 12px; color: var(--text); }
160
+ h3 { font-size: 1.1rem; font-weight: 600; margin: 24px 0 10px; color: var(--text); }
161
+ p { margin: 0 0 16px; color: #d4d4d8; }
162
+ a { color: var(--link); text-decoration: underline; text-underline-offset: 3px; }
163
+ a:hover { color: var(--accent); }
164
+ ul, ol { padding-left: 24px; margin: 0 0 16px; }
165
+ li { margin-bottom: 6px; color: #d4d4d8; }
166
+ blockquote {
167
+ border-left: 3px solid var(--accent); margin: 16px 0;
168
+ padding: 4px 16px; color: var(--muted); font-style: italic;
169
+ }
170
+ code {
171
+ background: var(--code-bg); padding: 2px 6px; border-radius: 4px;
172
+ font-family: 'Fira Code', 'Cascadia Code', monospace; font-size: 0.875em;
173
+ color: var(--accent);
174
+ }
175
+ pre {
176
+ background: var(--code-bg); padding: 16px; border-radius: 8px;
177
+ overflow-x: auto; margin: 16px 0; border: 1px solid var(--border);
178
+ }
179
+ pre code { background: none; padding: 0; color: #e4e4e7; }
180
+ img { max-width: 100%; border-radius: 6px; margin: 8px 0; }
181
+ hr { border: none; border-top: 1px solid var(--border); margin: 28px 0; }
182
+
183
+ /* Meta info */
184
+ .meta {
185
+ display: flex; gap: 16px; flex-wrap: wrap;
186
+ font-size: 12px; color: var(--muted);
187
+ margin-bottom: 28px; padding-bottom: 20px;
188
+ border-bottom: 1px solid var(--border);
189
+ }
190
+ .meta span { display: flex; align-items: center; gap: 4px; }
191
+
192
+ /* Footer */
193
+ footer {
194
+ max-width: var(--max-w);
195
+ margin: 48px auto 32px;
196
+ padding-top: 24px;
197
+ border-top: 1px solid var(--border);
198
+ display: flex;
199
+ align-items: center;
200
+ justify-content: space-between;
201
+ gap: 12px;
202
+ flex-wrap: wrap;
203
+ }
204
+ .footer-left { font-size: 13px; color: var(--muted); }
205
+ .cta-btn {
206
+ display: inline-flex; align-items: center; gap-6px;
207
+ padding: 8px 16px; border-radius: 8px;
208
+ background: var(--accent); color: #fff;
209
+ font-size: 13px; font-weight: 600;
210
+ text-decoration: none; transition: opacity 0.15s;
211
+ }
212
+ .cta-btn:hover { opacity: 0.85; color: #fff; }
213
+
214
+ @media (max-width: 600px) {
215
+ h1 { font-size: 1.4rem; }
216
+ .topbar { flex-direction: column; align-items: flex-start; }
217
+ }
218
+ </style>
219
+ </head>
220
+ <body>
221
+ <!-- Top bar -->
222
+ <div class="topbar">
223
+ <a class="logo" href="https://webpeel.dev" target="_blank" rel="noopener">
224
+ <div class="logo-mark">W</div>
225
+ <span class="logo-name">WebPeel</span>
226
+ </a>
227
+ <a class="source-link" href="${originalUrl}" target="_blank" rel="noopener noreferrer" title="${originalUrl}">
228
+ โ†— ${originalUrl}
229
+ </a>
230
+ </div>
231
+
232
+ <!-- Article -->
233
+ <main>
234
+ ${share.title ? `<h1>${share.title.replace(/</g, '&lt;').replace(/>/g, '&gt;')}</h1>` : ''}
235
+ <div class="meta">
236
+ ${share.tokens != null ? `<span>๐Ÿ“ ${share.tokens.toLocaleString()} tokens</span>` : ''}
237
+ <span>๐Ÿ‘ ${share.view_count.toLocaleString()} views</span>
238
+ <span>โฐ Expires ${new Date(share.expires_at).toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' })}</span>
239
+ </div>
240
+ <div class="content">
241
+ ${bodyHtml}
242
+ </div>
243
+ </main>
244
+
245
+ <!-- Footer -->
246
+ <footer>
247
+ <span class="footer-left">Powered by <a href="https://webpeel.dev" target="_blank" rel="noopener">WebPeel</a> โ€” clean web reading for humans &amp; AI</span>
248
+ <a class="cta-btn" href="https://app.webpeel.dev" target="_blank" rel="noopener">
249
+ Try WebPeel โ†’
250
+ </a>
251
+ </footer>
252
+ </body>
253
+ </html>`;
254
+ }
255
+ // โ”€โ”€โ”€ Rate limit: 50 shares per day per user โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
256
+ const shareRateMap = new Map();
257
+ const SHARE_DAY_LIMIT = 50;
258
+ const SHARE_DAY_MS = 24 * 60 * 60 * 1000;
259
+ function checkShareRateLimit(userId) {
260
+ const now = Date.now();
261
+ const entry = shareRateMap.get(userId);
262
+ if (!entry || entry.resetAt < now) {
263
+ shareRateMap.set(userId, { count: 1, resetAt: now + SHARE_DAY_MS });
264
+ return { allowed: true, remaining: SHARE_DAY_LIMIT - 1 };
265
+ }
266
+ entry.count++;
267
+ if (entry.count > SHARE_DAY_LIMIT) {
268
+ return { allowed: false, remaining: 0 };
269
+ }
270
+ return { allowed: true, remaining: SHARE_DAY_LIMIT - entry.count };
271
+ }
272
+ // โ”€โ”€โ”€ Public router: GET /s/:id โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
273
+ export function createSharePublicRouter(pool) {
274
+ const router = Router();
275
+ router.get('/s/:id', async (req, res, next) => {
276
+ const id = String(req.params['id'] || '');
277
+ // Only intercept valid-looking 9-char base64url IDs
278
+ if (!/^[A-Za-z0-9_-]{9}$/.test(id)) {
279
+ return next();
280
+ }
281
+ if (!pool) {
282
+ // No DB: fall through to reader's search handler
283
+ return next();
284
+ }
285
+ try {
286
+ // Fetch share and increment view count atomically
287
+ const result = await pool.query(`UPDATE shared_reads
288
+ SET view_count = view_count + 1
289
+ WHERE id = $1
290
+ AND expires_at > NOW()
291
+ RETURNING id, url, title, content, tokens, created_at, expires_at, view_count`, [id]);
292
+ if (result.rows.length === 0) {
293
+ // Not found or expired โ€” fall through to reader's /s/* search handler
294
+ return next();
295
+ }
296
+ const share = result.rows[0];
297
+ // Respond based on Accept header
298
+ const accept = req.headers.accept || '';
299
+ if (accept.includes('application/json')) {
300
+ return res.json({
301
+ success: true,
302
+ shareId: share.id,
303
+ url: share.url,
304
+ title: share.title,
305
+ content: share.content,
306
+ tokens: share.tokens,
307
+ viewCount: share.view_count,
308
+ createdAt: share.created_at,
309
+ expiresAt: share.expires_at,
310
+ });
311
+ }
312
+ if (accept.includes('text/markdown')) {
313
+ res.setHeader('Content-Type', 'text/markdown; charset=utf-8');
314
+ return res.send(share.content);
315
+ }
316
+ // Default: return HTML page (also covers text/html)
317
+ // Override CSP to allow inline styles for the share page
318
+ res.setHeader('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'; img-src https: data:; " +
319
+ "frame-ancestors 'none'; base-uri 'none'; form-action 'none'; " +
320
+ "script-src 'none'");
321
+ res.setHeader('Content-Type', 'text/html; charset=utf-8');
322
+ res.setHeader('Cache-Control', 'public, max-age=60, stale-while-revalidate=300');
323
+ return res.send(buildHtmlPage(share));
324
+ }
325
+ catch (err) {
326
+ log.error('Share GET error:', err.message);
327
+ return res.status(500).json({
328
+ success: false,
329
+ error: { type: 'server_error', message: 'Failed to retrieve share' },
330
+ });
331
+ }
332
+ });
333
+ return router;
334
+ }
335
+ // โ”€โ”€โ”€ Protected router: POST /v1/share โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
336
+ export function createShareRouter(pool) {
337
+ const router = Router();
338
+ router.post('/v1/share', async (req, res) => {
339
+ // Require auth
340
+ const userId = req.auth?.keyInfo?.accountId || req.user?.userId;
341
+ if (!userId) {
342
+ return res.status(401).json({
343
+ success: false,
344
+ error: {
345
+ type: 'unauthorized',
346
+ message: 'Authentication required to create share links.',
347
+ hint: 'Include an Authorization: Bearer <token> header.',
348
+ docs: 'https://webpeel.dev/docs/errors#unauthorized',
349
+ },
350
+ });
351
+ }
352
+ if (!pool) {
353
+ return res.status(503).json({
354
+ success: false,
355
+ error: {
356
+ type: 'unavailable',
357
+ message: 'Share links require a PostgreSQL database.',
358
+ },
359
+ });
360
+ }
361
+ // Rate limit: 50 shares per day per user
362
+ const { allowed, remaining } = checkShareRateLimit(userId);
363
+ res.setHeader('X-Share-Limit-Remaining', remaining.toString());
364
+ if (!allowed) {
365
+ return res.status(429).json({
366
+ success: false,
367
+ error: {
368
+ type: 'rate_limited',
369
+ message: 'Share limit exceeded. Maximum 50 shares per day.',
370
+ hint: 'Wait until tomorrow to create more share links.',
371
+ },
372
+ });
373
+ }
374
+ const { url, content, title } = req.body;
375
+ if (!url || typeof url !== 'string') {
376
+ return res.status(400).json({
377
+ success: false,
378
+ error: {
379
+ type: 'invalid_request',
380
+ message: 'url is required.',
381
+ },
382
+ });
383
+ }
384
+ // SECURITY: SSRF validation
385
+ try {
386
+ validateUrlForSSRF(url);
387
+ }
388
+ catch (err) {
389
+ if (err instanceof SSRFError) {
390
+ return res.status(400).json({
391
+ success: false,
392
+ error: { type: 'ssrf_blocked', message: err.message },
393
+ });
394
+ }
395
+ throw err;
396
+ }
397
+ let shareContent;
398
+ let shareTitle;
399
+ let tokens;
400
+ if (content && typeof content === 'string') {
401
+ // Content provided directly (user already fetched it in dashboard)
402
+ shareContent = content;
403
+ shareTitle = title;
404
+ tokens = content.split(/\s+/).filter(Boolean).length;
405
+ }
406
+ else {
407
+ // Fetch the URL via peel()
408
+ try {
409
+ const result = await peel(url, { timeout: 15000, noEscalate: true });
410
+ shareContent = result.content || '';
411
+ shareTitle = result.title;
412
+ tokens = result.tokens ?? undefined;
413
+ }
414
+ catch (err) {
415
+ log.error('Share: peel failed', { url, error: err.message });
416
+ return res.status(422).json({
417
+ success: false,
418
+ error: {
419
+ type: 'fetch_failed',
420
+ message: `Failed to fetch URL: ${err.message}`,
421
+ },
422
+ });
423
+ }
424
+ }
425
+ if (!shareContent) {
426
+ return res.status(422).json({
427
+ success: false,
428
+ error: {
429
+ type: 'empty_content',
430
+ message: 'No content could be extracted from the URL.',
431
+ },
432
+ });
433
+ }
434
+ // Generate a unique ID with retry for collisions (extremely rare)
435
+ let shareId = '';
436
+ for (let attempt = 0; attempt < 5; attempt++) {
437
+ const candidate = generateShareId();
438
+ const exists = await pool.query('SELECT 1 FROM shared_reads WHERE id = $1', [candidate]);
439
+ if (exists.rows.length === 0) {
440
+ shareId = candidate;
441
+ break;
442
+ }
443
+ }
444
+ if (!shareId) {
445
+ return res.status(500).json({
446
+ success: false,
447
+ error: { type: 'server_error', message: 'Failed to generate unique share ID.' },
448
+ });
449
+ }
450
+ // Insert share into DB
451
+ await pool.query(`INSERT INTO shared_reads (id, url, title, content, tokens, created_by)
452
+ VALUES ($1, $2, $3, $4, $5, $6)`, [shareId, url, shareTitle ?? null, shareContent, tokens ?? null, userId]);
453
+ const shareUrl = `${getBaseUrl()}/s/${shareId}`;
454
+ log.info('Share created', { shareId, url, userId });
455
+ return res.status(201).json({
456
+ success: true,
457
+ shareId,
458
+ shareUrl,
459
+ });
460
+ });
461
+ return router;
462
+ }
package/dist/server/routes/smart-search/handlers/cars.d.ts ADDED
@@ -0,0 +1,2 @@
1
+ import type { SearchIntent, SmartSearchResult } from '../types.js';
2
+ export declare function handleCarSearch(intent: SearchIntent): Promise<SmartSearchResult>;
package/dist/server/routes/smart-search/handlers/cars.js ADDED
@@ -0,0 +1,102 @@
1
+ import { peel } from '../../../../index.js';
2
+ import { getBestSearchProvider } from '../../../../core/search-provider.js';
3
+ import { addAffiliateTag, parsePrice } from '../utils.js';
4
+ import { callLLMQuick, sanitizeSearchQuery, PROMPT_INJECTION_DEFENSE } from '../llm.js';
5
+ export async function handleCarSearch(intent) {
6
+ const t0 = Date.now();
7
+ // Build a clean keyword: strip buying signals, price amounts, and common noise words
8
+ // NOTE: keep "car"/"cars" โ€” they're needed for Cars.com search!
9
+ const keyword = intent.query
10
+ .replace(/\b(buy|cheap|cheapest|under|budget|price|used|new|for sale|listing|deal|best|good|find|search|looking for|want|need|in|near|around)\b/gi, '')
11
+ .replace(/[$]\d[\d,]*/g, '') // strip $30000, $30,000 etc.
12
+ .replace(/\b\d{4,}\b/g, '') // strip standalone 4+ digit numbers (prices, not model years)
13
+ // Remove location words that were already extracted to zip
14
+ .replace(/\b(long island|nassau|suffolk|manhattan|brooklyn|queens|bronx|nyc|new york|los angeles|chicago|houston|miami|boston|seattle|san francisco|washington dc)\b/gi, '')
15
+ .replace(/\s+/g, ' ')
16
+ .trim();
17
+ const params = new URLSearchParams({
18
+ keyword,
19
+ sort: 'list_price',
20
+ stock_type: 'all',
21
+ zip: intent.params.zip || '10001',
22
+ maximum_distance: '50',
23
+ });
24
+ if (intent.params.maxPrice)
25
+ params.set('list_price_max', intent.params.maxPrice);
26
+ const carSearchUrl = `https://www.cars.com/shopping/results/?${params.toString()}`;
27
+ // Search MULTIPLE car sites in parallel โ€” first one with real listings wins
28
+ const { provider } = getBestSearchProvider();
29
+ const [carsComSettled, carGurusSettled, autotraderSettled, redditSettled] = await Promise.allSettled([
30
+ peel(carSearchUrl, { timeout: 15000 }),
31
+ provider.searchWeb(`${keyword} ${intent.params.maxPrice ? 'under $' + intent.params.maxPrice : ''} site:cargurus.com price listing`, { count: 3 }),
32
+ provider.searchWeb(`${keyword} ${intent.params.maxPrice ? 'under $' + intent.params.maxPrice : ''} site:autotrader.com price listing`, { count: 3 }),
33
+ provider.searchWeb(`${keyword} reddit review reliable problems`, { count: 3 }),
34
+ ]);
35
+ // Cars.com peel gives structured listings (best quality)
36
+ const carsComResult = carsComSettled.status === 'fulfilled' ? carsComSettled.value : null;
37
+ let carListings = carsComResult?.domainData?.structured?.listings || [];
38
+ // If Cars.com peel failed, combine results from CarGurus + Autotrader
39
+ if (carListings.length === 0) {
40
+ const carGurusResults = carGurusSettled.status === 'fulfilled' ? carGurusSettled.value : [];
41
+ const autotraderResults = autotraderSettled.status === 'fulfilled' ? autotraderSettled.value : [];
42
+ const allSearchResults = [...carGurusResults, ...autotraderResults];
43
+ carListings = allSearchResults
44
+ .filter(r => r.url && r.title)
45
+ .map(r => {
46
+ const textToSearch = `${r.title || ''} ${r.snippet || ''}`;
47
+ const price = parsePrice(textToSearch);
48
+ const isGenericPage = /\b(for sale near|cars for sale|search|browse|find)\b/i.test(r.title || '') && !price;
49
+ if (isGenericPage)
50
+ return null;
51
+ const yearMatch = (r.title || '').match(/\b(20\d{2}|19\d{2})\b/);
52
+ return {
53
+ title: r.title?.replace(/\s*[-|โ€“โ€”].*$/, '').trim() || 'Car Listing',
54
+ price,
55
+ year: yearMatch ? yearMatch[1] : '',
56
+ url: addAffiliateTag(r.url),
57
+ snippet: r.snippet || '',
58
+ source: (() => { try {
59
+ return new URL(r.url).hostname.replace('www.', '');
60
+ }
61
+ catch {
62
+ return '';
63
+ } })(),
64
+ };
65
+ })
66
+ .filter(Boolean)
67
+ .slice(0, 10);
68
+ }
69
+ const redditResults = redditSettled.status === 'fulfilled' ? redditSettled.value : [];
70
+ // AI synthesis: summarize top listings + Reddit input
71
+ let answer;
72
+ try {
73
+ const listingSummary = carListings.slice(0, 5).map((l) => `${l.title || l.name || 'Car'}: ${l.price || 'price N/A'}, ${l.mileage || ''} miles`).join(', ');
74
+ const redditSnippets = redditResults.slice(0, 2).map(r => r.snippet || '').join(' ');
75
+ const aiPrompt = `${PROMPT_INJECTION_DEFENSE}You are a car buying advisor. The user searched: "${sanitizeSearchQuery(intent.query)}". Here are the top listings: ${listingSummary || 'no listings found'}. Reddit says: ${redditSnippets || 'no community input'}. Give a 2-3 sentence recommendation about the best value. Mention specific prices and models. Cite sources inline as [1], [2], etc. if available. Max 200 words.`;
76
+ const aiText = await callLLMQuick(aiPrompt, { maxTokens: 250, timeoutMs: 8000, temperature: 0.3 });
77
+ if (aiText && aiText.length > 20)
78
+ answer = aiText;
79
+ }
80
+ catch (err) {
81
+ console.warn('[car-search] LLM synthesis failed (graceful fallback):', err.message);
82
+ }
83
+ const content = carListings.length > 0
84
+ ? `# ๐Ÿš— Cars โ€” ${intent.query}\n\n${carListings.map((l, i) => `${i + 1}. **${l.title || l.name}** โ€” ${l.price || 'see price'}${l.mileage ? ` ยท ${String(l.mileage).replace(/\s*mi$/i, '')} mi` : ''}\n ${l.snippet || ''}`).join('\n\n')}`
85
+ : (carsComResult?.content || `# ๐Ÿš— Cars โ€” ${intent.query}\n\nNo listings found. Try a different search.`);
86
+ return {
87
+ type: 'cars',
88
+ source: 'Cars.com + CarGurus + Autotrader + Reddit',
89
+ sourceUrl: carSearchUrl,
90
+ content,
91
+ title: carsComResult?.title || `Cars โ€” ${intent.query}`,
92
+ domainData: carsComResult?.domainData,
93
+ structured: carsComResult?.domainData?.structured || (carListings.length > 0 ? { listings: carListings } : undefined),
94
+ tokens: carsComResult?.tokens || content.split(/\s+/).length,
95
+ fetchTimeMs: Date.now() - t0,
96
+ ...(answer !== undefined ? { answer } : {}),
97
+ sources: [
98
+ { type: 'cars', url: carSearchUrl, count: carListings.length },
99
+ { type: 'reddit', threads: redditResults.map(r => ({ title: r.title, url: r.url, snippet: r.snippet })) },
100
+ ],
101
+ };
102
+ }
package/dist/server/routes/smart-search/handlers/flights.d.ts ADDED
@@ -0,0 +1,2 @@
1
+ import type { SearchIntent, SmartSearchResult } from '../types.js';
2
+ export declare function handleFlightSearch(intent: SearchIntent): Promise<SmartSearchResult>;
package/dist/server/routes/smart-search/handlers/flights.js ADDED
@@ -0,0 +1,72 @@
1
+ import { getBestSearchProvider } from '../../../../core/search-provider.js';
2
+ import { addAffiliateTag } from '../utils.js';
3
+ import { callLLMQuick, sanitizeSearchQuery, PROMPT_INJECTION_DEFENSE } from '../llm.js';
4
+ export async function handleFlightSearch(intent) {
5
+ const t0 = Date.now();
6
+ const gfUrl = `https://www.google.com/travel/flights?q=Flights+${encodeURIComponent(intent.query)}+one+way`;
7
+ // Search for actual flight prices + Reddit tips in parallel
8
+ const { provider: searchProvider } = getBestSearchProvider();
9
+ const [kayakSettled, skyscannerSettled, momondoSettled, googleSettled, redditSettled] = await Promise.allSettled([
10
+ searchProvider.searchWeb(`${intent.query} cheapest price site:kayak.com`, { count: 2 }),
11
+ searchProvider.searchWeb(`${intent.query} cheapest flights site:skyscanner.com`, { count: 3 }),
12
+ searchProvider.searchWeb(`${intent.query} cheap flights site:momondo.com OR site:cheapflights.com`, { count: 3 }),
13
+ searchProvider.searchWeb(`${intent.query} flights site:google.com/travel`, { count: 2 }),
14
+ searchProvider.searchWeb(`${intent.query} flights reddit tips cheap`, { count: 3 }),
15
+ ]);
16
+ const flightResults = [
17
+ ...(kayakSettled.status === 'fulfilled' ? kayakSettled.value : []),
18
+ ...(skyscannerSettled.status === 'fulfilled' ? skyscannerSettled.value : []),
19
+ ...(momondoSettled.status === 'fulfilled' ? momondoSettled.value : []),
20
+ ...(googleSettled.status === 'fulfilled' ? googleSettled.value : []),
21
+ ];
22
+ const redditResults = redditSettled.status === 'fulfilled' ? redditSettled.value : [];
23
+ // Build content from search results + static booking links as fallback
24
+ const searchSection = flightResults.length > 0
25
+ ? `## ๐Ÿ” Flight Results\n\n${flightResults.slice(0, 6).map((r, i) => `${i + 1}. **[${r.title}](${r.url})**\n ${r.snippet || ''}`).join('\n\n')}\n\n`
26
+ : '';
27
+ const content = `# โœˆ๏ธ Flights โ€” ${intent.query}
28
+
29
+ ${searchSection}## ๐Ÿ“Œ Book Directly
30
+
31
+ 1. **[Google Flights](${gfUrl})**
32
+ Direct link to Google Flights search
33
+
34
+ 2. **[Kayak](https://www.kayak.com/flights?a=help)**
35
+ Compare prices across all airlines
36
+
37
+ 3. **[Expedia](https://www.expedia.com/Flights)**
38
+ Flights, hotels, bundles
39
+
40
+ 4. **[Skyscanner](https://www.skyscanner.com/)**
41
+ Popular international flight search
42
+
43
+ 5. **[Momondo](https://www.momondo.com/)**
44
+ Meta-search with lowest prices
45
+
46
+ ---
47
+ `;
48
+ // AI synthesis from search results + Reddit tips
49
+ let answer;
50
+ try {
51
+ const flightInfo = flightResults.slice(0, 5).map(r => `${r.title}: ${r.snippet || ''}`).join('\n');
52
+ const redditSnippets = redditResults.slice(0, 2).map(r => `${r.title}: ${r.snippet || ''}`).join('\n');
53
+ const aiPrompt = `${PROMPT_INJECTION_DEFENSE}You are a flight booking advisor. ONLY use information from the sources below. Do NOT make up prices, airlines, or routes not mentioned. User searched: "${sanitizeSearchQuery(intent.query)}". Web results: ${flightInfo || 'no results found'}. Reddit tips: ${redditSnippets || 'none'}. Give a 2-3 sentence tip about cheapest flights for this route based ONLY on the sources. Mention actual prices found and booking sites. Max 200 words. Cite sources inline as [1], [2], [3].`;
54
+ const aiText = await callLLMQuick(aiPrompt, { maxTokens: 250, timeoutMs: 8000, temperature: 0.3 });
55
+ if (aiText && aiText.length > 20)
56
+ answer = aiText;
57
+ }
58
+ catch (err) {
59
+ console.warn('[flight-search] LLM synthesis failed (graceful fallback):', err.message);
60
+ }
61
+ return {
62
+ type: 'flights',
63
+ source: 'Flight Search',
64
+ sourceUrl: gfUrl,
65
+ content,
66
+ title: `Flights โ€” ${intent.query}`,
67
+ structured: { listings: flightResults.slice(0, 6).map(r => ({ title: r.title, url: addAffiliateTag(r.url), snippet: r.snippet })) },
68
+ tokens: content.split(' ').length,
69
+ fetchTimeMs: Date.now() - t0,
70
+ ...(answer !== undefined ? { answer } : {}),
71
+ };
72
+ }
package/dist/server/routes/smart-search/handlers/general.d.ts ADDED
@@ -0,0 +1,13 @@
1
+ import type { SmartSearchResult } from '../types.js';
2
+ /**
3
+ * Parse a transit/travel booking query to extract origin, destination, dates, and trip type.
4
+ */
5
+ export declare function parseTransitQuery(query: string): {
6
+ origin: string;
7
+ destination: string;
8
+ departDate: string;
9
+ returnDate: string;
10
+ isRoundTrip: boolean;
11
+ mode: string;
12
+ };
13
+ export declare function handleGeneralSearch(query: string): Promise<SmartSearchResult>;