@iflow-mcp/jakeliume-webpeel 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (547) hide show
  1. package/LICENSE +15 -0
  2. package/README.md +313 -0
  3. package/dist/cache.d.ts +30 -0
  4. package/dist/cache.js +139 -0
  5. package/dist/cli/commands/auth.d.ts +5 -0
  6. package/dist/cli/commands/auth.js +411 -0
  7. package/dist/cli/commands/doctor.d.ts +37 -0
  8. package/dist/cli/commands/doctor.js +371 -0
  9. package/dist/cli/commands/fetch.d.ts +6 -0
  10. package/dist/cli/commands/fetch.js +1345 -0
  11. package/dist/cli/commands/guide.d.ts +2 -0
  12. package/dist/cli/commands/guide.js +183 -0
  13. package/dist/cli/commands/interact.d.ts +5 -0
  14. package/dist/cli/commands/interact.js +840 -0
  15. package/dist/cli/commands/jobs.d.ts +5 -0
  16. package/dist/cli/commands/jobs.js +997 -0
  17. package/dist/cli/commands/monitor.d.ts +12 -0
  18. package/dist/cli/commands/monitor.js +197 -0
  19. package/dist/cli/commands/observe.d.ts +12 -0
  20. package/dist/cli/commands/observe.js +158 -0
  21. package/dist/cli/commands/screenshot.d.ts +5 -0
  22. package/dist/cli/commands/screenshot.js +282 -0
  23. package/dist/cli/commands/search.d.ts +5 -0
  24. package/dist/cli/commands/search.js +1021 -0
  25. package/dist/cli/commands/setup.d.ts +13 -0
  26. package/dist/cli/commands/setup.js +244 -0
  27. package/dist/cli/commands/skill.d.ts +15 -0
  28. package/dist/cli/commands/skill.js +195 -0
  29. package/dist/cli/utils.d.ts +84 -0
  30. package/dist/cli/utils.js +806 -0
  31. package/dist/cli-auth.d.ts +75 -0
  32. package/dist/cli-auth.js +369 -0
  33. package/dist/cli.d.ts +17 -0
  34. package/dist/cli.js +99 -0
  35. package/dist/core/actions.d.ts +69 -0
  36. package/dist/core/actions.js +495 -0
  37. package/dist/core/agent.d.ts +98 -0
  38. package/dist/core/agent.js +558 -0
  39. package/dist/core/answer.d.ts +42 -0
  40. package/dist/core/answer.js +395 -0
  41. package/dist/core/application-tracker.d.ts +84 -0
  42. package/dist/core/application-tracker.js +184 -0
  43. package/dist/core/apply.d.ts +162 -0
  44. package/dist/core/apply.js +816 -0
  45. package/dist/core/auth-detection.d.ts +35 -0
  46. package/dist/core/auth-detection.js +358 -0
  47. package/dist/core/auto-extract.d.ts +82 -0
  48. package/dist/core/auto-extract.js +604 -0
  49. package/dist/core/auto-interact.d.ts +23 -0
  50. package/dist/core/auto-interact.js +246 -0
  51. package/dist/core/bm25-filter.d.ts +66 -0
  52. package/dist/core/bm25-filter.js +288 -0
  53. package/dist/core/branding.d.ts +54 -0
  54. package/dist/core/branding.js +234 -0
  55. package/dist/core/browser-fetch.d.ts +323 -0
  56. package/dist/core/browser-fetch.js +1600 -0
  57. package/dist/core/browser-pool.d.ts +91 -0
  58. package/dist/core/browser-pool.js +550 -0
  59. package/dist/core/budget.d.ts +42 -0
  60. package/dist/core/budget.js +324 -0
  61. package/dist/core/business-intel.d.ts +47 -0
  62. package/dist/core/business-intel.js +279 -0
  63. package/dist/core/cache.d.ts +13 -0
  64. package/dist/core/cache.js +121 -0
  65. package/dist/core/cf-worker-proxy.d.ts +32 -0
  66. package/dist/core/cf-worker-proxy.js +87 -0
  67. package/dist/core/challenge-detection.d.ts +26 -0
  68. package/dist/core/challenge-detection.js +468 -0
  69. package/dist/core/change-tracking.d.ts +75 -0
  70. package/dist/core/change-tracking.js +276 -0
  71. package/dist/core/chunker.d.ts +46 -0
  72. package/dist/core/chunker.js +249 -0
  73. package/dist/core/chunking.d.ts +42 -0
  74. package/dist/core/chunking.js +181 -0
  75. package/dist/core/circuit-breaker.d.ts +44 -0
  76. package/dist/core/circuit-breaker.js +85 -0
  77. package/dist/core/content-pruner.d.ts +47 -0
  78. package/dist/core/content-pruner.js +425 -0
  79. package/dist/core/cookie-cache.d.ts +60 -0
  80. package/dist/core/cookie-cache.js +163 -0
  81. package/dist/core/crawl-checkpoint.d.ts +54 -0
  82. package/dist/core/crawl-checkpoint.js +104 -0
  83. package/dist/core/crawler.d.ts +84 -0
  84. package/dist/core/crawler.js +349 -0
  85. package/dist/core/cross-verify.d.ts +27 -0
  86. package/dist/core/cross-verify.js +93 -0
  87. package/dist/core/deep-fetch.d.ts +74 -0
  88. package/dist/core/deep-fetch.js +405 -0
  89. package/dist/core/deep-research.d.ts +141 -0
  90. package/dist/core/deep-research.js +972 -0
  91. package/dist/core/design-analysis.d.ts +70 -0
  92. package/dist/core/design-analysis.js +490 -0
  93. package/dist/core/design-compare.d.ts +38 -0
  94. package/dist/core/design-compare.js +264 -0
  95. package/dist/core/diff.d.ts +61 -0
  96. package/dist/core/diff.js +289 -0
  97. package/dist/core/dns-cache.d.ts +20 -0
  98. package/dist/core/dns-cache.js +198 -0
  99. package/dist/core/documents.d.ts +23 -0
  100. package/dist/core/documents.js +123 -0
  101. package/dist/core/domain-memory.d.ts +66 -0
  102. package/dist/core/domain-memory.js +163 -0
  103. package/dist/core/domain-verify.d.ts +40 -0
  104. package/dist/core/domain-verify.js +379 -0
  105. package/dist/core/engine-ranker.d.ts +112 -0
  106. package/dist/core/engine-ranker.js +395 -0
  107. package/dist/core/extract-inline.d.ts +38 -0
  108. package/dist/core/extract-inline.js +215 -0
  109. package/dist/core/extract-listings.d.ts +38 -0
  110. package/dist/core/extract-listings.js +461 -0
  111. package/dist/core/extract.d.ts +9 -0
  112. package/dist/core/extract.js +139 -0
  113. package/dist/core/fetch-cache.d.ts +57 -0
  114. package/dist/core/fetch-cache.js +95 -0
  115. package/dist/core/fetcher.d.ts +13 -0
  116. package/dist/core/fetcher.js +12 -0
  117. package/dist/core/google-cache.d.ts +29 -0
  118. package/dist/core/google-cache.js +180 -0
  119. package/dist/core/google-serp-parser.d.ts +82 -0
  120. package/dist/core/google-serp-parser.js +287 -0
  121. package/dist/core/hotel-search.d.ts +122 -0
  122. package/dist/core/hotel-search.js +382 -0
  123. package/dist/core/http-fetch.d.ts +72 -0
  124. package/dist/core/http-fetch.js +820 -0
  125. package/dist/core/human.d.ts +175 -0
  126. package/dist/core/human.js +680 -0
  127. package/dist/core/image-caption.d.ts +44 -0
  128. package/dist/core/image-caption.js +271 -0
  129. package/dist/core/jobs.d.ts +75 -0
  130. package/dist/core/jobs.js +634 -0
  131. package/dist/core/json-ld.d.ts +15 -0
  132. package/dist/core/json-ld.js +617 -0
  133. package/dist/core/language-detect.d.ts +18 -0
  134. package/dist/core/language-detect.js +135 -0
  135. package/dist/core/links.d.ts +10 -0
  136. package/dist/core/links.js +44 -0
  137. package/dist/core/llm-extract.d.ts +71 -0
  138. package/dist/core/llm-extract.js +507 -0
  139. package/dist/core/llm-provider.d.ts +100 -0
  140. package/dist/core/llm-provider.js +702 -0
  141. package/dist/core/local-search.d.ts +60 -0
  142. package/dist/core/local-search.js +308 -0
  143. package/dist/core/logger.d.ts +28 -0
  144. package/dist/core/logger.js +104 -0
  145. package/dist/core/map.d.ts +33 -0
  146. package/dist/core/map.js +127 -0
  147. package/dist/core/markdown.d.ts +92 -0
  148. package/dist/core/markdown.js +809 -0
  149. package/dist/core/metadata.d.ts +34 -0
  150. package/dist/core/metadata.js +422 -0
  151. package/dist/core/observe.d.ts +113 -0
  152. package/dist/core/observe.js +395 -0
  153. package/dist/core/ocr.d.ts +12 -0
  154. package/dist/core/ocr.js +33 -0
  155. package/dist/core/paginate.d.ts +31 -0
  156. package/dist/core/paginate.js +106 -0
  157. package/dist/core/pdf.d.ts +8 -0
  158. package/dist/core/pdf.js +25 -0
  159. package/dist/core/peel-tls.d.ts +25 -0
  160. package/dist/core/peel-tls.js +220 -0
  161. package/dist/core/pipeline.d.ts +132 -0
  162. package/dist/core/pipeline.js +1666 -0
  163. package/dist/core/profiles.d.ts +61 -0
  164. package/dist/core/profiles.js +350 -0
  165. package/dist/core/prompt-guard.d.ts +30 -0
  166. package/dist/core/prompt-guard.js +119 -0
  167. package/dist/core/proxy-config.d.ts +90 -0
  168. package/dist/core/proxy-config.js +172 -0
  169. package/dist/core/quick-answer.d.ts +53 -0
  170. package/dist/core/quick-answer.js +833 -0
  171. package/dist/core/rate-governor.d.ts +80 -0
  172. package/dist/core/rate-governor.js +238 -0
  173. package/dist/core/readability.d.ts +57 -0
  174. package/dist/core/readability.js +533 -0
  175. package/dist/core/research.d.ts +66 -0
  176. package/dist/core/research.js +270 -0
  177. package/dist/core/retry.d.ts +60 -0
  178. package/dist/core/retry.js +119 -0
  179. package/dist/core/safe-browsing.d.ts +30 -0
  180. package/dist/core/safe-browsing.js +206 -0
  181. package/dist/core/schema-extraction.d.ts +66 -0
  182. package/dist/core/schema-extraction.js +352 -0
  183. package/dist/core/schema-postprocess.d.ts +32 -0
  184. package/dist/core/schema-postprocess.js +469 -0
  185. package/dist/core/schema-templates.d.ts +19 -0
  186. package/dist/core/schema-templates.js +143 -0
  187. package/dist/core/screenshot.d.ts +224 -0
  188. package/dist/core/screenshot.js +207 -0
  189. package/dist/core/search-engines.d.ts +25 -0
  190. package/dist/core/search-engines.js +182 -0
  191. package/dist/core/search-provider.d.ts +243 -0
  192. package/dist/core/search-provider.js +1629 -0
  193. package/dist/core/searxng-provider.d.ts +35 -0
  194. package/dist/core/searxng-provider.js +105 -0
  195. package/dist/core/selective-evidence.d.ts +151 -0
  196. package/dist/core/selective-evidence.js +389 -0
  197. package/dist/core/site-search.d.ts +44 -0
  198. package/dist/core/site-search.js +252 -0
  199. package/dist/core/sitemap.d.ts +23 -0
  200. package/dist/core/sitemap.js +105 -0
  201. package/dist/core/source-credibility.d.ts +29 -0
  202. package/dist/core/source-credibility.js +584 -0
  203. package/dist/core/source-scoring.d.ts +166 -0
  204. package/dist/core/source-scoring.js +396 -0
  205. package/dist/core/stemmer.d.ts +38 -0
  206. package/dist/core/stemmer.js +509 -0
  207. package/dist/core/strategies.d.ts +104 -0
  208. package/dist/core/strategies.js +1044 -0
  209. package/dist/core/strategy-hooks.d.ts +145 -0
  210. package/dist/core/strategy-hooks.js +74 -0
  211. package/dist/core/structured-extract.d.ts +43 -0
  212. package/dist/core/structured-extract.js +550 -0
  213. package/dist/core/summarize.d.ts +17 -0
  214. package/dist/core/summarize.js +78 -0
  215. package/dist/core/synonyms.d.ts +42 -0
  216. package/dist/core/synonyms.js +184 -0
  217. package/dist/core/system-monitor.d.ts +61 -0
  218. package/dist/core/system-monitor.js +133 -0
  219. package/dist/core/table-format.d.ts +30 -0
  220. package/dist/core/table-format.js +146 -0
  221. package/dist/core/threat-feeds.d.ts +23 -0
  222. package/dist/core/threat-feeds.js +104 -0
  223. package/dist/core/timing.d.ts +21 -0
  224. package/dist/core/timing.js +33 -0
  225. package/dist/core/transcript-export.d.ts +47 -0
  226. package/dist/core/transcript-export.js +107 -0
  227. package/dist/core/user-agents.d.ts +82 -0
  228. package/dist/core/user-agents.js +239 -0
  229. package/dist/core/vertical-search.d.ts +54 -0
  230. package/dist/core/vertical-search.js +158 -0
  231. package/dist/core/watch-manager.d.ts +175 -0
  232. package/dist/core/watch-manager.js +416 -0
  233. package/dist/core/watch.d.ts +101 -0
  234. package/dist/core/watch.js +389 -0
  235. package/dist/core/youtube.d.ts +130 -0
  236. package/dist/core/youtube.js +1175 -0
  237. package/dist/ee/challenge-re-export.d.ts +1 -0
  238. package/dist/ee/challenge-re-export.js +1 -0
  239. package/dist/ee/challenge-solver.d.ts +72 -0
  240. package/dist/ee/challenge-solver.js +720 -0
  241. package/dist/ee/domain-extractors.d.ts +8 -0
  242. package/dist/ee/domain-extractors.js +8 -0
  243. package/dist/ee/domain-intel.d.ts +16 -0
  244. package/dist/ee/domain-intel.js +133 -0
  245. package/dist/ee/extractors/allrecipes.d.ts +2 -0
  246. package/dist/ee/extractors/allrecipes.js +120 -0
  247. package/dist/ee/extractors/amazon.d.ts +2 -0
  248. package/dist/ee/extractors/amazon.js +78 -0
  249. package/dist/ee/extractors/arxiv.d.ts +2 -0
  250. package/dist/ee/extractors/arxiv.js +137 -0
  251. package/dist/ee/extractors/bestbuy.d.ts +2 -0
  252. package/dist/ee/extractors/bestbuy.js +78 -0
  253. package/dist/ee/extractors/carscom.d.ts +2 -0
  254. package/dist/ee/extractors/carscom.js +121 -0
  255. package/dist/ee/extractors/coingecko.d.ts +2 -0
  256. package/dist/ee/extractors/coingecko.js +134 -0
  257. package/dist/ee/extractors/craigslist.d.ts +2 -0
  258. package/dist/ee/extractors/craigslist.js +92 -0
  259. package/dist/ee/extractors/devto.d.ts +2 -0
  260. package/dist/ee/extractors/devto.js +135 -0
  261. package/dist/ee/extractors/ebay.d.ts +2 -0
  262. package/dist/ee/extractors/ebay.js +90 -0
  263. package/dist/ee/extractors/espn.d.ts +2 -0
  264. package/dist/ee/extractors/espn.js +260 -0
  265. package/dist/ee/extractors/etsy.d.ts +2 -0
  266. package/dist/ee/extractors/etsy.js +52 -0
  267. package/dist/ee/extractors/facebook.d.ts +2 -0
  268. package/dist/ee/extractors/facebook.js +46 -0
  269. package/dist/ee/extractors/github.d.ts +2 -0
  270. package/dist/ee/extractors/github.js +196 -0
  271. package/dist/ee/extractors/google-flights.d.ts +2 -0
  272. package/dist/ee/extractors/google-flights.js +176 -0
  273. package/dist/ee/extractors/hackernews.d.ts +2 -0
  274. package/dist/ee/extractors/hackernews.js +147 -0
  275. package/dist/ee/extractors/imdb.d.ts +2 -0
  276. package/dist/ee/extractors/imdb.js +172 -0
  277. package/dist/ee/extractors/index.d.ts +26 -0
  278. package/dist/ee/extractors/index.js +247 -0
  279. package/dist/ee/extractors/instagram.d.ts +2 -0
  280. package/dist/ee/extractors/instagram.js +102 -0
  281. package/dist/ee/extractors/kalshi.d.ts +2 -0
  282. package/dist/ee/extractors/kalshi.js +121 -0
  283. package/dist/ee/extractors/kayak-cars.d.ts +2 -0
  284. package/dist/ee/extractors/kayak-cars.js +270 -0
  285. package/dist/ee/extractors/linkedin.d.ts +2 -0
  286. package/dist/ee/extractors/linkedin.js +113 -0
  287. package/dist/ee/extractors/medium.d.ts +2 -0
  288. package/dist/ee/extractors/medium.js +130 -0
  289. package/dist/ee/extractors/news.d.ts +4 -0
  290. package/dist/ee/extractors/news.js +173 -0
  291. package/dist/ee/extractors/npm.d.ts +2 -0
  292. package/dist/ee/extractors/npm.js +86 -0
  293. package/dist/ee/extractors/pdf.d.ts +2 -0
  294. package/dist/ee/extractors/pdf.js +108 -0
  295. package/dist/ee/extractors/pinterest.d.ts +2 -0
  296. package/dist/ee/extractors/pinterest.js +34 -0
  297. package/dist/ee/extractors/polymarket.d.ts +2 -0
  298. package/dist/ee/extractors/polymarket.js +358 -0
  299. package/dist/ee/extractors/producthunt.d.ts +2 -0
  300. package/dist/ee/extractors/producthunt.js +88 -0
  301. package/dist/ee/extractors/pubmed.d.ts +2 -0
  302. package/dist/ee/extractors/pubmed.js +162 -0
  303. package/dist/ee/extractors/pypi.d.ts +2 -0
  304. package/dist/ee/extractors/pypi.js +80 -0
  305. package/dist/ee/extractors/reddit.d.ts +2 -0
  306. package/dist/ee/extractors/reddit.js +438 -0
  307. package/dist/ee/extractors/redfin.d.ts +2 -0
  308. package/dist/ee/extractors/redfin.js +156 -0
  309. package/dist/ee/extractors/semanticscholar.d.ts +2 -0
  310. package/dist/ee/extractors/semanticscholar.js +131 -0
  311. package/dist/ee/extractors/shared.d.ts +12 -0
  312. package/dist/ee/extractors/shared.js +76 -0
  313. package/dist/ee/extractors/soundcloud.d.ts +2 -0
  314. package/dist/ee/extractors/soundcloud.js +34 -0
  315. package/dist/ee/extractors/sportsbetting.d.ts +2 -0
  316. package/dist/ee/extractors/sportsbetting.js +37 -0
  317. package/dist/ee/extractors/spotify.d.ts +2 -0
  318. package/dist/ee/extractors/spotify.js +34 -0
  319. package/dist/ee/extractors/stackoverflow.d.ts +2 -0
  320. package/dist/ee/extractors/stackoverflow.js +61 -0
  321. package/dist/ee/extractors/substack.d.ts +2 -0
  322. package/dist/ee/extractors/substack.js +115 -0
  323. package/dist/ee/extractors/substackroot.d.ts +2 -0
  324. package/dist/ee/extractors/substackroot.js +46 -0
  325. package/dist/ee/extractors/tiktok.d.ts +2 -0
  326. package/dist/ee/extractors/tiktok.js +29 -0
  327. package/dist/ee/extractors/tradingview.d.ts +2 -0
  328. package/dist/ee/extractors/tradingview.js +182 -0
  329. package/dist/ee/extractors/twitch.d.ts +2 -0
  330. package/dist/ee/extractors/twitch.js +36 -0
  331. package/dist/ee/extractors/twitter.d.ts +2 -0
  332. package/dist/ee/extractors/twitter.js +327 -0
  333. package/dist/ee/extractors/types.d.ts +14 -0
  334. package/dist/ee/extractors/types.js +1 -0
  335. package/dist/ee/extractors/walmart.d.ts +2 -0
  336. package/dist/ee/extractors/walmart.js +50 -0
  337. package/dist/ee/extractors/weather.d.ts +2 -0
  338. package/dist/ee/extractors/weather.js +133 -0
  339. package/dist/ee/extractors/wikipedia.d.ts +4 -0
  340. package/dist/ee/extractors/wikipedia.js +235 -0
  341. package/dist/ee/extractors/yelp.d.ts +2 -0
  342. package/dist/ee/extractors/yelp.js +216 -0
  343. package/dist/ee/extractors/youtube.d.ts +2 -0
  344. package/dist/ee/extractors/youtube.js +189 -0
  345. package/dist/ee/extractors/zillow.d.ts +54 -0
  346. package/dist/ee/extractors/zillow.js +247 -0
  347. package/dist/ee/extractors-re-export.d.ts +1 -0
  348. package/dist/ee/extractors-re-export.js +1 -0
  349. package/dist/ee/premium-hooks.d.ts +20 -0
  350. package/dist/ee/premium-hooks.js +50 -0
  351. package/dist/ee/spa-detection.d.ts +2 -0
  352. package/dist/ee/spa-detection.js +2 -0
  353. package/dist/ee/stability.d.ts +4 -0
  354. package/dist/ee/stability.js +29 -0
  355. package/dist/ee/swr-cache.d.ts +14 -0
  356. package/dist/ee/swr-cache.js +34 -0
  357. package/dist/index.d.ts +143 -0
  358. package/dist/index.js +291 -0
  359. package/dist/integrations/index.d.ts +2 -0
  360. package/dist/integrations/index.js +2 -0
  361. package/dist/integrations/langchain.d.ts +64 -0
  362. package/dist/integrations/langchain.js +115 -0
  363. package/dist/integrations/llamaindex.d.ts +50 -0
  364. package/dist/integrations/llamaindex.js +91 -0
  365. package/dist/mcp/handlers/act.d.ts +5 -0
  366. package/dist/mcp/handlers/act.js +34 -0
  367. package/dist/mcp/handlers/definitions.d.ts +6 -0
  368. package/dist/mcp/handlers/definitions.js +395 -0
  369. package/dist/mcp/handlers/extract.d.ts +7 -0
  370. package/dist/mcp/handlers/extract.js +135 -0
  371. package/dist/mcp/handlers/fetch.d.ts +6 -0
  372. package/dist/mcp/handlers/fetch.js +98 -0
  373. package/dist/mcp/handlers/find.d.ts +5 -0
  374. package/dist/mcp/handlers/find.js +137 -0
  375. package/dist/mcp/handlers/index.d.ts +13 -0
  376. package/dist/mcp/handlers/index.js +63 -0
  377. package/dist/mcp/handlers/legacy.d.ts +25 -0
  378. package/dist/mcp/handlers/legacy.js +450 -0
  379. package/dist/mcp/handlers/meta.d.ts +6 -0
  380. package/dist/mcp/handlers/meta.js +40 -0
  381. package/dist/mcp/handlers/monitor.d.ts +5 -0
  382. package/dist/mcp/handlers/monitor.js +41 -0
  383. package/dist/mcp/handlers/observe.d.ts +8 -0
  384. package/dist/mcp/handlers/observe.js +37 -0
  385. package/dist/mcp/handlers/read.d.ts +6 -0
  386. package/dist/mcp/handlers/read.js +78 -0
  387. package/dist/mcp/handlers/see.d.ts +5 -0
  388. package/dist/mcp/handlers/see.js +75 -0
  389. package/dist/mcp/handlers/types.d.ts +29 -0
  390. package/dist/mcp/handlers/types.js +28 -0
  391. package/dist/mcp/server.d.ts +7 -0
  392. package/dist/mcp/server.js +108 -0
  393. package/dist/mcp/smart-router.d.ts +23 -0
  394. package/dist/mcp/smart-router.js +178 -0
  395. package/dist/server/app.d.ts +14 -0
  396. package/dist/server/app.js +632 -0
  397. package/dist/server/auth-store.d.ts +28 -0
  398. package/dist/server/auth-store.js +88 -0
  399. package/dist/server/bull-queues.d.ts +60 -0
  400. package/dist/server/bull-queues.js +90 -0
  401. package/dist/server/email-service.d.ts +55 -0
  402. package/dist/server/email-service.js +291 -0
  403. package/dist/server/job-queue.d.ts +100 -0
  404. package/dist/server/job-queue.js +145 -0
  405. package/dist/server/logger.d.ts +10 -0
  406. package/dist/server/logger.js +37 -0
  407. package/dist/server/middleware/audit-log.d.ts +14 -0
  408. package/dist/server/middleware/audit-log.js +73 -0
  409. package/dist/server/middleware/auth.d.ts +35 -0
  410. package/dist/server/middleware/auth.js +225 -0
  411. package/dist/server/middleware/rate-limit.d.ts +50 -0
  412. package/dist/server/middleware/rate-limit.js +270 -0
  413. package/dist/server/middleware/scope-guard.d.ts +25 -0
  414. package/dist/server/middleware/scope-guard.js +45 -0
  415. package/dist/server/middleware/url-validator.d.ts +15 -0
  416. package/dist/server/middleware/url-validator.js +201 -0
  417. package/dist/server/openapi.yaml +6418 -0
  418. package/dist/server/pg-auth-store.d.ts +146 -0
  419. package/dist/server/pg-auth-store.js +576 -0
  420. package/dist/server/pg-job-queue.d.ts +59 -0
  421. package/dist/server/pg-job-queue.js +375 -0
  422. package/dist/server/routes/activity.d.ts +6 -0
  423. package/dist/server/routes/activity.js +79 -0
  424. package/dist/server/routes/admin-active.d.ts +7 -0
  425. package/dist/server/routes/admin-active.js +120 -0
  426. package/dist/server/routes/admin-stats.d.ts +7 -0
  427. package/dist/server/routes/admin-stats.js +176 -0
  428. package/dist/server/routes/agent.d.ts +24 -0
  429. package/dist/server/routes/agent.js +480 -0
  430. package/dist/server/routes/answer.d.ts +5 -0
  431. package/dist/server/routes/answer.js +125 -0
  432. package/dist/server/routes/ask.d.ts +28 -0
  433. package/dist/server/routes/ask.js +295 -0
  434. package/dist/server/routes/batch.d.ts +6 -0
  435. package/dist/server/routes/batch.js +493 -0
  436. package/dist/server/routes/cache-warm.d.ts +25 -0
  437. package/dist/server/routes/cache-warm.js +212 -0
  438. package/dist/server/routes/cli-usage.d.ts +6 -0
  439. package/dist/server/routes/cli-usage.js +127 -0
  440. package/dist/server/routes/compat.d.ts +23 -0
  441. package/dist/server/routes/compat.js +652 -0
  442. package/dist/server/routes/crawl.d.ts +13 -0
  443. package/dist/server/routes/crawl.js +287 -0
  444. package/dist/server/routes/deep-fetch.d.ts +8 -0
  445. package/dist/server/routes/deep-fetch.js +57 -0
  446. package/dist/server/routes/deep-research.d.ts +11 -0
  447. package/dist/server/routes/deep-research.js +232 -0
  448. package/dist/server/routes/demo.d.ts +24 -0
  449. package/dist/server/routes/demo.js +517 -0
  450. package/dist/server/routes/do.d.ts +8 -0
  451. package/dist/server/routes/do.js +72 -0
  452. package/dist/server/routes/extract.d.ts +14 -0
  453. package/dist/server/routes/extract.js +325 -0
  454. package/dist/server/routes/feed.d.ts +15 -0
  455. package/dist/server/routes/feed.js +311 -0
  456. package/dist/server/routes/fetch-queue.d.ts +13 -0
  457. package/dist/server/routes/fetch-queue.js +357 -0
  458. package/dist/server/routes/fetch.d.ts +7 -0
  459. package/dist/server/routes/fetch.js +1274 -0
  460. package/dist/server/routes/go.d.ts +14 -0
  461. package/dist/server/routes/go.js +81 -0
  462. package/dist/server/routes/health.d.ts +11 -0
  463. package/dist/server/routes/health.js +141 -0
  464. package/dist/server/routes/jobs.d.ts +7 -0
  465. package/dist/server/routes/jobs.js +574 -0
  466. package/dist/server/routes/map.d.ts +11 -0
  467. package/dist/server/routes/map.js +116 -0
  468. package/dist/server/routes/mcp.d.ts +14 -0
  469. package/dist/server/routes/mcp.js +197 -0
  470. package/dist/server/routes/metrics.d.ts +37 -0
  471. package/dist/server/routes/metrics.js +149 -0
  472. package/dist/server/routes/oauth.d.ts +9 -0
  473. package/dist/server/routes/oauth.js +396 -0
  474. package/dist/server/routes/playground.d.ts +17 -0
  475. package/dist/server/routes/playground.js +283 -0
  476. package/dist/server/routes/reader.d.ts +18 -0
  477. package/dist/server/routes/reader.js +192 -0
  478. package/dist/server/routes/research.d.ts +14 -0
  479. package/dist/server/routes/research.js +482 -0
  480. package/dist/server/routes/screenshot.d.ts +22 -0
  481. package/dist/server/routes/screenshot.js +820 -0
  482. package/dist/server/routes/search.d.ts +6 -0
  483. package/dist/server/routes/search.js +874 -0
  484. package/dist/server/routes/session.d.ts +17 -0
  485. package/dist/server/routes/session.js +548 -0
  486. package/dist/server/routes/share.d.ts +18 -0
  487. package/dist/server/routes/share.js +462 -0
  488. package/dist/server/routes/smart-search/handlers/cars.d.ts +2 -0
  489. package/dist/server/routes/smart-search/handlers/cars.js +102 -0
  490. package/dist/server/routes/smart-search/handlers/flights.d.ts +2 -0
  491. package/dist/server/routes/smart-search/handlers/flights.js +72 -0
  492. package/dist/server/routes/smart-search/handlers/general.d.ts +13 -0
  493. package/dist/server/routes/smart-search/handlers/general.js +717 -0
  494. package/dist/server/routes/smart-search/handlers/hotels.d.ts +2 -0
  495. package/dist/server/routes/smart-search/handlers/hotels.js +88 -0
  496. package/dist/server/routes/smart-search/handlers/products.d.ts +2 -0
  497. package/dist/server/routes/smart-search/handlers/products.js +1309 -0
  498. package/dist/server/routes/smart-search/handlers/rental.d.ts +2 -0
  499. package/dist/server/routes/smart-search/handlers/rental.js +154 -0
  500. package/dist/server/routes/smart-search/handlers/restaurants.d.ts +2 -0
  501. package/dist/server/routes/smart-search/handlers/restaurants.js +225 -0
  502. package/dist/server/routes/smart-search/handlers/transit-verdict.d.ts +41 -0
  503. package/dist/server/routes/smart-search/handlers/transit-verdict.js +224 -0
  504. package/dist/server/routes/smart-search/index.d.ts +19 -0
  505. package/dist/server/routes/smart-search/index.js +546 -0
  506. package/dist/server/routes/smart-search/intent.d.ts +3 -0
  507. package/dist/server/routes/smart-search/intent.js +264 -0
  508. package/dist/server/routes/smart-search/llm.d.ts +16 -0
  509. package/dist/server/routes/smart-search/llm.js +70 -0
  510. package/dist/server/routes/smart-search/sources/reddit.d.ts +18 -0
  511. package/dist/server/routes/smart-search/sources/reddit.js +34 -0
  512. package/dist/server/routes/smart-search/sources/yelp.d.ts +25 -0
  513. package/dist/server/routes/smart-search/sources/yelp.js +171 -0
  514. package/dist/server/routes/smart-search/sources/youtube.d.ts +8 -0
  515. package/dist/server/routes/smart-search/sources/youtube.js +9 -0
  516. package/dist/server/routes/smart-search/types.d.ts +81 -0
  517. package/dist/server/routes/smart-search/types.js +1 -0
  518. package/dist/server/routes/smart-search/utils.d.ts +20 -0
  519. package/dist/server/routes/smart-search/utils.js +146 -0
  520. package/dist/server/routes/stats.d.ts +6 -0
  521. package/dist/server/routes/stats.js +71 -0
  522. package/dist/server/routes/stripe.d.ts +15 -0
  523. package/dist/server/routes/stripe.js +296 -0
  524. package/dist/server/routes/transcript-export.d.ts +10 -0
  525. package/dist/server/routes/transcript-export.js +178 -0
  526. package/dist/server/routes/usage.d.ts +9 -0
  527. package/dist/server/routes/usage.js +279 -0
  528. package/dist/server/routes/users.d.ts +8 -0
  529. package/dist/server/routes/users.js +1867 -0
  530. package/dist/server/routes/watch.d.ts +15 -0
  531. package/dist/server/routes/watch.js +309 -0
  532. package/dist/server/routes/webhooks.d.ts +26 -0
  533. package/dist/server/routes/webhooks.js +170 -0
  534. package/dist/server/routes/youtube.d.ts +6 -0
  535. package/dist/server/routes/youtube.js +130 -0
  536. package/dist/server/sentry.d.ts +14 -0
  537. package/dist/server/sentry.js +104 -0
  538. package/dist/server/types.d.ts +15 -0
  539. package/dist/server/types.js +7 -0
  540. package/dist/server/utils/response.d.ts +44 -0
  541. package/dist/server/utils/response.js +69 -0
  542. package/dist/server/utils/sse.d.ts +22 -0
  543. package/dist/server/utils/sse.js +38 -0
  544. package/dist/types.d.ts +552 -0
  545. package/dist/types.js +39 -0
  546. package/llms.txt +105 -0
  547. package/package.json +189 -0
@@ -0,0 +1,576 @@
1
+ /**
2
+ * PostgreSQL-backed auth store for production deployments
3
+ * Uses SHA-256 hashing for API keys and tracks WEEKLY usage with burst limits
4
+ */
5
+ import pg from 'pg';
6
+ import crypto from 'crypto';
7
+ const { Pool } = pg;
8
+ // Extra usage cost constants
9
+ const EXTRA_USAGE_RATES = {
10
+ basic: 0.002, // $0.002 per basic fetch
11
+ stealth: 0.01, // $0.01 per stealth fetch
12
+ captcha: 0.02, // $0.02 per CAPTCHA solve
13
+ search: 0.001, // $0.001 per search
14
+ };
15
+ /**
16
+ * PostgreSQL auth store for production
17
+ */
18
+ export class PostgresAuthStore {
19
+ pool;
20
+ constructor(connectionString) {
21
+ const dbUrl = connectionString || process.env.DATABASE_URL;
22
+ if (!dbUrl) {
23
+ throw new Error('DATABASE_URL environment variable is required for PostgresAuthStore');
24
+ }
25
+ this.pool = new Pool({
26
+ connectionString: dbUrl,
27
+ // TLS: enabled when DATABASE_URL contains sslmode=require.
28
+ // Secure by default (rejectUnauthorized: true); set PG_REJECT_UNAUTHORIZED=false
29
+ // only for managed DBs (Render/Neon/Supabase) that use self-signed certs.
30
+ ssl: process.env.DATABASE_URL?.includes('sslmode=require')
31
+ ? { rejectUnauthorized: process.env.PG_REJECT_UNAUTHORIZED !== 'false' }
32
+ : undefined,
33
+ max: 20,
34
+ idleTimeoutMillis: 30000,
35
+ connectionTimeoutMillis: 10000,
36
+ });
37
+ // Run idempotent schema migrations on startup
38
+ this.ensureSchema().catch(err => console.error('[pg-auth-store] Schema migration failed:', err));
39
+ }
40
+ /**
41
+ * Run idempotent schema migrations.
42
+ * Safe to call on every startup — all statements use IF NOT EXISTS / IF EXISTS.
43
+ */
44
+ async ensureSchema() {
45
+ // ── Auto-fix schema mismatches on startup ──────────────────────────────
46
+ // These run as idempotent ALTER/CREATE IF NOT EXISTS so they're safe to
47
+ // run every boot. Prevents "column X does not exist" crashes after migrations.
48
+ const fixes = [
49
+ // api_keys
50
+ `ALTER TABLE api_keys ADD COLUMN IF NOT EXISTS scope VARCHAR(20) NOT NULL DEFAULT 'full'`,
51
+ // weekly_usage
52
+ `ALTER TABLE weekly_usage ADD COLUMN IF NOT EXISTS rollover_credits INTEGER DEFAULT 0`,
53
+ // burst_usage
54
+ `DO $$ BEGIN IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='burst_usage' AND column_name='hour') THEN ALTER TABLE burst_usage RENAME COLUMN hour TO hour_bucket; END IF; END $$`,
55
+ `ALTER TABLE burst_usage ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT now()`,
56
+ // usage_logs
57
+ `ALTER TABLE usage_logs ADD COLUMN IF NOT EXISTS processing_time_ms INTEGER`,
58
+ // oauth_accounts — ensure provider_id exists (not provider_account_id)
59
+ `DO $$ BEGIN IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='oauth_accounts' AND column_name='provider_account_id') THEN ALTER TABLE oauth_accounts RENAME COLUMN provider_account_id TO provider_id; END IF; END $$`,
60
+ // refresh_tokens — token_hash should be nullable (code uses id as JWT identifier)
61
+ `ALTER TABLE refresh_tokens ALTER COLUMN token_hash DROP NOT NULL`,
62
+ // users — ensure name/avatar columns exist
63
+ `ALTER TABLE users ADD COLUMN IF NOT EXISTS name TEXT`,
64
+ `ALTER TABLE users ADD COLUMN IF NOT EXISTS avatar_url TEXT`,
65
+ // shared_reads
66
+ `CREATE TABLE IF NOT EXISTS shared_reads (
67
+ id VARCHAR(12) PRIMARY KEY, url TEXT NOT NULL, title TEXT, content TEXT NOT NULL,
68
+ tokens INTEGER, created_by TEXT, created_at TIMESTAMPTZ DEFAULT NOW(),
69
+ expires_at TIMESTAMPTZ DEFAULT NOW() + INTERVAL '30 days', view_count INTEGER DEFAULT 0
70
+ )`,
71
+ `CREATE INDEX IF NOT EXISTS idx_shared_reads_url ON shared_reads(url)`,
72
+ `CREATE INDEX IF NOT EXISTS idx_shared_reads_created_by ON shared_reads(created_by)`,
73
+ // password_reset_tokens
74
+ `CREATE TABLE IF NOT EXISTS password_reset_tokens (
75
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
76
+ user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
77
+ token_hash TEXT NOT NULL,
78
+ expires_at TIMESTAMPTZ NOT NULL,
79
+ used BOOLEAN DEFAULT false,
80
+ created_at TIMESTAMPTZ DEFAULT now()
81
+ )`,
82
+ `CREATE INDEX IF NOT EXISTS idx_prt_token_hash ON password_reset_tokens(token_hash)`,
83
+ `CREATE INDEX IF NOT EXISTS idx_prt_user_id ON password_reset_tokens(user_id)`,
84
+ // usage_logs + api_keys + users indexes for admin analytics queries
85
+ `CREATE INDEX IF NOT EXISTS idx_usage_logs_user_id ON usage_logs(user_id)`,
86
+ `CREATE INDEX IF NOT EXISTS idx_usage_logs_created_at ON usage_logs(created_at)`,
87
+ `CREATE INDEX IF NOT EXISTS idx_usage_logs_endpoint ON usage_logs(endpoint)`,
88
+ `CREATE INDEX IF NOT EXISTS idx_api_keys_last_used ON api_keys(last_used_at)`,
89
+ `CREATE INDEX IF NOT EXISTS idx_users_created_at ON users(created_at)`,
90
+ ];
91
+ for (const sql of fixes) {
92
+ try {
93
+ await this.pool.query(sql);
94
+ }
95
+ catch (err) {
96
+ // Log but don't crash — individual fixes may fail on older schemas
97
+ console.warn('[webpeel] Schema fix warning:', err.message?.slice(0, 100));
98
+ }
99
+ }
100
+ }
101
+ /**
102
+ * Hash API key with SHA-256
103
+ * SECURITY: Never store raw API keys
104
+ */
105
+ hashKey(key) {
106
+ return crypto.createHash('sha256').update(key).digest('hex');
107
+ }
108
+ /**
109
+ * Get current ISO week in YYYY-WXX format (e.g., "2026-W07")
110
+ */
111
+ getCurrentWeek() {
112
+ const now = new Date();
113
+ const year = now.getUTCFullYear();
114
+ const jan4 = new Date(Date.UTC(year, 0, 4));
115
+ const weekNum = Math.ceil(((now.getTime() - jan4.getTime()) / 86400000 + jan4.getUTCDay() + 1) / 7);
116
+ return `${year}-W${String(weekNum).padStart(2, '0')}`;
117
+ }
118
+ /**
119
+ * Get previous ISO week in YYYY-WXX format
120
+ */
121
+ getPreviousWeek() {
122
+ const now = new Date();
123
+ const lastWeek = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000);
124
+ const year = lastWeek.getUTCFullYear();
125
+ const jan4 = new Date(Date.UTC(year, 0, 4));
126
+ const weekNum = Math.ceil(((lastWeek.getTime() - jan4.getTime()) / 86400000 + jan4.getUTCDay() + 1) / 7);
127
+ return `${year}-W${String(weekNum).padStart(2, '0')}`;
128
+ }
129
+ /**
130
+ * Get next Monday 00:00 UTC (week reset time)
131
+ */
132
+ getWeekResetTime() {
133
+ const now = new Date();
134
+ const dayOfWeek = now.getUTCDay();
135
+ const daysUntilMonday = dayOfWeek === 0 ? 1 : 8 - dayOfWeek;
136
+ const nextMonday = new Date(now);
137
+ nextMonday.setUTCDate(now.getUTCDate() + daysUntilMonday);
138
+ nextMonday.setUTCHours(0, 0, 0, 0);
139
+ return nextMonday;
140
+ }
141
+ /**
142
+ * Get current hour bucket in YYYY-MM-DDTHH format (UTC)
143
+ */
144
+ getCurrentHour() {
145
+ const now = new Date();
146
+ return now.toISOString().substring(0, 13); // "2026-02-12T20"
147
+ }
148
+ /**
149
+ * Get human-readable time until next hour
150
+ */
151
+ getTimeUntilNextHour() {
152
+ const now = new Date();
153
+ const minutesRemaining = 59 - now.getUTCMinutes();
154
+ if (minutesRemaining === 0) {
155
+ return '< 1 min';
156
+ }
157
+ return `${minutesRemaining} min`;
158
+ }
159
+ /**
160
+ * Validate API key and return user info
161
+ * SECURITY: Uses SHA-256 hash comparison, updates last_used_at
162
+ */
163
+ async validateKey(key) {
164
+ if (!key || typeof key !== 'string') {
165
+ return null;
166
+ }
167
+ const keyHash = this.hashKey(key);
168
+ try {
169
+ const result = await this.pool.query(`SELECT
170
+ ak.id,
171
+ ak.user_id,
172
+ ak.key_prefix,
173
+ ak.name,
174
+ ak.scope,
175
+ u.tier,
176
+ u.rate_limit,
177
+ u.weekly_limit,
178
+ u.burst_limit,
179
+ u.email
180
+ FROM api_keys ak
181
+ JOIN users u ON ak.user_id = u.id
182
+ WHERE ak.key_hash = $1 AND ak.is_active = true
183
+ AND (ak.expires_at IS NULL OR ak.expires_at > NOW())`, [keyHash]);
184
+ if (result.rows.length === 0) {
185
+ return null;
186
+ }
187
+ const row = result.rows[0];
188
+ // Update last_used_at (fire and forget, don't wait)
189
+ this.pool.query('UPDATE api_keys SET last_used_at = now() WHERE id = $1', [row.id]).catch(err => console.error('Failed to update last_used_at:', err));
190
+ return {
191
+ key,
192
+ tier: row.tier,
193
+ rateLimit: row.rate_limit,
194
+ accountId: row.user_id,
195
+ createdAt: new Date(),
196
+ scope: row.scope || 'full',
197
+ };
198
+ }
199
+ catch (error) {
200
+ console.error('Failed to validate API key:', error);
201
+ return null;
202
+ }
203
+ }
204
+ /**
205
+ * Check if a key exists but is expired (used to return specific 401 error)
206
+ */
207
+ async isKeyExpired(key) {
208
+ if (!key || typeof key !== 'string')
209
+ return false;
210
+ const keyHash = this.hashKey(key);
211
+ try {
212
+ const result = await this.pool.query(`SELECT 1 FROM api_keys WHERE key_hash = $1 AND is_active = true AND expires_at IS NOT NULL AND expires_at <= NOW()`, [keyHash]);
213
+ return result.rows.length > 0;
214
+ }
215
+ catch {
216
+ return false;
217
+ }
218
+ }
219
+ /**
220
+ * Track weekly usage for an API key
221
+ * SECURITY: Uses UPSERT to prevent race conditions
222
+ */
223
+ async trackUsage(key, fetchType) {
224
+ const keyHash = this.hashKey(key);
225
+ const week = this.getCurrentWeek();
226
+ try {
227
+ // Get API key ID and user ID
228
+ const keyResult = await this.pool.query('SELECT id, user_id FROM api_keys WHERE key_hash = $1', [keyHash]);
229
+ if (keyResult.rows.length === 0) {
230
+ return;
231
+ }
232
+ const { id: apiKeyId, user_id: userId } = keyResult.rows[0];
233
+ // Determine which counter to increment
234
+ const columnMap = {
235
+ basic: 'basic_count',
236
+ stealth: 'stealth_count',
237
+ captcha: 'captcha_count',
238
+ search: 'search_count',
239
+ };
240
+ const column = columnMap[fetchType];
241
+ // UPSERT usage record (total_count is GENERATED, don't touch it)
242
+ await this.pool.query(`INSERT INTO weekly_usage (user_id, api_key_id, week, ${column})
243
+ VALUES ($1, $2, $3, 1)
244
+ ON CONFLICT (api_key_id, week)
245
+ DO UPDATE SET
246
+ ${column} = weekly_usage.${column} + 1,
247
+ updated_at = now()`, [userId, apiKeyId, week]);
248
+ }
249
+ catch (error) {
250
+ console.error('Failed to track usage:', error);
251
+ throw error;
252
+ }
253
+ }
254
+ /**
255
+ * Track burst usage (hourly limit)
256
+ */
257
+ async trackBurstUsage(key) {
258
+ const keyHash = this.hashKey(key);
259
+ const hourBucket = this.getCurrentHour();
260
+ try {
261
+ const keyResult = await this.pool.query('SELECT id FROM api_keys WHERE key_hash = $1', [keyHash]);
262
+ if (keyResult.rows.length === 0) {
263
+ return;
264
+ }
265
+ const apiKeyId = keyResult.rows[0].id;
266
+ // UPSERT burst usage
267
+ await this.pool.query(`INSERT INTO burst_usage (api_key_id, hour_bucket, count)
268
+ VALUES ($1, $2, 1)
269
+ ON CONFLICT (api_key_id, hour_bucket)
270
+ DO UPDATE SET
271
+ count = burst_usage.count + 1,
272
+ updated_at = now()`, [apiKeyId, hourBucket]);
273
+ }
274
+ catch (error) {
275
+ console.error('Failed to track burst usage:', error);
276
+ throw error;
277
+ }
278
+ }
279
+ /**
280
+ * Check burst limit (hourly)
281
+ */
282
+ async checkBurstLimit(key) {
283
+ const keyHash = this.hashKey(key);
284
+ const hourBucket = this.getCurrentHour();
285
+ try {
286
+ const result = await this.pool.query(`SELECT
287
+ u.burst_limit,
288
+ COALESCE(bu.count, 0) as count
289
+ FROM api_keys ak
290
+ JOIN users u ON ak.user_id = u.id
291
+ LEFT JOIN burst_usage bu ON bu.api_key_id = ak.id AND bu.hour_bucket = $2
292
+ WHERE ak.key_hash = $1`, [keyHash, hourBucket]);
293
+ if (result.rows.length === 0) {
294
+ return {
295
+ allowed: false,
296
+ burst: {
297
+ hourBucket,
298
+ count: 0,
299
+ limit: 0,
300
+ remaining: 0,
301
+ resetsIn: this.getTimeUntilNextHour(),
302
+ },
303
+ };
304
+ }
305
+ const row = result.rows[0];
306
+ const allowed = row.count < row.burst_limit;
307
+ return {
308
+ allowed,
309
+ burst: {
310
+ hourBucket,
311
+ count: row.count,
312
+ limit: row.burst_limit,
313
+ remaining: Math.max(0, row.burst_limit - row.count),
314
+ resetsIn: this.getTimeUntilNextHour(),
315
+ },
316
+ };
317
+ }
318
+ catch (error) {
319
+ console.error('Failed to check burst limit:', error);
320
+ return {
321
+ allowed: false,
322
+ burst: {
323
+ hourBucket,
324
+ count: 0,
325
+ limit: 0,
326
+ remaining: 0,
327
+ resetsIn: this.getTimeUntilNextHour(),
328
+ },
329
+ };
330
+ }
331
+ }
332
+ /**
333
+ * Get weekly usage info for an API key with rollover calculation
334
+ */
335
+ async getUsage(key) {
336
+ const keyHash = this.hashKey(key);
337
+ const currentWeek = this.getCurrentWeek();
338
+ const previousWeek = this.getPreviousWeek();
339
+ try {
340
+ const result = await this.pool.query(`SELECT
341
+ u.weekly_limit,
342
+ COALESCE(curr.basic_count, 0) as basic_count,
343
+ COALESCE(curr.stealth_count, 0) as stealth_count,
344
+ COALESCE(curr.captcha_count, 0) as captcha_count,
345
+ COALESCE(curr.search_count, 0) as search_count,
346
+ COALESCE(curr.total_count, 0) as current_used,
347
+ COALESCE(prev.total_count, 0) as prev_used,
348
+ COALESCE(curr.rollover_credits, 0) as rollover_credits
349
+ FROM api_keys ak
350
+ JOIN users u ON ak.user_id = u.id
351
+ LEFT JOIN weekly_usage curr ON curr.api_key_id = ak.id AND curr.week = $2
352
+ LEFT JOIN weekly_usage prev ON prev.api_key_id = ak.id AND prev.week = $3
353
+ WHERE ak.key_hash = $1`, [keyHash, currentWeek, previousWeek]);
354
+ if (result.rows.length === 0) {
355
+ return null;
356
+ }
357
+ const row = result.rows[0];
358
+ const weeklyLimit = row.weekly_limit;
359
+ const currentUsed = row.current_used;
360
+ const prevUsed = row.prev_used;
361
+ const rolloverCredits = row.rollover_credits;
362
+ // Calculate rollover: MIN(unused_last_week, weekly_limit)
363
+ const prevUnused = Math.max(0, weeklyLimit - prevUsed);
364
+ const calculatedRollover = Math.min(prevUnused, weeklyLimit);
365
+ // Update rollover if it's the first access this week
366
+ if (rolloverCredits === 0 && calculatedRollover > 0) {
367
+ await this.pool.query(`INSERT INTO weekly_usage (user_id, api_key_id, week, rollover_credits, updated_at)
368
+ SELECT user_id, id, $2, $3, now()
369
+ FROM api_keys WHERE key_hash = $1
370
+ ON CONFLICT (api_key_id, week)
371
+ DO UPDATE SET rollover_credits = $3`, [keyHash, currentWeek, calculatedRollover]);
372
+ }
373
+ const effectiveRollover = rolloverCredits > 0 ? rolloverCredits : calculatedRollover;
374
+ const totalAvailable = weeklyLimit + effectiveRollover;
375
+ const remaining = Math.max(0, totalAvailable - currentUsed);
376
+ const percentUsed = totalAvailable > 0 ? Math.round((currentUsed / totalAvailable) * 100) : 0;
377
+ return {
378
+ week: currentWeek,
379
+ basicCount: row.basic_count,
380
+ stealthCount: row.stealth_count,
381
+ captchaCount: row.captcha_count,
382
+ searchCount: row.search_count,
383
+ totalUsed: currentUsed,
384
+ weeklyLimit,
385
+ rolloverCredits: effectiveRollover,
386
+ totalAvailable,
387
+ remaining,
388
+ percentUsed,
389
+ resetsAt: this.getWeekResetTime().toISOString(),
390
+ };
391
+ }
392
+ catch (error) {
393
+ console.error('Failed to get usage:', error);
394
+ return null;
395
+ }
396
+ }
397
+ /**
398
+ * Check if API key has exceeded weekly limit
399
+ */
400
+ async checkLimit(key) {
401
+ const usage = await this.getUsage(key);
402
+ if (!usage) {
403
+ return { allowed: false };
404
+ }
405
+ const allowed = usage.remaining > 0;
406
+ return { allowed, usage };
407
+ }
408
+ /**
409
+ * Get extra usage info for a user
410
+ */
411
+ async getExtraUsageInfo(key) {
412
+ const keyHash = this.hashKey(key);
413
+ try {
414
+ const result = await this.pool.query(`SELECT
415
+ u.extra_usage_enabled,
416
+ u.extra_usage_balance,
417
+ u.extra_usage_spent,
418
+ u.extra_usage_spending_limit,
419
+ u.auto_reload_enabled,
420
+ u.extra_usage_period_start
421
+ FROM api_keys ak
422
+ JOIN users u ON ak.user_id = u.id
423
+ WHERE ak.key_hash = $1`, [keyHash]);
424
+ if (result.rows.length === 0) {
425
+ return null;
426
+ }
427
+ const row = result.rows[0];
428
+ // Calculate next month reset (1st of next month, 00:00 UTC)
429
+ const now = new Date();
430
+ const nextMonth = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1, 0, 0, 0, 0));
431
+ const percentUsed = row.extra_usage_spending_limit > 0
432
+ ? Math.round((parseFloat(row.extra_usage_spent) / parseFloat(row.extra_usage_spending_limit)) * 100)
433
+ : 0;
434
+ return {
435
+ enabled: row.extra_usage_enabled,
436
+ balance: parseFloat(row.extra_usage_balance),
437
+ spent: parseFloat(row.extra_usage_spent),
438
+ spendingLimit: parseFloat(row.extra_usage_spending_limit),
439
+ autoReload: row.auto_reload_enabled,
440
+ percentUsed,
441
+ resetsAt: nextMonth.toISOString(),
442
+ };
443
+ }
444
+ catch (error) {
445
+ console.error('Failed to get extra usage info:', error);
446
+ return null;
447
+ }
448
+ }
449
+ /**
450
+ * Check if extra usage can be used
451
+ */
452
+ async canUseExtraUsage(key) {
453
+ const info = await this.getExtraUsageInfo(key);
454
+ if (!info || !info.enabled) {
455
+ return false;
456
+ }
457
+ // Check if under spending limit and has balance
458
+ return info.balance > 0 && info.spent < info.spendingLimit;
459
+ }
460
+ /**
461
+ * Track extra usage and deduct from balance
462
+ */
463
+ async trackExtraUsage(key, fetchType, url, processingTimeMs, statusCode) {
464
+ const keyHash = this.hashKey(key);
465
+ const cost = EXTRA_USAGE_RATES[fetchType];
466
+ try {
467
+ // Get API key and user info
468
+ const keyResult = await this.pool.query(`SELECT
469
+ ak.id as api_key_id,
470
+ ak.user_id,
471
+ u.extra_usage_balance,
472
+ u.extra_usage_spent
473
+ FROM api_keys ak
474
+ JOIN users u ON ak.user_id = u.id
475
+ WHERE ak.key_hash = $1`, [keyHash]);
476
+ if (keyResult.rows.length === 0) {
477
+ return { success: false, cost: 0, newBalance: 0 };
478
+ }
479
+ const { api_key_id, user_id, extra_usage_balance } = keyResult.rows[0];
480
+ const currentBalance = parseFloat(extra_usage_balance);
481
+ if (currentBalance < cost) {
482
+ return { success: false, cost, newBalance: currentBalance };
483
+ }
484
+ // Start transaction
485
+ const client = await this.pool.connect();
486
+ try {
487
+ await client.query('BEGIN');
488
+ // Deduct from balance and add to spent
489
+ const updateResult = await client.query(`UPDATE users
490
+ SET
491
+ extra_usage_balance = extra_usage_balance - $1,
492
+ extra_usage_spent = extra_usage_spent + $1,
493
+ updated_at = now()
494
+ WHERE id = $2
495
+ RETURNING extra_usage_balance`, [cost, user_id]);
496
+ const newBalance = parseFloat(updateResult.rows[0].extra_usage_balance);
497
+ // Log to extra_usage_logs
498
+ await client.query(`INSERT INTO extra_usage_logs
499
+ (user_id, api_key_id, fetch_type, url, cost, processing_time_ms, status_code)
500
+ VALUES ($1, $2, $3, $4, $5, $6, $7)`, [user_id, api_key_id, fetchType, url, cost, processingTimeMs, statusCode]);
501
+ await client.query('COMMIT');
502
+ return { success: true, cost, newBalance };
503
+ }
504
+ catch (error) {
505
+ await client.query('ROLLBACK');
506
+ throw error;
507
+ }
508
+ finally {
509
+ client.release();
510
+ }
511
+ }
512
+ catch (error) {
513
+ console.error('Failed to track extra usage:', error);
514
+ return { success: false, cost, newBalance: 0 };
515
+ }
516
+ }
517
+ /**
518
+ * Generate a cryptographically secure API key
519
+ * Format: wp_live_ + 32 random hex chars (total 40 chars)
520
+ */
521
+ static generateApiKey() {
522
+ const randomBytes = crypto.randomBytes(16).toString('hex');
523
+ return `wp_live_${randomBytes}`;
524
+ }
525
+ /**
526
+ * Get key prefix (first 12 characters for display)
527
+ */
528
+ static getKeyPrefix(key) {
529
+ return key.substring(0, 12);
530
+ }
531
+ /**
532
+ * GDPR: Delete all data associated with a user account (atomic transaction).
533
+ * Removes: api_keys, usage, weekly_usage, burst_usage, extra_usage_logs,
534
+ * usage_logs, watches, oauth_accounts, refresh_tokens, shared_reads,
535
+ * and finally the user row itself.
536
+ */
537
+ async deleteAccount(userId) {
538
+ const client = await this.pool.connect();
539
+ try {
540
+ await client.query('BEGIN');
541
+ // Delete per-key usage data first (FK → api_keys)
542
+ await client.query(`DELETE FROM weekly_usage WHERE api_key_id IN (SELECT id FROM api_keys WHERE user_id = $1)`, [userId]);
543
+ await client.query(`DELETE FROM burst_usage WHERE api_key_id IN (SELECT id FROM api_keys WHERE user_id = $1)`, [userId]);
544
+ await client.query(`DELETE FROM extra_usage_logs WHERE user_id = $1`, [userId]);
545
+ await client.query(`DELETE FROM usage_logs WHERE user_id = $1`, [userId]);
546
+ // usage table (legacy monthly usage)
547
+ await client.query(`DELETE FROM usage WHERE user_id = $1`, [userId]);
548
+ // watches (account_id column, no FK constraint to users in migration 007)
549
+ await client.query(`DELETE FROM watches WHERE account_id = $1`, [userId]);
550
+ // shared_reads created by this user (created_by is TEXT, not FK)
551
+ await client.query(`DELETE FROM shared_reads WHERE created_by = $1`, [userId]);
552
+ // oauth_accounts and refresh_tokens have ON DELETE CASCADE via users FK
553
+ // but we delete them explicitly to be safe and for auditability
554
+ await client.query(`DELETE FROM oauth_accounts WHERE user_id = $1`, [userId]);
555
+ await client.query(`DELETE FROM refresh_tokens WHERE user_id = $1`, [userId]);
556
+ // api_keys — has ON DELETE CASCADE but deleted explicitly here
557
+ await client.query(`DELETE FROM api_keys WHERE user_id = $1`, [userId]);
558
+ // Finally delete the user row
559
+ await client.query(`DELETE FROM users WHERE id = $1`, [userId]);
560
+ await client.query('COMMIT');
561
+ }
562
+ catch (error) {
563
+ await client.query('ROLLBACK');
564
+ throw error;
565
+ }
566
+ finally {
567
+ client.release();
568
+ }
569
+ }
570
+ /**
571
+ * Close the database pool
572
+ */
573
+ async close() {
574
+ await this.pool.end();
575
+ }
576
+ }
@@ -0,0 +1,59 @@
1
+ /**
2
+ * PostgreSQL-backed job queue for production deployments
3
+ * Uses same Pool pattern as pg-auth-store.ts
4
+ */
5
+ import type { Job, WebhookConfig } from './job-queue.js';
6
+ export declare class PostgresJobQueue {
7
+ private pool;
8
+ private cleanupInterval;
9
+ constructor(connectionString?: string);
10
+ /**
11
+ * Create jobs table if it doesn't exist
12
+ */
13
+ private initTable;
14
+ /**
15
+ * Create a new job
16
+ */
17
+ createJob(type: Job['type'], webhook?: WebhookConfig, ownerId?: string): Promise<Job>;
18
+ /**
19
+ * Get a job by ID
20
+ */
21
+ getJob(id: string): Promise<Job | null>;
22
+ /**
23
+ * Update a job
24
+ */
25
+ updateJob(id: string, update: Partial<Job>): Promise<void>;
26
+ /**
27
+ * Cancel a job
28
+ */
29
+ cancelJob(id: string): Promise<boolean>;
30
+ /**
31
+ * List jobs with optional filters
32
+ */
33
+ listJobs(options?: {
34
+ type?: string;
35
+ status?: string;
36
+ limit?: number;
37
+ ownerId?: string;
38
+ }): Promise<Job[]>;
39
+ /**
40
+ * Remove expired jobs (called periodically)
41
+ */
42
+ private cleanExpired;
43
+ /**
44
+ * Remove old completed/failed jobs (>7 days)
45
+ */
46
+ private cleanupOldJobs;
47
+ /**
48
+ * Map database row to Job object
49
+ */
50
+ private mapRowToJob;
51
+ /**
52
+ * Clean up interval on shutdown
53
+ */
54
+ destroy(): void;
55
+ /**
56
+ * Close the database pool
57
+ */
58
+ close(): Promise<void>;
59
+ }