@iflow-mcp/jakeliume-webpeel 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (547) hide show
  1. package/LICENSE +15 -0
  2. package/README.md +313 -0
  3. package/dist/cache.d.ts +30 -0
  4. package/dist/cache.js +139 -0
  5. package/dist/cli/commands/auth.d.ts +5 -0
  6. package/dist/cli/commands/auth.js +411 -0
  7. package/dist/cli/commands/doctor.d.ts +37 -0
  8. package/dist/cli/commands/doctor.js +371 -0
  9. package/dist/cli/commands/fetch.d.ts +6 -0
  10. package/dist/cli/commands/fetch.js +1345 -0
  11. package/dist/cli/commands/guide.d.ts +2 -0
  12. package/dist/cli/commands/guide.js +183 -0
  13. package/dist/cli/commands/interact.d.ts +5 -0
  14. package/dist/cli/commands/interact.js +840 -0
  15. package/dist/cli/commands/jobs.d.ts +5 -0
  16. package/dist/cli/commands/jobs.js +997 -0
  17. package/dist/cli/commands/monitor.d.ts +12 -0
  18. package/dist/cli/commands/monitor.js +197 -0
  19. package/dist/cli/commands/observe.d.ts +12 -0
  20. package/dist/cli/commands/observe.js +158 -0
  21. package/dist/cli/commands/screenshot.d.ts +5 -0
  22. package/dist/cli/commands/screenshot.js +282 -0
  23. package/dist/cli/commands/search.d.ts +5 -0
  24. package/dist/cli/commands/search.js +1021 -0
  25. package/dist/cli/commands/setup.d.ts +13 -0
  26. package/dist/cli/commands/setup.js +244 -0
  27. package/dist/cli/commands/skill.d.ts +15 -0
  28. package/dist/cli/commands/skill.js +195 -0
  29. package/dist/cli/utils.d.ts +84 -0
  30. package/dist/cli/utils.js +806 -0
  31. package/dist/cli-auth.d.ts +75 -0
  32. package/dist/cli-auth.js +369 -0
  33. package/dist/cli.d.ts +17 -0
  34. package/dist/cli.js +99 -0
  35. package/dist/core/actions.d.ts +69 -0
  36. package/dist/core/actions.js +495 -0
  37. package/dist/core/agent.d.ts +98 -0
  38. package/dist/core/agent.js +558 -0
  39. package/dist/core/answer.d.ts +42 -0
  40. package/dist/core/answer.js +395 -0
  41. package/dist/core/application-tracker.d.ts +84 -0
  42. package/dist/core/application-tracker.js +184 -0
  43. package/dist/core/apply.d.ts +162 -0
  44. package/dist/core/apply.js +816 -0
  45. package/dist/core/auth-detection.d.ts +35 -0
  46. package/dist/core/auth-detection.js +358 -0
  47. package/dist/core/auto-extract.d.ts +82 -0
  48. package/dist/core/auto-extract.js +604 -0
  49. package/dist/core/auto-interact.d.ts +23 -0
  50. package/dist/core/auto-interact.js +246 -0
  51. package/dist/core/bm25-filter.d.ts +66 -0
  52. package/dist/core/bm25-filter.js +288 -0
  53. package/dist/core/branding.d.ts +54 -0
  54. package/dist/core/branding.js +234 -0
  55. package/dist/core/browser-fetch.d.ts +323 -0
  56. package/dist/core/browser-fetch.js +1600 -0
  57. package/dist/core/browser-pool.d.ts +91 -0
  58. package/dist/core/browser-pool.js +550 -0
  59. package/dist/core/budget.d.ts +42 -0
  60. package/dist/core/budget.js +324 -0
  61. package/dist/core/business-intel.d.ts +47 -0
  62. package/dist/core/business-intel.js +279 -0
  63. package/dist/core/cache.d.ts +13 -0
  64. package/dist/core/cache.js +121 -0
  65. package/dist/core/cf-worker-proxy.d.ts +32 -0
  66. package/dist/core/cf-worker-proxy.js +87 -0
  67. package/dist/core/challenge-detection.d.ts +26 -0
  68. package/dist/core/challenge-detection.js +468 -0
  69. package/dist/core/change-tracking.d.ts +75 -0
  70. package/dist/core/change-tracking.js +276 -0
  71. package/dist/core/chunker.d.ts +46 -0
  72. package/dist/core/chunker.js +249 -0
  73. package/dist/core/chunking.d.ts +42 -0
  74. package/dist/core/chunking.js +181 -0
  75. package/dist/core/circuit-breaker.d.ts +44 -0
  76. package/dist/core/circuit-breaker.js +85 -0
  77. package/dist/core/content-pruner.d.ts +47 -0
  78. package/dist/core/content-pruner.js +425 -0
  79. package/dist/core/cookie-cache.d.ts +60 -0
  80. package/dist/core/cookie-cache.js +163 -0
  81. package/dist/core/crawl-checkpoint.d.ts +54 -0
  82. package/dist/core/crawl-checkpoint.js +104 -0
  83. package/dist/core/crawler.d.ts +84 -0
  84. package/dist/core/crawler.js +349 -0
  85. package/dist/core/cross-verify.d.ts +27 -0
  86. package/dist/core/cross-verify.js +93 -0
  87. package/dist/core/deep-fetch.d.ts +74 -0
  88. package/dist/core/deep-fetch.js +405 -0
  89. package/dist/core/deep-research.d.ts +141 -0
  90. package/dist/core/deep-research.js +972 -0
  91. package/dist/core/design-analysis.d.ts +70 -0
  92. package/dist/core/design-analysis.js +490 -0
  93. package/dist/core/design-compare.d.ts +38 -0
  94. package/dist/core/design-compare.js +264 -0
  95. package/dist/core/diff.d.ts +61 -0
  96. package/dist/core/diff.js +289 -0
  97. package/dist/core/dns-cache.d.ts +20 -0
  98. package/dist/core/dns-cache.js +198 -0
  99. package/dist/core/documents.d.ts +23 -0
  100. package/dist/core/documents.js +123 -0
  101. package/dist/core/domain-memory.d.ts +66 -0
  102. package/dist/core/domain-memory.js +163 -0
  103. package/dist/core/domain-verify.d.ts +40 -0
  104. package/dist/core/domain-verify.js +379 -0
  105. package/dist/core/engine-ranker.d.ts +112 -0
  106. package/dist/core/engine-ranker.js +395 -0
  107. package/dist/core/extract-inline.d.ts +38 -0
  108. package/dist/core/extract-inline.js +215 -0
  109. package/dist/core/extract-listings.d.ts +38 -0
  110. package/dist/core/extract-listings.js +461 -0
  111. package/dist/core/extract.d.ts +9 -0
  112. package/dist/core/extract.js +139 -0
  113. package/dist/core/fetch-cache.d.ts +57 -0
  114. package/dist/core/fetch-cache.js +95 -0
  115. package/dist/core/fetcher.d.ts +13 -0
  116. package/dist/core/fetcher.js +12 -0
  117. package/dist/core/google-cache.d.ts +29 -0
  118. package/dist/core/google-cache.js +180 -0
  119. package/dist/core/google-serp-parser.d.ts +82 -0
  120. package/dist/core/google-serp-parser.js +287 -0
  121. package/dist/core/hotel-search.d.ts +122 -0
  122. package/dist/core/hotel-search.js +382 -0
  123. package/dist/core/http-fetch.d.ts +72 -0
  124. package/dist/core/http-fetch.js +820 -0
  125. package/dist/core/human.d.ts +175 -0
  126. package/dist/core/human.js +680 -0
  127. package/dist/core/image-caption.d.ts +44 -0
  128. package/dist/core/image-caption.js +271 -0
  129. package/dist/core/jobs.d.ts +75 -0
  130. package/dist/core/jobs.js +634 -0
  131. package/dist/core/json-ld.d.ts +15 -0
  132. package/dist/core/json-ld.js +617 -0
  133. package/dist/core/language-detect.d.ts +18 -0
  134. package/dist/core/language-detect.js +135 -0
  135. package/dist/core/links.d.ts +10 -0
  136. package/dist/core/links.js +44 -0
  137. package/dist/core/llm-extract.d.ts +71 -0
  138. package/dist/core/llm-extract.js +507 -0
  139. package/dist/core/llm-provider.d.ts +100 -0
  140. package/dist/core/llm-provider.js +702 -0
  141. package/dist/core/local-search.d.ts +60 -0
  142. package/dist/core/local-search.js +308 -0
  143. package/dist/core/logger.d.ts +28 -0
  144. package/dist/core/logger.js +104 -0
  145. package/dist/core/map.d.ts +33 -0
  146. package/dist/core/map.js +127 -0
  147. package/dist/core/markdown.d.ts +92 -0
  148. package/dist/core/markdown.js +809 -0
  149. package/dist/core/metadata.d.ts +34 -0
  150. package/dist/core/metadata.js +422 -0
  151. package/dist/core/observe.d.ts +113 -0
  152. package/dist/core/observe.js +395 -0
  153. package/dist/core/ocr.d.ts +12 -0
  154. package/dist/core/ocr.js +33 -0
  155. package/dist/core/paginate.d.ts +31 -0
  156. package/dist/core/paginate.js +106 -0
  157. package/dist/core/pdf.d.ts +8 -0
  158. package/dist/core/pdf.js +25 -0
  159. package/dist/core/peel-tls.d.ts +25 -0
  160. package/dist/core/peel-tls.js +220 -0
  161. package/dist/core/pipeline.d.ts +132 -0
  162. package/dist/core/pipeline.js +1666 -0
  163. package/dist/core/profiles.d.ts +61 -0
  164. package/dist/core/profiles.js +350 -0
  165. package/dist/core/prompt-guard.d.ts +30 -0
  166. package/dist/core/prompt-guard.js +119 -0
  167. package/dist/core/proxy-config.d.ts +90 -0
  168. package/dist/core/proxy-config.js +172 -0
  169. package/dist/core/quick-answer.d.ts +53 -0
  170. package/dist/core/quick-answer.js +833 -0
  171. package/dist/core/rate-governor.d.ts +80 -0
  172. package/dist/core/rate-governor.js +238 -0
  173. package/dist/core/readability.d.ts +57 -0
  174. package/dist/core/readability.js +533 -0
  175. package/dist/core/research.d.ts +66 -0
  176. package/dist/core/research.js +270 -0
  177. package/dist/core/retry.d.ts +60 -0
  178. package/dist/core/retry.js +119 -0
  179. package/dist/core/safe-browsing.d.ts +30 -0
  180. package/dist/core/safe-browsing.js +206 -0
  181. package/dist/core/schema-extraction.d.ts +66 -0
  182. package/dist/core/schema-extraction.js +352 -0
  183. package/dist/core/schema-postprocess.d.ts +32 -0
  184. package/dist/core/schema-postprocess.js +469 -0
  185. package/dist/core/schema-templates.d.ts +19 -0
  186. package/dist/core/schema-templates.js +143 -0
  187. package/dist/core/screenshot.d.ts +224 -0
  188. package/dist/core/screenshot.js +207 -0
  189. package/dist/core/search-engines.d.ts +25 -0
  190. package/dist/core/search-engines.js +182 -0
  191. package/dist/core/search-provider.d.ts +243 -0
  192. package/dist/core/search-provider.js +1629 -0
  193. package/dist/core/searxng-provider.d.ts +35 -0
  194. package/dist/core/searxng-provider.js +105 -0
  195. package/dist/core/selective-evidence.d.ts +151 -0
  196. package/dist/core/selective-evidence.js +389 -0
  197. package/dist/core/site-search.d.ts +44 -0
  198. package/dist/core/site-search.js +252 -0
  199. package/dist/core/sitemap.d.ts +23 -0
  200. package/dist/core/sitemap.js +105 -0
  201. package/dist/core/source-credibility.d.ts +29 -0
  202. package/dist/core/source-credibility.js +584 -0
  203. package/dist/core/source-scoring.d.ts +166 -0
  204. package/dist/core/source-scoring.js +396 -0
  205. package/dist/core/stemmer.d.ts +38 -0
  206. package/dist/core/stemmer.js +509 -0
  207. package/dist/core/strategies.d.ts +104 -0
  208. package/dist/core/strategies.js +1044 -0
  209. package/dist/core/strategy-hooks.d.ts +145 -0
  210. package/dist/core/strategy-hooks.js +74 -0
  211. package/dist/core/structured-extract.d.ts +43 -0
  212. package/dist/core/structured-extract.js +550 -0
  213. package/dist/core/summarize.d.ts +17 -0
  214. package/dist/core/summarize.js +78 -0
  215. package/dist/core/synonyms.d.ts +42 -0
  216. package/dist/core/synonyms.js +184 -0
  217. package/dist/core/system-monitor.d.ts +61 -0
  218. package/dist/core/system-monitor.js +133 -0
  219. package/dist/core/table-format.d.ts +30 -0
  220. package/dist/core/table-format.js +146 -0
  221. package/dist/core/threat-feeds.d.ts +23 -0
  222. package/dist/core/threat-feeds.js +104 -0
  223. package/dist/core/timing.d.ts +21 -0
  224. package/dist/core/timing.js +33 -0
  225. package/dist/core/transcript-export.d.ts +47 -0
  226. package/dist/core/transcript-export.js +107 -0
  227. package/dist/core/user-agents.d.ts +82 -0
  228. package/dist/core/user-agents.js +239 -0
  229. package/dist/core/vertical-search.d.ts +54 -0
  230. package/dist/core/vertical-search.js +158 -0
  231. package/dist/core/watch-manager.d.ts +175 -0
  232. package/dist/core/watch-manager.js +416 -0
  233. package/dist/core/watch.d.ts +101 -0
  234. package/dist/core/watch.js +389 -0
  235. package/dist/core/youtube.d.ts +130 -0
  236. package/dist/core/youtube.js +1175 -0
  237. package/dist/ee/challenge-re-export.d.ts +1 -0
  238. package/dist/ee/challenge-re-export.js +1 -0
  239. package/dist/ee/challenge-solver.d.ts +72 -0
  240. package/dist/ee/challenge-solver.js +720 -0
  241. package/dist/ee/domain-extractors.d.ts +8 -0
  242. package/dist/ee/domain-extractors.js +8 -0
  243. package/dist/ee/domain-intel.d.ts +16 -0
  244. package/dist/ee/domain-intel.js +133 -0
  245. package/dist/ee/extractors/allrecipes.d.ts +2 -0
  246. package/dist/ee/extractors/allrecipes.js +120 -0
  247. package/dist/ee/extractors/amazon.d.ts +2 -0
  248. package/dist/ee/extractors/amazon.js +78 -0
  249. package/dist/ee/extractors/arxiv.d.ts +2 -0
  250. package/dist/ee/extractors/arxiv.js +137 -0
  251. package/dist/ee/extractors/bestbuy.d.ts +2 -0
  252. package/dist/ee/extractors/bestbuy.js +78 -0
  253. package/dist/ee/extractors/carscom.d.ts +2 -0
  254. package/dist/ee/extractors/carscom.js +121 -0
  255. package/dist/ee/extractors/coingecko.d.ts +2 -0
  256. package/dist/ee/extractors/coingecko.js +134 -0
  257. package/dist/ee/extractors/craigslist.d.ts +2 -0
  258. package/dist/ee/extractors/craigslist.js +92 -0
  259. package/dist/ee/extractors/devto.d.ts +2 -0
  260. package/dist/ee/extractors/devto.js +135 -0
  261. package/dist/ee/extractors/ebay.d.ts +2 -0
  262. package/dist/ee/extractors/ebay.js +90 -0
  263. package/dist/ee/extractors/espn.d.ts +2 -0
  264. package/dist/ee/extractors/espn.js +260 -0
  265. package/dist/ee/extractors/etsy.d.ts +2 -0
  266. package/dist/ee/extractors/etsy.js +52 -0
  267. package/dist/ee/extractors/facebook.d.ts +2 -0
  268. package/dist/ee/extractors/facebook.js +46 -0
  269. package/dist/ee/extractors/github.d.ts +2 -0
  270. package/dist/ee/extractors/github.js +196 -0
  271. package/dist/ee/extractors/google-flights.d.ts +2 -0
  272. package/dist/ee/extractors/google-flights.js +176 -0
  273. package/dist/ee/extractors/hackernews.d.ts +2 -0
  274. package/dist/ee/extractors/hackernews.js +147 -0
  275. package/dist/ee/extractors/imdb.d.ts +2 -0
  276. package/dist/ee/extractors/imdb.js +172 -0
  277. package/dist/ee/extractors/index.d.ts +26 -0
  278. package/dist/ee/extractors/index.js +247 -0
  279. package/dist/ee/extractors/instagram.d.ts +2 -0
  280. package/dist/ee/extractors/instagram.js +102 -0
  281. package/dist/ee/extractors/kalshi.d.ts +2 -0
  282. package/dist/ee/extractors/kalshi.js +121 -0
  283. package/dist/ee/extractors/kayak-cars.d.ts +2 -0
  284. package/dist/ee/extractors/kayak-cars.js +270 -0
  285. package/dist/ee/extractors/linkedin.d.ts +2 -0
  286. package/dist/ee/extractors/linkedin.js +113 -0
  287. package/dist/ee/extractors/medium.d.ts +2 -0
  288. package/dist/ee/extractors/medium.js +130 -0
  289. package/dist/ee/extractors/news.d.ts +4 -0
  290. package/dist/ee/extractors/news.js +173 -0
  291. package/dist/ee/extractors/npm.d.ts +2 -0
  292. package/dist/ee/extractors/npm.js +86 -0
  293. package/dist/ee/extractors/pdf.d.ts +2 -0
  294. package/dist/ee/extractors/pdf.js +108 -0
  295. package/dist/ee/extractors/pinterest.d.ts +2 -0
  296. package/dist/ee/extractors/pinterest.js +34 -0
  297. package/dist/ee/extractors/polymarket.d.ts +2 -0
  298. package/dist/ee/extractors/polymarket.js +358 -0
  299. package/dist/ee/extractors/producthunt.d.ts +2 -0
  300. package/dist/ee/extractors/producthunt.js +88 -0
  301. package/dist/ee/extractors/pubmed.d.ts +2 -0
  302. package/dist/ee/extractors/pubmed.js +162 -0
  303. package/dist/ee/extractors/pypi.d.ts +2 -0
  304. package/dist/ee/extractors/pypi.js +80 -0
  305. package/dist/ee/extractors/reddit.d.ts +2 -0
  306. package/dist/ee/extractors/reddit.js +438 -0
  307. package/dist/ee/extractors/redfin.d.ts +2 -0
  308. package/dist/ee/extractors/redfin.js +156 -0
  309. package/dist/ee/extractors/semanticscholar.d.ts +2 -0
  310. package/dist/ee/extractors/semanticscholar.js +131 -0
  311. package/dist/ee/extractors/shared.d.ts +12 -0
  312. package/dist/ee/extractors/shared.js +76 -0
  313. package/dist/ee/extractors/soundcloud.d.ts +2 -0
  314. package/dist/ee/extractors/soundcloud.js +34 -0
  315. package/dist/ee/extractors/sportsbetting.d.ts +2 -0
  316. package/dist/ee/extractors/sportsbetting.js +37 -0
  317. package/dist/ee/extractors/spotify.d.ts +2 -0
  318. package/dist/ee/extractors/spotify.js +34 -0
  319. package/dist/ee/extractors/stackoverflow.d.ts +2 -0
  320. package/dist/ee/extractors/stackoverflow.js +61 -0
  321. package/dist/ee/extractors/substack.d.ts +2 -0
  322. package/dist/ee/extractors/substack.js +115 -0
  323. package/dist/ee/extractors/substackroot.d.ts +2 -0
  324. package/dist/ee/extractors/substackroot.js +46 -0
  325. package/dist/ee/extractors/tiktok.d.ts +2 -0
  326. package/dist/ee/extractors/tiktok.js +29 -0
  327. package/dist/ee/extractors/tradingview.d.ts +2 -0
  328. package/dist/ee/extractors/tradingview.js +182 -0
  329. package/dist/ee/extractors/twitch.d.ts +2 -0
  330. package/dist/ee/extractors/twitch.js +36 -0
  331. package/dist/ee/extractors/twitter.d.ts +2 -0
  332. package/dist/ee/extractors/twitter.js +327 -0
  333. package/dist/ee/extractors/types.d.ts +14 -0
  334. package/dist/ee/extractors/types.js +1 -0
  335. package/dist/ee/extractors/walmart.d.ts +2 -0
  336. package/dist/ee/extractors/walmart.js +50 -0
  337. package/dist/ee/extractors/weather.d.ts +2 -0
  338. package/dist/ee/extractors/weather.js +133 -0
  339. package/dist/ee/extractors/wikipedia.d.ts +4 -0
  340. package/dist/ee/extractors/wikipedia.js +235 -0
  341. package/dist/ee/extractors/yelp.d.ts +2 -0
  342. package/dist/ee/extractors/yelp.js +216 -0
  343. package/dist/ee/extractors/youtube.d.ts +2 -0
  344. package/dist/ee/extractors/youtube.js +189 -0
  345. package/dist/ee/extractors/zillow.d.ts +54 -0
  346. package/dist/ee/extractors/zillow.js +247 -0
  347. package/dist/ee/extractors-re-export.d.ts +1 -0
  348. package/dist/ee/extractors-re-export.js +1 -0
  349. package/dist/ee/premium-hooks.d.ts +20 -0
  350. package/dist/ee/premium-hooks.js +50 -0
  351. package/dist/ee/spa-detection.d.ts +2 -0
  352. package/dist/ee/spa-detection.js +2 -0
  353. package/dist/ee/stability.d.ts +4 -0
  354. package/dist/ee/stability.js +29 -0
  355. package/dist/ee/swr-cache.d.ts +14 -0
  356. package/dist/ee/swr-cache.js +34 -0
  357. package/dist/index.d.ts +143 -0
  358. package/dist/index.js +291 -0
  359. package/dist/integrations/index.d.ts +2 -0
  360. package/dist/integrations/index.js +2 -0
  361. package/dist/integrations/langchain.d.ts +64 -0
  362. package/dist/integrations/langchain.js +115 -0
  363. package/dist/integrations/llamaindex.d.ts +50 -0
  364. package/dist/integrations/llamaindex.js +91 -0
  365. package/dist/mcp/handlers/act.d.ts +5 -0
  366. package/dist/mcp/handlers/act.js +34 -0
  367. package/dist/mcp/handlers/definitions.d.ts +6 -0
  368. package/dist/mcp/handlers/definitions.js +395 -0
  369. package/dist/mcp/handlers/extract.d.ts +7 -0
  370. package/dist/mcp/handlers/extract.js +135 -0
  371. package/dist/mcp/handlers/fetch.d.ts +6 -0
  372. package/dist/mcp/handlers/fetch.js +98 -0
  373. package/dist/mcp/handlers/find.d.ts +5 -0
  374. package/dist/mcp/handlers/find.js +137 -0
  375. package/dist/mcp/handlers/index.d.ts +13 -0
  376. package/dist/mcp/handlers/index.js +63 -0
  377. package/dist/mcp/handlers/legacy.d.ts +25 -0
  378. package/dist/mcp/handlers/legacy.js +450 -0
  379. package/dist/mcp/handlers/meta.d.ts +6 -0
  380. package/dist/mcp/handlers/meta.js +40 -0
  381. package/dist/mcp/handlers/monitor.d.ts +5 -0
  382. package/dist/mcp/handlers/monitor.js +41 -0
  383. package/dist/mcp/handlers/observe.d.ts +8 -0
  384. package/dist/mcp/handlers/observe.js +37 -0
  385. package/dist/mcp/handlers/read.d.ts +6 -0
  386. package/dist/mcp/handlers/read.js +78 -0
  387. package/dist/mcp/handlers/see.d.ts +5 -0
  388. package/dist/mcp/handlers/see.js +75 -0
  389. package/dist/mcp/handlers/types.d.ts +29 -0
  390. package/dist/mcp/handlers/types.js +28 -0
  391. package/dist/mcp/server.d.ts +7 -0
  392. package/dist/mcp/server.js +108 -0
  393. package/dist/mcp/smart-router.d.ts +23 -0
  394. package/dist/mcp/smart-router.js +178 -0
  395. package/dist/server/app.d.ts +14 -0
  396. package/dist/server/app.js +632 -0
  397. package/dist/server/auth-store.d.ts +28 -0
  398. package/dist/server/auth-store.js +88 -0
  399. package/dist/server/bull-queues.d.ts +60 -0
  400. package/dist/server/bull-queues.js +90 -0
  401. package/dist/server/email-service.d.ts +55 -0
  402. package/dist/server/email-service.js +291 -0
  403. package/dist/server/job-queue.d.ts +100 -0
  404. package/dist/server/job-queue.js +145 -0
  405. package/dist/server/logger.d.ts +10 -0
  406. package/dist/server/logger.js +37 -0
  407. package/dist/server/middleware/audit-log.d.ts +14 -0
  408. package/dist/server/middleware/audit-log.js +73 -0
  409. package/dist/server/middleware/auth.d.ts +35 -0
  410. package/dist/server/middleware/auth.js +225 -0
  411. package/dist/server/middleware/rate-limit.d.ts +50 -0
  412. package/dist/server/middleware/rate-limit.js +270 -0
  413. package/dist/server/middleware/scope-guard.d.ts +25 -0
  414. package/dist/server/middleware/scope-guard.js +45 -0
  415. package/dist/server/middleware/url-validator.d.ts +15 -0
  416. package/dist/server/middleware/url-validator.js +201 -0
  417. package/dist/server/openapi.yaml +6418 -0
  418. package/dist/server/pg-auth-store.d.ts +146 -0
  419. package/dist/server/pg-auth-store.js +576 -0
  420. package/dist/server/pg-job-queue.d.ts +59 -0
  421. package/dist/server/pg-job-queue.js +375 -0
  422. package/dist/server/routes/activity.d.ts +6 -0
  423. package/dist/server/routes/activity.js +79 -0
  424. package/dist/server/routes/admin-active.d.ts +7 -0
  425. package/dist/server/routes/admin-active.js +120 -0
  426. package/dist/server/routes/admin-stats.d.ts +7 -0
  427. package/dist/server/routes/admin-stats.js +176 -0
  428. package/dist/server/routes/agent.d.ts +24 -0
  429. package/dist/server/routes/agent.js +480 -0
  430. package/dist/server/routes/answer.d.ts +5 -0
  431. package/dist/server/routes/answer.js +125 -0
  432. package/dist/server/routes/ask.d.ts +28 -0
  433. package/dist/server/routes/ask.js +295 -0
  434. package/dist/server/routes/batch.d.ts +6 -0
  435. package/dist/server/routes/batch.js +493 -0
  436. package/dist/server/routes/cache-warm.d.ts +25 -0
  437. package/dist/server/routes/cache-warm.js +212 -0
  438. package/dist/server/routes/cli-usage.d.ts +6 -0
  439. package/dist/server/routes/cli-usage.js +127 -0
  440. package/dist/server/routes/compat.d.ts +23 -0
  441. package/dist/server/routes/compat.js +652 -0
  442. package/dist/server/routes/crawl.d.ts +13 -0
  443. package/dist/server/routes/crawl.js +287 -0
  444. package/dist/server/routes/deep-fetch.d.ts +8 -0
  445. package/dist/server/routes/deep-fetch.js +57 -0
  446. package/dist/server/routes/deep-research.d.ts +11 -0
  447. package/dist/server/routes/deep-research.js +232 -0
  448. package/dist/server/routes/demo.d.ts +24 -0
  449. package/dist/server/routes/demo.js +517 -0
  450. package/dist/server/routes/do.d.ts +8 -0
  451. package/dist/server/routes/do.js +72 -0
  452. package/dist/server/routes/extract.d.ts +14 -0
  453. package/dist/server/routes/extract.js +325 -0
  454. package/dist/server/routes/feed.d.ts +15 -0
  455. package/dist/server/routes/feed.js +311 -0
  456. package/dist/server/routes/fetch-queue.d.ts +13 -0
  457. package/dist/server/routes/fetch-queue.js +357 -0
  458. package/dist/server/routes/fetch.d.ts +7 -0
  459. package/dist/server/routes/fetch.js +1274 -0
  460. package/dist/server/routes/go.d.ts +14 -0
  461. package/dist/server/routes/go.js +81 -0
  462. package/dist/server/routes/health.d.ts +11 -0
  463. package/dist/server/routes/health.js +141 -0
  464. package/dist/server/routes/jobs.d.ts +7 -0
  465. package/dist/server/routes/jobs.js +574 -0
  466. package/dist/server/routes/map.d.ts +11 -0
  467. package/dist/server/routes/map.js +116 -0
  468. package/dist/server/routes/mcp.d.ts +14 -0
  469. package/dist/server/routes/mcp.js +197 -0
  470. package/dist/server/routes/metrics.d.ts +37 -0
  471. package/dist/server/routes/metrics.js +149 -0
  472. package/dist/server/routes/oauth.d.ts +9 -0
  473. package/dist/server/routes/oauth.js +396 -0
  474. package/dist/server/routes/playground.d.ts +17 -0
  475. package/dist/server/routes/playground.js +283 -0
  476. package/dist/server/routes/reader.d.ts +18 -0
  477. package/dist/server/routes/reader.js +192 -0
  478. package/dist/server/routes/research.d.ts +14 -0
  479. package/dist/server/routes/research.js +482 -0
  480. package/dist/server/routes/screenshot.d.ts +22 -0
  481. package/dist/server/routes/screenshot.js +820 -0
  482. package/dist/server/routes/search.d.ts +6 -0
  483. package/dist/server/routes/search.js +874 -0
  484. package/dist/server/routes/session.d.ts +17 -0
  485. package/dist/server/routes/session.js +548 -0
  486. package/dist/server/routes/share.d.ts +18 -0
  487. package/dist/server/routes/share.js +462 -0
  488. package/dist/server/routes/smart-search/handlers/cars.d.ts +2 -0
  489. package/dist/server/routes/smart-search/handlers/cars.js +102 -0
  490. package/dist/server/routes/smart-search/handlers/flights.d.ts +2 -0
  491. package/dist/server/routes/smart-search/handlers/flights.js +72 -0
  492. package/dist/server/routes/smart-search/handlers/general.d.ts +13 -0
  493. package/dist/server/routes/smart-search/handlers/general.js +717 -0
  494. package/dist/server/routes/smart-search/handlers/hotels.d.ts +2 -0
  495. package/dist/server/routes/smart-search/handlers/hotels.js +88 -0
  496. package/dist/server/routes/smart-search/handlers/products.d.ts +2 -0
  497. package/dist/server/routes/smart-search/handlers/products.js +1309 -0
  498. package/dist/server/routes/smart-search/handlers/rental.d.ts +2 -0
  499. package/dist/server/routes/smart-search/handlers/rental.js +154 -0
  500. package/dist/server/routes/smart-search/handlers/restaurants.d.ts +2 -0
  501. package/dist/server/routes/smart-search/handlers/restaurants.js +225 -0
  502. package/dist/server/routes/smart-search/handlers/transit-verdict.d.ts +41 -0
  503. package/dist/server/routes/smart-search/handlers/transit-verdict.js +224 -0
  504. package/dist/server/routes/smart-search/index.d.ts +19 -0
  505. package/dist/server/routes/smart-search/index.js +546 -0
  506. package/dist/server/routes/smart-search/intent.d.ts +3 -0
  507. package/dist/server/routes/smart-search/intent.js +264 -0
  508. package/dist/server/routes/smart-search/llm.d.ts +16 -0
  509. package/dist/server/routes/smart-search/llm.js +70 -0
  510. package/dist/server/routes/smart-search/sources/reddit.d.ts +18 -0
  511. package/dist/server/routes/smart-search/sources/reddit.js +34 -0
  512. package/dist/server/routes/smart-search/sources/yelp.d.ts +25 -0
  513. package/dist/server/routes/smart-search/sources/yelp.js +171 -0
  514. package/dist/server/routes/smart-search/sources/youtube.d.ts +8 -0
  515. package/dist/server/routes/smart-search/sources/youtube.js +9 -0
  516. package/dist/server/routes/smart-search/types.d.ts +81 -0
  517. package/dist/server/routes/smart-search/types.js +1 -0
  518. package/dist/server/routes/smart-search/utils.d.ts +20 -0
  519. package/dist/server/routes/smart-search/utils.js +146 -0
  520. package/dist/server/routes/stats.d.ts +6 -0
  521. package/dist/server/routes/stats.js +71 -0
  522. package/dist/server/routes/stripe.d.ts +15 -0
  523. package/dist/server/routes/stripe.js +296 -0
  524. package/dist/server/routes/transcript-export.d.ts +10 -0
  525. package/dist/server/routes/transcript-export.js +178 -0
  526. package/dist/server/routes/usage.d.ts +9 -0
  527. package/dist/server/routes/usage.js +279 -0
  528. package/dist/server/routes/users.d.ts +8 -0
  529. package/dist/server/routes/users.js +1867 -0
  530. package/dist/server/routes/watch.d.ts +15 -0
  531. package/dist/server/routes/watch.js +309 -0
  532. package/dist/server/routes/webhooks.d.ts +26 -0
  533. package/dist/server/routes/webhooks.js +170 -0
  534. package/dist/server/routes/youtube.d.ts +6 -0
  535. package/dist/server/routes/youtube.js +130 -0
  536. package/dist/server/sentry.d.ts +14 -0
  537. package/dist/server/sentry.js +104 -0
  538. package/dist/server/types.d.ts +15 -0
  539. package/dist/server/types.js +7 -0
  540. package/dist/server/utils/response.d.ts +44 -0
  541. package/dist/server/utils/response.js +69 -0
  542. package/dist/server/utils/sse.d.ts +22 -0
  543. package/dist/server/utils/sse.js +38 -0
  544. package/dist/types.d.ts +552 -0
  545. package/dist/types.js +39 -0
  546. package/llms.txt +105 -0
  547. package/package.json +189 -0
@@ -0,0 +1,12 @@
1
+ /**
2
+ * Monitor command: content change detection for URLs
3
+ *
4
+ * Usage:
5
+ * webpeel monitor <url> - Fetch & snapshot (or diff if prev exists)
6
+ * webpeel monitor <url> --interval 300 - Watch mode: re-check every 5 minutes
7
+ * webpeel monitor <url> --json - JSON output for automation
8
+ * webpeel monitor <url> --render - Use browser rendering
9
+ * webpeel monitor <url> --selector <css> - Monitor specific section only
10
+ */
11
+ import type { Command } from 'commander';
12
+ export declare function registerMonitorCommands(program: Command): void;
@@ -0,0 +1,197 @@
1
+ /**
2
+ * Monitor command: content change detection for URLs
3
+ *
4
+ * Usage:
5
+ * webpeel monitor <url> - Fetch & snapshot (or diff if prev exists)
6
+ * webpeel monitor <url> --interval 300 - Watch mode: re-check every 5 minutes
7
+ * webpeel monitor <url> --json - JSON output for automation
8
+ * webpeel monitor <url> --render - Use browser rendering
9
+ * webpeel monitor <url> --selector <css> - Monitor specific section only
10
+ */
11
+ import { createHash } from 'crypto';
12
+ import ora from 'ora';
13
+ import { peel, cleanup } from '../../index.js';
14
+ import { trackChange, getSnapshot } from '../../core/change-tracking.js';
15
+ // ─── helpers ─────────────────────────────────────────────────────────────────
16
+ function fingerprint(content) {
17
+ return createHash('sha256').update(content).digest('hex');
18
+ }
19
+ function formatTimestamp(iso) {
20
+ return new Date(iso).toLocaleString();
21
+ }
22
+ /** Run a single monitor check. Returns exit code (0 = ok, 1 = error). */
23
+ async function runMonitorCheck(url, options) {
24
+ const spinner = options.silent || options.json ? null : ora(`Checking ${url}...`).start();
25
+ try {
26
+ const peelOptions = {
27
+ render: options.render || false,
28
+ selector: options.selector,
29
+ format: 'markdown',
30
+ timeout: options.timeout,
31
+ raw: false,
32
+ };
33
+ const result = await peel(url, peelOptions);
34
+ const content = result.content ?? '';
35
+ const fp = fingerprint(content);
36
+ if (spinner)
37
+ spinner.succeed(`Fetched in ${result.elapsed ?? 0}ms`);
38
+ const changeResult = await trackChange(url, content, fp);
39
+ if (options.json) {
40
+ const out = {
41
+ success: true,
42
+ url,
43
+ changeStatus: changeResult.changeStatus,
44
+ previousScrapeAt: changeResult.previousScrapeAt,
45
+ checkedAt: new Date().toISOString(),
46
+ };
47
+ if (changeResult.diff) {
48
+ out.diff = {
49
+ additions: changeResult.diff.additions,
50
+ deletions: changeResult.diff.deletions,
51
+ text: changeResult.diff.text,
52
+ };
53
+ }
54
+ process.stdout.write(JSON.stringify(out, null, 2) + '\n');
55
+ return { exitCode: 0, changed: changeResult.changeStatus === 'changed' };
56
+ }
57
+ // Human-readable output
58
+ switch (changeResult.changeStatus) {
59
+ case 'new': {
60
+ console.log(`📸 Baseline snapshot saved for ${url}`);
61
+ break;
62
+ }
63
+ case 'same': {
64
+ const since = changeResult.previousScrapeAt
65
+ ? formatTimestamp(changeResult.previousScrapeAt)
66
+ : 'last check';
67
+ console.log(`✅ No changes detected since ${since}`);
68
+ break;
69
+ }
70
+ case 'changed': {
71
+ const since = changeResult.previousScrapeAt
72
+ ? formatTimestamp(changeResult.previousScrapeAt)
73
+ : 'last check';
74
+ console.log(`\n🔔 Changes detected since ${since}:`);
75
+ console.log('');
76
+ if (changeResult.diff) {
77
+ const { additions, deletions, text } = changeResult.diff;
78
+ // Print unified diff with color
79
+ if (text) {
80
+ for (const line of text.split('\n')) {
81
+ if (line.startsWith('+')) {
82
+ process.stdout.write(`\x1b[32m${line}\x1b[0m\n`);
83
+ }
84
+ else if (line.startsWith('-')) {
85
+ process.stdout.write(`\x1b[31m${line}\x1b[0m\n`);
86
+ }
87
+ else if (line.startsWith('@')) {
88
+ process.stdout.write(`\x1b[36m${line}\x1b[0m\n`);
89
+ }
90
+ else {
91
+ process.stdout.write(`${line}\n`);
92
+ }
93
+ }
94
+ }
95
+ console.log('');
96
+ console.log(`📊 Summary: \x1b[32m+${additions} lines added\x1b[0m, \x1b[31m-${deletions} lines removed\x1b[0m`);
97
+ }
98
+ break;
99
+ }
100
+ }
101
+ return { exitCode: 0, changed: changeResult.changeStatus === 'changed' };
102
+ }
103
+ catch (error) {
104
+ if (spinner)
105
+ spinner.fail('Monitor check failed');
106
+ const msg = error instanceof Error ? error.message : 'Unknown error';
107
+ if (options.json) {
108
+ process.stdout.write(JSON.stringify({ success: false, url, error: msg }) + '\n');
109
+ }
110
+ else {
111
+ console.error(`\nError: ${msg}`);
112
+ }
113
+ return { exitCode: 1, changed: false };
114
+ }
115
+ }
116
+ // ─── registerMonitorCommands ──────────────────────────────────────────────────
117
+ export function registerMonitorCommands(program) {
118
+ program
119
+ .command('monitor <url>')
120
+ .description('Monitor a URL for content changes (saves snapshots, shows diffs)')
121
+ .option('-r, --render', 'Use headless browser (for JS-heavy sites)')
122
+ .option('--selector <css>', 'CSS selector — only monitor this section of the page')
123
+ .option('-i, --interval <seconds>', 'Watch mode: re-check every N seconds', parseInt)
124
+ .option('--json', 'Output as JSON (for automation/scripting)')
125
+ .option('-s, --silent', 'Silent mode (no spinner)')
126
+ .option('-t, --timeout <ms>', 'Request timeout (ms)', (v) => parseInt(v, 10), 30000)
127
+ .addHelpText('after', `
128
+ Examples:
129
+ webpeel monitor https://example.com/pricing # First run: save baseline
130
+ webpeel monitor https://example.com/pricing # Second run: show diff
131
+ webpeel monitor https://example.com/pricing --interval 300 # Watch every 5 min
132
+ webpeel monitor https://example.com/pricing --render # JS-rendered page
133
+ webpeel monitor https://example.com/pricing --json # JSON output
134
+ webpeel monitor https://example.com/pricing --selector .price # Monitor prices only
135
+ `)
136
+ .action(async (url, options) => {
137
+ // Validate URL
138
+ try {
139
+ const parsed = new URL(url);
140
+ if (!['http:', 'https:'].includes(parsed.protocol)) {
141
+ console.error('Error: Only HTTP and HTTPS protocols are allowed');
142
+ process.exit(1);
143
+ }
144
+ }
145
+ catch {
146
+ console.error(`Error: Invalid URL format: ${url}`);
147
+ process.exit(1);
148
+ }
149
+ const checkOptions = {
150
+ json: options.json || false,
151
+ render: options.render || false,
152
+ selector: options.selector,
153
+ silent: options.silent || false,
154
+ timeout: options.timeout,
155
+ };
156
+ const intervalSec = options.interval;
157
+ if (!intervalSec) {
158
+ // Single run
159
+ const { exitCode } = await runMonitorCheck(url, checkOptions);
160
+ await cleanup();
161
+ process.exit(exitCode);
162
+ }
163
+ else {
164
+ // Watch mode
165
+ if (intervalSec < 5) {
166
+ console.error('Error: --interval must be at least 5 seconds');
167
+ process.exit(1);
168
+ }
169
+ if (!options.json && !options.silent) {
170
+ const prev = await getSnapshot(url);
171
+ const baselineNote = prev
172
+ ? `(baseline exists from ${formatTimestamp(new Date(prev.timestamp).toISOString())})`
173
+ : '(no baseline yet — first run will save it)';
174
+ console.log(`👁 Watching ${url} every ${intervalSec}s ${baselineNote}`);
175
+ console.log('Press Ctrl+C to stop.\n');
176
+ }
177
+ // Initial check immediately
178
+ await runMonitorCheck(url, checkOptions);
179
+ // Then loop
180
+ const loop = setInterval(async () => {
181
+ if (!options.json && !options.silent) {
182
+ console.log(`\n⏰ ${new Date().toLocaleTimeString()} — checking...`);
183
+ }
184
+ await runMonitorCheck(url, checkOptions);
185
+ }, intervalSec * 1000);
186
+ // Graceful shutdown
187
+ process.on('SIGINT', async () => {
188
+ clearInterval(loop);
189
+ if (!options.json && !options.silent) {
190
+ console.log('\n👋 Stopped monitoring.');
191
+ }
192
+ await cleanup();
193
+ process.exit(0);
194
+ });
195
+ }
196
+ });
197
+ }
@@ -0,0 +1,12 @@
1
+ /**
2
+ * Observe command: structured page observation for AI agents
3
+ *
4
+ * Usage:
5
+ * webpeel observe <url> - List all interactive elements
6
+ * webpeel observe <url> --json - JSON output for agents
7
+ * webpeel observe <url> --screenshot - Include screenshot
8
+ * webpeel observe <url> --selector main - Scope to a section
9
+ * webpeel observe <url> --viewport mobile - Mobile viewport
10
+ */
11
+ import type { Command } from 'commander';
12
+ export declare function registerObserveCommand(program: Command): void;
@@ -0,0 +1,158 @@
1
+ /**
2
+ * Observe command: structured page observation for AI agents
3
+ *
4
+ * Usage:
5
+ * webpeel observe <url> - List all interactive elements
6
+ * webpeel observe <url> --json - JSON output for agents
7
+ * webpeel observe <url> --screenshot - Include screenshot
8
+ * webpeel observe <url> --selector main - Scope to a section
9
+ * webpeel observe <url> --viewport mobile - Mobile viewport
10
+ */
11
+ import ora from 'ora';
12
+ import { writeFileSync } from 'fs';
13
+ import { cleanup } from '../../index.js';
14
+ import { writeStdout } from '../utils.js';
15
+ export function registerObserveCommand(program) {
16
+ program
17
+ .command('observe <url>')
18
+ .alias('obs')
19
+ .description('Observe a page — list interactive elements (links, buttons, inputs, forms) for agent use')
20
+ .option('--json', 'Output as JSON (default for piped output)')
21
+ .option('-s, --silent', 'Silent mode (no spinner)')
22
+ .option('--selector <css>', 'CSS selector to scope observation')
23
+ .option('--viewport <v>', 'Viewport: desktop (default), mobile, tablet, or WxH', 'desktop')
24
+ .option('--screenshot', 'Include a screenshot')
25
+ .option('--screenshot-full-page', 'Full-page screenshot')
26
+ .option('--screenshot-output <path>', 'Save screenshot to file instead of base64 in JSON')
27
+ .option('--max-elements <n>', 'Max elements per category (default: 50)', (v) => parseInt(v, 10), 50)
28
+ .option('-t, --timeout <ms>', 'Request timeout in ms', (v) => parseInt(v, 10), 30000)
29
+ .option('--stealth', 'Stealth mode for bot-protected sites')
30
+ .addHelpText('after', `
31
+ Examples:
32
+ webpeel observe https://news.ycombinator.com # See all interactive elements
33
+ webpeel observe https://github.com --json # JSON for agent consumption
34
+ webpeel observe https://amazon.com/dp/... --selector main # Scope to main content
35
+ webpeel observe https://example.com --viewport mobile # Mobile viewport
36
+ webpeel observe https://example.com --screenshot -o snap.png # With screenshot
37
+ `)
38
+ .action(async (url, options) => {
39
+ // Validate URL
40
+ try {
41
+ const parsed = new URL(url);
42
+ if (!['http:', 'https:'].includes(parsed.protocol)) {
43
+ console.error('Error: Only HTTP and HTTPS protocols are allowed');
44
+ process.exit(1);
45
+ }
46
+ }
47
+ catch {
48
+ console.error(`Error: Invalid URL format: ${url}`);
49
+ process.exit(1);
50
+ }
51
+ const spinner = options.silent ? null : ora('Observing page...').start();
52
+ try {
53
+ const { observe } = await import('../../core/observe.js');
54
+ // Parse viewport
55
+ let viewport = 'desktop';
56
+ const vpArg = options.viewport;
57
+ if (vpArg === 'mobile' || vpArg === 'tablet') {
58
+ viewport = vpArg;
59
+ }
60
+ else if (vpArg && vpArg.includes('x')) {
61
+ const [w, h] = vpArg.split('x').map(Number);
62
+ if (w && h)
63
+ viewport = { width: w, height: h };
64
+ }
65
+ const result = await observe({
66
+ url,
67
+ selector: options.selector,
68
+ viewport,
69
+ screenshot: options.screenshot || false,
70
+ screenshotFullPage: options.screenshotFullPage || false,
71
+ maxElements: options.maxElements,
72
+ timeout: options.timeout,
73
+ stealth: options.stealth || false,
74
+ });
75
+ if (spinner) {
76
+ spinner.succeed(`Observed ${result.totalElements} elements in ${result.elapsed}ms`);
77
+ }
78
+ // Save screenshot to file if requested
79
+ if (result.screenshot && options.screenshotOutput) {
80
+ const buf = Buffer.from(result.screenshot, 'base64');
81
+ writeFileSync(options.screenshotOutput, buf);
82
+ if (!options.silent) {
83
+ console.error(`Screenshot saved to: ${options.screenshotOutput} (${(buf.length / 1024).toFixed(1)} KB)`);
84
+ }
85
+ // Don't include screenshot in JSON output when saved to file
86
+ delete result.screenshot;
87
+ }
88
+ if (options.json) {
89
+ await writeStdout(JSON.stringify(result, null, 2) + '\n');
90
+ }
91
+ else {
92
+ // Human-readable output
93
+ console.log(`\n🔍 ${result.title}`);
94
+ console.log(` ${result.url}`);
95
+ console.log(` Viewport: ${result.viewport.width}×${result.viewport.height} | Page: ${result.scroll.width}×${result.scroll.height}\n`);
96
+ const { elements } = result;
97
+ if (elements.links.length > 0) {
98
+ console.log(`📎 Links (${elements.links.length}):`);
99
+ for (const el of elements.links.slice(0, 15)) {
100
+ const vp = el.inViewport ? '👁' : ' ';
101
+ const href = el.attributes.href ? ` → ${el.attributes.href.slice(0, 60)}` : '';
102
+ console.log(` ${vp} [${el.ref}] ${el.text || '(no text)'}${href}`);
103
+ }
104
+ if (elements.links.length > 15) {
105
+ console.log(` ... and ${elements.links.length - 15} more`);
106
+ }
107
+ console.log('');
108
+ }
109
+ if (elements.buttons.length > 0) {
110
+ console.log(`🔘 Buttons (${elements.buttons.length}):`);
111
+ for (const el of elements.buttons) {
112
+ const vp = el.inViewport ? '👁' : ' ';
113
+ console.log(` ${vp} [${el.ref}] "${el.text || '(no text)'}" — selector: ${el.selector}`);
114
+ }
115
+ console.log('');
116
+ }
117
+ if (elements.inputs.length > 0) {
118
+ console.log(`✏️ Inputs (${elements.inputs.length}):`);
119
+ for (const el of elements.inputs) {
120
+ const vp = el.inViewport ? '👁' : ' ';
121
+ const type = el.attributes.type || 'text';
122
+ const label = el.text || el.attributes.placeholder || el.attributes.name || 'unnamed';
123
+ console.log(` ${vp} [${el.ref}] ${label} (${type}) — selector: ${el.selector}`);
124
+ }
125
+ console.log('');
126
+ }
127
+ if (elements.selects.length > 0) {
128
+ console.log(`📋 Selects (${elements.selects.length}):`);
129
+ for (const el of elements.selects) {
130
+ const vp = el.inViewport ? '👁' : ' ';
131
+ const opts = el.attributes.options || '';
132
+ console.log(` ${vp} [${el.ref}] ${el.text || el.attributes.name || 'dropdown'}${opts ? ` [${opts}]` : ''}`);
133
+ }
134
+ console.log('');
135
+ }
136
+ if (elements.forms.length > 0) {
137
+ console.log(`📝 Forms (${elements.forms.length}):`);
138
+ for (const el of elements.forms) {
139
+ const fields = el.attributes.fields || '?';
140
+ const action = el.attributes.action || '';
141
+ console.log(` [${el.ref}] ${el.text || 'form'} — ${fields} fields${action ? ` → ${action.slice(0, 60)}` : ''}`);
142
+ }
143
+ console.log('');
144
+ }
145
+ console.log(`📊 Summary: ${result.summary}`);
146
+ }
147
+ await cleanup();
148
+ process.exit(0);
149
+ }
150
+ catch (error) {
151
+ if (spinner)
152
+ spinner.fail('Observation failed');
153
+ console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
154
+ await cleanup();
155
+ process.exit(1);
156
+ }
157
+ });
158
+ }
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Screenshot commands: screenshot, brand, design-compare
3
+ */
4
+ import type { Command } from 'commander';
5
+ export declare function registerScreenshotCommands(program: Command): void;
@@ -0,0 +1,282 @@
1
+ /**
2
+ * Screenshot commands: screenshot, brand, design-compare
3
+ */
4
+ import ora from 'ora';
5
+ import { writeFileSync } from 'fs';
6
+ import { cleanup } from '../../index.js';
7
+ import { checkUsage, showUsageFooter } from '../../cli-auth.js';
8
+ import { parseActions, extractColors, extractFonts } from '../utils.js';
9
+ import { peel } from '../../index.js';
10
+ export function registerScreenshotCommands(program) {
11
+ // ── screenshot command ────────────────────────────────────────────────────
12
+ program
13
+ .command('screenshot <url>')
14
+ .alias('snap')
15
+ .description('Take a screenshot of a URL and save as PNG/JPEG')
16
+ .option('--full-page', 'Capture full page (not just viewport)')
17
+ .option('--width <px>', 'Viewport width in pixels (default: 1280)', parseInt)
18
+ .option('--height <px>', 'Viewport height in pixels (default: 720)', parseInt)
19
+ .option('--format <fmt>', 'Image format: png (default) or jpeg', 'png')
20
+ .option('--quality <n>', 'JPEG quality 1-100 (ignored for PNG)', parseInt)
21
+ .option('-w, --wait <ms>', 'Wait time after page load (ms)', parseInt)
22
+ .option('-t, --timeout <ms>', 'Request timeout (ms)', (v) => parseInt(v, 10), 30000)
23
+ .option('--stealth', 'Use stealth mode to bypass bot detection')
24
+ .option('--action <actions...>', 'Page actions before screenshot (e.g., "click:.btn" "wait:2000")')
25
+ .option('--scroll-through', 'Auto-scroll page before screenshot (triggers lazy content + scroll animations)')
26
+ .option('-o, --output <path>', 'Output file path (default: screenshot.png)')
27
+ .option('-s, --silent', 'Silent mode (no spinner)')
28
+ .option('--json', 'Output base64 JSON instead of binary file')
29
+ .action(async (url, options) => {
30
+ // Validate URL
31
+ try {
32
+ const parsed = new URL(url);
33
+ if (!['http:', 'https:'].includes(parsed.protocol)) {
34
+ console.error('Error: Only HTTP and HTTPS protocols are allowed');
35
+ process.exit(1);
36
+ }
37
+ }
38
+ catch {
39
+ console.error(`Error: Invalid URL format: ${url}`);
40
+ process.exit(1);
41
+ }
42
+ // Check usage quota
43
+ const usageCheck = await checkUsage();
44
+ if (!usageCheck.allowed) {
45
+ console.error(usageCheck.message);
46
+ process.exit(1);
47
+ }
48
+ const spinner = options.silent ? null : ora('Taking screenshot...').start();
49
+ try {
50
+ // Validate format
51
+ const format = options.format?.toLowerCase();
52
+ if (format && !['png', 'jpeg', 'jpg'].includes(format)) {
53
+ console.error('Error: --format must be png, jpeg, or jpg');
54
+ process.exit(1);
55
+ }
56
+ // Parse actions
57
+ let actions;
58
+ if (options.action && options.action.length > 0) {
59
+ try {
60
+ actions = parseActions(options.action);
61
+ }
62
+ catch (e) {
63
+ console.error(`Error: ${e.message}`);
64
+ process.exit(1);
65
+ }
66
+ }
67
+ const { takeScreenshot } = await import('../../core/screenshot.js');
68
+ const result = await takeScreenshot(url, {
69
+ fullPage: options.fullPage || false,
70
+ width: options.width,
71
+ height: options.height,
72
+ format: format || 'png',
73
+ quality: options.quality,
74
+ waitFor: options.wait,
75
+ timeout: options.timeout,
76
+ stealth: options.stealth || false,
77
+ actions,
78
+ scrollThrough: options.scrollThrough || false,
79
+ });
80
+ if (spinner) {
81
+ spinner.succeed(`Screenshot taken (${result.format})`);
82
+ }
83
+ // Show usage footer for free/anonymous users
84
+ if (usageCheck.usageInfo && !options.silent) {
85
+ showUsageFooter(usageCheck.usageInfo, usageCheck.isAnonymous || false, true);
86
+ }
87
+ if (options.json) {
88
+ // Output JSON with base64
89
+ const jsonStr = JSON.stringify({
90
+ url: result.url,
91
+ format: result.format,
92
+ contentType: result.contentType,
93
+ screenshot: result.screenshot,
94
+ }, null, 2);
95
+ await new Promise((resolve, reject) => {
96
+ process.stdout.write(jsonStr + '\n', (err) => {
97
+ if (err)
98
+ reject(err);
99
+ else
100
+ resolve();
101
+ });
102
+ });
103
+ }
104
+ else {
105
+ // Save to file
106
+ const ext = result.format === 'jpeg' ? 'jpg' : 'png';
107
+ const outputPath = options.output || `screenshot.${ext}`;
108
+ const buffer = Buffer.from(result.screenshot, 'base64');
109
+ writeFileSync(outputPath, buffer);
110
+ if (!options.silent) {
111
+ console.error(`Screenshot saved to: ${outputPath} (${(buffer.length / 1024).toFixed(1)} KB)`);
112
+ }
113
+ }
114
+ await cleanup();
115
+ process.exit(0);
116
+ }
117
+ catch (error) {
118
+ if (spinner) {
119
+ spinner.fail('Screenshot failed');
120
+ }
121
+ if (error instanceof Error) {
122
+ const msg = error.message;
123
+ // Detect missing browser binary and give an actionable error
124
+ if (msg.includes("Executable doesn't exist") || msg.includes('browserType.launch') || msg.includes('Chromium is not installed')) {
125
+ console.error('\n\x1b[31m❌ Browser not installed.\x1b[0m');
126
+ console.error('\x1b[36m Run: npx playwright install chromium\x1b[0m');
127
+ console.error('\x1b[36m Then retry your screenshot command.\x1b[0m');
128
+ }
129
+ else {
130
+ console.error(`\nError: ${msg}`);
131
+ }
132
+ }
133
+ else {
134
+ console.error('\nError: Unknown error occurred');
135
+ }
136
+ await cleanup();
137
+ process.exit(1);
138
+ }
139
+ });
140
+ // ── brand command ─────────────────────────────────────────────────────────
141
+ program
142
+ .command('brand <url>')
143
+ .description('Extract branding and design system from a URL')
144
+ .option('-s, --silent', 'Silent mode (no spinner)')
145
+ .option('--json', 'Output as JSON (default)')
146
+ .action(async (url, options) => {
147
+ const spinner = options.silent ? null : ora('Extracting branding...').start();
148
+ try {
149
+ const result = await peel(url, {
150
+ extract: {
151
+ selectors: {
152
+ primaryColor: 'meta[name="theme-color"]',
153
+ title: 'title',
154
+ logo: 'img[class*="logo"], img[alt*="logo"]',
155
+ },
156
+ },
157
+ });
158
+ if (spinner) {
159
+ spinner.succeed(`Extracted branding in ${result.elapsed}ms`);
160
+ }
161
+ // Extract branding data from metadata and page
162
+ const branding = {
163
+ url: result.url,
164
+ title: result.title,
165
+ colors: extractColors(result.content),
166
+ fonts: extractFonts(result.content),
167
+ extracted: result.extracted,
168
+ metadata: result.metadata,
169
+ };
170
+ console.log(JSON.stringify(branding, null, 2));
171
+ await cleanup();
172
+ process.exit(0);
173
+ }
174
+ catch (error) {
175
+ if (spinner)
176
+ spinner.fail('Branding extraction failed');
177
+ console.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
178
+ await cleanup();
179
+ process.exit(1);
180
+ }
181
+ });
182
+ // ── design-compare command ────────────────────────────────────────────────
183
+ program
184
+ .command('design-compare <url>')
185
+ .description('Compare the design of a subject URL against a reference URL')
186
+ .option('--ref <url>', 'Reference URL to compare against (required)')
187
+ .option('--width <px>', 'Viewport width in pixels (default: 1440)', parseInt)
188
+ .option('--height <px>', 'Viewport height in pixels (default: 900)', parseInt)
189
+ .option('-o, --output <path>', 'Save comparison report to a JSON file')
190
+ .option('-s, --silent', 'Silent mode (no spinner)')
191
+ .option('--json', 'Output comparison as JSON to stdout')
192
+ .action(async (url, options) => {
193
+ // Validate subject URL
194
+ try {
195
+ const parsed = new URL(url);
196
+ if (!['http:', 'https:'].includes(parsed.protocol)) {
197
+ console.error('Error: Only HTTP and HTTPS protocols are allowed');
198
+ process.exit(1);
199
+ }
200
+ }
201
+ catch {
202
+ console.error(`Error: Invalid URL format: ${url}`);
203
+ process.exit(1);
204
+ }
205
+ // Validate --ref
206
+ if (!options.ref) {
207
+ console.error('Error: --ref <url> is required');
208
+ process.exit(1);
209
+ }
210
+ try {
211
+ const parsedRef = new URL(options.ref);
212
+ if (!['http:', 'https:'].includes(parsedRef.protocol)) {
213
+ console.error('Error: --ref must be an HTTP or HTTPS URL');
214
+ process.exit(1);
215
+ }
216
+ }
217
+ catch {
218
+ console.error(`Error: Invalid --ref URL format: ${options.ref}`);
219
+ process.exit(1);
220
+ }
221
+ const spinner = options.silent ? null : ora(`Comparing designs: ${url} vs ${options.ref}...`).start();
222
+ try {
223
+ const { takeDesignComparison } = await import('../../core/screenshot.js');
224
+ const result = await takeDesignComparison(url, options.ref, {
225
+ width: options.width,
226
+ height: options.height,
227
+ });
228
+ if (spinner)
229
+ spinner.succeed('Design comparison complete');
230
+ const { comparison } = result;
231
+ const output = {
232
+ subjectUrl: result.subjectUrl,
233
+ referenceUrl: result.referenceUrl,
234
+ score: comparison.score,
235
+ summary: comparison.summary,
236
+ gaps: comparison.gaps,
237
+ subjectAnalysis: comparison.subjectAnalysis,
238
+ referenceAnalysis: comparison.referenceAnalysis,
239
+ };
240
+ if (options.output) {
241
+ writeFileSync(options.output, JSON.stringify(output, null, 2));
242
+ if (!options.silent)
243
+ console.error(`Report saved to: ${options.output}`);
244
+ }
245
+ if (options.json || !options.output) {
246
+ const jsonStr = JSON.stringify(output, null, 2);
247
+ await new Promise((resolve, reject) => {
248
+ process.stdout.write(jsonStr + '\n', (err) => {
249
+ if (err)
250
+ reject(err);
251
+ else
252
+ resolve();
253
+ });
254
+ });
255
+ }
256
+ else if (!options.silent) {
257
+ // Human-readable summary
258
+ console.log(`\n🎨 Design Comparison`);
259
+ console.log(`Subject: ${result.subjectUrl}`);
260
+ console.log(`Reference: ${result.referenceUrl}`);
261
+ console.log(`Score: ${comparison.score}/10`);
262
+ console.log(`\n${comparison.summary}`);
263
+ if (comparison.gaps.length > 0) {
264
+ console.log(`\nGaps (${comparison.gaps.length}):`);
265
+ for (const gap of comparison.gaps) {
266
+ const sev = gap.severity === 'high' ? '🔴' : gap.severity === 'medium' ? '🟡' : '🟢';
267
+ console.log(` ${sev} ${gap.property}: ${gap.description}`);
268
+ console.log(` Subject: ${gap.subject}`);
269
+ console.log(` Reference: ${gap.reference}`);
270
+ console.log(` Suggestion: ${gap.suggestion}`);
271
+ }
272
+ }
273
+ }
274
+ }
275
+ catch (error) {
276
+ if (spinner)
277
+ spinner.fail('Design comparison failed');
278
+ console.error(`Error: ${error.message}`);
279
+ process.exit(1);
280
+ }
281
+ });
282
+ }
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Search commands: search, sites, batch, crawl, map
3
+ */
4
+ import type { Command } from 'commander';
5
+ export declare function registerSearchCommands(program: Command): void;