@iflow-mcp/jakeliume-webpeel 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (547) hide show
  1. package/LICENSE +15 -0
  2. package/README.md +313 -0
  3. package/dist/cache.d.ts +30 -0
  4. package/dist/cache.js +139 -0
  5. package/dist/cli/commands/auth.d.ts +5 -0
  6. package/dist/cli/commands/auth.js +411 -0
  7. package/dist/cli/commands/doctor.d.ts +37 -0
  8. package/dist/cli/commands/doctor.js +371 -0
  9. package/dist/cli/commands/fetch.d.ts +6 -0
  10. package/dist/cli/commands/fetch.js +1345 -0
  11. package/dist/cli/commands/guide.d.ts +2 -0
  12. package/dist/cli/commands/guide.js +183 -0
  13. package/dist/cli/commands/interact.d.ts +5 -0
  14. package/dist/cli/commands/interact.js +840 -0
  15. package/dist/cli/commands/jobs.d.ts +5 -0
  16. package/dist/cli/commands/jobs.js +997 -0
  17. package/dist/cli/commands/monitor.d.ts +12 -0
  18. package/dist/cli/commands/monitor.js +197 -0
  19. package/dist/cli/commands/observe.d.ts +12 -0
  20. package/dist/cli/commands/observe.js +158 -0
  21. package/dist/cli/commands/screenshot.d.ts +5 -0
  22. package/dist/cli/commands/screenshot.js +282 -0
  23. package/dist/cli/commands/search.d.ts +5 -0
  24. package/dist/cli/commands/search.js +1021 -0
  25. package/dist/cli/commands/setup.d.ts +13 -0
  26. package/dist/cli/commands/setup.js +244 -0
  27. package/dist/cli/commands/skill.d.ts +15 -0
  28. package/dist/cli/commands/skill.js +195 -0
  29. package/dist/cli/utils.d.ts +84 -0
  30. package/dist/cli/utils.js +806 -0
  31. package/dist/cli-auth.d.ts +75 -0
  32. package/dist/cli-auth.js +369 -0
  33. package/dist/cli.d.ts +17 -0
  34. package/dist/cli.js +99 -0
  35. package/dist/core/actions.d.ts +69 -0
  36. package/dist/core/actions.js +495 -0
  37. package/dist/core/agent.d.ts +98 -0
  38. package/dist/core/agent.js +558 -0
  39. package/dist/core/answer.d.ts +42 -0
  40. package/dist/core/answer.js +395 -0
  41. package/dist/core/application-tracker.d.ts +84 -0
  42. package/dist/core/application-tracker.js +184 -0
  43. package/dist/core/apply.d.ts +162 -0
  44. package/dist/core/apply.js +816 -0
  45. package/dist/core/auth-detection.d.ts +35 -0
  46. package/dist/core/auth-detection.js +358 -0
  47. package/dist/core/auto-extract.d.ts +82 -0
  48. package/dist/core/auto-extract.js +604 -0
  49. package/dist/core/auto-interact.d.ts +23 -0
  50. package/dist/core/auto-interact.js +246 -0
  51. package/dist/core/bm25-filter.d.ts +66 -0
  52. package/dist/core/bm25-filter.js +288 -0
  53. package/dist/core/branding.d.ts +54 -0
  54. package/dist/core/branding.js +234 -0
  55. package/dist/core/browser-fetch.d.ts +323 -0
  56. package/dist/core/browser-fetch.js +1600 -0
  57. package/dist/core/browser-pool.d.ts +91 -0
  58. package/dist/core/browser-pool.js +550 -0
  59. package/dist/core/budget.d.ts +42 -0
  60. package/dist/core/budget.js +324 -0
  61. package/dist/core/business-intel.d.ts +47 -0
  62. package/dist/core/business-intel.js +279 -0
  63. package/dist/core/cache.d.ts +13 -0
  64. package/dist/core/cache.js +121 -0
  65. package/dist/core/cf-worker-proxy.d.ts +32 -0
  66. package/dist/core/cf-worker-proxy.js +87 -0
  67. package/dist/core/challenge-detection.d.ts +26 -0
  68. package/dist/core/challenge-detection.js +468 -0
  69. package/dist/core/change-tracking.d.ts +75 -0
  70. package/dist/core/change-tracking.js +276 -0
  71. package/dist/core/chunker.d.ts +46 -0
  72. package/dist/core/chunker.js +249 -0
  73. package/dist/core/chunking.d.ts +42 -0
  74. package/dist/core/chunking.js +181 -0
  75. package/dist/core/circuit-breaker.d.ts +44 -0
  76. package/dist/core/circuit-breaker.js +85 -0
  77. package/dist/core/content-pruner.d.ts +47 -0
  78. package/dist/core/content-pruner.js +425 -0
  79. package/dist/core/cookie-cache.d.ts +60 -0
  80. package/dist/core/cookie-cache.js +163 -0
  81. package/dist/core/crawl-checkpoint.d.ts +54 -0
  82. package/dist/core/crawl-checkpoint.js +104 -0
  83. package/dist/core/crawler.d.ts +84 -0
  84. package/dist/core/crawler.js +349 -0
  85. package/dist/core/cross-verify.d.ts +27 -0
  86. package/dist/core/cross-verify.js +93 -0
  87. package/dist/core/deep-fetch.d.ts +74 -0
  88. package/dist/core/deep-fetch.js +405 -0
  89. package/dist/core/deep-research.d.ts +141 -0
  90. package/dist/core/deep-research.js +972 -0
  91. package/dist/core/design-analysis.d.ts +70 -0
  92. package/dist/core/design-analysis.js +490 -0
  93. package/dist/core/design-compare.d.ts +38 -0
  94. package/dist/core/design-compare.js +264 -0
  95. package/dist/core/diff.d.ts +61 -0
  96. package/dist/core/diff.js +289 -0
  97. package/dist/core/dns-cache.d.ts +20 -0
  98. package/dist/core/dns-cache.js +198 -0
  99. package/dist/core/documents.d.ts +23 -0
  100. package/dist/core/documents.js +123 -0
  101. package/dist/core/domain-memory.d.ts +66 -0
  102. package/dist/core/domain-memory.js +163 -0
  103. package/dist/core/domain-verify.d.ts +40 -0
  104. package/dist/core/domain-verify.js +379 -0
  105. package/dist/core/engine-ranker.d.ts +112 -0
  106. package/dist/core/engine-ranker.js +395 -0
  107. package/dist/core/extract-inline.d.ts +38 -0
  108. package/dist/core/extract-inline.js +215 -0
  109. package/dist/core/extract-listings.d.ts +38 -0
  110. package/dist/core/extract-listings.js +461 -0
  111. package/dist/core/extract.d.ts +9 -0
  112. package/dist/core/extract.js +139 -0
  113. package/dist/core/fetch-cache.d.ts +57 -0
  114. package/dist/core/fetch-cache.js +95 -0
  115. package/dist/core/fetcher.d.ts +13 -0
  116. package/dist/core/fetcher.js +12 -0
  117. package/dist/core/google-cache.d.ts +29 -0
  118. package/dist/core/google-cache.js +180 -0
  119. package/dist/core/google-serp-parser.d.ts +82 -0
  120. package/dist/core/google-serp-parser.js +287 -0
  121. package/dist/core/hotel-search.d.ts +122 -0
  122. package/dist/core/hotel-search.js +382 -0
  123. package/dist/core/http-fetch.d.ts +72 -0
  124. package/dist/core/http-fetch.js +820 -0
  125. package/dist/core/human.d.ts +175 -0
  126. package/dist/core/human.js +680 -0
  127. package/dist/core/image-caption.d.ts +44 -0
  128. package/dist/core/image-caption.js +271 -0
  129. package/dist/core/jobs.d.ts +75 -0
  130. package/dist/core/jobs.js +634 -0
  131. package/dist/core/json-ld.d.ts +15 -0
  132. package/dist/core/json-ld.js +617 -0
  133. package/dist/core/language-detect.d.ts +18 -0
  134. package/dist/core/language-detect.js +135 -0
  135. package/dist/core/links.d.ts +10 -0
  136. package/dist/core/links.js +44 -0
  137. package/dist/core/llm-extract.d.ts +71 -0
  138. package/dist/core/llm-extract.js +507 -0
  139. package/dist/core/llm-provider.d.ts +100 -0
  140. package/dist/core/llm-provider.js +702 -0
  141. package/dist/core/local-search.d.ts +60 -0
  142. package/dist/core/local-search.js +308 -0
  143. package/dist/core/logger.d.ts +28 -0
  144. package/dist/core/logger.js +104 -0
  145. package/dist/core/map.d.ts +33 -0
  146. package/dist/core/map.js +127 -0
  147. package/dist/core/markdown.d.ts +92 -0
  148. package/dist/core/markdown.js +809 -0
  149. package/dist/core/metadata.d.ts +34 -0
  150. package/dist/core/metadata.js +422 -0
  151. package/dist/core/observe.d.ts +113 -0
  152. package/dist/core/observe.js +395 -0
  153. package/dist/core/ocr.d.ts +12 -0
  154. package/dist/core/ocr.js +33 -0
  155. package/dist/core/paginate.d.ts +31 -0
  156. package/dist/core/paginate.js +106 -0
  157. package/dist/core/pdf.d.ts +8 -0
  158. package/dist/core/pdf.js +25 -0
  159. package/dist/core/peel-tls.d.ts +25 -0
  160. package/dist/core/peel-tls.js +220 -0
  161. package/dist/core/pipeline.d.ts +132 -0
  162. package/dist/core/pipeline.js +1666 -0
  163. package/dist/core/profiles.d.ts +61 -0
  164. package/dist/core/profiles.js +350 -0
  165. package/dist/core/prompt-guard.d.ts +30 -0
  166. package/dist/core/prompt-guard.js +119 -0
  167. package/dist/core/proxy-config.d.ts +90 -0
  168. package/dist/core/proxy-config.js +172 -0
  169. package/dist/core/quick-answer.d.ts +53 -0
  170. package/dist/core/quick-answer.js +833 -0
  171. package/dist/core/rate-governor.d.ts +80 -0
  172. package/dist/core/rate-governor.js +238 -0
  173. package/dist/core/readability.d.ts +57 -0
  174. package/dist/core/readability.js +533 -0
  175. package/dist/core/research.d.ts +66 -0
  176. package/dist/core/research.js +270 -0
  177. package/dist/core/retry.d.ts +60 -0
  178. package/dist/core/retry.js +119 -0
  179. package/dist/core/safe-browsing.d.ts +30 -0
  180. package/dist/core/safe-browsing.js +206 -0
  181. package/dist/core/schema-extraction.d.ts +66 -0
  182. package/dist/core/schema-extraction.js +352 -0
  183. package/dist/core/schema-postprocess.d.ts +32 -0
  184. package/dist/core/schema-postprocess.js +469 -0
  185. package/dist/core/schema-templates.d.ts +19 -0
  186. package/dist/core/schema-templates.js +143 -0
  187. package/dist/core/screenshot.d.ts +224 -0
  188. package/dist/core/screenshot.js +207 -0
  189. package/dist/core/search-engines.d.ts +25 -0
  190. package/dist/core/search-engines.js +182 -0
  191. package/dist/core/search-provider.d.ts +243 -0
  192. package/dist/core/search-provider.js +1629 -0
  193. package/dist/core/searxng-provider.d.ts +35 -0
  194. package/dist/core/searxng-provider.js +105 -0
  195. package/dist/core/selective-evidence.d.ts +151 -0
  196. package/dist/core/selective-evidence.js +389 -0
  197. package/dist/core/site-search.d.ts +44 -0
  198. package/dist/core/site-search.js +252 -0
  199. package/dist/core/sitemap.d.ts +23 -0
  200. package/dist/core/sitemap.js +105 -0
  201. package/dist/core/source-credibility.d.ts +29 -0
  202. package/dist/core/source-credibility.js +584 -0
  203. package/dist/core/source-scoring.d.ts +166 -0
  204. package/dist/core/source-scoring.js +396 -0
  205. package/dist/core/stemmer.d.ts +38 -0
  206. package/dist/core/stemmer.js +509 -0
  207. package/dist/core/strategies.d.ts +104 -0
  208. package/dist/core/strategies.js +1044 -0
  209. package/dist/core/strategy-hooks.d.ts +145 -0
  210. package/dist/core/strategy-hooks.js +74 -0
  211. package/dist/core/structured-extract.d.ts +43 -0
  212. package/dist/core/structured-extract.js +550 -0
  213. package/dist/core/summarize.d.ts +17 -0
  214. package/dist/core/summarize.js +78 -0
  215. package/dist/core/synonyms.d.ts +42 -0
  216. package/dist/core/synonyms.js +184 -0
  217. package/dist/core/system-monitor.d.ts +61 -0
  218. package/dist/core/system-monitor.js +133 -0
  219. package/dist/core/table-format.d.ts +30 -0
  220. package/dist/core/table-format.js +146 -0
  221. package/dist/core/threat-feeds.d.ts +23 -0
  222. package/dist/core/threat-feeds.js +104 -0
  223. package/dist/core/timing.d.ts +21 -0
  224. package/dist/core/timing.js +33 -0
  225. package/dist/core/transcript-export.d.ts +47 -0
  226. package/dist/core/transcript-export.js +107 -0
  227. package/dist/core/user-agents.d.ts +82 -0
  228. package/dist/core/user-agents.js +239 -0
  229. package/dist/core/vertical-search.d.ts +54 -0
  230. package/dist/core/vertical-search.js +158 -0
  231. package/dist/core/watch-manager.d.ts +175 -0
  232. package/dist/core/watch-manager.js +416 -0
  233. package/dist/core/watch.d.ts +101 -0
  234. package/dist/core/watch.js +389 -0
  235. package/dist/core/youtube.d.ts +130 -0
  236. package/dist/core/youtube.js +1175 -0
  237. package/dist/ee/challenge-re-export.d.ts +1 -0
  238. package/dist/ee/challenge-re-export.js +1 -0
  239. package/dist/ee/challenge-solver.d.ts +72 -0
  240. package/dist/ee/challenge-solver.js +720 -0
  241. package/dist/ee/domain-extractors.d.ts +8 -0
  242. package/dist/ee/domain-extractors.js +8 -0
  243. package/dist/ee/domain-intel.d.ts +16 -0
  244. package/dist/ee/domain-intel.js +133 -0
  245. package/dist/ee/extractors/allrecipes.d.ts +2 -0
  246. package/dist/ee/extractors/allrecipes.js +120 -0
  247. package/dist/ee/extractors/amazon.d.ts +2 -0
  248. package/dist/ee/extractors/amazon.js +78 -0
  249. package/dist/ee/extractors/arxiv.d.ts +2 -0
  250. package/dist/ee/extractors/arxiv.js +137 -0
  251. package/dist/ee/extractors/bestbuy.d.ts +2 -0
  252. package/dist/ee/extractors/bestbuy.js +78 -0
  253. package/dist/ee/extractors/carscom.d.ts +2 -0
  254. package/dist/ee/extractors/carscom.js +121 -0
  255. package/dist/ee/extractors/coingecko.d.ts +2 -0
  256. package/dist/ee/extractors/coingecko.js +134 -0
  257. package/dist/ee/extractors/craigslist.d.ts +2 -0
  258. package/dist/ee/extractors/craigslist.js +92 -0
  259. package/dist/ee/extractors/devto.d.ts +2 -0
  260. package/dist/ee/extractors/devto.js +135 -0
  261. package/dist/ee/extractors/ebay.d.ts +2 -0
  262. package/dist/ee/extractors/ebay.js +90 -0
  263. package/dist/ee/extractors/espn.d.ts +2 -0
  264. package/dist/ee/extractors/espn.js +260 -0
  265. package/dist/ee/extractors/etsy.d.ts +2 -0
  266. package/dist/ee/extractors/etsy.js +52 -0
  267. package/dist/ee/extractors/facebook.d.ts +2 -0
  268. package/dist/ee/extractors/facebook.js +46 -0
  269. package/dist/ee/extractors/github.d.ts +2 -0
  270. package/dist/ee/extractors/github.js +196 -0
  271. package/dist/ee/extractors/google-flights.d.ts +2 -0
  272. package/dist/ee/extractors/google-flights.js +176 -0
  273. package/dist/ee/extractors/hackernews.d.ts +2 -0
  274. package/dist/ee/extractors/hackernews.js +147 -0
  275. package/dist/ee/extractors/imdb.d.ts +2 -0
  276. package/dist/ee/extractors/imdb.js +172 -0
  277. package/dist/ee/extractors/index.d.ts +26 -0
  278. package/dist/ee/extractors/index.js +247 -0
  279. package/dist/ee/extractors/instagram.d.ts +2 -0
  280. package/dist/ee/extractors/instagram.js +102 -0
  281. package/dist/ee/extractors/kalshi.d.ts +2 -0
  282. package/dist/ee/extractors/kalshi.js +121 -0
  283. package/dist/ee/extractors/kayak-cars.d.ts +2 -0
  284. package/dist/ee/extractors/kayak-cars.js +270 -0
  285. package/dist/ee/extractors/linkedin.d.ts +2 -0
  286. package/dist/ee/extractors/linkedin.js +113 -0
  287. package/dist/ee/extractors/medium.d.ts +2 -0
  288. package/dist/ee/extractors/medium.js +130 -0
  289. package/dist/ee/extractors/news.d.ts +4 -0
  290. package/dist/ee/extractors/news.js +173 -0
  291. package/dist/ee/extractors/npm.d.ts +2 -0
  292. package/dist/ee/extractors/npm.js +86 -0
  293. package/dist/ee/extractors/pdf.d.ts +2 -0
  294. package/dist/ee/extractors/pdf.js +108 -0
  295. package/dist/ee/extractors/pinterest.d.ts +2 -0
  296. package/dist/ee/extractors/pinterest.js +34 -0
  297. package/dist/ee/extractors/polymarket.d.ts +2 -0
  298. package/dist/ee/extractors/polymarket.js +358 -0
  299. package/dist/ee/extractors/producthunt.d.ts +2 -0
  300. package/dist/ee/extractors/producthunt.js +88 -0
  301. package/dist/ee/extractors/pubmed.d.ts +2 -0
  302. package/dist/ee/extractors/pubmed.js +162 -0
  303. package/dist/ee/extractors/pypi.d.ts +2 -0
  304. package/dist/ee/extractors/pypi.js +80 -0
  305. package/dist/ee/extractors/reddit.d.ts +2 -0
  306. package/dist/ee/extractors/reddit.js +438 -0
  307. package/dist/ee/extractors/redfin.d.ts +2 -0
  308. package/dist/ee/extractors/redfin.js +156 -0
  309. package/dist/ee/extractors/semanticscholar.d.ts +2 -0
  310. package/dist/ee/extractors/semanticscholar.js +131 -0
  311. package/dist/ee/extractors/shared.d.ts +12 -0
  312. package/dist/ee/extractors/shared.js +76 -0
  313. package/dist/ee/extractors/soundcloud.d.ts +2 -0
  314. package/dist/ee/extractors/soundcloud.js +34 -0
  315. package/dist/ee/extractors/sportsbetting.d.ts +2 -0
  316. package/dist/ee/extractors/sportsbetting.js +37 -0
  317. package/dist/ee/extractors/spotify.d.ts +2 -0
  318. package/dist/ee/extractors/spotify.js +34 -0
  319. package/dist/ee/extractors/stackoverflow.d.ts +2 -0
  320. package/dist/ee/extractors/stackoverflow.js +61 -0
  321. package/dist/ee/extractors/substack.d.ts +2 -0
  322. package/dist/ee/extractors/substack.js +115 -0
  323. package/dist/ee/extractors/substackroot.d.ts +2 -0
  324. package/dist/ee/extractors/substackroot.js +46 -0
  325. package/dist/ee/extractors/tiktok.d.ts +2 -0
  326. package/dist/ee/extractors/tiktok.js +29 -0
  327. package/dist/ee/extractors/tradingview.d.ts +2 -0
  328. package/dist/ee/extractors/tradingview.js +182 -0
  329. package/dist/ee/extractors/twitch.d.ts +2 -0
  330. package/dist/ee/extractors/twitch.js +36 -0
  331. package/dist/ee/extractors/twitter.d.ts +2 -0
  332. package/dist/ee/extractors/twitter.js +327 -0
  333. package/dist/ee/extractors/types.d.ts +14 -0
  334. package/dist/ee/extractors/types.js +1 -0
  335. package/dist/ee/extractors/walmart.d.ts +2 -0
  336. package/dist/ee/extractors/walmart.js +50 -0
  337. package/dist/ee/extractors/weather.d.ts +2 -0
  338. package/dist/ee/extractors/weather.js +133 -0
  339. package/dist/ee/extractors/wikipedia.d.ts +4 -0
  340. package/dist/ee/extractors/wikipedia.js +235 -0
  341. package/dist/ee/extractors/yelp.d.ts +2 -0
  342. package/dist/ee/extractors/yelp.js +216 -0
  343. package/dist/ee/extractors/youtube.d.ts +2 -0
  344. package/dist/ee/extractors/youtube.js +189 -0
  345. package/dist/ee/extractors/zillow.d.ts +54 -0
  346. package/dist/ee/extractors/zillow.js +247 -0
  347. package/dist/ee/extractors-re-export.d.ts +1 -0
  348. package/dist/ee/extractors-re-export.js +1 -0
  349. package/dist/ee/premium-hooks.d.ts +20 -0
  350. package/dist/ee/premium-hooks.js +50 -0
  351. package/dist/ee/spa-detection.d.ts +2 -0
  352. package/dist/ee/spa-detection.js +2 -0
  353. package/dist/ee/stability.d.ts +4 -0
  354. package/dist/ee/stability.js +29 -0
  355. package/dist/ee/swr-cache.d.ts +14 -0
  356. package/dist/ee/swr-cache.js +34 -0
  357. package/dist/index.d.ts +143 -0
  358. package/dist/index.js +291 -0
  359. package/dist/integrations/index.d.ts +2 -0
  360. package/dist/integrations/index.js +2 -0
  361. package/dist/integrations/langchain.d.ts +64 -0
  362. package/dist/integrations/langchain.js +115 -0
  363. package/dist/integrations/llamaindex.d.ts +50 -0
  364. package/dist/integrations/llamaindex.js +91 -0
  365. package/dist/mcp/handlers/act.d.ts +5 -0
  366. package/dist/mcp/handlers/act.js +34 -0
  367. package/dist/mcp/handlers/definitions.d.ts +6 -0
  368. package/dist/mcp/handlers/definitions.js +395 -0
  369. package/dist/mcp/handlers/extract.d.ts +7 -0
  370. package/dist/mcp/handlers/extract.js +135 -0
  371. package/dist/mcp/handlers/fetch.d.ts +6 -0
  372. package/dist/mcp/handlers/fetch.js +98 -0
  373. package/dist/mcp/handlers/find.d.ts +5 -0
  374. package/dist/mcp/handlers/find.js +137 -0
  375. package/dist/mcp/handlers/index.d.ts +13 -0
  376. package/dist/mcp/handlers/index.js +63 -0
  377. package/dist/mcp/handlers/legacy.d.ts +25 -0
  378. package/dist/mcp/handlers/legacy.js +450 -0
  379. package/dist/mcp/handlers/meta.d.ts +6 -0
  380. package/dist/mcp/handlers/meta.js +40 -0
  381. package/dist/mcp/handlers/monitor.d.ts +5 -0
  382. package/dist/mcp/handlers/monitor.js +41 -0
  383. package/dist/mcp/handlers/observe.d.ts +8 -0
  384. package/dist/mcp/handlers/observe.js +37 -0
  385. package/dist/mcp/handlers/read.d.ts +6 -0
  386. package/dist/mcp/handlers/read.js +78 -0
  387. package/dist/mcp/handlers/see.d.ts +5 -0
  388. package/dist/mcp/handlers/see.js +75 -0
  389. package/dist/mcp/handlers/types.d.ts +29 -0
  390. package/dist/mcp/handlers/types.js +28 -0
  391. package/dist/mcp/server.d.ts +7 -0
  392. package/dist/mcp/server.js +108 -0
  393. package/dist/mcp/smart-router.d.ts +23 -0
  394. package/dist/mcp/smart-router.js +178 -0
  395. package/dist/server/app.d.ts +14 -0
  396. package/dist/server/app.js +632 -0
  397. package/dist/server/auth-store.d.ts +28 -0
  398. package/dist/server/auth-store.js +88 -0
  399. package/dist/server/bull-queues.d.ts +60 -0
  400. package/dist/server/bull-queues.js +90 -0
  401. package/dist/server/email-service.d.ts +55 -0
  402. package/dist/server/email-service.js +291 -0
  403. package/dist/server/job-queue.d.ts +100 -0
  404. package/dist/server/job-queue.js +145 -0
  405. package/dist/server/logger.d.ts +10 -0
  406. package/dist/server/logger.js +37 -0
  407. package/dist/server/middleware/audit-log.d.ts +14 -0
  408. package/dist/server/middleware/audit-log.js +73 -0
  409. package/dist/server/middleware/auth.d.ts +35 -0
  410. package/dist/server/middleware/auth.js +225 -0
  411. package/dist/server/middleware/rate-limit.d.ts +50 -0
  412. package/dist/server/middleware/rate-limit.js +270 -0
  413. package/dist/server/middleware/scope-guard.d.ts +25 -0
  414. package/dist/server/middleware/scope-guard.js +45 -0
  415. package/dist/server/middleware/url-validator.d.ts +15 -0
  416. package/dist/server/middleware/url-validator.js +201 -0
  417. package/dist/server/openapi.yaml +6418 -0
  418. package/dist/server/pg-auth-store.d.ts +146 -0
  419. package/dist/server/pg-auth-store.js +576 -0
  420. package/dist/server/pg-job-queue.d.ts +59 -0
  421. package/dist/server/pg-job-queue.js +375 -0
  422. package/dist/server/routes/activity.d.ts +6 -0
  423. package/dist/server/routes/activity.js +79 -0
  424. package/dist/server/routes/admin-active.d.ts +7 -0
  425. package/dist/server/routes/admin-active.js +120 -0
  426. package/dist/server/routes/admin-stats.d.ts +7 -0
  427. package/dist/server/routes/admin-stats.js +176 -0
  428. package/dist/server/routes/agent.d.ts +24 -0
  429. package/dist/server/routes/agent.js +480 -0
  430. package/dist/server/routes/answer.d.ts +5 -0
  431. package/dist/server/routes/answer.js +125 -0
  432. package/dist/server/routes/ask.d.ts +28 -0
  433. package/dist/server/routes/ask.js +295 -0
  434. package/dist/server/routes/batch.d.ts +6 -0
  435. package/dist/server/routes/batch.js +493 -0
  436. package/dist/server/routes/cache-warm.d.ts +25 -0
  437. package/dist/server/routes/cache-warm.js +212 -0
  438. package/dist/server/routes/cli-usage.d.ts +6 -0
  439. package/dist/server/routes/cli-usage.js +127 -0
  440. package/dist/server/routes/compat.d.ts +23 -0
  441. package/dist/server/routes/compat.js +652 -0
  442. package/dist/server/routes/crawl.d.ts +13 -0
  443. package/dist/server/routes/crawl.js +287 -0
  444. package/dist/server/routes/deep-fetch.d.ts +8 -0
  445. package/dist/server/routes/deep-fetch.js +57 -0
  446. package/dist/server/routes/deep-research.d.ts +11 -0
  447. package/dist/server/routes/deep-research.js +232 -0
  448. package/dist/server/routes/demo.d.ts +24 -0
  449. package/dist/server/routes/demo.js +517 -0
  450. package/dist/server/routes/do.d.ts +8 -0
  451. package/dist/server/routes/do.js +72 -0
  452. package/dist/server/routes/extract.d.ts +14 -0
  453. package/dist/server/routes/extract.js +325 -0
  454. package/dist/server/routes/feed.d.ts +15 -0
  455. package/dist/server/routes/feed.js +311 -0
  456. package/dist/server/routes/fetch-queue.d.ts +13 -0
  457. package/dist/server/routes/fetch-queue.js +357 -0
  458. package/dist/server/routes/fetch.d.ts +7 -0
  459. package/dist/server/routes/fetch.js +1274 -0
  460. package/dist/server/routes/go.d.ts +14 -0
  461. package/dist/server/routes/go.js +81 -0
  462. package/dist/server/routes/health.d.ts +11 -0
  463. package/dist/server/routes/health.js +141 -0
  464. package/dist/server/routes/jobs.d.ts +7 -0
  465. package/dist/server/routes/jobs.js +574 -0
  466. package/dist/server/routes/map.d.ts +11 -0
  467. package/dist/server/routes/map.js +116 -0
  468. package/dist/server/routes/mcp.d.ts +14 -0
  469. package/dist/server/routes/mcp.js +197 -0
  470. package/dist/server/routes/metrics.d.ts +37 -0
  471. package/dist/server/routes/metrics.js +149 -0
  472. package/dist/server/routes/oauth.d.ts +9 -0
  473. package/dist/server/routes/oauth.js +396 -0
  474. package/dist/server/routes/playground.d.ts +17 -0
  475. package/dist/server/routes/playground.js +283 -0
  476. package/dist/server/routes/reader.d.ts +18 -0
  477. package/dist/server/routes/reader.js +192 -0
  478. package/dist/server/routes/research.d.ts +14 -0
  479. package/dist/server/routes/research.js +482 -0
  480. package/dist/server/routes/screenshot.d.ts +22 -0
  481. package/dist/server/routes/screenshot.js +820 -0
  482. package/dist/server/routes/search.d.ts +6 -0
  483. package/dist/server/routes/search.js +874 -0
  484. package/dist/server/routes/session.d.ts +17 -0
  485. package/dist/server/routes/session.js +548 -0
  486. package/dist/server/routes/share.d.ts +18 -0
  487. package/dist/server/routes/share.js +462 -0
  488. package/dist/server/routes/smart-search/handlers/cars.d.ts +2 -0
  489. package/dist/server/routes/smart-search/handlers/cars.js +102 -0
  490. package/dist/server/routes/smart-search/handlers/flights.d.ts +2 -0
  491. package/dist/server/routes/smart-search/handlers/flights.js +72 -0
  492. package/dist/server/routes/smart-search/handlers/general.d.ts +13 -0
  493. package/dist/server/routes/smart-search/handlers/general.js +717 -0
  494. package/dist/server/routes/smart-search/handlers/hotels.d.ts +2 -0
  495. package/dist/server/routes/smart-search/handlers/hotels.js +88 -0
  496. package/dist/server/routes/smart-search/handlers/products.d.ts +2 -0
  497. package/dist/server/routes/smart-search/handlers/products.js +1309 -0
  498. package/dist/server/routes/smart-search/handlers/rental.d.ts +2 -0
  499. package/dist/server/routes/smart-search/handlers/rental.js +154 -0
  500. package/dist/server/routes/smart-search/handlers/restaurants.d.ts +2 -0
  501. package/dist/server/routes/smart-search/handlers/restaurants.js +225 -0
  502. package/dist/server/routes/smart-search/handlers/transit-verdict.d.ts +41 -0
  503. package/dist/server/routes/smart-search/handlers/transit-verdict.js +224 -0
  504. package/dist/server/routes/smart-search/index.d.ts +19 -0
  505. package/dist/server/routes/smart-search/index.js +546 -0
  506. package/dist/server/routes/smart-search/intent.d.ts +3 -0
  507. package/dist/server/routes/smart-search/intent.js +264 -0
  508. package/dist/server/routes/smart-search/llm.d.ts +16 -0
  509. package/dist/server/routes/smart-search/llm.js +70 -0
  510. package/dist/server/routes/smart-search/sources/reddit.d.ts +18 -0
  511. package/dist/server/routes/smart-search/sources/reddit.js +34 -0
  512. package/dist/server/routes/smart-search/sources/yelp.d.ts +25 -0
  513. package/dist/server/routes/smart-search/sources/yelp.js +171 -0
  514. package/dist/server/routes/smart-search/sources/youtube.d.ts +8 -0
  515. package/dist/server/routes/smart-search/sources/youtube.js +9 -0
  516. package/dist/server/routes/smart-search/types.d.ts +81 -0
  517. package/dist/server/routes/smart-search/types.js +1 -0
  518. package/dist/server/routes/smart-search/utils.d.ts +20 -0
  519. package/dist/server/routes/smart-search/utils.js +146 -0
  520. package/dist/server/routes/stats.d.ts +6 -0
  521. package/dist/server/routes/stats.js +71 -0
  522. package/dist/server/routes/stripe.d.ts +15 -0
  523. package/dist/server/routes/stripe.js +296 -0
  524. package/dist/server/routes/transcript-export.d.ts +10 -0
  525. package/dist/server/routes/transcript-export.js +178 -0
  526. package/dist/server/routes/usage.d.ts +9 -0
  527. package/dist/server/routes/usage.js +279 -0
  528. package/dist/server/routes/users.d.ts +8 -0
  529. package/dist/server/routes/users.js +1867 -0
  530. package/dist/server/routes/watch.d.ts +15 -0
  531. package/dist/server/routes/watch.js +309 -0
  532. package/dist/server/routes/webhooks.d.ts +26 -0
  533. package/dist/server/routes/webhooks.js +170 -0
  534. package/dist/server/routes/youtube.d.ts +6 -0
  535. package/dist/server/routes/youtube.js +130 -0
  536. package/dist/server/sentry.d.ts +14 -0
  537. package/dist/server/sentry.js +104 -0
  538. package/dist/server/types.d.ts +15 -0
  539. package/dist/server/types.js +7 -0
  540. package/dist/server/utils/response.d.ts +44 -0
  541. package/dist/server/utils/response.js +69 -0
  542. package/dist/server/utils/sse.d.ts +22 -0
  543. package/dist/server/utils/sse.js +38 -0
  544. package/dist/types.d.ts +552 -0
  545. package/dist/types.js +39 -0
  546. package/llms.txt +105 -0
  547. package/package.json +189 -0
@@ -0,0 +1,264 @@
1
+ /**
2
+ * Design Compare — Structural diff between two pages' design tokens.
3
+ *
4
+ * Compares design quality of a subject URL against a reference URL,
5
+ * returning a structured gap list with severity ratings and CSS suggestions.
6
+ */
7
+ import { extractDesignAnalysis } from './design-analysis.js';
8
+ // ── Constants ──────────────────────────────────────────────────────────────────
9
+ const SEVERITY_DEDUCTIONS = {
10
+ high: 1.5,
11
+ medium: 0.8,
12
+ low: 0.3,
13
+ };
14
+ // ── Pure comparison logic ──────────────────────────────────────────────────────
15
+ /**
16
+ * Build a DesignComparison from two pre-extracted DesignAnalysis objects.
17
+ *
18
+ * This is a pure synchronous function — straightforward to test and reuse
19
+ * from both the HTTP route and the CLI command.
20
+ */
21
+ export function buildDesignComparison(subjectUrl, referenceUrl, subjectAnalysis, referenceAnalysis) {
22
+ const gaps = [];
23
+ // ── 1. Color scheme ────────────────────────────────────────────────────────
24
+ if (subjectAnalysis.palette.scheme !== referenceAnalysis.palette.scheme) {
25
+ gaps.push({
26
+ property: 'palette.scheme',
27
+ description: 'Color scheme differs',
28
+ subject: subjectAnalysis.palette.scheme,
29
+ reference: referenceAnalysis.palette.scheme,
30
+ severity: 'medium',
31
+ suggestion: `Switch to a ${referenceAnalysis.palette.scheme} color scheme. Update background and text colors accordingly.`,
32
+ });
33
+ }
34
+ // ── 2. Primary background color ────────────────────────────────────────────
35
+ const subjectBg = subjectAnalysis.palette.dominant[0] ?? '';
36
+ const referenceBg = referenceAnalysis.palette.dominant[0] ?? '';
37
+ if (subjectBg && referenceBg && subjectBg.toLowerCase() !== referenceBg.toLowerCase()) {
38
+ gaps.push({
39
+ property: 'palette.dominant[0]',
40
+ description: 'Primary background color differs',
41
+ subject: subjectBg,
42
+ reference: referenceBg,
43
+ severity: 'low',
44
+ suggestion: `Consider using ${referenceBg} as the primary background color.`,
45
+ });
46
+ }
47
+ // ── 3. Layout system ───────────────────────────────────────────────────────
48
+ if (subjectAnalysis.layout.gridSystem !== referenceAnalysis.layout.gridSystem) {
49
+ gaps.push({
50
+ property: 'layout.gridSystem',
51
+ description: 'Layout system differs',
52
+ subject: subjectAnalysis.layout.gridSystem,
53
+ reference: referenceAnalysis.layout.gridSystem,
54
+ severity: 'medium',
55
+ suggestion: `Migrate to CSS ${referenceAnalysis.layout.gridSystem} to match the reference layout system.`,
56
+ });
57
+ }
58
+ // ── 4. Container max-width ─────────────────────────────────────────────────
59
+ if (subjectAnalysis.layout.maxWidth !== 'none' &&
60
+ referenceAnalysis.layout.maxWidth !== 'none' &&
61
+ subjectAnalysis.layout.maxWidth !== referenceAnalysis.layout.maxWidth) {
62
+ gaps.push({
63
+ property: 'layout.maxWidth',
64
+ description: 'Container max-width differs',
65
+ subject: subjectAnalysis.layout.maxWidth,
66
+ reference: referenceAnalysis.layout.maxWidth,
67
+ severity: 'low',
68
+ suggestion: `Set max-width to ${referenceAnalysis.layout.maxWidth} to match the reference layout.`,
69
+ });
70
+ }
71
+ // ── 5. Base font size ──────────────────────────────────────────────────────
72
+ const subjectBaseSize = parseFloat(subjectAnalysis.typeScale.baseSize);
73
+ const referenceBaseSize = parseFloat(referenceAnalysis.typeScale.baseSize);
74
+ if (!isNaN(subjectBaseSize) &&
75
+ !isNaN(referenceBaseSize) &&
76
+ Math.abs(subjectBaseSize - referenceBaseSize) >= 2) {
77
+ gaps.push({
78
+ property: 'typeScale.baseSize',
79
+ description: 'Base font size differs significantly',
80
+ subject: subjectAnalysis.typeScale.baseSize,
81
+ reference: referenceAnalysis.typeScale.baseSize,
82
+ severity: 'medium',
83
+ suggestion: `Set the root font-size to ${referenceAnalysis.typeScale.baseSize} (e.g., html { font-size: ${referenceAnalysis.typeScale.baseSize}; }).`,
84
+ });
85
+ }
86
+ // ── 6. Heading font family ─────────────────────────────────────────────────
87
+ const subjectHeadingFamily = subjectAnalysis.typeScale.headingStyle.family.toLowerCase();
88
+ const referenceHeadingFamily = referenceAnalysis.typeScale.headingStyle.family.toLowerCase();
89
+ if (subjectHeadingFamily && referenceHeadingFamily && subjectHeadingFamily !== referenceHeadingFamily) {
90
+ gaps.push({
91
+ property: 'typeScale.headingStyle.family',
92
+ description: 'Heading font family differs',
93
+ subject: subjectAnalysis.typeScale.headingStyle.family,
94
+ reference: referenceAnalysis.typeScale.headingStyle.family,
95
+ severity: 'high',
96
+ suggestion: `Use "${referenceAnalysis.typeScale.headingStyle.family}" as the heading font family. Add it via Google Fonts or your font provider.`,
97
+ });
98
+ }
99
+ // ── 7. Body font family ────────────────────────────────────────────────────
100
+ const subjectBodyFamily = subjectAnalysis.typeScale.bodyStyle.family.toLowerCase();
101
+ const referenceBodyFamily = referenceAnalysis.typeScale.bodyStyle.family.toLowerCase();
102
+ if (subjectBodyFamily && referenceBodyFamily && subjectBodyFamily !== referenceBodyFamily) {
103
+ gaps.push({
104
+ property: 'typeScale.bodyStyle.family',
105
+ description: 'Body font family differs',
106
+ subject: subjectAnalysis.typeScale.bodyStyle.family,
107
+ reference: referenceAnalysis.typeScale.bodyStyle.family,
108
+ severity: 'high',
109
+ suggestion: `Set body { font-family: "${referenceAnalysis.typeScale.bodyStyle.family}", sans-serif; }.`,
110
+ });
111
+ }
112
+ // ── 8. Body font weight ────────────────────────────────────────────────────
113
+ if (subjectAnalysis.typeScale.bodyStyle.weight !== referenceAnalysis.typeScale.bodyStyle.weight) {
114
+ gaps.push({
115
+ property: 'typeScale.bodyStyle.weight',
116
+ description: 'Body font weight differs',
117
+ subject: String(subjectAnalysis.typeScale.bodyStyle.weight),
118
+ reference: String(referenceAnalysis.typeScale.bodyStyle.weight),
119
+ severity: 'low',
120
+ suggestion: `Set body { font-weight: ${referenceAnalysis.typeScale.bodyStyle.weight}; }.`,
121
+ });
122
+ }
123
+ // ── 9. Modular type scale ──────────────────────────────────────────────────
124
+ if (!subjectAnalysis.typeScale.isModular && referenceAnalysis.typeScale.isModular) {
125
+ const referenceRatio = referenceAnalysis.typeScale.ratio;
126
+ gaps.push({
127
+ property: 'typeScale.isModular',
128
+ description: 'Reference uses a modular type scale; subject does not',
129
+ subject: 'non-modular',
130
+ reference: referenceRatio !== undefined ? `modular (ratio: ${referenceRatio})` : 'modular',
131
+ severity: 'medium',
132
+ suggestion: referenceRatio !== undefined
133
+ ? `Adopt a modular type scale with ratio ${referenceRatio}. Use a tool like https://type-scale.com to generate sizes.`
134
+ : 'Adopt a modular type scale for consistent typography.',
135
+ });
136
+ }
137
+ // ── 10. Box shadow presence ────────────────────────────────────────────────
138
+ const subjectHasShadows = subjectAnalysis.visualEffects.shadows.length > 0;
139
+ const referenceHasShadows = referenceAnalysis.visualEffects.shadows.length > 0;
140
+ if (!subjectHasShadows && referenceHasShadows) {
141
+ gaps.push({
142
+ property: 'visualEffects.shadows',
143
+ description: 'Reference uses box shadows; subject has none',
144
+ subject: 'no shadows',
145
+ reference: `${referenceAnalysis.visualEffects.shadows.length} shadow(s)`,
146
+ severity: 'low',
147
+ suggestion: 'Add subtle box-shadow to cards and interactive elements (e.g., box-shadow: 0 2px 8px rgba(0,0,0,0.1)).',
148
+ });
149
+ }
150
+ // ── 11. Gradient usage ─────────────────────────────────────────────────────
151
+ const subjectHasGradients = subjectAnalysis.visualEffects.gradients.length > 0;
152
+ const referenceHasGradients = referenceAnalysis.visualEffects.gradients.length > 0;
153
+ if (!subjectHasGradients && referenceHasGradients) {
154
+ gaps.push({
155
+ property: 'visualEffects.gradients',
156
+ description: 'Reference uses gradients; subject has none',
157
+ subject: 'no gradients',
158
+ reference: `${referenceAnalysis.visualEffects.gradients.length} gradient(s)`,
159
+ severity: 'low',
160
+ suggestion: 'Add CSS gradients to hero sections or accent elements to add visual depth.',
161
+ });
162
+ }
163
+ // ── 12. Spacing consistency quality signal ─────────────────────────────────
164
+ const spacingDiff = referenceAnalysis.qualitySignals.spacingConsistency -
165
+ subjectAnalysis.qualitySignals.spacingConsistency;
166
+ if (spacingDiff >= 0.2) {
167
+ gaps.push({
168
+ property: 'qualitySignals.spacingConsistency',
169
+ description: 'Spacing consistency is notably lower than the reference',
170
+ subject: String(subjectAnalysis.qualitySignals.spacingConsistency),
171
+ reference: String(referenceAnalysis.qualitySignals.spacingConsistency),
172
+ severity: spacingDiff >= 0.4 ? 'high' : 'medium',
173
+ suggestion: 'Align margin and padding values to a 4px or 8px grid for consistent spacing.',
174
+ });
175
+ }
176
+ // ── 13. Typography consistency quality signal ──────────────────────────────
177
+ const typoDiff = referenceAnalysis.qualitySignals.typographyConsistency -
178
+ subjectAnalysis.qualitySignals.typographyConsistency;
179
+ if (typoDiff >= 0.2) {
180
+ gaps.push({
181
+ property: 'qualitySignals.typographyConsistency',
182
+ description: 'Typography consistency is notably lower than the reference',
183
+ subject: String(subjectAnalysis.qualitySignals.typographyConsistency),
184
+ reference: String(referenceAnalysis.qualitySignals.typographyConsistency),
185
+ severity: typoDiff >= 0.4 ? 'high' : 'medium',
186
+ suggestion: 'Reduce the number of distinct font sizes and establish a clear type hierarchy.',
187
+ });
188
+ }
189
+ // ── 14. Color harmony quality signal ───────────────────────────────────────
190
+ const colorDiff = referenceAnalysis.qualitySignals.colorHarmony -
191
+ subjectAnalysis.qualitySignals.colorHarmony;
192
+ if (colorDiff >= 0.2) {
193
+ gaps.push({
194
+ property: 'qualitySignals.colorHarmony',
195
+ description: 'Color harmony is notably lower than the reference',
196
+ subject: String(subjectAnalysis.qualitySignals.colorHarmony),
197
+ reference: String(referenceAnalysis.qualitySignals.colorHarmony),
198
+ severity: colorDiff >= 0.4 ? 'high' : 'medium',
199
+ suggestion: 'Reduce the color palette to 3–5 primary colors. Use tints/shades instead of completely different hues.',
200
+ });
201
+ }
202
+ // ── 15. Visual hierarchy quality signal ───────────────────────────────────
203
+ const hierarchyDiff = referenceAnalysis.qualitySignals.visualHierarchy -
204
+ subjectAnalysis.qualitySignals.visualHierarchy;
205
+ if (hierarchyDiff >= 0.2) {
206
+ gaps.push({
207
+ property: 'qualitySignals.visualHierarchy',
208
+ description: 'Visual hierarchy is weaker than the reference',
209
+ subject: String(subjectAnalysis.qualitySignals.visualHierarchy),
210
+ reference: String(referenceAnalysis.qualitySignals.visualHierarchy),
211
+ severity: hierarchyDiff >= 0.4 ? 'high' : 'medium',
212
+ suggestion: 'Increase the h1 font size relative to body text (aim for 2× or more) to strengthen visual hierarchy.',
213
+ });
214
+ }
215
+ // ── Score calculation ──────────────────────────────────────────────────────
216
+ let score = 10;
217
+ for (const gap of gaps) {
218
+ score -= SEVERITY_DEDUCTIONS[gap.severity];
219
+ }
220
+ score = Math.max(1, Math.round(score * 10) / 10);
221
+ // ── Summary ───────────────────────────────────────────────────────────────
222
+ const highCount = gaps.filter((g) => g.severity === 'high').length;
223
+ const mediumCount = gaps.filter((g) => g.severity === 'medium').length;
224
+ const lowCount = gaps.filter((g) => g.severity === 'low').length;
225
+ let summary;
226
+ if (gaps.length === 0) {
227
+ summary = 'Subject closely matches the reference design. No significant gaps detected.';
228
+ }
229
+ else {
230
+ const parts = [];
231
+ if (highCount > 0)
232
+ parts.push(`${highCount} high-severity`);
233
+ if (mediumCount > 0)
234
+ parts.push(`${mediumCount} medium-severity`);
235
+ if (lowCount > 0)
236
+ parts.push(`${lowCount} low-severity`);
237
+ summary = `Found ${gaps.length} design gap${gaps.length === 1 ? '' : 's'} (${parts.join(', ')}) with a design score of ${score}/10.`;
238
+ if (highCount > 0) {
239
+ const highGaps = gaps.filter((g) => g.severity === 'high');
240
+ summary += ` Priority: ${highGaps.map((g) => g.property).join(', ')}.`;
241
+ }
242
+ }
243
+ return {
244
+ subjectUrl,
245
+ referenceUrl,
246
+ score,
247
+ gaps,
248
+ subjectAnalysis,
249
+ referenceAnalysis,
250
+ summary,
251
+ };
252
+ }
253
+ // ── Page-based wrapper ────────────────────────────────────────────────────────
254
+ /**
255
+ * Compare the design of two Playwright pages by extracting design tokens from
256
+ * both in parallel, then diffing the tokens into a structured DesignComparison.
257
+ */
258
+ export async function compareDesigns(subjectPage, referencePage) {
259
+ const [subjectAnalysis, referenceAnalysis] = await Promise.all([
260
+ extractDesignAnalysis(subjectPage),
261
+ extractDesignAnalysis(referencePage),
262
+ ]);
263
+ return buildDesignComparison(subjectPage.url(), referencePage.url(), subjectAnalysis, referenceAnalysis);
264
+ }
@@ -0,0 +1,61 @@
1
+ /**
2
+ * WebPeel Diff - Semantic content diff against stored snapshots
3
+ *
4
+ * Fetches the current content of a URL, loads the previous snapshot from the
5
+ * change-tracking store, computes a structured diff (field-by-field for JSON,
6
+ * section-by-section for text/markdown), saves the new snapshot, and returns
7
+ * a structured {@link DiffResult}.
8
+ */
9
+ export interface DiffOptions {
10
+ /** Compare against the last tracked snapshot (default behaviour). */
11
+ last?: boolean;
12
+ /** Only compare these dot-notation fields (for JSON responses only). */
13
+ fields?: string[];
14
+ /** Use browser rendering for the fetch. */
15
+ render?: boolean;
16
+ /** Per-request timeout in milliseconds (default: 30 000). */
17
+ timeout?: number;
18
+ }
19
+ export interface DiffResult {
20
+ url: string;
21
+ /** Whether any changes were detected. */
22
+ changed: boolean;
23
+ /** ISO-8601 timestamp of the current fetch. */
24
+ timestamp: string;
25
+ /** ISO-8601 timestamp of the previous snapshot (undefined if none). */
26
+ previousTimestamp?: string;
27
+ changes: DiffChange[];
28
+ /** Human-readable summary sentence. */
29
+ summary: string;
30
+ }
31
+ export interface DiffChange {
32
+ type: 'added' | 'removed' | 'modified';
33
+ /** For JSON diffs: dot-notation path to the changed field. */
34
+ field?: string;
35
+ /** For text diffs: nearest section heading or line reference. */
36
+ path?: string;
37
+ before?: string;
38
+ after?: string;
39
+ }
40
+ /**
41
+ * Fetch a URL and compute a semantic diff against the last tracked snapshot.
42
+ *
43
+ * The function:
44
+ * 1. Loads the previous snapshot (if any) from the change-tracking store.
45
+ * 2. Fetches the current content via {@link peel}.
46
+ * 3. Saves the current content as the new snapshot (auto-tracking).
47
+ * 4. Computes a structured diff — field-by-field for JSON, section-by-section
48
+ * for text/markdown.
49
+ * 5. Returns a {@link DiffResult}.
50
+ *
51
+ * @example
52
+ * ```typescript
53
+ * const result = await diffUrl('https://api.example.com/health', { last: true });
54
+ * console.log(result.summary);
55
+ * result.changes.forEach(c => {
56
+ * if (c.field) console.log(`${c.type}: ${c.field} ${c.before} → ${c.after}`);
57
+ * });
58
+ * ```
59
+ */
60
+ export declare function diffUrl(url: string, options?: DiffOptions): Promise<DiffResult>;
61
+ export { trackChange, getSnapshot } from './change-tracking.js';
@@ -0,0 +1,289 @@
1
+ /**
2
+ * WebPeel Diff - Semantic content diff against stored snapshots
3
+ *
4
+ * Fetches the current content of a URL, loads the previous snapshot from the
5
+ * change-tracking store, computes a structured diff (field-by-field for JSON,
6
+ * section-by-section for text/markdown), saves the new snapshot, and returns
7
+ * a structured {@link DiffResult}.
8
+ */
9
+ import { peel } from '../index.js';
10
+ import { getSnapshot } from './change-tracking.js';
11
+ // ─── JSON diffing ──────────────────────────────────────────────────────────────
12
+ /**
13
+ * Compare two JSON values recursively, returning structured {@link DiffChange}
14
+ * objects. Non-object values (numbers, strings, arrays) are reported as atomic
15
+ * modifications.
16
+ *
17
+ * @param before - Previous JSON value
18
+ * @param after - Current JSON value
19
+ * @param path - Current dot-notation path (for recursion; start with "")
20
+ * @param fields - Optional allowlist of dot-notation paths to compare
21
+ */
22
+ function diffJson(before, after, path = '', fields) {
23
+ const changes = [];
24
+ const isPlainObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
25
+ if (!isPlainObject(before) || !isPlainObject(after)) {
26
+ // Atomic comparison.
27
+ if (JSON.stringify(before) !== JSON.stringify(after)) {
28
+ if (!fields || fields.length === 0 || fields.some(f => f === path || path.startsWith(f + '.'))) {
29
+ changes.push({
30
+ type: 'modified',
31
+ field: path || '(root)',
32
+ before: stringify(before),
33
+ after: stringify(after),
34
+ });
35
+ }
36
+ }
37
+ return changes;
38
+ }
39
+ const allKeys = new Set([...Object.keys(before), ...Object.keys(after)]);
40
+ for (const key of allKeys) {
41
+ const fullPath = path ? `${path}.${key}` : key;
42
+ // Apply field filter when specified.
43
+ if (fields && fields.length > 0) {
44
+ const inScope = fields.some(f => fullPath === f || fullPath.startsWith(f + '.') || f.startsWith(fullPath + '.'));
45
+ if (!inScope)
46
+ continue;
47
+ }
48
+ const bVal = before[key];
49
+ const aVal = after[key];
50
+ if (bVal === undefined && aVal !== undefined) {
51
+ changes.push({ type: 'added', field: fullPath, after: stringify(aVal) });
52
+ }
53
+ else if (bVal !== undefined && aVal === undefined) {
54
+ changes.push({ type: 'removed', field: fullPath, before: stringify(bVal) });
55
+ }
56
+ else if (JSON.stringify(bVal) !== JSON.stringify(aVal)) {
57
+ // Recurse into nested objects.
58
+ if (isPlainObject(bVal) && isPlainObject(aVal)) {
59
+ changes.push(...diffJson(bVal, aVal, fullPath, fields));
60
+ }
61
+ else {
62
+ changes.push({
63
+ type: 'modified',
64
+ field: fullPath,
65
+ before: stringify(bVal),
66
+ after: stringify(aVal),
67
+ });
68
+ }
69
+ }
70
+ }
71
+ return changes;
72
+ }
73
+ /** Serialize a JSON value concisely for display. */
74
+ function stringify(v) {
75
+ if (typeof v === 'string')
76
+ return v;
77
+ return JSON.stringify(v) ?? 'undefined';
78
+ }
79
+ /**
80
+ * Diff two text blobs line-by-line using LCS (Longest Common Subsequence).
81
+ * Groups consecutive additions/deletions into sections keyed by the nearest
82
+ * Markdown heading.
83
+ */
84
+ function diffText(beforeText, afterText) {
85
+ const MAX_LINES = 5_000;
86
+ const bLines = beforeText.split('\n').slice(0, MAX_LINES);
87
+ const aLines = afterText.split('\n').slice(0, MAX_LINES);
88
+ // Build LCS table.
89
+ const m = bLines.length;
90
+ const n = aLines.length;
91
+ const lcs = Array.from({ length: m + 1 }, () => new Array(n + 1).fill(0));
92
+ for (let i = 1; i <= m; i++) {
93
+ for (let j = 1; j <= n; j++) {
94
+ if (bLines[i - 1] === aLines[j - 1]) {
95
+ lcs[i][j] = lcs[i - 1][j - 1] + 1;
96
+ }
97
+ else {
98
+ lcs[i][j] = Math.max(lcs[i - 1][j], lcs[i][j - 1]);
99
+ }
100
+ }
101
+ }
102
+ const ops = [];
103
+ let bi = m;
104
+ let ai = n;
105
+ while (bi > 0 || ai > 0) {
106
+ if (bi > 0 && ai > 0 && bLines[bi - 1] === aLines[ai - 1]) {
107
+ ops.unshift({ op: 'same', line: aLines[ai - 1] });
108
+ bi--;
109
+ ai--;
110
+ }
111
+ else if (ai > 0 && (bi === 0 || lcs[bi][ai - 1] >= lcs[bi - 1][ai])) {
112
+ ops.unshift({ op: 'add', line: aLines[ai - 1] });
113
+ ai--;
114
+ }
115
+ else {
116
+ ops.unshift({ op: 'del', line: bLines[bi - 1] });
117
+ bi--;
118
+ }
119
+ }
120
+ // Group consecutive non-same ops into sections.
121
+ const changes = [];
122
+ let linesAdded = 0;
123
+ let linesRemoved = 0;
124
+ let i = 0;
125
+ while (i < ops.length) {
126
+ if (ops[i].op === 'same') {
127
+ i++;
128
+ continue;
129
+ }
130
+ // Collect the run of changes.
131
+ const added = [];
132
+ const removed = [];
133
+ while (i < ops.length && ops[i].op !== 'same') {
134
+ if (ops[i].op === 'add')
135
+ added.push(ops[i].line);
136
+ if (ops[i].op === 'del')
137
+ removed.push(ops[i].line);
138
+ i++;
139
+ }
140
+ linesAdded += added.length;
141
+ linesRemoved += removed.length;
142
+ // Find nearest Markdown heading in the 'same' lines before this block.
143
+ let sectionLabel = `line ~${i}`;
144
+ for (let k = i - added.length - removed.length - 1; k >= 0; k--) {
145
+ const prevOp = ops[k];
146
+ if (prevOp && prevOp.op === 'same' && /^#{1,6}\s/.test(prevOp.line)) {
147
+ sectionLabel = prevOp.line.trim();
148
+ break;
149
+ }
150
+ }
151
+ if (removed.length > 0 && added.length > 0) {
152
+ changes.push({ type: 'modified', path: sectionLabel, before: removed.join('\n'), after: added.join('\n') });
153
+ }
154
+ else if (added.length > 0) {
155
+ changes.push({ type: 'added', path: sectionLabel, after: added.join('\n') });
156
+ }
157
+ else if (removed.length > 0) {
158
+ changes.push({ type: 'removed', path: sectionLabel, before: removed.join('\n') });
159
+ }
160
+ }
161
+ return { changes, stats: { linesAdded, linesRemoved } };
162
+ }
163
+ // ─── Summary generation ────────────────────────────────────────────────────────
164
+ function buildSummary(changes, mode, jsonTotalFields, textStats) {
165
+ if (changes.length === 0)
166
+ return 'No changes detected.';
167
+ if (mode === 'json') {
168
+ const unchanged = Math.max(0, (jsonTotalFields ?? 0) - changes.length);
169
+ const parts = [
170
+ `${changes.length} field${changes.length === 1 ? '' : 's'} changed`,
171
+ ];
172
+ if (unchanged > 0)
173
+ parts.push(`${unchanged} unchanged`);
174
+ return parts.join(', ') + '.';
175
+ }
176
+ // Text mode.
177
+ const sections = changes.length;
178
+ const added = textStats?.linesAdded ?? 0;
179
+ const removed = textStats?.linesRemoved ?? 0;
180
+ return (`${sections} section${sections === 1 ? '' : 's'} changed` +
181
+ (added > 0 ? `, ${added} lines added` : '') +
182
+ (removed > 0 ? `, ${removed} removed` : '') +
183
+ '.');
184
+ }
185
+ // ─── Key counting helpers ──────────────────────────────────────────────────────
186
+ /** Count the total number of leaf-level keys (dot-notation) in two JSON objects combined. */
187
+ function countTotalFields(a, b) {
188
+ const keys = new Set();
189
+ collectKeys(a, '', keys);
190
+ collectKeys(b, '', keys);
191
+ return keys.size;
192
+ }
193
+ function collectKeys(obj, prefix, acc) {
194
+ if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) {
195
+ if (prefix)
196
+ acc.add(prefix);
197
+ return;
198
+ }
199
+ for (const key of Object.keys(obj)) {
200
+ const path = prefix ? `${prefix}.${key}` : key;
201
+ acc.add(path);
202
+ collectKeys(obj[key], path, acc);
203
+ }
204
+ }
205
+ // ─── Public API ────────────────────────────────────────────────────────────────
206
+ /**
207
+ * Fetch a URL and compute a semantic diff against the last tracked snapshot.
208
+ *
209
+ * The function:
210
+ * 1. Loads the previous snapshot (if any) from the change-tracking store.
211
+ * 2. Fetches the current content via {@link peel}.
212
+ * 3. Saves the current content as the new snapshot (auto-tracking).
213
+ * 4. Computes a structured diff — field-by-field for JSON, section-by-section
214
+ * for text/markdown.
215
+ * 5. Returns a {@link DiffResult}.
216
+ *
217
+ * @example
218
+ * ```typescript
219
+ * const result = await diffUrl('https://api.example.com/health', { last: true });
220
+ * console.log(result.summary);
221
+ * result.changes.forEach(c => {
222
+ * if (c.field) console.log(`${c.type}: ${c.field} ${c.before} → ${c.after}`);
223
+ * });
224
+ * ```
225
+ */
226
+ export async function diffUrl(url, options = {}) {
227
+ const { fields, render = false, timeout = 30_000 } = options;
228
+ // 1. Load previous snapshot before fetching (fetch overwrites it).
229
+ const prevSnapshot = await getSnapshot(url);
230
+ // 2. Fetch current content. changeTracking: true auto-saves the new snapshot.
231
+ const result = await peel(url, {
232
+ render,
233
+ timeout,
234
+ format: 'markdown',
235
+ changeTracking: true,
236
+ });
237
+ const now = new Date().toISOString();
238
+ // 3. No baseline → return an informational result.
239
+ if (!prevSnapshot) {
240
+ return {
241
+ url: result.url,
242
+ changed: false,
243
+ timestamp: now,
244
+ changes: [],
245
+ summary: 'No previous snapshot found. Current content saved as baseline.',
246
+ };
247
+ }
248
+ const previousTimestamp = new Date(prevSnapshot.timestamp).toISOString();
249
+ const previousContent = prevSnapshot.content;
250
+ const currentContent = result.content;
251
+ // 4. Detect content type and compute appropriate diff.
252
+ let mode = 'text';
253
+ let changes = [];
254
+ let jsonTotalFields = 0;
255
+ let textStats;
256
+ let prevJson = null;
257
+ let currJson = null;
258
+ try {
259
+ prevJson = JSON.parse(previousContent);
260
+ currJson = JSON.parse(currentContent);
261
+ mode = 'json';
262
+ }
263
+ catch (e) {
264
+ if (process.env.DEBUG)
265
+ console.debug('[webpeel]', 'json parse failed:', e instanceof Error ? e.message : e);
266
+ }
267
+ if (mode === 'json' && prevJson !== null && currJson !== null) {
268
+ changes = diffJson(prevJson, currJson, '', fields);
269
+ jsonTotalFields = countTotalFields(prevJson, currJson);
270
+ }
271
+ else {
272
+ const { changes: textChanges, stats } = diffText(previousContent, currentContent);
273
+ changes = textChanges;
274
+ textStats = stats;
275
+ }
276
+ // 5. Handle edge case: content changed but we couldn't detect it (e.g. fingerprint
277
+ // mismatch recorded by peel, but diff shows no changes at field level).
278
+ const changed = changes.length > 0;
279
+ return {
280
+ url: result.url,
281
+ changed,
282
+ timestamp: now,
283
+ previousTimestamp,
284
+ changes,
285
+ summary: buildSummary(changes, mode, jsonTotalFields, textStats),
286
+ };
287
+ }
288
+ // ─── Re-export trackChange for CLI convenience ─────────────────────────────────
289
+ export { trackChange, getSnapshot } from './change-tracking.js';
@@ -0,0 +1,20 @@
1
+ /**
2
+ * DNS Pre-Resolution Cache
3
+ *
4
+ * Warms a local Map<hostname, ip[]> on startup for the top ~50 popular domains
5
+ * and exposes a custom lookup function compatible with undici's Agent `connect.lookup`.
6
+ */
7
+ import dns from 'node:dns';
8
+ export declare function getCachedDns(hostname: string): string[] | null;
9
+ export declare function resolveAndCache(hostname: string): Promise<string[]>;
10
+ /**
11
+ * Custom lookup function compatible with undici's Agent `connect.lookup`.
12
+ *
13
+ * undici passes `{ hints: 1024, all: true }` — so when `all` is true the
14
+ * callback must receive `(err, entries: { address, family }[])`.
15
+ * When `all` is false (or absent), the callback is `(err, address, family)`.
16
+ */
17
+ export declare function cachedLookup(hostname: string, options: dns.LookupOptions, callback: (...args: any[]) => void): void;
18
+ export declare function warmupDnsCache(domains?: string[]): Promise<void>;
19
+ export declare function startDnsWarmup(): void;
20
+ export declare function clearDnsCache(): void;