@taodigital/crawler 0.0.0-stage → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (494) hide show
  1. package/Dockerfile +82 -0
  2. package/README.md +10 -2
  3. package/dist/accessibility-auditor.d.ts +29 -0
  4. package/dist/accessibility-auditor.js +57 -0
  5. package/dist/accessibility-auditor.js.map +7 -0
  6. package/dist/accessibility-tree-crawler.d.ts +44 -0
  7. package/dist/accessibility-tree-crawler.js +156 -0
  8. package/dist/accessibility-tree-crawler.js.map +7 -0
  9. package/dist/advanced-ui-probes.d.ts +47 -0
  10. package/dist/advanced-ui-probes.js +310 -0
  11. package/dist/advanced-ui-probes.js.map +7 -0
  12. package/dist/ai-login.d.ts +32 -0
  13. package/dist/ai-login.js +250 -0
  14. package/dist/ai-login.js.map +7 -0
  15. package/dist/auth-token-rotator.d.ts +47 -0
  16. package/dist/auth-token-rotator.js +58 -0
  17. package/dist/auth-token-rotator.js.map +7 -0
  18. package/dist/auth.d.ts +20 -0
  19. package/dist/auth.js +348 -0
  20. package/dist/auth.js.map +7 -0
  21. package/dist/browser-extension.d.ts +37 -0
  22. package/dist/browser-extension.js +113 -0
  23. package/dist/browser-extension.js.map +7 -0
  24. package/dist/captcha-solver.d.ts +16 -0
  25. package/dist/captcha-solver.js +105 -0
  26. package/dist/captcha-solver.js.map +7 -0
  27. package/dist/color-contrast.d.ts +30 -0
  28. package/dist/color-contrast.js +78 -0
  29. package/dist/color-contrast.js.map +7 -0
  30. package/dist/content-chunker.d.ts +18 -0
  31. package/dist/content-chunker.js +152 -0
  32. package/dist/content-chunker.js.map +7 -0
  33. package/dist/content-dedup.d.ts +23 -0
  34. package/dist/content-dedup.js +60 -0
  35. package/dist/content-dedup.js.map +7 -0
  36. package/dist/cookie-consent.d.ts +25 -0
  37. package/dist/cookie-consent.js +78 -0
  38. package/dist/cookie-consent.js.map +7 -0
  39. package/dist/cookie-jar.d.ts +20 -0
  40. package/dist/cookie-jar.js +61 -0
  41. package/dist/cookie-jar.js.map +7 -0
  42. package/dist/crawl-session-manager.d.ts +44 -0
  43. package/dist/crawl-session-manager.js +145 -0
  44. package/dist/crawl-session-manager.js.map +7 -0
  45. package/dist/crawler.d.ts +233 -0
  46. package/dist/crawler.js +6239 -0
  47. package/dist/crawler.js.map +7 -0
  48. package/dist/crud-interaction.d.ts +31 -0
  49. package/dist/crud-interaction.js +264 -0
  50. package/dist/crud-interaction.js.map +7 -0
  51. package/dist/custom-js-injector.d.ts +42 -0
  52. package/dist/custom-js-injector.js +50 -0
  53. package/dist/custom-js-injector.js.map +7 -0
  54. package/dist/dark-mode-capture.d.ts +23 -0
  55. package/dist/dark-mode-capture.js +75 -0
  56. package/dist/dark-mode-capture.js.map +7 -0
  57. package/dist/dead-link-detector.d.ts +37 -0
  58. package/dist/dead-link-detector.js +50 -0
  59. package/dist/dead-link-detector.js.map +7 -0
  60. package/dist/dropdown-enumerator.d.ts +20 -0
  61. package/dist/dropdown-enumerator.js +230 -0
  62. package/dist/dropdown-enumerator.js.map +7 -0
  63. package/dist/element-hash.d.ts +9 -0
  64. package/dist/element-hash.js +9 -0
  65. package/dist/element-hash.js.map +7 -0
  66. package/dist/email-otp.d.ts +42 -0
  67. package/dist/email-otp.js +177 -0
  68. package/dist/email-otp.js.map +7 -0
  69. package/dist/engine-router.d.ts +47 -0
  70. package/dist/engine-router.js +94 -0
  71. package/dist/engine-router.js.map +7 -0
  72. package/dist/export/bulk-export.d.ts +26 -0
  73. package/dist/export/bulk-export.js +76 -0
  74. package/dist/export/bulk-export.js.map +7 -0
  75. package/dist/export/vector-export.d.ts +76 -0
  76. package/dist/export/vector-export.js +85 -0
  77. package/dist/export/vector-export.js.map +7 -0
  78. package/dist/extractors/advanced.d.ts +64 -0
  79. package/dist/extractors/advanced.js +265 -0
  80. package/dist/extractors/advanced.js.map +7 -0
  81. package/dist/extractors/artifact-envelope.d.ts +63 -0
  82. package/dist/extractors/artifact-envelope.js +63 -0
  83. package/dist/extractors/artifact-envelope.js.map +7 -0
  84. package/dist/extractors/branding-extractor.d.ts +46 -0
  85. package/dist/extractors/branding-extractor.js +104 -0
  86. package/dist/extractors/branding-extractor.js.map +7 -0
  87. package/dist/extractors/summary-extractor.d.ts +10 -0
  88. package/dist/extractors/summary-extractor.js +36 -0
  89. package/dist/extractors/summary-extractor.js.map +7 -0
  90. package/dist/failure-analysis-agent.d.ts +16 -0
  91. package/dist/failure-analysis-agent.js +42 -0
  92. package/dist/failure-analysis-agent.js.map +7 -0
  93. package/dist/feature-flag-detector.d.ts +30 -0
  94. package/dist/feature-flag-detector.js +55 -0
  95. package/dist/feature-flag-detector.js.map +7 -0
  96. package/dist/filter-pagination-crawler.d.ts +52 -0
  97. package/dist/filter-pagination-crawler.js +299 -0
  98. package/dist/filter-pagination-crawler.js.map +7 -0
  99. package/dist/form-detector.d.ts +34 -0
  100. package/dist/form-detector.js +58 -0
  101. package/dist/form-detector.js.map +7 -0
  102. package/dist/frontier.d.ts +40 -0
  103. package/dist/frontier.js +103 -0
  104. package/dist/frontier.js.map +7 -0
  105. package/dist/gap-pass.d.ts +36 -0
  106. package/dist/gap-pass.js +115 -0
  107. package/dist/gap-pass.js.map +7 -0
  108. package/dist/graphql-detector.d.ts +39 -0
  109. package/dist/graphql-detector.js +86 -0
  110. package/dist/graphql-detector.js.map +7 -0
  111. package/dist/http-engine.d.ts +80 -0
  112. package/dist/http-engine.js +318 -0
  113. package/dist/http-engine.js.map +7 -0
  114. package/dist/incremental-diff.d.ts +116 -0
  115. package/dist/incremental-diff.js +137 -0
  116. package/dist/incremental-diff.js.map +7 -0
  117. package/dist/index.d.ts +52 -0
  118. package/dist/index.js +83 -0
  119. package/dist/index.js.map +7 -0
  120. package/dist/input-fuzzer.d.ts +30 -0
  121. package/dist/input-fuzzer.js +105 -0
  122. package/dist/input-fuzzer.js.map +7 -0
  123. package/dist/interactive-probe.d.ts +144 -0
  124. package/dist/interactive-probe.js +865 -0
  125. package/dist/interactive-probe.js.map +7 -0
  126. package/dist/job-queue.d.ts +297 -0
  127. package/dist/job-queue.js +3812 -0
  128. package/dist/job-queue.js.map +7 -0
  129. package/dist/js-coverage.d.ts +29 -0
  130. package/dist/js-coverage.js +56 -0
  131. package/dist/js-coverage.js.map +7 -0
  132. package/dist/js-router-detector.d.ts +17 -0
  133. package/dist/js-router-detector.js +90 -0
  134. package/dist/js-router-detector.js.map +7 -0
  135. package/dist/language-detector.d.ts +22 -0
  136. package/dist/language-detector.js +57 -0
  137. package/dist/language-detector.js.map +7 -0
  138. package/dist/link-classifier.d.ts +23 -0
  139. package/dist/link-classifier.js +40 -0
  140. package/dist/link-classifier.js.map +7 -0
  141. package/dist/live-view.d.ts +36 -0
  142. package/dist/live-view.js +105 -0
  143. package/dist/live-view.js.map +7 -0
  144. package/dist/locator-extractor.d.ts +46 -0
  145. package/dist/locator-extractor.js +220 -0
  146. package/dist/locator-extractor.js.map +7 -0
  147. package/dist/logger.d.ts +2 -0
  148. package/dist/logger.js +10 -0
  149. package/dist/logger.js.map +7 -0
  150. package/dist/markdown-extract.d.ts +8 -0
  151. package/dist/markdown-extract.js +54 -0
  152. package/dist/markdown-extract.js.map +7 -0
  153. package/dist/multi-context-handler.d.ts +55 -0
  154. package/dist/multi-context-handler.js +182 -0
  155. package/dist/multi-context-handler.js.map +7 -0
  156. package/dist/native-crawler.d.ts +69 -0
  157. package/dist/native-crawler.js +839 -0
  158. package/dist/native-crawler.js.map +7 -0
  159. package/dist/nested-window-crawler.d.ts +41 -0
  160. package/dist/nested-window-crawler.js +172 -0
  161. package/dist/nested-window-crawler.js.map +7 -0
  162. package/dist/network-action-correlator.d.ts +47 -0
  163. package/dist/network-action-correlator.js +209 -0
  164. package/dist/network-action-correlator.js.map +7 -0
  165. package/dist/network-observer.d.ts +49 -0
  166. package/dist/network-observer.js +149 -0
  167. package/dist/network-observer.js.map +7 -0
  168. package/dist/network-throttle.d.ts +25 -0
  169. package/dist/network-throttle.js +34 -0
  170. package/dist/network-throttle.js.map +7 -0
  171. package/dist/openapi-generator.d.ts +49 -0
  172. package/dist/openapi-generator.js +89 -0
  173. package/dist/openapi-generator.js.map +7 -0
  174. package/dist/pdf-extractor.d.ts +28 -0
  175. package/dist/pdf-extractor.js +81 -0
  176. package/dist/pdf-extractor.js.map +7 -0
  177. package/dist/performance-metrics-collector.d.ts +50 -0
  178. package/dist/performance-metrics-collector.js +155 -0
  179. package/dist/performance-metrics-collector.js.map +7 -0
  180. package/dist/performance-metrics.d.ts +52 -0
  181. package/dist/performance-metrics.js +83 -0
  182. package/dist/performance-metrics.js.map +7 -0
  183. package/dist/phash.d.ts +50 -0
  184. package/dist/phash.js +136 -0
  185. package/dist/phash.js.map +7 -0
  186. package/dist/pii-scrubber.d.ts +21 -0
  187. package/dist/pii-scrubber.js +75 -0
  188. package/dist/pii-scrubber.js.map +7 -0
  189. package/dist/playwright-recorder.d.ts +39 -0
  190. package/dist/playwright-recorder.js +123 -0
  191. package/dist/playwright-recorder.js.map +7 -0
  192. package/dist/policies/crawl-safety.d.ts +9 -0
  193. package/dist/policies/crawl-safety.js +41 -0
  194. package/dist/policies/crawl-safety.js.map +7 -0
  195. package/dist/profiles/domain-profile.d.ts +19 -0
  196. package/dist/profiles/domain-profile.js +160 -0
  197. package/dist/profiles/domain-profile.js.map +7 -0
  198. package/dist/proxy.d.ts +50 -0
  199. package/dist/proxy.js +165 -0
  200. package/dist/proxy.js.map +7 -0
  201. package/dist/rate-limiter.d.ts +31 -0
  202. package/dist/rate-limiter.js +64 -0
  203. package/dist/rate-limiter.js.map +7 -0
  204. package/dist/redirect-tracker.d.ts +37 -0
  205. package/dist/redirect-tracker.js +65 -0
  206. package/dist/redirect-tracker.js.map +7 -0
  207. package/dist/redis-frontier.d.ts +48 -0
  208. package/dist/redis-frontier.js +126 -0
  209. package/dist/redis-frontier.js.map +7 -0
  210. package/dist/redis-queue.d.ts +48 -0
  211. package/dist/redis-queue.js +165 -0
  212. package/dist/redis-queue.js.map +7 -0
  213. package/dist/response-cache.d.ts +38 -0
  214. package/dist/response-cache.js +140 -0
  215. package/dist/response-cache.js.map +7 -0
  216. package/dist/responsive-detector.d.ts +29 -0
  217. package/dist/responsive-detector.js +46 -0
  218. package/dist/responsive-detector.js.map +7 -0
  219. package/dist/retry-policy.d.ts +20 -0
  220. package/dist/retry-policy.js +54 -0
  221. package/dist/retry-policy.js.map +7 -0
  222. package/dist/robots-txt.d.ts +9 -0
  223. package/dist/robots-txt.js +77 -0
  224. package/dist/robots-txt.js.map +7 -0
  225. package/dist/screen-reader.d.ts +15 -0
  226. package/dist/screen-reader.js +48 -0
  227. package/dist/screen-reader.js.map +7 -0
  228. package/dist/screenshot-masker.d.ts +28 -0
  229. package/dist/screenshot-masker.js +62 -0
  230. package/dist/screenshot-masker.js.map +7 -0
  231. package/dist/seo-checker.d.ts +33 -0
  232. package/dist/seo-checker.js +91 -0
  233. package/dist/seo-checker.js.map +7 -0
  234. package/dist/service-worker-interceptor.d.ts +36 -0
  235. package/dist/service-worker-interceptor.js +136 -0
  236. package/dist/service-worker-interceptor.js.map +7 -0
  237. package/dist/session-capture.d.ts +39 -0
  238. package/dist/session-capture.js +274 -0
  239. package/dist/session-capture.js.map +7 -0
  240. package/dist/session-export.d.ts +49 -0
  241. package/dist/session-export.js +117 -0
  242. package/dist/session-export.js.map +7 -0
  243. package/dist/shadow-dom-crawler.d.ts +35 -0
  244. package/dist/shadow-dom-crawler.js +132 -0
  245. package/dist/shadow-dom-crawler.js.map +7 -0
  246. package/dist/sitemap-writer.d.ts +21 -0
  247. package/dist/sitemap-writer.js +47 -0
  248. package/dist/sitemap-writer.js.map +7 -0
  249. package/dist/sitemap.d.ts +51 -0
  250. package/dist/sitemap.js +160 -0
  251. package/dist/sitemap.js.map +7 -0
  252. package/dist/skyvern.d.ts +56 -0
  253. package/dist/skyvern.js +129 -0
  254. package/dist/skyvern.js.map +7 -0
  255. package/dist/sqs-queue.d.ts +13 -0
  256. package/dist/sqs-queue.js +115 -0
  257. package/dist/sqs-queue.js.map +7 -0
  258. package/dist/sso-token.d.ts +29 -0
  259. package/dist/sso-token.js +76 -0
  260. package/dist/sso-token.js.map +7 -0
  261. package/dist/state-graph.d.ts +59 -0
  262. package/dist/state-graph.js +193 -0
  263. package/dist/state-graph.js.map +7 -0
  264. package/dist/stealth/human-behavior.d.ts +20 -0
  265. package/dist/stealth/human-behavior.js +202 -0
  266. package/dist/stealth/human-behavior.js.map +7 -0
  267. package/dist/telemetry.d.ts +34 -0
  268. package/dist/telemetry.js +77 -0
  269. package/dist/telemetry.js.map +7 -0
  270. package/dist/test-executor.d.ts +81 -0
  271. package/dist/test-executor.js +443 -0
  272. package/dist/test-executor.js.map +7 -0
  273. package/dist/tls-fingerprint.d.ts +50 -0
  274. package/dist/tls-fingerprint.js +81 -0
  275. package/dist/tls-fingerprint.js.map +7 -0
  276. package/dist/url-pattern-completer.d.ts +28 -0
  277. package/dist/url-pattern-completer.js +122 -0
  278. package/dist/url-pattern-completer.js.map +7 -0
  279. package/dist/visual-diff-verdict.d.ts +40 -0
  280. package/dist/visual-diff-verdict.js +27 -0
  281. package/dist/visual-diff-verdict.js.map +7 -0
  282. package/dist/webhook-notifier.d.ts +30 -0
  283. package/dist/webhook-notifier.js +76 -0
  284. package/dist/webhook-notifier.js.map +7 -0
  285. package/dist/websocket-detector.d.ts +24 -0
  286. package/dist/websocket-detector.js +60 -0
  287. package/dist/websocket-detector.js.map +7 -0
  288. package/dist/worker.d.ts +1 -0
  289. package/dist/worker.js +33 -0
  290. package/dist/worker.js.map +7 -0
  291. package/package.json +49 -4
  292. package/src/accessibility-auditor.ts +102 -0
  293. package/src/accessibility-tree-crawler.ts +246 -0
  294. package/src/advanced-ui-probes.ts +470 -0
  295. package/src/ai-login.ts +298 -0
  296. package/src/auth-token-rotator.ts +89 -0
  297. package/src/auth.ts +418 -0
  298. package/src/browser-extension.ts +186 -0
  299. package/src/captcha-solver.ts +143 -0
  300. package/src/color-contrast.ts +116 -0
  301. package/src/content-chunker.ts +170 -0
  302. package/src/content-dedup.ts +99 -0
  303. package/src/cookie-consent.ts +106 -0
  304. package/src/cookie-jar.ts +91 -0
  305. package/src/crawl-session-manager.ts +231 -0
  306. package/src/crawler.ts +7123 -0
  307. package/src/crawler.ts.rej +12 -0
  308. package/src/crud-interaction.ts +386 -0
  309. package/src/custom-js-injector.ts +93 -0
  310. package/src/dark-mode-capture.ts +110 -0
  311. package/src/dead-link-detector.ts +76 -0
  312. package/src/dropdown-enumerator.ts +312 -0
  313. package/src/element-hash.ts +16 -0
  314. package/src/email-otp.ts +252 -0
  315. package/src/engine-router.ts +141 -0
  316. package/src/export/bulk-export.ts +85 -0
  317. package/src/export/vector-export.ts +160 -0
  318. package/src/extractors/advanced.ts +309 -0
  319. package/src/extractors/artifact-envelope.ts +111 -0
  320. package/src/extractors/branding-extractor.ts +159 -0
  321. package/src/extractors/summary-extractor.ts +49 -0
  322. package/src/failure-analysis-agent.ts +51 -0
  323. package/src/feature-flag-detector.ts +97 -0
  324. package/src/filter-pagination-crawler.ts +500 -0
  325. package/src/form-detector.ts +103 -0
  326. package/src/frontier.test.ts +64 -0
  327. package/src/frontier.ts +144 -0
  328. package/src/gap-pass.ts +165 -0
  329. package/src/graphql-detector.ts +129 -0
  330. package/src/http-engine.ts +453 -0
  331. package/src/incremental-diff.ts +259 -0
  332. package/src/index.ts +52 -0
  333. package/src/input-fuzzer.ts +129 -0
  334. package/src/interactive-probe.ts +1136 -0
  335. package/src/job-queue.test.ts +187 -0
  336. package/src/job-queue.ts +4504 -0
  337. package/src/js-coverage.ts +87 -0
  338. package/src/js-router-detector.ts +112 -0
  339. package/src/language-detector.ts +82 -0
  340. package/src/link-classifier.ts +65 -0
  341. package/src/live-view.ts +146 -0
  342. package/src/locator-extractor.ts +309 -0
  343. package/src/logger.ts +9 -0
  344. package/src/markdown-extract.test.ts +73 -0
  345. package/src/markdown-extract.ts +76 -0
  346. package/src/multi-context-handler.ts +296 -0
  347. package/src/native-crawler.ts +1108 -0
  348. package/src/nested-window-crawler.ts +310 -0
  349. package/src/network-action-correlator.ts +320 -0
  350. package/src/network-observer.ts +204 -0
  351. package/src/network-throttle.ts +69 -0
  352. package/src/openapi-generator.ts +135 -0
  353. package/src/pdf-extractor.ts +123 -0
  354. package/src/performance-metrics-collector.ts +243 -0
  355. package/src/performance-metrics.ts +134 -0
  356. package/src/phash.ts +184 -0
  357. package/src/pii-scrubber.ts +99 -0
  358. package/src/playwright-recorder.ts +170 -0
  359. package/src/policies/crawl-safety.ts +52 -0
  360. package/src/profiles/domain-profile.ts +190 -0
  361. package/src/proxy.ts +238 -0
  362. package/src/rate-limiter.ts +87 -0
  363. package/src/redirect-tracker.ts +97 -0
  364. package/src/redis-frontier.ts +143 -0
  365. package/src/redis-queue.ts +208 -0
  366. package/src/response-cache.ts +150 -0
  367. package/src/responsive-detector.ts +92 -0
  368. package/src/retry-policy.ts +87 -0
  369. package/src/robots-txt.ts +96 -0
  370. package/src/screen-reader.ts +98 -0
  371. package/src/screenshot-masker.ts +98 -0
  372. package/src/seo-checker.ts +134 -0
  373. package/src/service-worker-interceptor.ts +185 -0
  374. package/src/session-capture.ts +335 -0
  375. package/src/session-export.test.ts +75 -0
  376. package/src/session-export.ts +172 -0
  377. package/src/shadow-dom-crawler.ts +198 -0
  378. package/src/sitemap-writer.ts +65 -0
  379. package/src/sitemap.test.ts +127 -0
  380. package/src/sitemap.ts +233 -0
  381. package/src/skyvern.ts +199 -0
  382. package/src/sqs-queue.ts +129 -0
  383. package/src/sso-token.ts +120 -0
  384. package/src/state-graph.ts +280 -0
  385. package/src/stealth/human-behavior.ts +227 -0
  386. package/src/telemetry.ts +122 -0
  387. package/src/test-executor.test.ts +72 -0
  388. package/src/test-executor.ts +578 -0
  389. package/src/tls-fingerprint.ts +111 -0
  390. package/src/url-pattern-completer.ts +161 -0
  391. package/src/vendor.d.ts +20 -0
  392. package/src/visual-diff-verdict.test.ts +80 -0
  393. package/src/visual-diff-verdict.ts +59 -0
  394. package/src/webhook-notifier.ts +109 -0
  395. package/src/websocket-detector.ts +83 -0
  396. package/src/worker.ts +50 -0
  397. package/tsconfig.json +13 -0
  398. package/types/accessibility-auditor.d.ts +29 -0
  399. package/types/accessibility-tree-crawler.d.ts +44 -0
  400. package/types/advanced-ui-probes.d.ts +47 -0
  401. package/types/ai-login.d.ts +32 -0
  402. package/types/auth-token-rotator.d.ts +47 -0
  403. package/types/auth.d.ts +20 -0
  404. package/types/browser-extension.d.ts +37 -0
  405. package/types/bull-worker.d.ts +6 -0
  406. package/types/captcha-solver.d.ts +16 -0
  407. package/types/color-contrast.d.ts +30 -0
  408. package/types/content-chunker.d.ts +18 -0
  409. package/types/content-dedup.d.ts +23 -0
  410. package/types/cookie-consent.d.ts +25 -0
  411. package/types/cookie-jar.d.ts +20 -0
  412. package/types/crawl-session-manager.d.ts +44 -0
  413. package/types/crawler.d.ts +233 -0
  414. package/types/crud-interaction.d.ts +31 -0
  415. package/types/custom-js-injector.d.ts +42 -0
  416. package/types/dark-mode-capture.d.ts +23 -0
  417. package/types/dead-link-detector.d.ts +37 -0
  418. package/types/dropdown-enumerator.d.ts +20 -0
  419. package/types/element-hash.d.ts +9 -0
  420. package/types/email-otp.d.ts +42 -0
  421. package/types/engine-router.d.ts +47 -0
  422. package/types/export/bulk-export.d.ts +26 -0
  423. package/types/export/vector-export.d.ts +76 -0
  424. package/types/extractors/advanced.d.ts +64 -0
  425. package/types/extractors/artifact-envelope.d.ts +63 -0
  426. package/types/extractors/branding-extractor.d.ts +46 -0
  427. package/types/extractors/summary-extractor.d.ts +10 -0
  428. package/types/failure-analysis-agent.d.ts +16 -0
  429. package/types/feature-flag-detector.d.ts +30 -0
  430. package/types/filter-pagination-crawler.d.ts +52 -0
  431. package/types/form-detector.d.ts +34 -0
  432. package/types/frontier.d.ts +40 -0
  433. package/types/gap-pass.d.ts +36 -0
  434. package/types/graphql-detector.d.ts +39 -0
  435. package/types/http-engine.d.ts +80 -0
  436. package/types/incremental-diff.d.ts +116 -0
  437. package/types/index.d.ts +52 -0
  438. package/types/input-fuzzer.d.ts +30 -0
  439. package/types/interactive-probe.d.ts +144 -0
  440. package/types/job-queue.d.ts +308 -0
  441. package/types/js-coverage.d.ts +29 -0
  442. package/types/js-router-detector.d.ts +17 -0
  443. package/types/language-detector.d.ts +22 -0
  444. package/types/link-classifier.d.ts +23 -0
  445. package/types/live-view.d.ts +36 -0
  446. package/types/locator-extractor.d.ts +46 -0
  447. package/types/logger.d.ts +2 -0
  448. package/types/markdown-extract.d.ts +8 -0
  449. package/types/multi-context-handler.d.ts +55 -0
  450. package/types/native-crawler.d.ts +69 -0
  451. package/types/nested-window-crawler.d.ts +41 -0
  452. package/types/network-action-correlator.d.ts +47 -0
  453. package/types/network-observer.d.ts +49 -0
  454. package/types/network-throttle.d.ts +25 -0
  455. package/types/openapi-generator.d.ts +49 -0
  456. package/types/pdf-extractor.d.ts +28 -0
  457. package/types/performance-metrics-collector.d.ts +50 -0
  458. package/types/performance-metrics.d.ts +52 -0
  459. package/types/phash.d.ts +50 -0
  460. package/types/pii-scrubber.d.ts +21 -0
  461. package/types/playwright-recorder.d.ts +39 -0
  462. package/types/policies/crawl-safety.d.ts +9 -0
  463. package/types/profiles/domain-profile.d.ts +19 -0
  464. package/types/proxy.d.ts +50 -0
  465. package/types/rate-limiter.d.ts +31 -0
  466. package/types/redirect-tracker.d.ts +37 -0
  467. package/types/redis-frontier.d.ts +48 -0
  468. package/types/redis-queue.d.ts +48 -0
  469. package/types/response-cache.d.ts +38 -0
  470. package/types/responsive-detector.d.ts +29 -0
  471. package/types/retry-policy.d.ts +20 -0
  472. package/types/robots-txt.d.ts +9 -0
  473. package/types/screen-reader.d.ts +15 -0
  474. package/types/screenshot-masker.d.ts +28 -0
  475. package/types/seo-checker.d.ts +33 -0
  476. package/types/service-worker-interceptor.d.ts +36 -0
  477. package/types/session-capture.d.ts +39 -0
  478. package/types/session-export.d.ts +49 -0
  479. package/types/shadow-dom-crawler.d.ts +35 -0
  480. package/types/sitemap-writer.d.ts +21 -0
  481. package/types/sitemap.d.ts +51 -0
  482. package/types/skyvern.d.ts +56 -0
  483. package/types/sqs-queue.d.ts +13 -0
  484. package/types/sso-token.d.ts +29 -0
  485. package/types/state-graph.d.ts +59 -0
  486. package/types/stealth/human-behavior.d.ts +20 -0
  487. package/types/telemetry.d.ts +34 -0
  488. package/types/test-executor.d.ts +84 -0
  489. package/types/tls-fingerprint.d.ts +50 -0
  490. package/types/url-pattern-completer.d.ts +28 -0
  491. package/types/visual-diff-verdict.d.ts +40 -0
  492. package/types/webhook-notifier.d.ts +30 -0
  493. package/types/websocket-detector.d.ts +24 -0
  494. package/types/worker.d.ts +1 -0
@@ -0,0 +1,92 @@
1
+ /**
2
+ * Responsive design detector — captures page at multiple breakpoints
3
+ * and detects layout differences between desktop and mobile viewports.
4
+ *
5
+ * Breakpoints: mobile(375), tablet(768), desktop(1280), wide(1920)
6
+ * Detects: viewport-specific content visibility, menu collapse state,
7
+ * font size changes, hidden/shown elements.
8
+ */
9
+
10
+ import type { Page } from 'playwright';
11
+
12
+ export interface ViewportCapture {
13
+ width: number;
14
+ height: number;
15
+ label: 'mobile' | 'tablet' | 'desktop' | 'wide';
16
+ screenshotBuffer?: Buffer;
17
+ visibleTextLength: number;
18
+ hasHamburgerMenu: boolean;
19
+ hiddenElements: number; // display:none element count
20
+ scrollHeight: number;
21
+ }
22
+
23
+ export interface ResponsiveReport {
24
+ url: string;
25
+ captures: ViewportCapture[];
26
+ isResponsive: boolean; // true if layout changes between viewports
27
+ hasMobileMenu: boolean;
28
+ issues: string[];
29
+ }
30
+
31
+ const BREAKPOINTS: Array<{ width: number; height: number; label: ViewportCapture['label'] }> = [
32
+ { width: 375, height: 812, label: 'mobile' },
33
+ { width: 768, height: 1024, label: 'tablet' },
34
+ { width: 1280, height: 800, label: 'desktop' },
35
+ ];
36
+
37
+ export async function detectResponsive(
38
+ page: Page,
39
+ url: string,
40
+ opts: { captureScreenshots?: boolean } = {},
41
+ ): Promise<ResponsiveReport> {
42
+ const captures: ViewportCapture[] = [];
43
+ const issues: string[] = [];
44
+
45
+ for (const bp of BREAKPOINTS) {
46
+ try {
47
+ await page.setViewportSize({ width: bp.width, height: bp.height });
48
+ await page.waitForTimeout(500); // allow CSS reflow
49
+
50
+ const data = await page.evaluate(() => ({
51
+ visibleTextLength: (document.body.innerText ?? '').length,
52
+ hasHamburgerMenu: !!(
53
+ document.querySelector('[class*="hamburger"],[class*="menu-toggle"],[class*="nav-toggle"],[aria-label*="menu"],[aria-label*="Menu"]')
54
+ ),
55
+ hiddenElements: Array.from(document.querySelectorAll('*')).filter(
56
+ (el: any) => window.getComputedStyle(el).display === 'none'
57
+ ).length,
58
+ scrollHeight: document.body.scrollHeight,
59
+ })).catch(() => ({ visibleTextLength: 0, hasHamburgerMenu: false, hiddenElements: 0, scrollHeight: 0 }));
60
+
61
+ let screenshotBuffer: Buffer | undefined;
62
+ if (opts.captureScreenshots) {
63
+ screenshotBuffer = await page.screenshot({ type: 'jpeg', quality: 60 }).catch(() => undefined);
64
+ }
65
+
66
+ captures.push({ ...bp, ...data, screenshotBuffer });
67
+ } catch { /* viewport failed — skip */ }
68
+ }
69
+
70
+ // Restore to desktop
71
+ await page.setViewportSize({ width: 1280, height: 800 }).catch(() => {});
72
+
73
+ const mobile = captures.find(c => c.label === 'mobile');
74
+ const desktop = captures.find(c => c.label === 'desktop');
75
+
76
+ const isResponsive = !!(mobile && desktop && (
77
+ mobile.hiddenElements !== desktop.hiddenElements ||
78
+ mobile.scrollHeight !== desktop.scrollHeight ||
79
+ mobile.hasHamburgerMenu
80
+ ));
81
+
82
+ const hasMobileMenu = captures.some(c => c.label === 'mobile' && c.hasHamburgerMenu);
83
+
84
+ if (mobile && desktop && mobile.visibleTextLength > desktop.visibleTextLength * 1.3) {
85
+ issues.push('Mobile shows significantly more text than desktop — possible responsive issue');
86
+ }
87
+ if (!isResponsive && captures.length > 1) {
88
+ issues.push('Layout appears identical across breakpoints — may not be responsive');
89
+ }
90
+
91
+ return { url, captures, isResponsive, hasMobileMenu, issues };
92
+ }
@@ -0,0 +1,87 @@
1
+ /**
2
+ * Retry policy — configurable exponential backoff with jitter.
3
+ * Covers: 5xx server errors, network timeouts, ECONNREFUSED, ECONNRESET.
4
+ * NOT retried: 4xx client errors (except 429 — handled separately), redirects.
5
+ */
6
+
7
+ export interface RetryPolicyOpts {
8
+ maxRetries?: number; // default 3
9
+ baseDelayMs?: number; // default 1000
10
+ maxDelayMs?: number; // default 30000
11
+ jitterMs?: number; // default 500
12
+ retryOn?: number[]; // HTTP status codes to retry (default: [429, 500, 502, 503, 504])
13
+ }
14
+
15
+ export interface RetryResult<T> {
16
+ value: T;
17
+ attempts: number;
18
+ totalDelayMs: number;
19
+ }
20
+
21
+ const DEFAULT_RETRY_STATUSES = [429, 500, 502, 503, 504];
22
+
23
+ function shouldRetry(error: any, retryOn: number[]): boolean {
24
+ // Network-level errors
25
+ if (error?.code && ['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'ENOTFOUND', 'EPIPE'].includes(error.code)) return true;
26
+ // Playwright navigation timeout
27
+ if (error?.message && /timeout|net::ERR_/i.test(error.message)) return true;
28
+ // HTTP status errors
29
+ if (typeof error?.status === 'number') return retryOn.includes(error.status);
30
+ return false;
31
+ }
32
+
33
+ function delay(ms: number): Promise<void> {
34
+ return new Promise(r => setTimeout(r, ms));
35
+ }
36
+
37
+ export async function withRetry<T>(
38
+ fn: (attempt: number) => Promise<T>,
39
+ opts: RetryPolicyOpts = {},
40
+ ): Promise<RetryResult<T>> {
41
+ const {
42
+ maxRetries = 3,
43
+ baseDelayMs = 1_000,
44
+ maxDelayMs = 30_000,
45
+ jitterMs = 500,
46
+ retryOn = DEFAULT_RETRY_STATUSES,
47
+ } = opts;
48
+
49
+ let lastError: any;
50
+ let totalDelayMs = 0;
51
+
52
+ for (let attempt = 0; attempt <= maxRetries; attempt++) {
53
+ try {
54
+ const value = await fn(attempt);
55
+ return { value, attempts: attempt + 1, totalDelayMs };
56
+ } catch (err) {
57
+ lastError = err;
58
+ if (attempt === maxRetries) break;
59
+ if (!shouldRetry(err, retryOn)) throw err; // non-retryable — fail fast
60
+
61
+ const backoff = Math.min(maxDelayMs, baseDelayMs * Math.pow(2, attempt));
62
+ const jitter = Math.random() * jitterMs;
63
+ const waitMs = Math.round(backoff + jitter);
64
+ totalDelayMs += waitMs;
65
+ await delay(waitMs);
66
+ }
67
+ }
68
+ throw lastError;
69
+ }
70
+
71
+ /** Wraps fetch() with retry policy */
72
+ export async function fetchWithRetry(
73
+ url: string,
74
+ init: RequestInit = {},
75
+ retryOpts: RetryPolicyOpts = {},
76
+ ): Promise<Response> {
77
+ const result = await withRetry(async () => {
78
+ const res = await fetch(url, init);
79
+ if (retryOpts.retryOn?.includes(res.status) ?? DEFAULT_RETRY_STATUSES.includes(res.status)) {
80
+ const err: any = new Error(`HTTP ${res.status}`);
81
+ err.status = res.status;
82
+ throw err;
83
+ }
84
+ return res;
85
+ }, retryOpts);
86
+ return result.value;
87
+ }
@@ -0,0 +1,96 @@
1
+ /**
2
+ * Minimal robots.txt fetcher + parser. No external deps. Caches per origin.
3
+ * Implements the subset of robots.txt spec used in practice:
4
+ * User-agent, Disallow, Allow — longest match wins (RFC 9309).
5
+ */
6
+
7
+ interface RobotsRule {
8
+ userAgent: string;
9
+ disallow: string[];
10
+ allow: string[];
11
+ }
12
+
13
+ function parseRobots(text: string): RobotsRule[] {
14
+ const rules: RobotsRule[] = [];
15
+ let current: RobotsRule | null = null;
16
+
17
+ for (const rawLine of text.split('\n')) {
18
+ const line = rawLine.split('#')[0].trim();
19
+ if (!line) { current = null; continue; }
20
+ const colonIdx = line.indexOf(':');
21
+ if (colonIdx === -1) continue;
22
+ const key = line.slice(0, colonIdx).trim().toLowerCase();
23
+ const value = line.slice(colonIdx + 1).trim();
24
+
25
+ if (key === 'user-agent') {
26
+ // Start new block only when current block already has rules, otherwise
27
+ // accumulate multiple User-agent lines into one block.
28
+ if (!current || current.disallow.length > 0 || current.allow.length > 0) {
29
+ current = { userAgent: value.toLowerCase(), disallow: [], allow: [] };
30
+ rules.push(current);
31
+ } else {
32
+ current.userAgent = value.toLowerCase();
33
+ }
34
+ } else if (key === 'disallow' && current && value) {
35
+ current.disallow.push(value);
36
+ } else if (key === 'allow' && current && value) {
37
+ current.allow.push(value);
38
+ }
39
+ }
40
+
41
+ return rules;
42
+ }
43
+
44
+ function isAllowedByRules(rules: RobotsRule[], userAgent: string, pathname: string): boolean {
45
+ const ua = userAgent.toLowerCase();
46
+ const exactRules = rules.filter(r => r.userAgent === ua);
47
+ const wildcardRules = rules.filter(r => r.userAgent === '*');
48
+ const applicable = exactRules.length > 0 ? exactRules : wildcardRules;
49
+
50
+ if (applicable.length === 0) return true;
51
+
52
+ // Collect all matched allow/disallow directives; longest path wins.
53
+ const matched: Array<{ path: string; allow: boolean }> = [];
54
+ for (const rule of applicable) {
55
+ for (const p of rule.allow) {
56
+ if (pathname.startsWith(p)) matched.push({ path: p, allow: true });
57
+ }
58
+ for (const p of rule.disallow) {
59
+ if (pathname.startsWith(p)) matched.push({ path: p, allow: false });
60
+ }
61
+ }
62
+
63
+ if (matched.length === 0) return true;
64
+ matched.sort((a, b) => b.path.length - a.path.length);
65
+ return matched[0].allow;
66
+ }
67
+
68
+ export class RobotsCache {
69
+ private readonly cache = new Map<string, RobotsRule[]>();
70
+
71
+ async isAllowed(url: string, userAgent = 'ZeTa-Crawler'): Promise<boolean> {
72
+ let origin: string;
73
+ let pathname: string;
74
+ try {
75
+ const u = new URL(url);
76
+ origin = u.origin;
77
+ pathname = u.pathname || '/';
78
+ } catch {
79
+ return true;
80
+ }
81
+
82
+ if (!this.cache.has(origin)) {
83
+ try {
84
+ const res = await fetch(`${origin}/robots.txt`, {
85
+ headers: { 'User-Agent': userAgent },
86
+ signal: AbortSignal.timeout(5_000),
87
+ });
88
+ this.cache.set(origin, res.ok ? parseRobots(await res.text()) : []);
89
+ } catch {
90
+ this.cache.set(origin, []); // network error or timeout = no restrictions
91
+ }
92
+ }
93
+
94
+ return isAllowedByRules(this.cache.get(origin)!, userAgent, pathname);
95
+ }
96
+ }
@@ -0,0 +1,98 @@
1
+ import type { Page } from 'playwright';
2
+
3
+ /**
4
+ * Reads a page as a structured outline (cheap + reliable), the way
5
+ * Playwright MCP / the accessibility tree works — NOT by sending pixels to a
6
+ * vision model. Vision is a fallback for image-only pages (added later).
7
+ *
8
+ * Returns the interactive elements found, which the Screen Reader Agent then
9
+ * enriches with MEANING and saves to the Zeta Core.
10
+ */
11
+ export interface RawElement {
12
+ role: string;
13
+ label: string;
14
+ selectorHint: string;
15
+ expectedData?: string;
16
+ }
17
+
18
+ export async function readScreen(page: any): Promise<RawElement[]> {
19
+ return page.evaluate(() => {
20
+ // esbuild injects __name() for named arrow-function constants (e.g. `const priority = ...`).
21
+ // Polyfill prevents ReferenceError when the serialized callback runs in the browser.
22
+ if (typeof (globalThis as any).__name === 'undefined') (globalThis as any).__name = (fn: any) => fn;
23
+ const out: { role: string; label: string; selectorHint: string; expectedData?: string }[] = [];
24
+ const seen = new Set<string>();
25
+
26
+ // Noise patterns to skip — cookie banners, footer chrome, version strings
27
+ // Intentionally excludes standalone "close" and "copy" — too many false positives
28
+ // (Close account, Close preview, Copy link, Copy to clipboard are all valid test targets)
29
+ const noiseLabel =
30
+ /^(cookie|consent|privacy policy|terms of use|copyright|©|\bv\d+\.\d+|got it|dismiss|skip to|powered by|copied|\s*$)/i;
31
+
32
+ // Include ARIA-role elements — Headless UI, Radix, Ant Design, Shadcn all use these instead of native tags
33
+ const nodes = document.querySelectorAll(
34
+ 'input:not([type=hidden]), select, textarea, button, a[href], ' +
35
+ '[role="button"], [role="combobox"], [role="slider"], [role="switch"], ' +
36
+ '[role="tab"], [role="menuitem"], [role="searchbox"], [role="spinbutton"]'
37
+ );
38
+
39
+ nodes.forEach((el) => {
40
+ const htmlEl = el as HTMLElement;
41
+
42
+ // Skip invisible elements
43
+ const style = window.getComputedStyle(htmlEl);
44
+ if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return;
45
+ const rect = el.getBoundingClientRect();
46
+ // Allow zero-size elements that carry an accessible label (sr-only icon buttons, skip-nav links)
47
+ const hasAccessibleLabel = !!(el.getAttribute('aria-label') || el.getAttribute('aria-labelledby'));
48
+ if (rect.width === 0 && rect.height === 0 && !hasAccessibleLabel) return;
49
+
50
+ // Derive role
51
+ const tag = el.tagName.toLowerCase();
52
+ const role =
53
+ el.getAttribute('role') ||
54
+ (tag === 'input' ? ((el as HTMLInputElement).type || 'text') : tag);
55
+
56
+ // Derive the best human-readable label (prefer aria > placeholder > inner text)
57
+ const raw = (
58
+ el.getAttribute('aria-label') ||
59
+ el.getAttribute('placeholder') ||
60
+ (el.getAttribute('title') ?? '') ||
61
+ htmlEl.innerText?.replace(/\s+/g, ' ').trim() ||
62
+ el.getAttribute('name') ||
63
+ ''
64
+ ).substring(0, 80);
65
+
66
+ // Skip empty, noise patterns, or very long labels (likely container text)
67
+ if (!raw || noiseLabel.test(raw) || raw.length > 70) return;
68
+
69
+ // Deduplicate by role+normalised label
70
+ const key = `${role}:${raw.toLowerCase()}`;
71
+ if (seen.has(key)) return;
72
+ seen.add(key);
73
+
74
+ let expectedData: string | undefined;
75
+ if (tag === 'input') {
76
+ const input = el as HTMLInputElement;
77
+ if (input.type === 'email' || /email/i.test(raw)) expectedData = 'valid_email';
78
+ else if (input.type === 'password' || /password/i.test(raw)) expectedData = 'password';
79
+ else if (input.type === 'tel' || /phone|mobile/i.test(raw)) expectedData = 'phone';
80
+ else if (input.type === 'date' || /\bdate\b/i.test(raw)) expectedData = 'date';
81
+ else if (/\b(code|otp|token)\b/i.test(raw)) expectedData = 'one_time_code';
82
+ else if (/\b(search|query)\b/i.test(raw)) expectedData = 'search_term';
83
+ }
84
+
85
+ out.push({ role, label: raw, selectorHint: tag, expectedData });
86
+ });
87
+
88
+ // Prioritise: form fields > buttons > links; within each group keep order
89
+ const priority = (e: typeof out[0]) => {
90
+ if (['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea'].includes(e.role)) return 0;
91
+ if (e.role === 'button' || e.role === 'submit') return 1;
92
+ return 2;
93
+ };
94
+ out.sort((a, b) => priority(a) - priority(b));
95
+
96
+ return out.slice(0, 25); // cap at 25 — enough context, not overwhelming
97
+ });
98
+ }
@@ -0,0 +1,98 @@
1
+ /**
2
+ * Screenshot masker — blur or black-box PII regions in captured screenshots.
3
+ *
4
+ * Strategy: inject a CSS overlay into the page before screenshot capture that
5
+ * covers detected PII-likely elements (input[type=email], input[type=password],
6
+ * elements with PII-containing text) with a blur or solid black mask.
7
+ *
8
+ * This is a CSS-based approach (no image processing library needed):
9
+ * 1. Identify likely-PII DOM elements via selectors
10
+ * 2. Add an overlaid <div> with blur/black filter at each element's bounding box
11
+ * 3. Take screenshot
12
+ * 4. Remove overlays
13
+ *
14
+ * Falls back gracefully if elements are not found or overlay injection fails.
15
+ */
16
+
17
+ import type { Page } from 'playwright';
18
+
19
+ export interface MaskingOpts {
20
+ /** Additional CSS selectors to mask (in addition to defaults) */
21
+ extraSelectors?: string[];
22
+ /** Masking style: 'blur' or 'blackout' */
23
+ style?: 'blur' | 'blackout';
24
+ /** Blur strength in px (default 8) */
25
+ blurPx?: number;
26
+ }
27
+
28
+ // Default PII-sensitive selectors
29
+ const DEFAULT_PII_SELECTORS = [
30
+ 'input[type="email"]',
31
+ 'input[type="password"]',
32
+ 'input[type="tel"]',
33
+ 'input[autocomplete="cc-number"]',
34
+ 'input[autocomplete="cc-csc"]',
35
+ 'input[autocomplete="ssn"]',
36
+ '[data-pii="true"]',
37
+ '[data-sensitive="true"]',
38
+ ];
39
+
40
+ export async function maskScreenshot(
41
+ page: Page,
42
+ opts: MaskingOpts = {},
43
+ ): Promise<{ maskedCount: number; screenshot: Buffer }> {
44
+ const { extraSelectors = [], style = 'blur', blurPx = 8 } = opts;
45
+ const selectors = [...DEFAULT_PII_SELECTORS, ...extraSelectors];
46
+
47
+ let maskedCount = 0;
48
+ const overlayIds: string[] = [];
49
+
50
+ // Inject CSS overlays
51
+ try {
52
+ const result = await page.evaluate(({ selectors, style, blurPx }) => {
53
+ const ids: string[] = [];
54
+ for (const sel of selectors) {
55
+ try {
56
+ const elements = Array.from(document.querySelectorAll(sel));
57
+ for (const el of elements) {
58
+ const rect = el.getBoundingClientRect();
59
+ if (rect.width === 0 || rect.height === 0) continue;
60
+ const overlay = document.createElement('div');
61
+ const id = `__zeta_mask_${Math.random().toString(36).slice(2)}`;
62
+ overlay.id = id;
63
+ overlay.style.cssText = [
64
+ 'position: fixed',
65
+ `top: ${rect.top + window.scrollY}px`,
66
+ `left: ${rect.left + window.scrollX}px`,
67
+ `width: ${rect.width}px`,
68
+ `height: ${rect.height}px`,
69
+ 'z-index: 2147483647',
70
+ style === 'blur'
71
+ ? `backdrop-filter: blur(${blurPx}px); -webkit-backdrop-filter: blur(${blurPx}px); background: rgba(0,0,0,0.1);`
72
+ : 'background: #000000;',
73
+ 'pointer-events: none',
74
+ ].join('; ');
75
+ document.body.appendChild(overlay);
76
+ ids.push(id);
77
+ }
78
+ } catch { /* selector error */ }
79
+ }
80
+ return ids;
81
+ }, { selectors, style, blurPx });
82
+
83
+ overlayIds.push(...result);
84
+ maskedCount = result.length;
85
+ } catch { /* non-fatal */ }
86
+
87
+ // Take screenshot with overlays active
88
+ const screenshot = await page.screenshot({ type: 'png', fullPage: false });
89
+
90
+ // Remove overlays
91
+ if (overlayIds.length > 0) {
92
+ await page.evaluate((ids) => {
93
+ for (const id of ids) document.getElementById(id)?.remove();
94
+ }, overlayIds).catch(() => {});
95
+ }
96
+
97
+ return { maskedCount, screenshot };
98
+ }
@@ -0,0 +1,134 @@
1
+ /**
2
+ * SEO indexability checker — determines if a page is crawlable and indexable.
3
+ *
4
+ * Checks:
5
+ * 1. X-Robots-Tag response header (noindex, nofollow, noarchive)
6
+ * 2. <meta name="robots"> tag (noindex, nofollow)
7
+ * 3. Canonical URL (self-canonical vs cross-canonical)
8
+ * 4. rel="nofollow" on outbound links
9
+ * 5. hreflang tags (multilingual)
10
+ * 6. Open Graph completeness (og:title, og:description, og:image)
11
+ * 7. Structured data presence (JSON-LD)
12
+ * 8. Meta description
13
+ * 9. Title tag length (< 60 chars recommended)
14
+ * 10. URL structure (query params, fragments, excessive depth)
15
+ */
16
+
17
+ import type { Page } from 'playwright';
18
+
19
+ export interface SeoIssue {
20
+ rule: string;
21
+ severity: 'critical' | 'warning' | 'info';
22
+ message: string;
23
+ value?: string;
24
+ }
25
+
26
+ export interface SeoCheckResult {
27
+ url: string;
28
+ isIndexable: boolean;
29
+ canonicalUrl?: string;
30
+ isSelfCanonical?: boolean;
31
+ hreflang: Record<string, string>; // lang → url
32
+ issues: SeoIssue[];
33
+ score: number; // 0-100
34
+ checkedAt: string;
35
+ }
36
+
37
+ export async function checkSeo(page: Page, url: string, responseHeaders: Record<string, string> = {}): Promise<SeoCheckResult> {
38
+ const issues: SeoIssue[] = [];
39
+ const checkedAt = new Date().toISOString();
40
+ let isIndexable = true;
41
+
42
+ // 1. X-Robots-Tag
43
+ const robotsHeader = responseHeaders['x-robots-tag'] ?? '';
44
+ if (/noindex/i.test(robotsHeader)) {
45
+ isIndexable = false;
46
+ issues.push({ rule: 'x-robots-noindex', severity: 'critical', message: 'X-Robots-Tag: noindex found in response header', value: robotsHeader });
47
+ }
48
+
49
+ // 2-9. Page-level checks
50
+ const pageData = await page.evaluate(() => {
51
+ const meta = (name: string) => document.querySelector(`meta[name="${name}"]`)?.getAttribute('content') ?? '';
52
+ const og = (prop: string) => document.querySelector(`meta[property="og:${prop}"]`)?.getAttribute('content') ?? '';
53
+ const canonical = document.querySelector('link[rel="canonical"]')?.getAttribute('href') ?? '';
54
+ const hreflang: Record<string, string> = {};
55
+ document.querySelectorAll('link[rel="alternate"][hreflang]').forEach((el: any) => {
56
+ hreflang[el.getAttribute('hreflang')] = el.getAttribute('href') ?? '';
57
+ });
58
+ const jsonLd = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).map((s: any) => s.textContent ?? '');
59
+ return {
60
+ robotsMeta: meta('robots'),
61
+ description: meta('description'),
62
+ title: document.title ?? '',
63
+ canonical,
64
+ hreflang,
65
+ ogTitle: og('title'),
66
+ ogDescription: og('description'),
67
+ ogImage: og('image'),
68
+ hasJsonLd: jsonLd.length > 0,
69
+ jsonLd,
70
+ };
71
+ }).catch(() => ({
72
+ robotsMeta: '', description: '', title: '', canonical: '', hreflang: {},
73
+ ogTitle: '', ogDescription: '', ogImage: '', hasJsonLd: false, jsonLd: [],
74
+ }));
75
+
76
+ // 2. Meta robots
77
+ if (/noindex/i.test(pageData.robotsMeta)) {
78
+ isIndexable = false;
79
+ issues.push({ rule: 'meta-robots-noindex', severity: 'critical', message: 'Meta robots noindex found', value: pageData.robotsMeta });
80
+ }
81
+
82
+ // 3. Canonical
83
+ let isSelfCanonical: boolean | undefined;
84
+ if (pageData.canonical) {
85
+ isSelfCanonical = pageData.canonical === url || pageData.canonical.replace(/\/$/, '') === url.replace(/\/$/, '');
86
+ if (!isSelfCanonical) {
87
+ issues.push({ rule: 'cross-canonical', severity: 'warning', message: 'Cross-canonical points elsewhere', value: pageData.canonical });
88
+ }
89
+ } else {
90
+ issues.push({ rule: 'missing-canonical', severity: 'warning', message: 'No canonical link tag' });
91
+ }
92
+
93
+ // 4. Title
94
+ if (!pageData.title) {
95
+ issues.push({ rule: 'missing-title', severity: 'critical', message: 'Page has no title tag' });
96
+ } else if (pageData.title.length > 60) {
97
+ issues.push({ rule: 'title-too-long', severity: 'warning', message: `Title ${pageData.title.length} chars (>60)`, value: pageData.title });
98
+ }
99
+
100
+ // 5. Meta description
101
+ if (!pageData.description) {
102
+ issues.push({ rule: 'missing-description', severity: 'warning', message: 'Missing meta description' });
103
+ } else if (pageData.description.length > 160) {
104
+ issues.push({ rule: 'description-too-long', severity: 'info', message: `Description ${pageData.description.length} chars (>160)` });
105
+ }
106
+
107
+ // 6. OG tags
108
+ if (!pageData.ogTitle) issues.push({ rule: 'missing-og-title', severity: 'info', message: 'Missing og:title' });
109
+ if (!pageData.ogImage) issues.push({ rule: 'missing-og-image', severity: 'info', message: 'Missing og:image' });
110
+
111
+ // 7. Structured data
112
+ if (!pageData.hasJsonLd) issues.push({ rule: 'no-structured-data', severity: 'info', message: 'No JSON-LD structured data found' });
113
+
114
+ // 8. URL depth
115
+ try {
116
+ const depth = new URL(url).pathname.split('/').filter(Boolean).length;
117
+ if (depth > 5) issues.push({ rule: 'deep-url', severity: 'info', message: `URL depth ${depth} (>5 levels)`, value: url });
118
+ } catch {}
119
+
120
+ const criticals = issues.filter(i => i.severity === 'critical').length;
121
+ const warnings = issues.filter(i => i.severity === 'warning').length;
122
+ const score = Math.max(0, 100 - criticals * 20 - warnings * 5 - issues.filter(i => i.severity === 'info').length * 2);
123
+
124
+ return {
125
+ url,
126
+ isIndexable,
127
+ canonicalUrl: pageData.canonical || undefined,
128
+ isSelfCanonical,
129
+ hreflang: pageData.hreflang,
130
+ issues,
131
+ score,
132
+ checkedAt,
133
+ };
134
+ }