@taodigital/crawler 0.0.0-stage → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (494) hide show
  1. package/Dockerfile +82 -0
  2. package/README.md +10 -2
  3. package/dist/accessibility-auditor.d.ts +29 -0
  4. package/dist/accessibility-auditor.js +57 -0
  5. package/dist/accessibility-auditor.js.map +7 -0
  6. package/dist/accessibility-tree-crawler.d.ts +44 -0
  7. package/dist/accessibility-tree-crawler.js +156 -0
  8. package/dist/accessibility-tree-crawler.js.map +7 -0
  9. package/dist/advanced-ui-probes.d.ts +47 -0
  10. package/dist/advanced-ui-probes.js +310 -0
  11. package/dist/advanced-ui-probes.js.map +7 -0
  12. package/dist/ai-login.d.ts +32 -0
  13. package/dist/ai-login.js +250 -0
  14. package/dist/ai-login.js.map +7 -0
  15. package/dist/auth-token-rotator.d.ts +47 -0
  16. package/dist/auth-token-rotator.js +58 -0
  17. package/dist/auth-token-rotator.js.map +7 -0
  18. package/dist/auth.d.ts +20 -0
  19. package/dist/auth.js +348 -0
  20. package/dist/auth.js.map +7 -0
  21. package/dist/browser-extension.d.ts +37 -0
  22. package/dist/browser-extension.js +113 -0
  23. package/dist/browser-extension.js.map +7 -0
  24. package/dist/captcha-solver.d.ts +16 -0
  25. package/dist/captcha-solver.js +105 -0
  26. package/dist/captcha-solver.js.map +7 -0
  27. package/dist/color-contrast.d.ts +30 -0
  28. package/dist/color-contrast.js +78 -0
  29. package/dist/color-contrast.js.map +7 -0
  30. package/dist/content-chunker.d.ts +18 -0
  31. package/dist/content-chunker.js +152 -0
  32. package/dist/content-chunker.js.map +7 -0
  33. package/dist/content-dedup.d.ts +23 -0
  34. package/dist/content-dedup.js +60 -0
  35. package/dist/content-dedup.js.map +7 -0
  36. package/dist/cookie-consent.d.ts +25 -0
  37. package/dist/cookie-consent.js +78 -0
  38. package/dist/cookie-consent.js.map +7 -0
  39. package/dist/cookie-jar.d.ts +20 -0
  40. package/dist/cookie-jar.js +61 -0
  41. package/dist/cookie-jar.js.map +7 -0
  42. package/dist/crawl-session-manager.d.ts +44 -0
  43. package/dist/crawl-session-manager.js +145 -0
  44. package/dist/crawl-session-manager.js.map +7 -0
  45. package/dist/crawler.d.ts +233 -0
  46. package/dist/crawler.js +6239 -0
  47. package/dist/crawler.js.map +7 -0
  48. package/dist/crud-interaction.d.ts +31 -0
  49. package/dist/crud-interaction.js +264 -0
  50. package/dist/crud-interaction.js.map +7 -0
  51. package/dist/custom-js-injector.d.ts +42 -0
  52. package/dist/custom-js-injector.js +50 -0
  53. package/dist/custom-js-injector.js.map +7 -0
  54. package/dist/dark-mode-capture.d.ts +23 -0
  55. package/dist/dark-mode-capture.js +75 -0
  56. package/dist/dark-mode-capture.js.map +7 -0
  57. package/dist/dead-link-detector.d.ts +37 -0
  58. package/dist/dead-link-detector.js +50 -0
  59. package/dist/dead-link-detector.js.map +7 -0
  60. package/dist/dropdown-enumerator.d.ts +20 -0
  61. package/dist/dropdown-enumerator.js +230 -0
  62. package/dist/dropdown-enumerator.js.map +7 -0
  63. package/dist/element-hash.d.ts +9 -0
  64. package/dist/element-hash.js +9 -0
  65. package/dist/element-hash.js.map +7 -0
  66. package/dist/email-otp.d.ts +42 -0
  67. package/dist/email-otp.js +177 -0
  68. package/dist/email-otp.js.map +7 -0
  69. package/dist/engine-router.d.ts +47 -0
  70. package/dist/engine-router.js +94 -0
  71. package/dist/engine-router.js.map +7 -0
  72. package/dist/export/bulk-export.d.ts +26 -0
  73. package/dist/export/bulk-export.js +76 -0
  74. package/dist/export/bulk-export.js.map +7 -0
  75. package/dist/export/vector-export.d.ts +76 -0
  76. package/dist/export/vector-export.js +85 -0
  77. package/dist/export/vector-export.js.map +7 -0
  78. package/dist/extractors/advanced.d.ts +64 -0
  79. package/dist/extractors/advanced.js +265 -0
  80. package/dist/extractors/advanced.js.map +7 -0
  81. package/dist/extractors/artifact-envelope.d.ts +63 -0
  82. package/dist/extractors/artifact-envelope.js +63 -0
  83. package/dist/extractors/artifact-envelope.js.map +7 -0
  84. package/dist/extractors/branding-extractor.d.ts +46 -0
  85. package/dist/extractors/branding-extractor.js +104 -0
  86. package/dist/extractors/branding-extractor.js.map +7 -0
  87. package/dist/extractors/summary-extractor.d.ts +10 -0
  88. package/dist/extractors/summary-extractor.js +36 -0
  89. package/dist/extractors/summary-extractor.js.map +7 -0
  90. package/dist/failure-analysis-agent.d.ts +16 -0
  91. package/dist/failure-analysis-agent.js +42 -0
  92. package/dist/failure-analysis-agent.js.map +7 -0
  93. package/dist/feature-flag-detector.d.ts +30 -0
  94. package/dist/feature-flag-detector.js +55 -0
  95. package/dist/feature-flag-detector.js.map +7 -0
  96. package/dist/filter-pagination-crawler.d.ts +52 -0
  97. package/dist/filter-pagination-crawler.js +299 -0
  98. package/dist/filter-pagination-crawler.js.map +7 -0
  99. package/dist/form-detector.d.ts +34 -0
  100. package/dist/form-detector.js +58 -0
  101. package/dist/form-detector.js.map +7 -0
  102. package/dist/frontier.d.ts +40 -0
  103. package/dist/frontier.js +103 -0
  104. package/dist/frontier.js.map +7 -0
  105. package/dist/gap-pass.d.ts +36 -0
  106. package/dist/gap-pass.js +115 -0
  107. package/dist/gap-pass.js.map +7 -0
  108. package/dist/graphql-detector.d.ts +39 -0
  109. package/dist/graphql-detector.js +86 -0
  110. package/dist/graphql-detector.js.map +7 -0
  111. package/dist/http-engine.d.ts +80 -0
  112. package/dist/http-engine.js +318 -0
  113. package/dist/http-engine.js.map +7 -0
  114. package/dist/incremental-diff.d.ts +116 -0
  115. package/dist/incremental-diff.js +137 -0
  116. package/dist/incremental-diff.js.map +7 -0
  117. package/dist/index.d.ts +52 -0
  118. package/dist/index.js +83 -0
  119. package/dist/index.js.map +7 -0
  120. package/dist/input-fuzzer.d.ts +30 -0
  121. package/dist/input-fuzzer.js +105 -0
  122. package/dist/input-fuzzer.js.map +7 -0
  123. package/dist/interactive-probe.d.ts +144 -0
  124. package/dist/interactive-probe.js +865 -0
  125. package/dist/interactive-probe.js.map +7 -0
  126. package/dist/job-queue.d.ts +297 -0
  127. package/dist/job-queue.js +3812 -0
  128. package/dist/job-queue.js.map +7 -0
  129. package/dist/js-coverage.d.ts +29 -0
  130. package/dist/js-coverage.js +56 -0
  131. package/dist/js-coverage.js.map +7 -0
  132. package/dist/js-router-detector.d.ts +17 -0
  133. package/dist/js-router-detector.js +90 -0
  134. package/dist/js-router-detector.js.map +7 -0
  135. package/dist/language-detector.d.ts +22 -0
  136. package/dist/language-detector.js +57 -0
  137. package/dist/language-detector.js.map +7 -0
  138. package/dist/link-classifier.d.ts +23 -0
  139. package/dist/link-classifier.js +40 -0
  140. package/dist/link-classifier.js.map +7 -0
  141. package/dist/live-view.d.ts +36 -0
  142. package/dist/live-view.js +105 -0
  143. package/dist/live-view.js.map +7 -0
  144. package/dist/locator-extractor.d.ts +46 -0
  145. package/dist/locator-extractor.js +220 -0
  146. package/dist/locator-extractor.js.map +7 -0
  147. package/dist/logger.d.ts +2 -0
  148. package/dist/logger.js +10 -0
  149. package/dist/logger.js.map +7 -0
  150. package/dist/markdown-extract.d.ts +8 -0
  151. package/dist/markdown-extract.js +54 -0
  152. package/dist/markdown-extract.js.map +7 -0
  153. package/dist/multi-context-handler.d.ts +55 -0
  154. package/dist/multi-context-handler.js +182 -0
  155. package/dist/multi-context-handler.js.map +7 -0
  156. package/dist/native-crawler.d.ts +69 -0
  157. package/dist/native-crawler.js +839 -0
  158. package/dist/native-crawler.js.map +7 -0
  159. package/dist/nested-window-crawler.d.ts +41 -0
  160. package/dist/nested-window-crawler.js +172 -0
  161. package/dist/nested-window-crawler.js.map +7 -0
  162. package/dist/network-action-correlator.d.ts +47 -0
  163. package/dist/network-action-correlator.js +209 -0
  164. package/dist/network-action-correlator.js.map +7 -0
  165. package/dist/network-observer.d.ts +49 -0
  166. package/dist/network-observer.js +149 -0
  167. package/dist/network-observer.js.map +7 -0
  168. package/dist/network-throttle.d.ts +25 -0
  169. package/dist/network-throttle.js +34 -0
  170. package/dist/network-throttle.js.map +7 -0
  171. package/dist/openapi-generator.d.ts +49 -0
  172. package/dist/openapi-generator.js +89 -0
  173. package/dist/openapi-generator.js.map +7 -0
  174. package/dist/pdf-extractor.d.ts +28 -0
  175. package/dist/pdf-extractor.js +81 -0
  176. package/dist/pdf-extractor.js.map +7 -0
  177. package/dist/performance-metrics-collector.d.ts +50 -0
  178. package/dist/performance-metrics-collector.js +155 -0
  179. package/dist/performance-metrics-collector.js.map +7 -0
  180. package/dist/performance-metrics.d.ts +52 -0
  181. package/dist/performance-metrics.js +83 -0
  182. package/dist/performance-metrics.js.map +7 -0
  183. package/dist/phash.d.ts +50 -0
  184. package/dist/phash.js +136 -0
  185. package/dist/phash.js.map +7 -0
  186. package/dist/pii-scrubber.d.ts +21 -0
  187. package/dist/pii-scrubber.js +75 -0
  188. package/dist/pii-scrubber.js.map +7 -0
  189. package/dist/playwright-recorder.d.ts +39 -0
  190. package/dist/playwright-recorder.js +123 -0
  191. package/dist/playwright-recorder.js.map +7 -0
  192. package/dist/policies/crawl-safety.d.ts +9 -0
  193. package/dist/policies/crawl-safety.js +41 -0
  194. package/dist/policies/crawl-safety.js.map +7 -0
  195. package/dist/profiles/domain-profile.d.ts +19 -0
  196. package/dist/profiles/domain-profile.js +160 -0
  197. package/dist/profiles/domain-profile.js.map +7 -0
  198. package/dist/proxy.d.ts +50 -0
  199. package/dist/proxy.js +165 -0
  200. package/dist/proxy.js.map +7 -0
  201. package/dist/rate-limiter.d.ts +31 -0
  202. package/dist/rate-limiter.js +64 -0
  203. package/dist/rate-limiter.js.map +7 -0
  204. package/dist/redirect-tracker.d.ts +37 -0
  205. package/dist/redirect-tracker.js +65 -0
  206. package/dist/redirect-tracker.js.map +7 -0
  207. package/dist/redis-frontier.d.ts +48 -0
  208. package/dist/redis-frontier.js +126 -0
  209. package/dist/redis-frontier.js.map +7 -0
  210. package/dist/redis-queue.d.ts +48 -0
  211. package/dist/redis-queue.js +165 -0
  212. package/dist/redis-queue.js.map +7 -0
  213. package/dist/response-cache.d.ts +38 -0
  214. package/dist/response-cache.js +140 -0
  215. package/dist/response-cache.js.map +7 -0
  216. package/dist/responsive-detector.d.ts +29 -0
  217. package/dist/responsive-detector.js +46 -0
  218. package/dist/responsive-detector.js.map +7 -0
  219. package/dist/retry-policy.d.ts +20 -0
  220. package/dist/retry-policy.js +54 -0
  221. package/dist/retry-policy.js.map +7 -0
  222. package/dist/robots-txt.d.ts +9 -0
  223. package/dist/robots-txt.js +77 -0
  224. package/dist/robots-txt.js.map +7 -0
  225. package/dist/screen-reader.d.ts +15 -0
  226. package/dist/screen-reader.js +48 -0
  227. package/dist/screen-reader.js.map +7 -0
  228. package/dist/screenshot-masker.d.ts +28 -0
  229. package/dist/screenshot-masker.js +62 -0
  230. package/dist/screenshot-masker.js.map +7 -0
  231. package/dist/seo-checker.d.ts +33 -0
  232. package/dist/seo-checker.js +91 -0
  233. package/dist/seo-checker.js.map +7 -0
  234. package/dist/service-worker-interceptor.d.ts +36 -0
  235. package/dist/service-worker-interceptor.js +136 -0
  236. package/dist/service-worker-interceptor.js.map +7 -0
  237. package/dist/session-capture.d.ts +39 -0
  238. package/dist/session-capture.js +274 -0
  239. package/dist/session-capture.js.map +7 -0
  240. package/dist/session-export.d.ts +49 -0
  241. package/dist/session-export.js +117 -0
  242. package/dist/session-export.js.map +7 -0
  243. package/dist/shadow-dom-crawler.d.ts +35 -0
  244. package/dist/shadow-dom-crawler.js +132 -0
  245. package/dist/shadow-dom-crawler.js.map +7 -0
  246. package/dist/sitemap-writer.d.ts +21 -0
  247. package/dist/sitemap-writer.js +47 -0
  248. package/dist/sitemap-writer.js.map +7 -0
  249. package/dist/sitemap.d.ts +51 -0
  250. package/dist/sitemap.js +160 -0
  251. package/dist/sitemap.js.map +7 -0
  252. package/dist/skyvern.d.ts +56 -0
  253. package/dist/skyvern.js +129 -0
  254. package/dist/skyvern.js.map +7 -0
  255. package/dist/sqs-queue.d.ts +13 -0
  256. package/dist/sqs-queue.js +115 -0
  257. package/dist/sqs-queue.js.map +7 -0
  258. package/dist/sso-token.d.ts +29 -0
  259. package/dist/sso-token.js +76 -0
  260. package/dist/sso-token.js.map +7 -0
  261. package/dist/state-graph.d.ts +59 -0
  262. package/dist/state-graph.js +193 -0
  263. package/dist/state-graph.js.map +7 -0
  264. package/dist/stealth/human-behavior.d.ts +20 -0
  265. package/dist/stealth/human-behavior.js +202 -0
  266. package/dist/stealth/human-behavior.js.map +7 -0
  267. package/dist/telemetry.d.ts +34 -0
  268. package/dist/telemetry.js +77 -0
  269. package/dist/telemetry.js.map +7 -0
  270. package/dist/test-executor.d.ts +81 -0
  271. package/dist/test-executor.js +443 -0
  272. package/dist/test-executor.js.map +7 -0
  273. package/dist/tls-fingerprint.d.ts +50 -0
  274. package/dist/tls-fingerprint.js +81 -0
  275. package/dist/tls-fingerprint.js.map +7 -0
  276. package/dist/url-pattern-completer.d.ts +28 -0
  277. package/dist/url-pattern-completer.js +122 -0
  278. package/dist/url-pattern-completer.js.map +7 -0
  279. package/dist/visual-diff-verdict.d.ts +40 -0
  280. package/dist/visual-diff-verdict.js +27 -0
  281. package/dist/visual-diff-verdict.js.map +7 -0
  282. package/dist/webhook-notifier.d.ts +30 -0
  283. package/dist/webhook-notifier.js +76 -0
  284. package/dist/webhook-notifier.js.map +7 -0
  285. package/dist/websocket-detector.d.ts +24 -0
  286. package/dist/websocket-detector.js +60 -0
  287. package/dist/websocket-detector.js.map +7 -0
  288. package/dist/worker.d.ts +1 -0
  289. package/dist/worker.js +33 -0
  290. package/dist/worker.js.map +7 -0
  291. package/package.json +49 -4
  292. package/src/accessibility-auditor.ts +102 -0
  293. package/src/accessibility-tree-crawler.ts +246 -0
  294. package/src/advanced-ui-probes.ts +470 -0
  295. package/src/ai-login.ts +298 -0
  296. package/src/auth-token-rotator.ts +89 -0
  297. package/src/auth.ts +418 -0
  298. package/src/browser-extension.ts +186 -0
  299. package/src/captcha-solver.ts +143 -0
  300. package/src/color-contrast.ts +116 -0
  301. package/src/content-chunker.ts +170 -0
  302. package/src/content-dedup.ts +99 -0
  303. package/src/cookie-consent.ts +106 -0
  304. package/src/cookie-jar.ts +91 -0
  305. package/src/crawl-session-manager.ts +231 -0
  306. package/src/crawler.ts +7123 -0
  307. package/src/crawler.ts.rej +12 -0
  308. package/src/crud-interaction.ts +386 -0
  309. package/src/custom-js-injector.ts +93 -0
  310. package/src/dark-mode-capture.ts +110 -0
  311. package/src/dead-link-detector.ts +76 -0
  312. package/src/dropdown-enumerator.ts +312 -0
  313. package/src/element-hash.ts +16 -0
  314. package/src/email-otp.ts +252 -0
  315. package/src/engine-router.ts +141 -0
  316. package/src/export/bulk-export.ts +85 -0
  317. package/src/export/vector-export.ts +160 -0
  318. package/src/extractors/advanced.ts +309 -0
  319. package/src/extractors/artifact-envelope.ts +111 -0
  320. package/src/extractors/branding-extractor.ts +159 -0
  321. package/src/extractors/summary-extractor.ts +49 -0
  322. package/src/failure-analysis-agent.ts +51 -0
  323. package/src/feature-flag-detector.ts +97 -0
  324. package/src/filter-pagination-crawler.ts +500 -0
  325. package/src/form-detector.ts +103 -0
  326. package/src/frontier.test.ts +64 -0
  327. package/src/frontier.ts +144 -0
  328. package/src/gap-pass.ts +165 -0
  329. package/src/graphql-detector.ts +129 -0
  330. package/src/http-engine.ts +453 -0
  331. package/src/incremental-diff.ts +259 -0
  332. package/src/index.ts +52 -0
  333. package/src/input-fuzzer.ts +129 -0
  334. package/src/interactive-probe.ts +1136 -0
  335. package/src/job-queue.test.ts +187 -0
  336. package/src/job-queue.ts +4504 -0
  337. package/src/js-coverage.ts +87 -0
  338. package/src/js-router-detector.ts +112 -0
  339. package/src/language-detector.ts +82 -0
  340. package/src/link-classifier.ts +65 -0
  341. package/src/live-view.ts +146 -0
  342. package/src/locator-extractor.ts +309 -0
  343. package/src/logger.ts +9 -0
  344. package/src/markdown-extract.test.ts +73 -0
  345. package/src/markdown-extract.ts +76 -0
  346. package/src/multi-context-handler.ts +296 -0
  347. package/src/native-crawler.ts +1108 -0
  348. package/src/nested-window-crawler.ts +310 -0
  349. package/src/network-action-correlator.ts +320 -0
  350. package/src/network-observer.ts +204 -0
  351. package/src/network-throttle.ts +69 -0
  352. package/src/openapi-generator.ts +135 -0
  353. package/src/pdf-extractor.ts +123 -0
  354. package/src/performance-metrics-collector.ts +243 -0
  355. package/src/performance-metrics.ts +134 -0
  356. package/src/phash.ts +184 -0
  357. package/src/pii-scrubber.ts +99 -0
  358. package/src/playwright-recorder.ts +170 -0
  359. package/src/policies/crawl-safety.ts +52 -0
  360. package/src/profiles/domain-profile.ts +190 -0
  361. package/src/proxy.ts +238 -0
  362. package/src/rate-limiter.ts +87 -0
  363. package/src/redirect-tracker.ts +97 -0
  364. package/src/redis-frontier.ts +143 -0
  365. package/src/redis-queue.ts +208 -0
  366. package/src/response-cache.ts +150 -0
  367. package/src/responsive-detector.ts +92 -0
  368. package/src/retry-policy.ts +87 -0
  369. package/src/robots-txt.ts +96 -0
  370. package/src/screen-reader.ts +98 -0
  371. package/src/screenshot-masker.ts +98 -0
  372. package/src/seo-checker.ts +134 -0
  373. package/src/service-worker-interceptor.ts +185 -0
  374. package/src/session-capture.ts +335 -0
  375. package/src/session-export.test.ts +75 -0
  376. package/src/session-export.ts +172 -0
  377. package/src/shadow-dom-crawler.ts +198 -0
  378. package/src/sitemap-writer.ts +65 -0
  379. package/src/sitemap.test.ts +127 -0
  380. package/src/sitemap.ts +233 -0
  381. package/src/skyvern.ts +199 -0
  382. package/src/sqs-queue.ts +129 -0
  383. package/src/sso-token.ts +120 -0
  384. package/src/state-graph.ts +280 -0
  385. package/src/stealth/human-behavior.ts +227 -0
  386. package/src/telemetry.ts +122 -0
  387. package/src/test-executor.test.ts +72 -0
  388. package/src/test-executor.ts +578 -0
  389. package/src/tls-fingerprint.ts +111 -0
  390. package/src/url-pattern-completer.ts +161 -0
  391. package/src/vendor.d.ts +20 -0
  392. package/src/visual-diff-verdict.test.ts +80 -0
  393. package/src/visual-diff-verdict.ts +59 -0
  394. package/src/webhook-notifier.ts +109 -0
  395. package/src/websocket-detector.ts +83 -0
  396. package/src/worker.ts +50 -0
  397. package/tsconfig.json +13 -0
  398. package/types/accessibility-auditor.d.ts +29 -0
  399. package/types/accessibility-tree-crawler.d.ts +44 -0
  400. package/types/advanced-ui-probes.d.ts +47 -0
  401. package/types/ai-login.d.ts +32 -0
  402. package/types/auth-token-rotator.d.ts +47 -0
  403. package/types/auth.d.ts +20 -0
  404. package/types/browser-extension.d.ts +37 -0
  405. package/types/bull-worker.d.ts +6 -0
  406. package/types/captcha-solver.d.ts +16 -0
  407. package/types/color-contrast.d.ts +30 -0
  408. package/types/content-chunker.d.ts +18 -0
  409. package/types/content-dedup.d.ts +23 -0
  410. package/types/cookie-consent.d.ts +25 -0
  411. package/types/cookie-jar.d.ts +20 -0
  412. package/types/crawl-session-manager.d.ts +44 -0
  413. package/types/crawler.d.ts +233 -0
  414. package/types/crud-interaction.d.ts +31 -0
  415. package/types/custom-js-injector.d.ts +42 -0
  416. package/types/dark-mode-capture.d.ts +23 -0
  417. package/types/dead-link-detector.d.ts +37 -0
  418. package/types/dropdown-enumerator.d.ts +20 -0
  419. package/types/element-hash.d.ts +9 -0
  420. package/types/email-otp.d.ts +42 -0
  421. package/types/engine-router.d.ts +47 -0
  422. package/types/export/bulk-export.d.ts +26 -0
  423. package/types/export/vector-export.d.ts +76 -0
  424. package/types/extractors/advanced.d.ts +64 -0
  425. package/types/extractors/artifact-envelope.d.ts +63 -0
  426. package/types/extractors/branding-extractor.d.ts +46 -0
  427. package/types/extractors/summary-extractor.d.ts +10 -0
  428. package/types/failure-analysis-agent.d.ts +16 -0
  429. package/types/feature-flag-detector.d.ts +30 -0
  430. package/types/filter-pagination-crawler.d.ts +52 -0
  431. package/types/form-detector.d.ts +34 -0
  432. package/types/frontier.d.ts +40 -0
  433. package/types/gap-pass.d.ts +36 -0
  434. package/types/graphql-detector.d.ts +39 -0
  435. package/types/http-engine.d.ts +80 -0
  436. package/types/incremental-diff.d.ts +116 -0
  437. package/types/index.d.ts +52 -0
  438. package/types/input-fuzzer.d.ts +30 -0
  439. package/types/interactive-probe.d.ts +144 -0
  440. package/types/job-queue.d.ts +308 -0
  441. package/types/js-coverage.d.ts +29 -0
  442. package/types/js-router-detector.d.ts +17 -0
  443. package/types/language-detector.d.ts +22 -0
  444. package/types/link-classifier.d.ts +23 -0
  445. package/types/live-view.d.ts +36 -0
  446. package/types/locator-extractor.d.ts +46 -0
  447. package/types/logger.d.ts +2 -0
  448. package/types/markdown-extract.d.ts +8 -0
  449. package/types/multi-context-handler.d.ts +55 -0
  450. package/types/native-crawler.d.ts +69 -0
  451. package/types/nested-window-crawler.d.ts +41 -0
  452. package/types/network-action-correlator.d.ts +47 -0
  453. package/types/network-observer.d.ts +49 -0
  454. package/types/network-throttle.d.ts +25 -0
  455. package/types/openapi-generator.d.ts +49 -0
  456. package/types/pdf-extractor.d.ts +28 -0
  457. package/types/performance-metrics-collector.d.ts +50 -0
  458. package/types/performance-metrics.d.ts +52 -0
  459. package/types/phash.d.ts +50 -0
  460. package/types/pii-scrubber.d.ts +21 -0
  461. package/types/playwright-recorder.d.ts +39 -0
  462. package/types/policies/crawl-safety.d.ts +9 -0
  463. package/types/profiles/domain-profile.d.ts +19 -0
  464. package/types/proxy.d.ts +50 -0
  465. package/types/rate-limiter.d.ts +31 -0
  466. package/types/redirect-tracker.d.ts +37 -0
  467. package/types/redis-frontier.d.ts +48 -0
  468. package/types/redis-queue.d.ts +48 -0
  469. package/types/response-cache.d.ts +38 -0
  470. package/types/responsive-detector.d.ts +29 -0
  471. package/types/retry-policy.d.ts +20 -0
  472. package/types/robots-txt.d.ts +9 -0
  473. package/types/screen-reader.d.ts +15 -0
  474. package/types/screenshot-masker.d.ts +28 -0
  475. package/types/seo-checker.d.ts +33 -0
  476. package/types/service-worker-interceptor.d.ts +36 -0
  477. package/types/session-capture.d.ts +39 -0
  478. package/types/session-export.d.ts +49 -0
  479. package/types/shadow-dom-crawler.d.ts +35 -0
  480. package/types/sitemap-writer.d.ts +21 -0
  481. package/types/sitemap.d.ts +51 -0
  482. package/types/skyvern.d.ts +56 -0
  483. package/types/sqs-queue.d.ts +13 -0
  484. package/types/sso-token.d.ts +29 -0
  485. package/types/state-graph.d.ts +59 -0
  486. package/types/stealth/human-behavior.d.ts +20 -0
  487. package/types/telemetry.d.ts +34 -0
  488. package/types/test-executor.d.ts +84 -0
  489. package/types/tls-fingerprint.d.ts +50 -0
  490. package/types/url-pattern-completer.d.ts +28 -0
  491. package/types/visual-diff-verdict.d.ts +40 -0
  492. package/types/webhook-notifier.d.ts +30 -0
  493. package/types/websocket-detector.d.ts +24 -0
  494. package/types/worker.d.ts +1 -0
@@ -0,0 +1,38 @@
1
+ export interface ResponseCacheEntry {
2
+ url: string;
3
+ etag?: string;
4
+ lastModified?: string;
5
+ cacheControl?: string;
6
+ body: string;
7
+ contentType: string;
8
+ statusCode: number;
9
+ cachedAt: number;
10
+ hitCount: number;
11
+ }
12
+ export declare class ResponseCache {
13
+ private readonly dir;
14
+ private totalHits;
15
+ private totalRequests;
16
+ constructor(cacheDir: string);
17
+ private key;
18
+ private path;
19
+ get(url: string): ResponseCacheEntry | null;
20
+ private read;
21
+ isFresh(url: string): boolean;
22
+ staleIfError(url: string): boolean;
23
+ set(url: string, entry: Omit<ResponseCacheEntry, 'cachedAt' | 'hitCount'>): void;
24
+ getValidationHeaders(url: string): {
25
+ 'If-None-Match'?: string;
26
+ 'If-Modified-Since'?: string;
27
+ };
28
+ markHit(url: string): void;
29
+ invalidate(url: string): void;
30
+ clear(): void;
31
+ stats(): {
32
+ entries: number;
33
+ totalSizeBytes: number;
34
+ hitRate: number;
35
+ totalHits: number;
36
+ };
37
+ prune(maxAgeMs: number): number;
38
+ }
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Responsive design detector — captures page at multiple breakpoints
3
+ * and detects layout differences between desktop and mobile viewports.
4
+ *
5
+ * Breakpoints: mobile(375), tablet(768), desktop(1280), wide(1920)
6
+ * Detects: viewport-specific content visibility, menu collapse state,
7
+ * font size changes, hidden/shown elements.
8
+ */
9
+ import type { Page } from 'playwright';
10
+ export interface ViewportCapture {
11
+ width: number;
12
+ height: number;
13
+ label: 'mobile' | 'tablet' | 'desktop' | 'wide';
14
+ screenshotBuffer?: Buffer;
15
+ visibleTextLength: number;
16
+ hasHamburgerMenu: boolean;
17
+ hiddenElements: number;
18
+ scrollHeight: number;
19
+ }
20
+ export interface ResponsiveReport {
21
+ url: string;
22
+ captures: ViewportCapture[];
23
+ isResponsive: boolean;
24
+ hasMobileMenu: boolean;
25
+ issues: string[];
26
+ }
27
+ export declare function detectResponsive(page: Page, url: string, opts?: {
28
+ captureScreenshots?: boolean;
29
+ }): Promise<ResponsiveReport>;
@@ -0,0 +1,20 @@
1
+ /**
2
+ * Retry policy — configurable exponential backoff with jitter.
3
+ * Covers: 5xx server errors, network timeouts, ECONNREFUSED, ECONNRESET.
4
+ * NOT retried: 4xx client errors (except 429 — handled separately), redirects.
5
+ */
6
+ export interface RetryPolicyOpts {
7
+ maxRetries?: number;
8
+ baseDelayMs?: number;
9
+ maxDelayMs?: number;
10
+ jitterMs?: number;
11
+ retryOn?: number[];
12
+ }
13
+ export interface RetryResult<T> {
14
+ value: T;
15
+ attempts: number;
16
+ totalDelayMs: number;
17
+ }
18
+ export declare function withRetry<T>(fn: (attempt: number) => Promise<T>, opts?: RetryPolicyOpts): Promise<RetryResult<T>>;
19
+ /** Wraps fetch() with retry policy */
20
+ export declare function fetchWithRetry(url: string, init?: RequestInit, retryOpts?: RetryPolicyOpts): Promise<Response>;
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Minimal robots.txt fetcher + parser. No external deps. Caches per origin.
3
+ * Implements the subset of robots.txt spec used in practice:
4
+ * User-agent, Disallow, Allow — longest match wins (RFC 9309).
5
+ */
6
+ export declare class RobotsCache {
7
+ private readonly cache;
8
+ isAllowed(url: string, userAgent?: string): Promise<boolean>;
9
+ }
@@ -0,0 +1,15 @@
1
+ /**
2
+ * Reads a page as a structured outline (cheap + reliable), the way
3
+ * Playwright MCP / the accessibility tree works — NOT by sending pixels to a
4
+ * vision model. Vision is a fallback for image-only pages (added later).
5
+ *
6
+ * Returns the interactive elements found, which the Screen Reader Agent then
7
+ * enriches with MEANING and saves to the Zeta Core.
8
+ */
9
+ export interface RawElement {
10
+ role: string;
11
+ label: string;
12
+ selectorHint: string;
13
+ expectedData?: string;
14
+ }
15
+ export declare function readScreen(page: any): Promise<RawElement[]>;
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Screenshot masker — blur or black-box PII regions in captured screenshots.
3
+ *
4
+ * Strategy: inject a CSS overlay into the page before screenshot capture that
5
+ * covers detected PII-likely elements (input[type=email], input[type=password],
6
+ * elements with PII-containing text) with a blur or solid black mask.
7
+ *
8
+ * This is a CSS-based approach (no image processing library needed):
9
+ * 1. Identify likely-PII DOM elements via selectors
10
+ * 2. Add an overlaid <div> with blur/black filter at each element's bounding box
11
+ * 3. Take screenshot
12
+ * 4. Remove overlays
13
+ *
14
+ * Falls back gracefully if elements are not found or overlay injection fails.
15
+ */
16
+ import type { Page } from 'playwright';
17
+ export interface MaskingOpts {
18
+ /** Additional CSS selectors to mask (in addition to defaults) */
19
+ extraSelectors?: string[];
20
+ /** Masking style: 'blur' or 'blackout' */
21
+ style?: 'blur' | 'blackout';
22
+ /** Blur strength in px (default 8) */
23
+ blurPx?: number;
24
+ }
25
+ export declare function maskScreenshot(page: Page, opts?: MaskingOpts): Promise<{
26
+ maskedCount: number;
27
+ screenshot: Buffer;
28
+ }>;
@@ -0,0 +1,33 @@
1
+ /**
2
+ * SEO indexability checker — determines if a page is crawlable and indexable.
3
+ *
4
+ * Checks:
5
+ * 1. X-Robots-Tag response header (noindex, nofollow, noarchive)
6
+ * 2. <meta name="robots"> tag (noindex, nofollow)
7
+ * 3. Canonical URL (self-canonical vs cross-canonical)
8
+ * 4. rel="nofollow" on outbound links
9
+ * 5. hreflang tags (multilingual)
10
+ * 6. Open Graph completeness (og:title, og:description, og:image)
11
+ * 7. Structured data presence (JSON-LD)
12
+ * 8. Meta description
13
+ * 9. Title tag length (< 60 chars recommended)
14
+ * 10. URL structure (query params, fragments, excessive depth)
15
+ */
16
+ import type { Page } from 'playwright';
17
+ export interface SeoIssue {
18
+ rule: string;
19
+ severity: 'critical' | 'warning' | 'info';
20
+ message: string;
21
+ value?: string;
22
+ }
23
+ export interface SeoCheckResult {
24
+ url: string;
25
+ isIndexable: boolean;
26
+ canonicalUrl?: string;
27
+ isSelfCanonical?: boolean;
28
+ hreflang: Record<string, string>;
29
+ issues: SeoIssue[];
30
+ score: number;
31
+ checkedAt: string;
32
+ }
33
+ export declare function checkSeo(page: Page, url: string, responseHeaders?: Record<string, string>): Promise<SeoCheckResult>;
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Service Worker detection — inspects SW registrations, cache storage, background
3
+ * sync queues, and PWA installability for a crawled page.
4
+ * Surfaces offline-first contracts and push-capable surfaces that HTTP-only
5
+ * crawlers cannot observe.
6
+ */
7
+ export interface SwRegistration {
8
+ scope: string;
9
+ scriptURL: string | null;
10
+ state: 'active' | 'installing' | 'waiting' | 'none';
11
+ updateViaCache: string;
12
+ }
13
+ export interface SwCacheEntry {
14
+ cacheName: string;
15
+ entryCount: number;
16
+ sampleUrls: string[];
17
+ }
18
+ export interface BackgroundSyncInfo {
19
+ scope: string;
20
+ pendingTags: string[];
21
+ }
22
+ export interface PwaInfo {
23
+ isInstallable: boolean;
24
+ manifestUrl: string | null;
25
+ hasPushPermission: boolean;
26
+ }
27
+ export interface ServiceWorkerResult {
28
+ supported: boolean;
29
+ active: boolean;
30
+ registrations: SwRegistration[];
31
+ cacheStorages: SwCacheEntry[];
32
+ backgroundSync: BackgroundSyncInfo[];
33
+ pwa: PwaInfo;
34
+ totalCachedEntries: number;
35
+ }
36
+ export declare function inspectServiceWorker(page: any): Promise<ServiceWorkerResult>;
@@ -0,0 +1,39 @@
1
+ /**
2
+ * Opens a headed (visible) browser at appUrl and returns a captureId.
3
+ * The user logs in manually, then calls completeSessionCapture() to harvest storageState.
4
+ * Sessions expire after 10 minutes if not completed.
5
+ *
6
+ * Throws 'HEADLESS_ENV' if running in Docker/Linux without a display server.
7
+ * The API route surfaces this as a 503 so the frontend can switch to manual JSON paste.
8
+ */
9
+ export declare function startSessionCapture(appUrl: string): Promise<string>;
10
+ /** Captures storageState from the open browser, closes it, and returns the JSON string. */
11
+ export declare function completeSessionCapture(captureId: string): Promise<string>;
12
+ /** Closes the browser without capturing — called on Cancel. */
13
+ export declare function cancelSessionCapture(captureId: string): Promise<void>;
14
+ /**
15
+ * Headless login + session capture — works on Docker/cloud (no display needed).
16
+ * Tries DOM-based login first (fast, free). If it fails and llmConfig is provided,
17
+ * falls back to Stagehand AI-guided login (handles modals, multi-step, marketing pages).
18
+ *
19
+ * Returns { storageState, cookies, localStorage } on success.
20
+ * Throws a human-readable error on failure.
21
+ */
22
+ export declare function validateAndCaptureSession(appUrl: string, creds: {
23
+ username: string;
24
+ password: string;
25
+ totpSecret?: string;
26
+ httpBasicUsername?: string;
27
+ httpBasicPassword?: string;
28
+ }, llmConfig?: {
29
+ modelName: string;
30
+ apiKey: string;
31
+ }, captchaOpts?: {
32
+ apiKey?: string;
33
+ provider?: '2captcha' | 'capsolver';
34
+ }, loginInstructions?: string): Promise<{
35
+ storageState: string;
36
+ cookies: number;
37
+ localStorage: number;
38
+ finalUrl?: string;
39
+ }>;
@@ -0,0 +1,49 @@
1
+ /**
2
+ * Converts a Playwright storageState (the format ZeTa already captures via
3
+ * session-capture.ts/live-view.ts and stores per-project) into snippets for
4
+ * other automation frameworks. Playwright's storageState is the de-facto
5
+ * standard for browser session export — see docs/UNIVERSAL-SESSION-CAPTURE-
6
+ * STRATEGY.md — so every other framework's import mechanism is a small,
7
+ * mechanical conversion away, not a new capture flow.
8
+ *
9
+ * Pure, dependency-free functions — no I/O, no network, fully unit-testable.
10
+ */
11
+ export interface StorageStateCookie {
12
+ name: string;
13
+ value: string;
14
+ domain: string;
15
+ path: string;
16
+ expires?: number;
17
+ httpOnly?: boolean;
18
+ secure?: boolean;
19
+ sameSite?: 'Strict' | 'Lax' | 'None';
20
+ }
21
+ export interface StorageStateOrigin {
22
+ origin: string;
23
+ localStorage: Array<{
24
+ name: string;
25
+ value: string;
26
+ }>;
27
+ }
28
+ export interface StorageState {
29
+ cookies: StorageStateCookie[];
30
+ origins: StorageStateOrigin[];
31
+ }
32
+ export type SessionExportFormat = 'cypress' | 'selenium-python' | 'webdriverio';
33
+ /** cy.setCookie() + localStorage.setItem() inside a reusable custom command,
34
+ * following Cypress's documented cy.session() pattern (see references in
35
+ * docs/UNIVERSAL-SESSION-CAPTURE-STRATEGY.md). */
36
+ export declare function toCypressSnippet(state: StorageState): string;
37
+ /** driver.add_cookie() (Selenium requires the browser already be on the
38
+ * cookie's domain before adding it) + execute_script() for localStorage,
39
+ * per Selenium's documented Cookie API. */
40
+ export declare function toSeleniumPythonDict(state: StorageState): string;
41
+ /** browser.setCookies() accepts an array shape very close to Playwright's own
42
+ * cookie objects (per WebdriverIO's documented API) — closest 1:1 mapping
43
+ * of the three target formats. */
44
+ export declare function toWebdriverIOSnippet(state: StorageState): string;
45
+ export declare function exportSessionAs(format: SessionExportFormat, state: StorageState): {
46
+ content: string;
47
+ filename: string;
48
+ mimeType: string;
49
+ };
@@ -0,0 +1,35 @@
1
+ /**
2
+ * Shadow DOM crawler — enumerate shadow hosts and detect web component libraries.
3
+ *
4
+ * Walks the live DOM recursively (up to depth 8) to find all open shadow roots
5
+ * and custom elements with closed shadow roots, then classifies the component
6
+ * framework in use. Results feed ZeTa's test generator so it can account for
7
+ * elements that are invisible to standard Playwright locators.
8
+ */
9
+ export interface ShadowInteractiveEl {
10
+ role: string;
11
+ label: string;
12
+ type: string;
13
+ tagName: string;
14
+ }
15
+ export interface ShadowHostInfo {
16
+ tagName: string;
17
+ hostId: string;
18
+ shadowMode: 'open' | 'closed';
19
+ depth: number;
20
+ interactiveCount: number;
21
+ interactiveElements: ShadowInteractiveEl[];
22
+ slotNames: string[];
23
+ childShadowHosts: number;
24
+ }
25
+ export interface ShadowDomCrawlResult {
26
+ shadowHostCount: number;
27
+ openShadowCount: number;
28
+ closedShadowCount: number;
29
+ maxNestingDepth: number;
30
+ hosts: ShadowHostInfo[];
31
+ totalInteractiveInShadow: number;
32
+ detectedLibraries: string[];
33
+ blindSpots: string[];
34
+ }
35
+ export declare function crawlShadowDom(page: any): Promise<ShadowDomCrawlResult>;
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Sitemap writer — generate XML sitemap from crawled screens.
3
+ *
4
+ * Compliant with sitemaps.org protocol (sitemap index + individual sitemap files).
5
+ * Splits into 50k-URL files per the spec when needed.
6
+ * Used for: SEO audit export, future crawl seed, and crawler/robots.txt comparison.
7
+ */
8
+ export interface SitemapUrl {
9
+ loc: string;
10
+ lastmod?: string;
11
+ changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
12
+ priority?: number;
13
+ }
14
+ export declare function writeSitemap(urls: SitemapUrl[], outputPath: string): Promise<void>;
15
+ /** Convert crawled screen records to SitemapUrl objects */
16
+ export declare function screensToSitemapUrls(screens: Array<{
17
+ url: string;
18
+ updatedAt?: Date;
19
+ depth?: number;
20
+ }>): SitemapUrl[];
21
+ export declare function buildSitemapString(urls: SitemapUrl[]): string;
@@ -0,0 +1,51 @@
1
+ import { ProxyAgent } from 'undici';
2
+ /**
3
+ * Same-site check that tolerates www/non-www and http/https differences — a sitemap on
4
+ * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).
5
+ */
6
+ export declare function sameSite(candidateUrl: string, siteOrigin: string): boolean;
7
+ /**
8
+ * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap
9
+ * document (either a <urlset> or a <sitemapindex>), filtered to same-origin
10
+ * and deduped. Pure function — no I/O — for direct unit testing without a
11
+ * real HTTP fetch.
12
+ */
13
+ export declare function extractUrlsFromParsedSitemap(parsed: any, origin: string, maxUrls?: number): {
14
+ pageUrls: string[];
15
+ subSitemapUrls: string[];
16
+ };
17
+ /**
18
+ * Fetches and parses a sitemap (or sitemap index, recursed up to a small
19
+ * depth) into a flat, deduped list of same-origin URLs.
20
+ */
21
+ export declare function fetchSitemapUrls(sitemapUrl: string, origin: string, depth?: number, dispatcher?: any, maxUrls?: number): Promise<string[]>;
22
+ /**
23
+ * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an
24
+ * explicit override) and returns the URLs it lists. Never throws.
25
+ */
26
+ export declare function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]>;
27
+ /**
28
+ * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because
29
+ * DataImpulse-style usernames contain commas (country codes) that break new URL().
30
+ * Returns undefined for no/auto proxy — the caller then fetches directly.
31
+ */
32
+ export declare function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined;
33
+ /** Extract same-origin `Sitemap:` directives from a robots.txt body. */
34
+ export declare function parseRobotsSitemaps(robotsTxt: string, origin: string): string[];
35
+ export interface UrlCandidate {
36
+ url: string;
37
+ source: 'sitemap' | 'robots' | 'link';
38
+ }
39
+ /**
40
+ * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap
41
+ * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped
42
+ * candidate list the user can review and seed before launching the Playwright crawl.
43
+ * Never throws — bot-protected sites (Akamai/Cloudflare) simply yield an empty list.
44
+ */
45
+ export declare function discoverProjectUrls(appUrl: string, opts?: {
46
+ sitemapUrl?: string;
47
+ proxyUrl?: string;
48
+ }): Promise<{
49
+ candidates: UrlCandidate[];
50
+ sitemapFound: boolean;
51
+ }>;
@@ -0,0 +1,56 @@
1
+ /**
2
+ * Skyvern Vision Fallback
3
+ *
4
+ * Skyvern combines computer vision + LLM reasoning to navigate sites that
5
+ * defeat DOM-based approaches: CAPTCHAs, legacy apps, visual-only login flows,
6
+ * government portals. It's the "break glass" option when Stagehand fails.
7
+ *
8
+ * Production setup (Phase 4+):
9
+ * 1. Self-host: https://github.com/Skyvern-AI/skyvern
10
+ * 2. Or use cloud: https://app.skyvern.com
11
+ * 3. Set SKYVERN_API_URL + SKYVERN_API_KEY in .env
12
+ *
13
+ * Pricing note: vision tasks cost more per step than DOM-based Stagehand.
14
+ * Use only when the fast path fails.
15
+ */
16
+ export interface SkyvernTask {
17
+ taskDescription: string;
18
+ url: string;
19
+ credentials?: {
20
+ username: string;
21
+ password: string;
22
+ oneTimeCode?: string;
23
+ };
24
+ }
25
+ export interface SkyvernConfig {
26
+ apiUrl: string;
27
+ apiKey: string;
28
+ llmKey?: string;
29
+ llmApiKey?: string;
30
+ }
31
+ export interface SkyvernStep {
32
+ order: number;
33
+ action_type: string;
34
+ element_description?: string;
35
+ reasoning?: string;
36
+ success: boolean;
37
+ screenshot_url?: string;
38
+ duration_ms?: number;
39
+ error?: string;
40
+ }
41
+ export interface SkyvernNavigateResult {
42
+ screenshotUrls: string[];
43
+ visitedUrls: string[];
44
+ status: 'success' | 'failed' | 'partial';
45
+ steps?: SkyvernStep[];
46
+ taskId?: string;
47
+ durationMs?: number;
48
+ triggerReason?: string;
49
+ }
50
+ /**
51
+ * Gap 5: Post-login navigation via Skyvern vision for apps where DOM analysis
52
+ * cannot discover screens (canvas, WebGL, legacy Flash-replacement stacks).
53
+ * Skyvern drives the browser via screenshot + LLM, returning visited page list.
54
+ */
55
+ export declare function skyvernNavigate(startUrl: string, navigationGoals: string[], tenantConfig?: SkyvernConfig, triggerReason?: string): Promise<SkyvernNavigateResult>;
56
+ export declare function skyvernFallback(task: SkyvernTask, tenantConfig?: SkyvernConfig): Promise<'success' | 'failed'>;
@@ -0,0 +1,13 @@
1
+ export declare function sendJobToSqs(jobId: string): Promise<void>;
2
+ /** Returns the approximate number of messages in the SQS DLQ. */
3
+ export declare function getDlqStats(): Promise<{
4
+ approximateMessageCount: number;
5
+ dlqUrl: string | null;
6
+ }>;
7
+ /**
8
+ * Redrive up to `maxMessages` messages from the SQS DLQ back to the main queue.
9
+ * Each message is received from the DLQ, re-sent to the main queue, then deleted
10
+ * from the DLQ. Returns the number of messages successfully redriven.
11
+ */
12
+ export declare function redriveFromDlq(maxMessages?: number): Promise<number>;
13
+ export declare function startSqsWorker(): Promise<void>;
@@ -0,0 +1,29 @@
1
+ /**
2
+ * SSO token retrieval for Okta and Azure AD.
3
+ * Uses OAuth2 ROPC (Resource Owner Password Credentials) flow to get access tokens
4
+ * without browser-based login.
5
+ *
6
+ * Note: ROPC is deprecated in OAuth 2.1 but still supported by Okta and Azure AD
7
+ * for service accounts and automated testing scenarios.
8
+ */
9
+ export type SsoProvider = 'okta' | 'azure_ad' | 'generic_oidc';
10
+ export interface SsoConfig {
11
+ provider: SsoProvider;
12
+ /** Okta domain (e.g. 'mycompany.okta.com') or Azure tenant ID */
13
+ domain: string;
14
+ /** OAuth2 client ID */
15
+ clientId: string;
16
+ /** OAuth2 client secret (for client_credentials or confidential ROPC) */
17
+ clientSecret?: string;
18
+ /** Scopes to request (space-separated). Default: 'openid profile' */
19
+ scope?: string;
20
+ }
21
+ /**
22
+ * Get an access token using ROPC flow (username + password → token).
23
+ * Returns the access token string.
24
+ */
25
+ export declare function getSsoToken(config: SsoConfig, username: string, password: string): Promise<string>;
26
+ /**
27
+ * Get a client credentials token (no user — for service accounts).
28
+ */
29
+ export declare function getSsoClientCredentialsToken(config: SsoConfig): Promise<string>;
@@ -0,0 +1,59 @@
1
+ import type { Page } from 'playwright';
2
+ export declare function simHash32(text: string): number;
3
+ export declare function hammingDistance32(a: number, b: number): number;
4
+ export declare function normalizeUrl(raw: string): string;
5
+ export declare function computeStateId(normalizedUrl: string, hash: number): string;
6
+ export interface UIStateNode {
7
+ id: string;
8
+ url: string;
9
+ normalizedUrl: string;
10
+ title: string;
11
+ interactiveElementCount: number;
12
+ domFingerprint: string;
13
+ simHashValue: number;
14
+ visitedAt: string;
15
+ }
16
+ export interface StateTransition {
17
+ fromId: string;
18
+ toId: string;
19
+ action: string;
20
+ label: string;
21
+ triggeredAt: string;
22
+ }
23
+ export interface CoverageReport {
24
+ totalStates: number;
25
+ totalTransitions: number;
26
+ uniqueUrls: number;
27
+ interactiveElementsTotal: number;
28
+ estimatedCoveragePercent: number;
29
+ }
30
+ export declare function fingerprintPage(page: Page, visitedAt: string): Promise<{
31
+ fingerprint: string;
32
+ visibleText: string;
33
+ interactiveCount: number;
34
+ simHashValue: number;
35
+ }>;
36
+ export declare class ScreenTransitionGraph {
37
+ private states;
38
+ private transitions;
39
+ /** Maps simHash value → array of state ids with that hash (for near-dup lookup) */
40
+ private hashIndex;
41
+ addState(node: UIStateNode): {
42
+ added: boolean;
43
+ duplicateOf?: string;
44
+ };
45
+ addTransition(fromId: string, toId: string, action: string, label: string, triggeredAt: string): void;
46
+ isDuplicate(url: string, visibleText: string, threshold?: number): boolean;
47
+ getCoverageReport(): CoverageReport;
48
+ toJSON(): {
49
+ states: UIStateNode[];
50
+ transitions: StateTransition[];
51
+ };
52
+ static fromJSON(data: {
53
+ states: UIStateNode[];
54
+ transitions: StateTransition[];
55
+ }): ScreenTransitionGraph;
56
+ getState(id: string): UIStateNode | undefined;
57
+ getAllStates(): UIStateNode[];
58
+ getTransitionsFrom(id: string): StateTransition[];
59
+ }
@@ -0,0 +1,20 @@
1
+ import type { Page } from 'playwright';
2
+ export declare function humanDelay(min?: number, max?: number): Promise<void>;
3
+ export declare function simulateMouseMovement(page: Page, targetX: number, targetY: number): Promise<void>;
4
+ export declare function simulateMouseHover(page: Page, selector: string): Promise<void>;
5
+ export declare function simulateClick(page: Page, selector: string): Promise<void>;
6
+ export declare function injectFingerprintOverrides(page: Page): Promise<void>;
7
+ export declare function infiniteScrollToBottom(page: Page, opts?: {
8
+ maxScrolls?: number;
9
+ scrollDelay?: number;
10
+ stabilizeMs?: number;
11
+ }, rawPage?: any): Promise<{
12
+ scrollCount: number;
13
+ heightReached: number;
14
+ }>;
15
+ export declare function extractIframeContent(page: Page): Promise<Array<{
16
+ src: string;
17
+ content: string;
18
+ markdown: string;
19
+ }>>;
20
+ export declare function pierceShadowDom(page: Page): Promise<string>;
@@ -0,0 +1,34 @@
1
+ /**
2
+ * Lightweight OTel-compatible tracing for Zeta crawler.
3
+ *
4
+ * Emits spans as structured JSON logs compatible with OpenTelemetry format.
5
+ * Drop-in upgrade path: replace this module with @opentelemetry/sdk-node when needed.
6
+ *
7
+ * Usage:
8
+ * const span = startSpan('crawl.url', { url, engine });
9
+ * // ... do work ...
10
+ * endSpan(span, { screensFound: 3 });
11
+ */
12
+ export interface Span {
13
+ traceId: string;
14
+ spanId: string;
15
+ parentSpanId?: string;
16
+ name: string;
17
+ startTimeMs: number;
18
+ attributes: Record<string, string | number | boolean>;
19
+ }
20
+ export interface FinishedSpan extends Span {
21
+ endTimeMs: number;
22
+ durationMs: number;
23
+ status: 'ok' | 'error';
24
+ error?: string;
25
+ }
26
+ /** Start a new span. If a trace is already active, the new span is a child. */
27
+ export declare function startSpan(name: string, attributes?: Record<string, string | number | boolean>): Span;
28
+ /** End a span and emit it as a structured log line. */
29
+ export declare function endSpan(span: Span, extraAttributes?: Record<string, string | number | boolean>, error?: Error): FinishedSpan;
30
+ /** Convenience: wrap an async function in a span. */
31
+ export declare function withSpan<T>(name: string, attributes: Record<string, string | number | boolean>, fn: (span: Span) => Promise<T>): Promise<T>;
32
+ export declare function increment(metric: string, value?: number, tags?: Record<string, string>): void;
33
+ /** Flush counters as a structured log — call periodically (e.g. at job end). */
34
+ export declare function flushMetrics(): Record<string, number>;