@taodigital/crawler 0.0.0-stage → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +82 -0
- package/README.md +10 -2
- package/dist/accessibility-auditor.d.ts +29 -0
- package/dist/accessibility-auditor.js +57 -0
- package/dist/accessibility-auditor.js.map +7 -0
- package/dist/accessibility-tree-crawler.d.ts +44 -0
- package/dist/accessibility-tree-crawler.js +156 -0
- package/dist/accessibility-tree-crawler.js.map +7 -0
- package/dist/advanced-ui-probes.d.ts +47 -0
- package/dist/advanced-ui-probes.js +310 -0
- package/dist/advanced-ui-probes.js.map +7 -0
- package/dist/ai-login.d.ts +32 -0
- package/dist/ai-login.js +250 -0
- package/dist/ai-login.js.map +7 -0
- package/dist/auth-token-rotator.d.ts +47 -0
- package/dist/auth-token-rotator.js +58 -0
- package/dist/auth-token-rotator.js.map +7 -0
- package/dist/auth.d.ts +20 -0
- package/dist/auth.js +348 -0
- package/dist/auth.js.map +7 -0
- package/dist/browser-extension.d.ts +37 -0
- package/dist/browser-extension.js +113 -0
- package/dist/browser-extension.js.map +7 -0
- package/dist/captcha-solver.d.ts +16 -0
- package/dist/captcha-solver.js +105 -0
- package/dist/captcha-solver.js.map +7 -0
- package/dist/color-contrast.d.ts +30 -0
- package/dist/color-contrast.js +78 -0
- package/dist/color-contrast.js.map +7 -0
- package/dist/content-chunker.d.ts +18 -0
- package/dist/content-chunker.js +152 -0
- package/dist/content-chunker.js.map +7 -0
- package/dist/content-dedup.d.ts +23 -0
- package/dist/content-dedup.js +60 -0
- package/dist/content-dedup.js.map +7 -0
- package/dist/cookie-consent.d.ts +25 -0
- package/dist/cookie-consent.js +78 -0
- package/dist/cookie-consent.js.map +7 -0
- package/dist/cookie-jar.d.ts +20 -0
- package/dist/cookie-jar.js +61 -0
- package/dist/cookie-jar.js.map +7 -0
- package/dist/crawl-session-manager.d.ts +44 -0
- package/dist/crawl-session-manager.js +145 -0
- package/dist/crawl-session-manager.js.map +7 -0
- package/dist/crawler.d.ts +233 -0
- package/dist/crawler.js +6239 -0
- package/dist/crawler.js.map +7 -0
- package/dist/crud-interaction.d.ts +31 -0
- package/dist/crud-interaction.js +264 -0
- package/dist/crud-interaction.js.map +7 -0
- package/dist/custom-js-injector.d.ts +42 -0
- package/dist/custom-js-injector.js +50 -0
- package/dist/custom-js-injector.js.map +7 -0
- package/dist/dark-mode-capture.d.ts +23 -0
- package/dist/dark-mode-capture.js +75 -0
- package/dist/dark-mode-capture.js.map +7 -0
- package/dist/dead-link-detector.d.ts +37 -0
- package/dist/dead-link-detector.js +50 -0
- package/dist/dead-link-detector.js.map +7 -0
- package/dist/dropdown-enumerator.d.ts +20 -0
- package/dist/dropdown-enumerator.js +230 -0
- package/dist/dropdown-enumerator.js.map +7 -0
- package/dist/element-hash.d.ts +9 -0
- package/dist/element-hash.js +9 -0
- package/dist/element-hash.js.map +7 -0
- package/dist/email-otp.d.ts +42 -0
- package/dist/email-otp.js +177 -0
- package/dist/email-otp.js.map +7 -0
- package/dist/engine-router.d.ts +47 -0
- package/dist/engine-router.js +94 -0
- package/dist/engine-router.js.map +7 -0
- package/dist/export/bulk-export.d.ts +26 -0
- package/dist/export/bulk-export.js +76 -0
- package/dist/export/bulk-export.js.map +7 -0
- package/dist/export/vector-export.d.ts +76 -0
- package/dist/export/vector-export.js +85 -0
- package/dist/export/vector-export.js.map +7 -0
- package/dist/extractors/advanced.d.ts +64 -0
- package/dist/extractors/advanced.js +265 -0
- package/dist/extractors/advanced.js.map +7 -0
- package/dist/extractors/artifact-envelope.d.ts +63 -0
- package/dist/extractors/artifact-envelope.js +63 -0
- package/dist/extractors/artifact-envelope.js.map +7 -0
- package/dist/extractors/branding-extractor.d.ts +46 -0
- package/dist/extractors/branding-extractor.js +104 -0
- package/dist/extractors/branding-extractor.js.map +7 -0
- package/dist/extractors/summary-extractor.d.ts +10 -0
- package/dist/extractors/summary-extractor.js +36 -0
- package/dist/extractors/summary-extractor.js.map +7 -0
- package/dist/failure-analysis-agent.d.ts +16 -0
- package/dist/failure-analysis-agent.js +42 -0
- package/dist/failure-analysis-agent.js.map +7 -0
- package/dist/feature-flag-detector.d.ts +30 -0
- package/dist/feature-flag-detector.js +55 -0
- package/dist/feature-flag-detector.js.map +7 -0
- package/dist/filter-pagination-crawler.d.ts +52 -0
- package/dist/filter-pagination-crawler.js +299 -0
- package/dist/filter-pagination-crawler.js.map +7 -0
- package/dist/form-detector.d.ts +34 -0
- package/dist/form-detector.js +58 -0
- package/dist/form-detector.js.map +7 -0
- package/dist/frontier.d.ts +40 -0
- package/dist/frontier.js +103 -0
- package/dist/frontier.js.map +7 -0
- package/dist/gap-pass.d.ts +36 -0
- package/dist/gap-pass.js +115 -0
- package/dist/gap-pass.js.map +7 -0
- package/dist/graphql-detector.d.ts +39 -0
- package/dist/graphql-detector.js +86 -0
- package/dist/graphql-detector.js.map +7 -0
- package/dist/http-engine.d.ts +80 -0
- package/dist/http-engine.js +318 -0
- package/dist/http-engine.js.map +7 -0
- package/dist/incremental-diff.d.ts +116 -0
- package/dist/incremental-diff.js +137 -0
- package/dist/incremental-diff.js.map +7 -0
- package/dist/index.d.ts +52 -0
- package/dist/index.js +83 -0
- package/dist/index.js.map +7 -0
- package/dist/input-fuzzer.d.ts +30 -0
- package/dist/input-fuzzer.js +105 -0
- package/dist/input-fuzzer.js.map +7 -0
- package/dist/interactive-probe.d.ts +144 -0
- package/dist/interactive-probe.js +865 -0
- package/dist/interactive-probe.js.map +7 -0
- package/dist/job-queue.d.ts +297 -0
- package/dist/job-queue.js +3812 -0
- package/dist/job-queue.js.map +7 -0
- package/dist/js-coverage.d.ts +29 -0
- package/dist/js-coverage.js +56 -0
- package/dist/js-coverage.js.map +7 -0
- package/dist/js-router-detector.d.ts +17 -0
- package/dist/js-router-detector.js +90 -0
- package/dist/js-router-detector.js.map +7 -0
- package/dist/language-detector.d.ts +22 -0
- package/dist/language-detector.js +57 -0
- package/dist/language-detector.js.map +7 -0
- package/dist/link-classifier.d.ts +23 -0
- package/dist/link-classifier.js +40 -0
- package/dist/link-classifier.js.map +7 -0
- package/dist/live-view.d.ts +36 -0
- package/dist/live-view.js +105 -0
- package/dist/live-view.js.map +7 -0
- package/dist/locator-extractor.d.ts +46 -0
- package/dist/locator-extractor.js +220 -0
- package/dist/locator-extractor.js.map +7 -0
- package/dist/logger.d.ts +2 -0
- package/dist/logger.js +10 -0
- package/dist/logger.js.map +7 -0
- package/dist/markdown-extract.d.ts +8 -0
- package/dist/markdown-extract.js +54 -0
- package/dist/markdown-extract.js.map +7 -0
- package/dist/multi-context-handler.d.ts +55 -0
- package/dist/multi-context-handler.js +182 -0
- package/dist/multi-context-handler.js.map +7 -0
- package/dist/native-crawler.d.ts +69 -0
- package/dist/native-crawler.js +839 -0
- package/dist/native-crawler.js.map +7 -0
- package/dist/nested-window-crawler.d.ts +41 -0
- package/dist/nested-window-crawler.js +172 -0
- package/dist/nested-window-crawler.js.map +7 -0
- package/dist/network-action-correlator.d.ts +47 -0
- package/dist/network-action-correlator.js +209 -0
- package/dist/network-action-correlator.js.map +7 -0
- package/dist/network-observer.d.ts +49 -0
- package/dist/network-observer.js +149 -0
- package/dist/network-observer.js.map +7 -0
- package/dist/network-throttle.d.ts +25 -0
- package/dist/network-throttle.js +34 -0
- package/dist/network-throttle.js.map +7 -0
- package/dist/openapi-generator.d.ts +49 -0
- package/dist/openapi-generator.js +89 -0
- package/dist/openapi-generator.js.map +7 -0
- package/dist/pdf-extractor.d.ts +28 -0
- package/dist/pdf-extractor.js +81 -0
- package/dist/pdf-extractor.js.map +7 -0
- package/dist/performance-metrics-collector.d.ts +50 -0
- package/dist/performance-metrics-collector.js +155 -0
- package/dist/performance-metrics-collector.js.map +7 -0
- package/dist/performance-metrics.d.ts +52 -0
- package/dist/performance-metrics.js +83 -0
- package/dist/performance-metrics.js.map +7 -0
- package/dist/phash.d.ts +50 -0
- package/dist/phash.js +136 -0
- package/dist/phash.js.map +7 -0
- package/dist/pii-scrubber.d.ts +21 -0
- package/dist/pii-scrubber.js +75 -0
- package/dist/pii-scrubber.js.map +7 -0
- package/dist/playwright-recorder.d.ts +39 -0
- package/dist/playwright-recorder.js +123 -0
- package/dist/playwright-recorder.js.map +7 -0
- package/dist/policies/crawl-safety.d.ts +9 -0
- package/dist/policies/crawl-safety.js +41 -0
- package/dist/policies/crawl-safety.js.map +7 -0
- package/dist/profiles/domain-profile.d.ts +19 -0
- package/dist/profiles/domain-profile.js +160 -0
- package/dist/profiles/domain-profile.js.map +7 -0
- package/dist/proxy.d.ts +50 -0
- package/dist/proxy.js +165 -0
- package/dist/proxy.js.map +7 -0
- package/dist/rate-limiter.d.ts +31 -0
- package/dist/rate-limiter.js +64 -0
- package/dist/rate-limiter.js.map +7 -0
- package/dist/redirect-tracker.d.ts +37 -0
- package/dist/redirect-tracker.js +65 -0
- package/dist/redirect-tracker.js.map +7 -0
- package/dist/redis-frontier.d.ts +48 -0
- package/dist/redis-frontier.js +126 -0
- package/dist/redis-frontier.js.map +7 -0
- package/dist/redis-queue.d.ts +48 -0
- package/dist/redis-queue.js +165 -0
- package/dist/redis-queue.js.map +7 -0
- package/dist/response-cache.d.ts +38 -0
- package/dist/response-cache.js +140 -0
- package/dist/response-cache.js.map +7 -0
- package/dist/responsive-detector.d.ts +29 -0
- package/dist/responsive-detector.js +46 -0
- package/dist/responsive-detector.js.map +7 -0
- package/dist/retry-policy.d.ts +20 -0
- package/dist/retry-policy.js +54 -0
- package/dist/retry-policy.js.map +7 -0
- package/dist/robots-txt.d.ts +9 -0
- package/dist/robots-txt.js +77 -0
- package/dist/robots-txt.js.map +7 -0
- package/dist/screen-reader.d.ts +15 -0
- package/dist/screen-reader.js +48 -0
- package/dist/screen-reader.js.map +7 -0
- package/dist/screenshot-masker.d.ts +28 -0
- package/dist/screenshot-masker.js +62 -0
- package/dist/screenshot-masker.js.map +7 -0
- package/dist/seo-checker.d.ts +33 -0
- package/dist/seo-checker.js +91 -0
- package/dist/seo-checker.js.map +7 -0
- package/dist/service-worker-interceptor.d.ts +36 -0
- package/dist/service-worker-interceptor.js +136 -0
- package/dist/service-worker-interceptor.js.map +7 -0
- package/dist/session-capture.d.ts +39 -0
- package/dist/session-capture.js +274 -0
- package/dist/session-capture.js.map +7 -0
- package/dist/session-export.d.ts +49 -0
- package/dist/session-export.js +117 -0
- package/dist/session-export.js.map +7 -0
- package/dist/shadow-dom-crawler.d.ts +35 -0
- package/dist/shadow-dom-crawler.js +132 -0
- package/dist/shadow-dom-crawler.js.map +7 -0
- package/dist/sitemap-writer.d.ts +21 -0
- package/dist/sitemap-writer.js +47 -0
- package/dist/sitemap-writer.js.map +7 -0
- package/dist/sitemap.d.ts +51 -0
- package/dist/sitemap.js +160 -0
- package/dist/sitemap.js.map +7 -0
- package/dist/skyvern.d.ts +56 -0
- package/dist/skyvern.js +129 -0
- package/dist/skyvern.js.map +7 -0
- package/dist/sqs-queue.d.ts +13 -0
- package/dist/sqs-queue.js +115 -0
- package/dist/sqs-queue.js.map +7 -0
- package/dist/sso-token.d.ts +29 -0
- package/dist/sso-token.js +76 -0
- package/dist/sso-token.js.map +7 -0
- package/dist/state-graph.d.ts +59 -0
- package/dist/state-graph.js +193 -0
- package/dist/state-graph.js.map +7 -0
- package/dist/stealth/human-behavior.d.ts +20 -0
- package/dist/stealth/human-behavior.js +202 -0
- package/dist/stealth/human-behavior.js.map +7 -0
- package/dist/telemetry.d.ts +34 -0
- package/dist/telemetry.js +77 -0
- package/dist/telemetry.js.map +7 -0
- package/dist/test-executor.d.ts +81 -0
- package/dist/test-executor.js +443 -0
- package/dist/test-executor.js.map +7 -0
- package/dist/tls-fingerprint.d.ts +50 -0
- package/dist/tls-fingerprint.js +81 -0
- package/dist/tls-fingerprint.js.map +7 -0
- package/dist/url-pattern-completer.d.ts +28 -0
- package/dist/url-pattern-completer.js +122 -0
- package/dist/url-pattern-completer.js.map +7 -0
- package/dist/visual-diff-verdict.d.ts +40 -0
- package/dist/visual-diff-verdict.js +27 -0
- package/dist/visual-diff-verdict.js.map +7 -0
- package/dist/webhook-notifier.d.ts +30 -0
- package/dist/webhook-notifier.js +76 -0
- package/dist/webhook-notifier.js.map +7 -0
- package/dist/websocket-detector.d.ts +24 -0
- package/dist/websocket-detector.js +60 -0
- package/dist/websocket-detector.js.map +7 -0
- package/dist/worker.d.ts +1 -0
- package/dist/worker.js +33 -0
- package/dist/worker.js.map +7 -0
- package/package.json +49 -4
- package/src/accessibility-auditor.ts +102 -0
- package/src/accessibility-tree-crawler.ts +246 -0
- package/src/advanced-ui-probes.ts +470 -0
- package/src/ai-login.ts +298 -0
- package/src/auth-token-rotator.ts +89 -0
- package/src/auth.ts +418 -0
- package/src/browser-extension.ts +186 -0
- package/src/captcha-solver.ts +143 -0
- package/src/color-contrast.ts +116 -0
- package/src/content-chunker.ts +170 -0
- package/src/content-dedup.ts +99 -0
- package/src/cookie-consent.ts +106 -0
- package/src/cookie-jar.ts +91 -0
- package/src/crawl-session-manager.ts +231 -0
- package/src/crawler.ts +7123 -0
- package/src/crawler.ts.rej +12 -0
- package/src/crud-interaction.ts +386 -0
- package/src/custom-js-injector.ts +93 -0
- package/src/dark-mode-capture.ts +110 -0
- package/src/dead-link-detector.ts +76 -0
- package/src/dropdown-enumerator.ts +312 -0
- package/src/element-hash.ts +16 -0
- package/src/email-otp.ts +252 -0
- package/src/engine-router.ts +141 -0
- package/src/export/bulk-export.ts +85 -0
- package/src/export/vector-export.ts +160 -0
- package/src/extractors/advanced.ts +309 -0
- package/src/extractors/artifact-envelope.ts +111 -0
- package/src/extractors/branding-extractor.ts +159 -0
- package/src/extractors/summary-extractor.ts +49 -0
- package/src/failure-analysis-agent.ts +51 -0
- package/src/feature-flag-detector.ts +97 -0
- package/src/filter-pagination-crawler.ts +500 -0
- package/src/form-detector.ts +103 -0
- package/src/frontier.test.ts +64 -0
- package/src/frontier.ts +144 -0
- package/src/gap-pass.ts +165 -0
- package/src/graphql-detector.ts +129 -0
- package/src/http-engine.ts +453 -0
- package/src/incremental-diff.ts +259 -0
- package/src/index.ts +52 -0
- package/src/input-fuzzer.ts +129 -0
- package/src/interactive-probe.ts +1136 -0
- package/src/job-queue.test.ts +187 -0
- package/src/job-queue.ts +4504 -0
- package/src/js-coverage.ts +87 -0
- package/src/js-router-detector.ts +112 -0
- package/src/language-detector.ts +82 -0
- package/src/link-classifier.ts +65 -0
- package/src/live-view.ts +146 -0
- package/src/locator-extractor.ts +309 -0
- package/src/logger.ts +9 -0
- package/src/markdown-extract.test.ts +73 -0
- package/src/markdown-extract.ts +76 -0
- package/src/multi-context-handler.ts +296 -0
- package/src/native-crawler.ts +1108 -0
- package/src/nested-window-crawler.ts +310 -0
- package/src/network-action-correlator.ts +320 -0
- package/src/network-observer.ts +204 -0
- package/src/network-throttle.ts +69 -0
- package/src/openapi-generator.ts +135 -0
- package/src/pdf-extractor.ts +123 -0
- package/src/performance-metrics-collector.ts +243 -0
- package/src/performance-metrics.ts +134 -0
- package/src/phash.ts +184 -0
- package/src/pii-scrubber.ts +99 -0
- package/src/playwright-recorder.ts +170 -0
- package/src/policies/crawl-safety.ts +52 -0
- package/src/profiles/domain-profile.ts +190 -0
- package/src/proxy.ts +238 -0
- package/src/rate-limiter.ts +87 -0
- package/src/redirect-tracker.ts +97 -0
- package/src/redis-frontier.ts +143 -0
- package/src/redis-queue.ts +208 -0
- package/src/response-cache.ts +150 -0
- package/src/responsive-detector.ts +92 -0
- package/src/retry-policy.ts +87 -0
- package/src/robots-txt.ts +96 -0
- package/src/screen-reader.ts +98 -0
- package/src/screenshot-masker.ts +98 -0
- package/src/seo-checker.ts +134 -0
- package/src/service-worker-interceptor.ts +185 -0
- package/src/session-capture.ts +335 -0
- package/src/session-export.test.ts +75 -0
- package/src/session-export.ts +172 -0
- package/src/shadow-dom-crawler.ts +198 -0
- package/src/sitemap-writer.ts +65 -0
- package/src/sitemap.test.ts +127 -0
- package/src/sitemap.ts +233 -0
- package/src/skyvern.ts +199 -0
- package/src/sqs-queue.ts +129 -0
- package/src/sso-token.ts +120 -0
- package/src/state-graph.ts +280 -0
- package/src/stealth/human-behavior.ts +227 -0
- package/src/telemetry.ts +122 -0
- package/src/test-executor.test.ts +72 -0
- package/src/test-executor.ts +578 -0
- package/src/tls-fingerprint.ts +111 -0
- package/src/url-pattern-completer.ts +161 -0
- package/src/vendor.d.ts +20 -0
- package/src/visual-diff-verdict.test.ts +80 -0
- package/src/visual-diff-verdict.ts +59 -0
- package/src/webhook-notifier.ts +109 -0
- package/src/websocket-detector.ts +83 -0
- package/src/worker.ts +50 -0
- package/tsconfig.json +13 -0
- package/types/accessibility-auditor.d.ts +29 -0
- package/types/accessibility-tree-crawler.d.ts +44 -0
- package/types/advanced-ui-probes.d.ts +47 -0
- package/types/ai-login.d.ts +32 -0
- package/types/auth-token-rotator.d.ts +47 -0
- package/types/auth.d.ts +20 -0
- package/types/browser-extension.d.ts +37 -0
- package/types/bull-worker.d.ts +6 -0
- package/types/captcha-solver.d.ts +16 -0
- package/types/color-contrast.d.ts +30 -0
- package/types/content-chunker.d.ts +18 -0
- package/types/content-dedup.d.ts +23 -0
- package/types/cookie-consent.d.ts +25 -0
- package/types/cookie-jar.d.ts +20 -0
- package/types/crawl-session-manager.d.ts +44 -0
- package/types/crawler.d.ts +233 -0
- package/types/crud-interaction.d.ts +31 -0
- package/types/custom-js-injector.d.ts +42 -0
- package/types/dark-mode-capture.d.ts +23 -0
- package/types/dead-link-detector.d.ts +37 -0
- package/types/dropdown-enumerator.d.ts +20 -0
- package/types/element-hash.d.ts +9 -0
- package/types/email-otp.d.ts +42 -0
- package/types/engine-router.d.ts +47 -0
- package/types/export/bulk-export.d.ts +26 -0
- package/types/export/vector-export.d.ts +76 -0
- package/types/extractors/advanced.d.ts +64 -0
- package/types/extractors/artifact-envelope.d.ts +63 -0
- package/types/extractors/branding-extractor.d.ts +46 -0
- package/types/extractors/summary-extractor.d.ts +10 -0
- package/types/failure-analysis-agent.d.ts +16 -0
- package/types/feature-flag-detector.d.ts +30 -0
- package/types/filter-pagination-crawler.d.ts +52 -0
- package/types/form-detector.d.ts +34 -0
- package/types/frontier.d.ts +40 -0
- package/types/gap-pass.d.ts +36 -0
- package/types/graphql-detector.d.ts +39 -0
- package/types/http-engine.d.ts +80 -0
- package/types/incremental-diff.d.ts +116 -0
- package/types/index.d.ts +52 -0
- package/types/input-fuzzer.d.ts +30 -0
- package/types/interactive-probe.d.ts +144 -0
- package/types/job-queue.d.ts +308 -0
- package/types/js-coverage.d.ts +29 -0
- package/types/js-router-detector.d.ts +17 -0
- package/types/language-detector.d.ts +22 -0
- package/types/link-classifier.d.ts +23 -0
- package/types/live-view.d.ts +36 -0
- package/types/locator-extractor.d.ts +46 -0
- package/types/logger.d.ts +2 -0
- package/types/markdown-extract.d.ts +8 -0
- package/types/multi-context-handler.d.ts +55 -0
- package/types/native-crawler.d.ts +69 -0
- package/types/nested-window-crawler.d.ts +41 -0
- package/types/network-action-correlator.d.ts +47 -0
- package/types/network-observer.d.ts +49 -0
- package/types/network-throttle.d.ts +25 -0
- package/types/openapi-generator.d.ts +49 -0
- package/types/pdf-extractor.d.ts +28 -0
- package/types/performance-metrics-collector.d.ts +50 -0
- package/types/performance-metrics.d.ts +52 -0
- package/types/phash.d.ts +50 -0
- package/types/pii-scrubber.d.ts +21 -0
- package/types/playwright-recorder.d.ts +39 -0
- package/types/policies/crawl-safety.d.ts +9 -0
- package/types/profiles/domain-profile.d.ts +19 -0
- package/types/proxy.d.ts +50 -0
- package/types/rate-limiter.d.ts +31 -0
- package/types/redirect-tracker.d.ts +37 -0
- package/types/redis-frontier.d.ts +48 -0
- package/types/redis-queue.d.ts +48 -0
- package/types/response-cache.d.ts +38 -0
- package/types/responsive-detector.d.ts +29 -0
- package/types/retry-policy.d.ts +20 -0
- package/types/robots-txt.d.ts +9 -0
- package/types/screen-reader.d.ts +15 -0
- package/types/screenshot-masker.d.ts +28 -0
- package/types/seo-checker.d.ts +33 -0
- package/types/service-worker-interceptor.d.ts +36 -0
- package/types/session-capture.d.ts +39 -0
- package/types/session-export.d.ts +49 -0
- package/types/shadow-dom-crawler.d.ts +35 -0
- package/types/sitemap-writer.d.ts +21 -0
- package/types/sitemap.d.ts +51 -0
- package/types/skyvern.d.ts +56 -0
- package/types/sqs-queue.d.ts +13 -0
- package/types/sso-token.d.ts +29 -0
- package/types/state-graph.d.ts +59 -0
- package/types/stealth/human-behavior.d.ts +20 -0
- package/types/telemetry.d.ts +34 -0
- package/types/test-executor.d.ts +84 -0
- package/types/tls-fingerprint.d.ts +50 -0
- package/types/url-pattern-completer.d.ts +28 -0
- package/types/visual-diff-verdict.d.ts +40 -0
- package/types/webhook-notifier.d.ts +30 -0
- package/types/websocket-detector.d.ts +24 -0
- package/types/worker.d.ts +1 -0
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
import type { Page } from 'playwright';
|
|
2
|
+
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// Internal: FNV-1a 32-bit SimHash
|
|
5
|
+
// ---------------------------------------------------------------------------
|
|
6
|
+
|
|
7
|
+
function fnv1a32(token: string): number {
|
|
8
|
+
let h = 2166136261;
|
|
9
|
+
for (let i = 0; i < token.length; i++) {
|
|
10
|
+
h = Math.imul(h ^ token.charCodeAt(i), 16777619);
|
|
11
|
+
h = h >>> 0;
|
|
12
|
+
}
|
|
13
|
+
return h;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function simHash32(text: string): number {
|
|
17
|
+
const tokens = text
|
|
18
|
+
.split(/\W+/)
|
|
19
|
+
.filter((t) => t.length > 2)
|
|
20
|
+
.slice(0, 500);
|
|
21
|
+
|
|
22
|
+
const vec = new Array<number>(32).fill(0);
|
|
23
|
+
|
|
24
|
+
for (const token of tokens) {
|
|
25
|
+
const h = fnv1a32(token);
|
|
26
|
+
for (let b = 0; b < 32; b++) {
|
|
27
|
+
vec[b] += (h >>> b) & 1 ? 1 : -1;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
let result = 0;
|
|
32
|
+
for (let b = 0; b < 32; b++) {
|
|
33
|
+
if (vec[b] > 0) result |= 1 << b;
|
|
34
|
+
}
|
|
35
|
+
return result >>> 0;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function hammingDistance32(a: number, b: number): number {
|
|
39
|
+
let n = (a ^ b) >>> 0;
|
|
40
|
+
n = n - ((n >>> 1) & 0x55555555);
|
|
41
|
+
n = (n & 0x33333333) + ((n >>> 2) & 0x33333333);
|
|
42
|
+
n = (n + (n >>> 4)) & 0x0f0f0f0f;
|
|
43
|
+
return Math.imul(n, 0x01010101) >>> 24;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// ---------------------------------------------------------------------------
|
|
47
|
+
// URL normalisation
|
|
48
|
+
// ---------------------------------------------------------------------------
|
|
49
|
+
|
|
50
|
+
const STRIP_PARAMS = new Set([
|
|
51
|
+
'utm_source', 'utm_medium', 'utm_campaign',
|
|
52
|
+
'ref', '_ga', 'fbclid', 'session_id',
|
|
53
|
+
]);
|
|
54
|
+
|
|
55
|
+
export function normalizeUrl(raw: string): string {
|
|
56
|
+
try {
|
|
57
|
+
const u = new URL(raw);
|
|
58
|
+
u.hash = '';
|
|
59
|
+
const keep: Array<[string, string]> = [];
|
|
60
|
+
u.searchParams.forEach((v, k) => {
|
|
61
|
+
if (!STRIP_PARAMS.has(k)) keep.push([k, v]);
|
|
62
|
+
});
|
|
63
|
+
keep.sort((a, b) => a[0].localeCompare(b[0]));
|
|
64
|
+
u.search = keep.length
|
|
65
|
+
? '?' + keep.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join('&')
|
|
66
|
+
: '';
|
|
67
|
+
return u.toString();
|
|
68
|
+
} catch {
|
|
69
|
+
return raw;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function computeStateId(normalizedUrl: string, hash: number): string {
|
|
74
|
+
let path = normalizedUrl.slice(0, 40).replace(/[^a-zA-Z0-9]/g, '_');
|
|
75
|
+
const hex = hash.toString(16).padStart(8, '0');
|
|
76
|
+
return `${path}_${hex}`;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// ---------------------------------------------------------------------------
|
|
80
|
+
// Public types
|
|
81
|
+
// ---------------------------------------------------------------------------
|
|
82
|
+
|
|
83
|
+
export interface UIStateNode {
|
|
84
|
+
id: string;
|
|
85
|
+
url: string;
|
|
86
|
+
normalizedUrl: string;
|
|
87
|
+
title: string;
|
|
88
|
+
interactiveElementCount: number;
|
|
89
|
+
domFingerprint: string;
|
|
90
|
+
simHashValue: number;
|
|
91
|
+
visitedAt: string;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export interface StateTransition {
|
|
95
|
+
fromId: string;
|
|
96
|
+
toId: string;
|
|
97
|
+
action: string;
|
|
98
|
+
label: string;
|
|
99
|
+
triggeredAt: string;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export interface CoverageReport {
|
|
103
|
+
totalStates: number;
|
|
104
|
+
totalTransitions: number;
|
|
105
|
+
uniqueUrls: number;
|
|
106
|
+
interactiveElementsTotal: number;
|
|
107
|
+
estimatedCoveragePercent: number;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// ---------------------------------------------------------------------------
|
|
111
|
+
// Page fingerprinting
|
|
112
|
+
// ---------------------------------------------------------------------------
|
|
113
|
+
|
|
114
|
+
export async function fingerprintPage(
|
|
115
|
+
page: Page,
|
|
116
|
+
visitedAt: string,
|
|
117
|
+
): Promise<{
|
|
118
|
+
fingerprint: string;
|
|
119
|
+
visibleText: string;
|
|
120
|
+
interactiveCount: number;
|
|
121
|
+
simHashValue: number;
|
|
122
|
+
}> {
|
|
123
|
+
let visibleText = '';
|
|
124
|
+
let interactiveCount = 0;
|
|
125
|
+
|
|
126
|
+
try {
|
|
127
|
+
visibleText = await page.evaluate(() => {
|
|
128
|
+
const el = document.querySelector('main') ?? document.body;
|
|
129
|
+
return (el as HTMLElement).innerText ?? '';
|
|
130
|
+
});
|
|
131
|
+
} catch {
|
|
132
|
+
visibleText = '';
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
try {
|
|
136
|
+
interactiveCount = await page.evaluate(() =>
|
|
137
|
+
document.querySelectorAll(
|
|
138
|
+
'a[href], button, input, select, textarea, [role="button"], [role="link"], [tabindex]',
|
|
139
|
+
).length,
|
|
140
|
+
);
|
|
141
|
+
} catch {
|
|
142
|
+
interactiveCount = 0;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const simHashValue = simHash32(visibleText + visitedAt.slice(0, 10));
|
|
146
|
+
const fingerprint = simHashValue.toString(16).padStart(8, '0');
|
|
147
|
+
|
|
148
|
+
return { fingerprint, visibleText, interactiveCount, simHashValue };
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// ---------------------------------------------------------------------------
|
|
152
|
+
// ScreenTransitionGraph
|
|
153
|
+
// ---------------------------------------------------------------------------
|
|
154
|
+
|
|
155
|
+
export class ScreenTransitionGraph {
|
|
156
|
+
private states = new Map<string, UIStateNode>();
|
|
157
|
+
private transitions: StateTransition[] = [];
|
|
158
|
+
/** Maps simHash value → array of state ids with that hash (for near-dup lookup) */
|
|
159
|
+
private hashIndex = new Map<number, string[]>();
|
|
160
|
+
|
|
161
|
+
// -------------------------------------------------------------------------
|
|
162
|
+
// State management
|
|
163
|
+
// -------------------------------------------------------------------------
|
|
164
|
+
|
|
165
|
+
addState(node: UIStateNode): { added: boolean; duplicateOf?: string } {
|
|
166
|
+
if (this.states.has(node.id)) {
|
|
167
|
+
return { added: false, duplicateOf: node.id };
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// Check near-duplicate via hamming distance across all indexed hashes
|
|
171
|
+
for (const [indexedHash, ids] of this.hashIndex) {
|
|
172
|
+
if (hammingDistance32(node.simHashValue, indexedHash) < 3) {
|
|
173
|
+
return { added: false, duplicateOf: ids[0] };
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
this.states.set(node.id, node);
|
|
178
|
+
|
|
179
|
+
const bucket = this.hashIndex.get(node.simHashValue);
|
|
180
|
+
if (bucket) {
|
|
181
|
+
bucket.push(node.id);
|
|
182
|
+
} else {
|
|
183
|
+
this.hashIndex.set(node.simHashValue, [node.id]);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
return { added: true };
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
addTransition(
|
|
190
|
+
fromId: string,
|
|
191
|
+
toId: string,
|
|
192
|
+
action: string,
|
|
193
|
+
label: string,
|
|
194
|
+
triggeredAt: string,
|
|
195
|
+
): void {
|
|
196
|
+
this.transitions.push({ fromId, toId, action, label, triggeredAt });
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
isDuplicate(url: string, visibleText: string, threshold = 3): boolean {
|
|
200
|
+
const normalized = normalizeUrl(url);
|
|
201
|
+
const hash = simHash32(visibleText);
|
|
202
|
+
for (const [indexedHash] of this.hashIndex) {
|
|
203
|
+
if (hammingDistance32(hash, indexedHash) < threshold) return true;
|
|
204
|
+
}
|
|
205
|
+
// Also check exact normalized URL match
|
|
206
|
+
for (const node of this.states.values()) {
|
|
207
|
+
if (node.normalizedUrl === normalized) return true;
|
|
208
|
+
}
|
|
209
|
+
return false;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// -------------------------------------------------------------------------
|
|
213
|
+
// Reporting
|
|
214
|
+
// -------------------------------------------------------------------------
|
|
215
|
+
|
|
216
|
+
getCoverageReport(): CoverageReport {
|
|
217
|
+
const totalStates = this.states.size;
|
|
218
|
+
const totalTransitions = this.transitions.length;
|
|
219
|
+
const uniqueUrls = new Set(
|
|
220
|
+
Array.from(this.states.values()).map((n) => n.normalizedUrl),
|
|
221
|
+
).size;
|
|
222
|
+
const interactiveElementsTotal = Array.from(this.states.values()).reduce(
|
|
223
|
+
(sum, n) => sum + n.interactiveElementCount,
|
|
224
|
+
0,
|
|
225
|
+
);
|
|
226
|
+
const estimatedCoveragePercent = Math.min(
|
|
227
|
+
100,
|
|
228
|
+
(totalTransitions / Math.max(1, totalStates)) * 50,
|
|
229
|
+
);
|
|
230
|
+
return {
|
|
231
|
+
totalStates,
|
|
232
|
+
totalTransitions,
|
|
233
|
+
uniqueUrls,
|
|
234
|
+
interactiveElementsTotal,
|
|
235
|
+
estimatedCoveragePercent,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// -------------------------------------------------------------------------
|
|
240
|
+
// Serialisation
|
|
241
|
+
// -------------------------------------------------------------------------
|
|
242
|
+
|
|
243
|
+
toJSON(): { states: UIStateNode[]; transitions: StateTransition[] } {
|
|
244
|
+
return {
|
|
245
|
+
states: Array.from(this.states.values()),
|
|
246
|
+
transitions: this.transitions,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
static fromJSON(data: { states: UIStateNode[]; transitions: StateTransition[] }): ScreenTransitionGraph {
|
|
251
|
+
const graph = new ScreenTransitionGraph();
|
|
252
|
+
for (const node of data.states) {
|
|
253
|
+
graph.states.set(node.id, node);
|
|
254
|
+
const bucket = graph.hashIndex.get(node.simHashValue);
|
|
255
|
+
if (bucket) {
|
|
256
|
+
bucket.push(node.id);
|
|
257
|
+
} else {
|
|
258
|
+
graph.hashIndex.set(node.simHashValue, [node.id]);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
graph.transitions = [...data.transitions];
|
|
262
|
+
return graph;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// -------------------------------------------------------------------------
|
|
266
|
+
// Accessors
|
|
267
|
+
// -------------------------------------------------------------------------
|
|
268
|
+
|
|
269
|
+
getState(id: string): UIStateNode | undefined {
|
|
270
|
+
return this.states.get(id);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
getAllStates(): UIStateNode[] {
|
|
274
|
+
return Array.from(this.states.values());
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
getTransitionsFrom(id: string): StateTransition[] {
|
|
278
|
+
return this.transitions.filter((t) => t.fromId === id);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
import type { Page } from 'playwright';
|
|
2
|
+
|
|
3
|
+
// Gaussian random using Box-Muller transform
|
|
4
|
+
function gaussian(mean: number, sigma: number): number {
|
|
5
|
+
const u1 = Math.random();
|
|
6
|
+
const u2 = Math.random();
|
|
7
|
+
const z = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
|
|
8
|
+
return mean + z * sigma;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export async function humanDelay(min = 80, max = 450): Promise<void> {
|
|
12
|
+
const mean = (min + max) / 2;
|
|
13
|
+
const sigma = (max - min) / 6;
|
|
14
|
+
const delay = Math.max(min, Math.min(max, gaussian(mean, sigma)));
|
|
15
|
+
await new Promise(r => setTimeout(r, Math.round(delay)));
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export async function simulateMouseMovement(page: Page, targetX: number, targetY: number): Promise<void> {
|
|
19
|
+
try {
|
|
20
|
+
const mouse = page.mouse;
|
|
21
|
+
const steps = 8 + Math.floor(Math.random() * 8);
|
|
22
|
+
// Generate Bezier control points with slight perpendicular offset
|
|
23
|
+
const startX = targetX - 100 + Math.random() * 200;
|
|
24
|
+
const startY = targetY - 100 + Math.random() * 200;
|
|
25
|
+
const cpX = (startX + targetX) / 2 + (Math.random() - 0.5) * 80;
|
|
26
|
+
const cpY = (startY + targetY) / 2 + (Math.random() - 0.5) * 80;
|
|
27
|
+
|
|
28
|
+
for (let i = 0; i <= steps; i++) {
|
|
29
|
+
const t = i / steps;
|
|
30
|
+
// Quadratic Bezier: B(t) = (1-t)^2 * P0 + 2(1-t)t * P1 + t^2 * P2
|
|
31
|
+
const x = (1 - t) * (1 - t) * startX + 2 * (1 - t) * t * cpX + t * t * targetX;
|
|
32
|
+
const y = (1 - t) * (1 - t) * startY + 2 * (1 - t) * t * cpY + t * t * targetY;
|
|
33
|
+
await mouse.move(x + (Math.random() - 0.5) * 2, y + (Math.random() - 0.5) * 2);
|
|
34
|
+
if (i < steps) await humanDelay(8, 25);
|
|
35
|
+
}
|
|
36
|
+
// Small overshoot correction
|
|
37
|
+
await mouse.move(targetX + (Math.random() - 0.5) * 4, targetY + (Math.random() - 0.5) * 4);
|
|
38
|
+
await humanDelay(30, 80);
|
|
39
|
+
await mouse.move(targetX, targetY);
|
|
40
|
+
} catch { /* non-fatal */ }
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export async function simulateMouseHover(page: Page, selector: string): Promise<void> {
|
|
44
|
+
try {
|
|
45
|
+
const el = await page.$(selector);
|
|
46
|
+
if (!el) return;
|
|
47
|
+
const box = await el.boundingBox();
|
|
48
|
+
if (!box) return;
|
|
49
|
+
const cx = box.x + box.width / 2;
|
|
50
|
+
const cy = box.y + box.height / 2;
|
|
51
|
+
await simulateMouseMovement(page, cx, cy);
|
|
52
|
+
await humanDelay(100, 300);
|
|
53
|
+
} catch { /* non-fatal */ }
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export async function simulateClick(page: Page, selector: string): Promise<void> {
|
|
57
|
+
try {
|
|
58
|
+
await simulateMouseHover(page, selector);
|
|
59
|
+
await humanDelay(50, 150);
|
|
60
|
+
await page.click(selector, { delay: Math.round(gaussian(80, 30)) });
|
|
61
|
+
} catch { /* non-fatal */ }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export async function injectFingerprintOverrides(page: Page): Promise<void> {
|
|
65
|
+
await page.addInitScript(() => {
|
|
66
|
+
// Canvas noise — subtle per-pixel perturbation
|
|
67
|
+
const origToDataURL = HTMLCanvasElement.prototype.toDataURL;
|
|
68
|
+
HTMLCanvasElement.prototype.toDataURL = function(this: HTMLCanvasElement, ...args: any[]) {
|
|
69
|
+
const ctx = this.getContext('2d');
|
|
70
|
+
if (ctx) {
|
|
71
|
+
const imageData = ctx.getImageData(0, 0, this.width, this.height);
|
|
72
|
+
for (let i = 0; i < imageData.data.length; i += 4) {
|
|
73
|
+
imageData.data[i] = Math.max(0, Math.min(255, imageData.data[i] + (Math.random() < 0.01 ? (Math.random() > 0.5 ? 1 : -1) : 0)));
|
|
74
|
+
}
|
|
75
|
+
ctx.putImageData(imageData, 0, 0);
|
|
76
|
+
}
|
|
77
|
+
return origToDataURL.apply(this, args as [type?: string, quality?: number]);
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
// WebGL parameter spoofing
|
|
81
|
+
const origGetParam = WebGLRenderingContext.prototype.getParameter;
|
|
82
|
+
WebGLRenderingContext.prototype.getParameter = function(this: WebGLRenderingContext, parameter: number) {
|
|
83
|
+
const UNMASKED_VENDOR_WEBGL = 0x9245;
|
|
84
|
+
const UNMASKED_RENDERER_WEBGL = 0x9246;
|
|
85
|
+
if (parameter === UNMASKED_VENDOR_WEBGL) return 'Intel Inc.';
|
|
86
|
+
if (parameter === UNMASKED_RENDERER_WEBGL) return 'Intel Iris OpenGL Engine';
|
|
87
|
+
return origGetParam.call(this, parameter);
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
// Hardware concurrency — pick from common values
|
|
91
|
+
const cpuCounts = [4, 6, 8, 8];
|
|
92
|
+
Object.defineProperty(navigator, 'hardwareConcurrency', {
|
|
93
|
+
get: () => cpuCounts[Math.floor(Math.random() * cpuCounts.length)],
|
|
94
|
+
configurable: true,
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
// Device memory
|
|
98
|
+
Object.defineProperty(navigator, 'deviceMemory', {
|
|
99
|
+
get: () => [4, 8][Math.floor(Math.random() * 2)],
|
|
100
|
+
configurable: true,
|
|
101
|
+
});
|
|
102
|
+
}).catch(() => {});
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export async function infiniteScrollToBottom(
|
|
106
|
+
page: Page,
|
|
107
|
+
opts: { maxScrolls?: number; scrollDelay?: number; stabilizeMs?: number } = {},
|
|
108
|
+
rawPage?: any,
|
|
109
|
+
): Promise<{ scrollCount: number; heightReached: number }> {
|
|
110
|
+
const { maxScrolls = 50, scrollDelay = 400, stabilizeMs = 600 } = opts;
|
|
111
|
+
let scrollCount = 0;
|
|
112
|
+
let stableScrollCount = 0;
|
|
113
|
+
let previousHeight = 0;
|
|
114
|
+
|
|
115
|
+
// One CDP session for wheel events (triggers IntersectionObserver-based virtual scrollers).
|
|
116
|
+
// React-window / react-virtualized / TanStack Virtual ignore window.scrollBy but respond to
|
|
117
|
+
// wheel events because they attach their own scroll listeners to the list container.
|
|
118
|
+
let cdpForWheel: any = null;
|
|
119
|
+
let viewW = 1280;
|
|
120
|
+
let viewH = 800;
|
|
121
|
+
if (rawPage && typeof rawPage.context === 'function') {
|
|
122
|
+
try {
|
|
123
|
+
const ctx = rawPage.context();
|
|
124
|
+
if (typeof ctx?.newCDPSession === 'function') {
|
|
125
|
+
cdpForWheel = await ctx.newCDPSession(rawPage);
|
|
126
|
+
const { result } = await cdpForWheel.send('Runtime.evaluate', {
|
|
127
|
+
expression: `({"w":window.innerWidth,"h":window.innerHeight})`,
|
|
128
|
+
returnByValue: true,
|
|
129
|
+
}).catch(() => ({ result: { value: null } }));
|
|
130
|
+
if (result?.value) { viewW = result.value.w ?? 1280; viewH = result.value.h ?? 800; }
|
|
131
|
+
}
|
|
132
|
+
} catch { cdpForWheel = null; }
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
try {
|
|
136
|
+
while (scrollCount < maxScrolls) {
|
|
137
|
+
previousHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
|
|
138
|
+
|
|
139
|
+
const scrollBy = 500 + Math.floor(Math.random() * 300);
|
|
140
|
+
// window.scrollBy handles standard scroll containers
|
|
141
|
+
await page.evaluate((by: number) => window.scrollBy(0, by), scrollBy).catch(() => {});
|
|
142
|
+
// CDP wheel event triggers IntersectionObserver + virtual list scroll listeners
|
|
143
|
+
if (cdpForWheel) {
|
|
144
|
+
await cdpForWheel.send('Input.dispatchMouseEvent', {
|
|
145
|
+
type: 'mouseWheel',
|
|
146
|
+
x: Math.floor(viewW / 2),
|
|
147
|
+
y: Math.floor(viewH / 2),
|
|
148
|
+
deltaX: 0,
|
|
149
|
+
deltaY: scrollBy,
|
|
150
|
+
modifiers: 0,
|
|
151
|
+
}).catch(() => {});
|
|
152
|
+
}
|
|
153
|
+
await humanDelay(scrollDelay - 100, scrollDelay + 200);
|
|
154
|
+
|
|
155
|
+
// Use Node.js setTimeout — page.waitForTimeout may not be available on Stagehand proxy
|
|
156
|
+
await new Promise<void>(r => setTimeout(r, stabilizeMs));
|
|
157
|
+
scrollCount++;
|
|
158
|
+
|
|
159
|
+
const newHeight: number = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
|
|
160
|
+
if (newHeight - previousHeight < 10) {
|
|
161
|
+
stableScrollCount++;
|
|
162
|
+
if (stableScrollCount >= 3) break; // no new content after 3 consecutive non-growing scrolls
|
|
163
|
+
} else {
|
|
164
|
+
stableScrollCount = 0;
|
|
165
|
+
}
|
|
166
|
+
previousHeight = newHeight;
|
|
167
|
+
}
|
|
168
|
+
} catch { /* non-fatal */ }
|
|
169
|
+
|
|
170
|
+
if (cdpForWheel) cdpForWheel.detach().catch(() => {});
|
|
171
|
+
return { scrollCount, heightReached: previousHeight };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export async function extractIframeContent(page: Page): Promise<Array<{ src: string; content: string; markdown: string }>> {
|
|
175
|
+
const results: Array<{ src: string; content: string; markdown: string }> = [];
|
|
176
|
+
try {
|
|
177
|
+
const iframeSrcs: string[] = await page.evaluate(() =>
|
|
178
|
+
Array.from(document.querySelectorAll('iframe')).map(f => f.getAttribute('src') ?? f.getAttribute('data-src') ?? '')
|
|
179
|
+
).catch(() => []);
|
|
180
|
+
|
|
181
|
+
for (const frame of page.frames()) {
|
|
182
|
+
const src = frame.url();
|
|
183
|
+
if (!src || src === 'about:blank' || src === page.url()) continue;
|
|
184
|
+
try {
|
|
185
|
+
const content = await frame.content().catch(() => '');
|
|
186
|
+
if (!content || content.length < 50) continue;
|
|
187
|
+
// Simple text extraction from iframe HTML
|
|
188
|
+
const text = content.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
|
|
189
|
+
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
|
|
190
|
+
.replace(/<[^>]+>/g, ' ')
|
|
191
|
+
.replace(/\s+/g, ' ')
|
|
192
|
+
.trim();
|
|
193
|
+
results.push({ src, content: content.slice(0, 10_000), markdown: text.slice(0, 5_000) });
|
|
194
|
+
} catch { /* skip this frame */ }
|
|
195
|
+
}
|
|
196
|
+
} catch { /* non-fatal */ }
|
|
197
|
+
return results;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
export async function pierceShadowDom(page: Page): Promise<string> {
|
|
201
|
+
try {
|
|
202
|
+
return await page.evaluate(() => {
|
|
203
|
+
const parts: string[] = [];
|
|
204
|
+
function walkShadow(root: Element | ShadowRoot): void {
|
|
205
|
+
const shadow = (root as any).shadowRoot;
|
|
206
|
+
if (shadow) {
|
|
207
|
+
parts.push(shadow.innerHTML ?? '');
|
|
208
|
+
for (const child of [...shadow.querySelectorAll('*')]) walkShadow(child);
|
|
209
|
+
}
|
|
210
|
+
for (const child of Array.from(root.querySelectorAll(':not(script):not(style)'))) {
|
|
211
|
+
if ((child as any).shadowRoot) walkShadow(child);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
walkShadow(document.body);
|
|
215
|
+
return parts
|
|
216
|
+
.join('\n')
|
|
217
|
+
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
|
|
218
|
+
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
|
|
219
|
+
.replace(/<[^>]+>/g, ' ')
|
|
220
|
+
.replace(/\s+/g, ' ')
|
|
221
|
+
.trim()
|
|
222
|
+
.slice(0, 10_000);
|
|
223
|
+
}).catch(() => '');
|
|
224
|
+
} catch {
|
|
225
|
+
return '';
|
|
226
|
+
}
|
|
227
|
+
}
|
package/src/telemetry.ts
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight OTel-compatible tracing for Zeta crawler.
|
|
3
|
+
*
|
|
4
|
+
* Emits spans as structured JSON logs compatible with OpenTelemetry format.
|
|
5
|
+
* Drop-in upgrade path: replace this module with @opentelemetry/sdk-node when needed.
|
|
6
|
+
*
|
|
7
|
+
* Usage:
|
|
8
|
+
* const span = startSpan('crawl.url', { url, engine });
|
|
9
|
+
* // ... do work ...
|
|
10
|
+
* endSpan(span, { screensFound: 3 });
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { randomBytes } from 'node:crypto';
|
|
14
|
+
|
|
15
|
+
export interface Span {
|
|
16
|
+
traceId: string;
|
|
17
|
+
spanId: string;
|
|
18
|
+
parentSpanId?: string;
|
|
19
|
+
name: string;
|
|
20
|
+
startTimeMs: number;
|
|
21
|
+
attributes: Record<string, string | number | boolean>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface FinishedSpan extends Span {
|
|
25
|
+
endTimeMs: number;
|
|
26
|
+
durationMs: number;
|
|
27
|
+
status: 'ok' | 'error';
|
|
28
|
+
error?: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Active trace context — per async execution context (simplified: per-process global)
|
|
32
|
+
let currentTraceId: string | null = null;
|
|
33
|
+
let currentSpanId: string | null = null;
|
|
34
|
+
|
|
35
|
+
function genId(bytes: number): string {
|
|
36
|
+
return randomBytes(bytes).toString('hex');
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Start a new span. If a trace is already active, the new span is a child. */
|
|
40
|
+
export function startSpan(name: string, attributes: Record<string, string | number | boolean> = {}): Span {
|
|
41
|
+
const traceId = currentTraceId ?? genId(16);
|
|
42
|
+
const spanId = genId(8);
|
|
43
|
+
const parentSpanId = currentSpanId ?? undefined;
|
|
44
|
+
|
|
45
|
+
if (!currentTraceId) currentTraceId = traceId;
|
|
46
|
+
currentSpanId = spanId;
|
|
47
|
+
|
|
48
|
+
return { traceId, spanId, parentSpanId, name, startTimeMs: Date.now(), attributes };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** End a span and emit it as a structured log line. */
|
|
52
|
+
export function endSpan(span: Span, extraAttributes: Record<string, string | number | boolean> = {}, error?: Error): FinishedSpan {
|
|
53
|
+
const endTimeMs = Date.now();
|
|
54
|
+
const durationMs = endTimeMs - span.startTimeMs;
|
|
55
|
+
const finished: FinishedSpan = {
|
|
56
|
+
...span,
|
|
57
|
+
attributes: { ...span.attributes, ...extraAttributes },
|
|
58
|
+
endTimeMs,
|
|
59
|
+
durationMs,
|
|
60
|
+
status: error ? 'error' : 'ok',
|
|
61
|
+
error: error?.message,
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
// Reset context when root span ends
|
|
65
|
+
if (!span.parentSpanId) {
|
|
66
|
+
currentTraceId = null;
|
|
67
|
+
currentSpanId = span.parentSpanId ?? null;
|
|
68
|
+
} else {
|
|
69
|
+
currentSpanId = span.parentSpanId ?? null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Emit as structured JSON — compatible with Datadog/Grafana log parsing
|
|
73
|
+
console.log(JSON.stringify({
|
|
74
|
+
ts: new Date(endTimeMs).toISOString(),
|
|
75
|
+
level: error ? 'error' : 'info',
|
|
76
|
+
msg: `span:${finished.name}`,
|
|
77
|
+
trace_id: finished.traceId,
|
|
78
|
+
span_id: finished.spanId,
|
|
79
|
+
parent_span_id: finished.parentSpanId,
|
|
80
|
+
duration_ms: durationMs,
|
|
81
|
+
status: finished.status,
|
|
82
|
+
error: finished.error,
|
|
83
|
+
...finished.attributes,
|
|
84
|
+
}));
|
|
85
|
+
|
|
86
|
+
return finished;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Convenience: wrap an async function in a span. */
|
|
90
|
+
export async function withSpan<T>(
|
|
91
|
+
name: string,
|
|
92
|
+
attributes: Record<string, string | number | boolean>,
|
|
93
|
+
fn: (span: Span) => Promise<T>,
|
|
94
|
+
): Promise<T> {
|
|
95
|
+
const span = startSpan(name, attributes);
|
|
96
|
+
try {
|
|
97
|
+
const result = await fn(span);
|
|
98
|
+
endSpan(span);
|
|
99
|
+
return result;
|
|
100
|
+
} catch (err: any) {
|
|
101
|
+
endSpan(span, {}, err);
|
|
102
|
+
throw err;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Counter metrics — increment and log on flush. */
|
|
107
|
+
const counters: Map<string, number> = new Map();
|
|
108
|
+
|
|
109
|
+
export function increment(metric: string, value = 1, tags: Record<string, string> = {}): void {
|
|
110
|
+
const key = metric + (Object.keys(tags).length ? ':' + Object.entries(tags).map(([k, v]) => `${k}=${v}`).join(',') : '');
|
|
111
|
+
counters.set(key, (counters.get(key) ?? 0) + value);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Flush counters as a structured log — call periodically (e.g. at job end). */
|
|
115
|
+
export function flushMetrics(): Record<string, number> {
|
|
116
|
+
const snapshot = Object.fromEntries(counters);
|
|
117
|
+
if (Object.keys(snapshot).length > 0) {
|
|
118
|
+
console.log(JSON.stringify({ ts: new Date().toISOString(), level: 'info', msg: 'metrics:flush', ...snapshot }));
|
|
119
|
+
counters.clear();
|
|
120
|
+
}
|
|
121
|
+
return snapshot;
|
|
122
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import http from 'node:http';
|
|
2
|
+
import test from 'node:test';
|
|
3
|
+
import assert from 'node:assert/strict';
|
|
4
|
+
import { executeTestPlan } from './test-executor.js';
|
|
5
|
+
import type { ExecutionStep } from '@taodigital/agents';
|
|
6
|
+
|
|
7
|
+
async function startFixtureServer(html: string): Promise<{ url: string; close: () => Promise<void> }> {
|
|
8
|
+
const server = http.createServer((_req, res) => {
|
|
9
|
+
res.writeHead(200, { 'Content-Type': 'text/html' });
|
|
10
|
+
res.end(html);
|
|
11
|
+
});
|
|
12
|
+
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
|
|
13
|
+
const address = server.address();
|
|
14
|
+
const port = typeof address === 'object' && address ? address.port : 0;
|
|
15
|
+
return {
|
|
16
|
+
url: `http://127.0.0.1:${port}`,
|
|
17
|
+
close: () => new Promise<void>((resolve) => server.close(() => resolve())),
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// No AI healing context is passed in these tests (healCtx omitted), so the AI tier
|
|
22
|
+
// (which needs a real LLM call) never engages — this is regression coverage for the
|
|
23
|
+
// pre-existing mechanical strategy-fallback tier that self-healing wiring must not break.
|
|
24
|
+
|
|
25
|
+
test('resolveLocator: primary strategy succeeds when the selector/strategy pair is correct', async () => {
|
|
26
|
+
const fixture = await startFixtureServer('<html><body><button>Submit</button></body></html>');
|
|
27
|
+
try {
|
|
28
|
+
const plan: ExecutionStep[] = [
|
|
29
|
+
{ stepNumber: 1, description: 'navigate', action: 'navigate', url: fixture.url },
|
|
30
|
+
{ stepNumber: 2, description: 'click Submit', action: 'click', selector: 'Submit', selectorStrategy: 'text' },
|
|
31
|
+
];
|
|
32
|
+
const result = await executeTestPlan(plan, { screenshotDir: '/tmp', testRunId: 'test-primary-ok', allowUnsafeLocalUrlsForTests: true });
|
|
33
|
+
assert.equal(result.status, 'PASSED');
|
|
34
|
+
assert.equal(result.stepResults[1].healed, false);
|
|
35
|
+
} finally {
|
|
36
|
+
await fixture.close();
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test('resolveLocator: mechanical tier heals when the strategy is wrong but the text still matches', async () => {
|
|
41
|
+
// "label" strategy looks for a form control associated with a <label> — a bare
|
|
42
|
+
// <button> has no such association, so the primary lookup must fail and fall
|
|
43
|
+
// through to the "text" strategy, which does find it.
|
|
44
|
+
const fixture = await startFixtureServer('<html><body><button>Submit</button></body></html>');
|
|
45
|
+
try {
|
|
46
|
+
const plan: ExecutionStep[] = [
|
|
47
|
+
{ stepNumber: 1, description: 'navigate', action: 'navigate', url: fixture.url },
|
|
48
|
+
{ stepNumber: 2, description: 'click Submit', action: 'click', selector: 'Submit', selectorStrategy: 'label' },
|
|
49
|
+
];
|
|
50
|
+
const result = await executeTestPlan(plan, { screenshotDir: '/tmp', testRunId: 'test-mechanical-heal', allowUnsafeLocalUrlsForTests: true });
|
|
51
|
+
assert.equal(result.status, 'PASSED');
|
|
52
|
+
assert.equal(result.stepResults[1].healed, true);
|
|
53
|
+
assert.equal(result.stepResults[1].healedVia, 'mechanical');
|
|
54
|
+
} finally {
|
|
55
|
+
await fixture.close();
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test('resolveLocator: fails cleanly (no healCtx configured) when no strategy matches at all', async () => {
|
|
60
|
+
const fixture = await startFixtureServer('<html><body><button>Submit</button></body></html>');
|
|
61
|
+
try {
|
|
62
|
+
const plan: ExecutionStep[] = [
|
|
63
|
+
{ stepNumber: 1, description: 'navigate', action: 'navigate', url: fixture.url },
|
|
64
|
+
{ stepNumber: 2, description: 'click Nonexistent', action: 'click', selector: 'Nonexistent Button', selectorStrategy: 'text', timeout: 500 },
|
|
65
|
+
];
|
|
66
|
+
const result = await executeTestPlan(plan, { screenshotDir: '/tmp', testRunId: 'test-no-match', allowUnsafeLocalUrlsForTests: true });
|
|
67
|
+
assert.equal(result.status, 'FAILED');
|
|
68
|
+
assert.equal(result.stepResults[1].healed, undefined);
|
|
69
|
+
} finally {
|
|
70
|
+
await fixture.close();
|
|
71
|
+
}
|
|
72
|
+
});
|