@taodigital/crawler 0.0.0-stage → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +82 -0
- package/README.md +10 -2
- package/dist/accessibility-auditor.d.ts +29 -0
- package/dist/accessibility-auditor.js +57 -0
- package/dist/accessibility-auditor.js.map +7 -0
- package/dist/accessibility-tree-crawler.d.ts +44 -0
- package/dist/accessibility-tree-crawler.js +156 -0
- package/dist/accessibility-tree-crawler.js.map +7 -0
- package/dist/advanced-ui-probes.d.ts +47 -0
- package/dist/advanced-ui-probes.js +310 -0
- package/dist/advanced-ui-probes.js.map +7 -0
- package/dist/ai-login.d.ts +32 -0
- package/dist/ai-login.js +250 -0
- package/dist/ai-login.js.map +7 -0
- package/dist/auth-token-rotator.d.ts +47 -0
- package/dist/auth-token-rotator.js +58 -0
- package/dist/auth-token-rotator.js.map +7 -0
- package/dist/auth.d.ts +20 -0
- package/dist/auth.js +348 -0
- package/dist/auth.js.map +7 -0
- package/dist/browser-extension.d.ts +37 -0
- package/dist/browser-extension.js +113 -0
- package/dist/browser-extension.js.map +7 -0
- package/dist/captcha-solver.d.ts +16 -0
- package/dist/captcha-solver.js +105 -0
- package/dist/captcha-solver.js.map +7 -0
- package/dist/color-contrast.d.ts +30 -0
- package/dist/color-contrast.js +78 -0
- package/dist/color-contrast.js.map +7 -0
- package/dist/content-chunker.d.ts +18 -0
- package/dist/content-chunker.js +152 -0
- package/dist/content-chunker.js.map +7 -0
- package/dist/content-dedup.d.ts +23 -0
- package/dist/content-dedup.js +60 -0
- package/dist/content-dedup.js.map +7 -0
- package/dist/cookie-consent.d.ts +25 -0
- package/dist/cookie-consent.js +78 -0
- package/dist/cookie-consent.js.map +7 -0
- package/dist/cookie-jar.d.ts +20 -0
- package/dist/cookie-jar.js +61 -0
- package/dist/cookie-jar.js.map +7 -0
- package/dist/crawl-session-manager.d.ts +44 -0
- package/dist/crawl-session-manager.js +145 -0
- package/dist/crawl-session-manager.js.map +7 -0
- package/dist/crawler.d.ts +233 -0
- package/dist/crawler.js +6239 -0
- package/dist/crawler.js.map +7 -0
- package/dist/crud-interaction.d.ts +31 -0
- package/dist/crud-interaction.js +264 -0
- package/dist/crud-interaction.js.map +7 -0
- package/dist/custom-js-injector.d.ts +42 -0
- package/dist/custom-js-injector.js +50 -0
- package/dist/custom-js-injector.js.map +7 -0
- package/dist/dark-mode-capture.d.ts +23 -0
- package/dist/dark-mode-capture.js +75 -0
- package/dist/dark-mode-capture.js.map +7 -0
- package/dist/dead-link-detector.d.ts +37 -0
- package/dist/dead-link-detector.js +50 -0
- package/dist/dead-link-detector.js.map +7 -0
- package/dist/dropdown-enumerator.d.ts +20 -0
- package/dist/dropdown-enumerator.js +230 -0
- package/dist/dropdown-enumerator.js.map +7 -0
- package/dist/element-hash.d.ts +9 -0
- package/dist/element-hash.js +9 -0
- package/dist/element-hash.js.map +7 -0
- package/dist/email-otp.d.ts +42 -0
- package/dist/email-otp.js +177 -0
- package/dist/email-otp.js.map +7 -0
- package/dist/engine-router.d.ts +47 -0
- package/dist/engine-router.js +94 -0
- package/dist/engine-router.js.map +7 -0
- package/dist/export/bulk-export.d.ts +26 -0
- package/dist/export/bulk-export.js +76 -0
- package/dist/export/bulk-export.js.map +7 -0
- package/dist/export/vector-export.d.ts +76 -0
- package/dist/export/vector-export.js +85 -0
- package/dist/export/vector-export.js.map +7 -0
- package/dist/extractors/advanced.d.ts +64 -0
- package/dist/extractors/advanced.js +265 -0
- package/dist/extractors/advanced.js.map +7 -0
- package/dist/extractors/artifact-envelope.d.ts +63 -0
- package/dist/extractors/artifact-envelope.js +63 -0
- package/dist/extractors/artifact-envelope.js.map +7 -0
- package/dist/extractors/branding-extractor.d.ts +46 -0
- package/dist/extractors/branding-extractor.js +104 -0
- package/dist/extractors/branding-extractor.js.map +7 -0
- package/dist/extractors/summary-extractor.d.ts +10 -0
- package/dist/extractors/summary-extractor.js +36 -0
- package/dist/extractors/summary-extractor.js.map +7 -0
- package/dist/failure-analysis-agent.d.ts +16 -0
- package/dist/failure-analysis-agent.js +42 -0
- package/dist/failure-analysis-agent.js.map +7 -0
- package/dist/feature-flag-detector.d.ts +30 -0
- package/dist/feature-flag-detector.js +55 -0
- package/dist/feature-flag-detector.js.map +7 -0
- package/dist/filter-pagination-crawler.d.ts +52 -0
- package/dist/filter-pagination-crawler.js +299 -0
- package/dist/filter-pagination-crawler.js.map +7 -0
- package/dist/form-detector.d.ts +34 -0
- package/dist/form-detector.js +58 -0
- package/dist/form-detector.js.map +7 -0
- package/dist/frontier.d.ts +40 -0
- package/dist/frontier.js +103 -0
- package/dist/frontier.js.map +7 -0
- package/dist/gap-pass.d.ts +36 -0
- package/dist/gap-pass.js +115 -0
- package/dist/gap-pass.js.map +7 -0
- package/dist/graphql-detector.d.ts +39 -0
- package/dist/graphql-detector.js +86 -0
- package/dist/graphql-detector.js.map +7 -0
- package/dist/http-engine.d.ts +80 -0
- package/dist/http-engine.js +318 -0
- package/dist/http-engine.js.map +7 -0
- package/dist/incremental-diff.d.ts +116 -0
- package/dist/incremental-diff.js +137 -0
- package/dist/incremental-diff.js.map +7 -0
- package/dist/index.d.ts +52 -0
- package/dist/index.js +83 -0
- package/dist/index.js.map +7 -0
- package/dist/input-fuzzer.d.ts +30 -0
- package/dist/input-fuzzer.js +105 -0
- package/dist/input-fuzzer.js.map +7 -0
- package/dist/interactive-probe.d.ts +144 -0
- package/dist/interactive-probe.js +865 -0
- package/dist/interactive-probe.js.map +7 -0
- package/dist/job-queue.d.ts +297 -0
- package/dist/job-queue.js +3812 -0
- package/dist/job-queue.js.map +7 -0
- package/dist/js-coverage.d.ts +29 -0
- package/dist/js-coverage.js +56 -0
- package/dist/js-coverage.js.map +7 -0
- package/dist/js-router-detector.d.ts +17 -0
- package/dist/js-router-detector.js +90 -0
- package/dist/js-router-detector.js.map +7 -0
- package/dist/language-detector.d.ts +22 -0
- package/dist/language-detector.js +57 -0
- package/dist/language-detector.js.map +7 -0
- package/dist/link-classifier.d.ts +23 -0
- package/dist/link-classifier.js +40 -0
- package/dist/link-classifier.js.map +7 -0
- package/dist/live-view.d.ts +36 -0
- package/dist/live-view.js +105 -0
- package/dist/live-view.js.map +7 -0
- package/dist/locator-extractor.d.ts +46 -0
- package/dist/locator-extractor.js +220 -0
- package/dist/locator-extractor.js.map +7 -0
- package/dist/logger.d.ts +2 -0
- package/dist/logger.js +10 -0
- package/dist/logger.js.map +7 -0
- package/dist/markdown-extract.d.ts +8 -0
- package/dist/markdown-extract.js +54 -0
- package/dist/markdown-extract.js.map +7 -0
- package/dist/multi-context-handler.d.ts +55 -0
- package/dist/multi-context-handler.js +182 -0
- package/dist/multi-context-handler.js.map +7 -0
- package/dist/native-crawler.d.ts +69 -0
- package/dist/native-crawler.js +839 -0
- package/dist/native-crawler.js.map +7 -0
- package/dist/nested-window-crawler.d.ts +41 -0
- package/dist/nested-window-crawler.js +172 -0
- package/dist/nested-window-crawler.js.map +7 -0
- package/dist/network-action-correlator.d.ts +47 -0
- package/dist/network-action-correlator.js +209 -0
- package/dist/network-action-correlator.js.map +7 -0
- package/dist/network-observer.d.ts +49 -0
- package/dist/network-observer.js +149 -0
- package/dist/network-observer.js.map +7 -0
- package/dist/network-throttle.d.ts +25 -0
- package/dist/network-throttle.js +34 -0
- package/dist/network-throttle.js.map +7 -0
- package/dist/openapi-generator.d.ts +49 -0
- package/dist/openapi-generator.js +89 -0
- package/dist/openapi-generator.js.map +7 -0
- package/dist/pdf-extractor.d.ts +28 -0
- package/dist/pdf-extractor.js +81 -0
- package/dist/pdf-extractor.js.map +7 -0
- package/dist/performance-metrics-collector.d.ts +50 -0
- package/dist/performance-metrics-collector.js +155 -0
- package/dist/performance-metrics-collector.js.map +7 -0
- package/dist/performance-metrics.d.ts +52 -0
- package/dist/performance-metrics.js +83 -0
- package/dist/performance-metrics.js.map +7 -0
- package/dist/phash.d.ts +50 -0
- package/dist/phash.js +136 -0
- package/dist/phash.js.map +7 -0
- package/dist/pii-scrubber.d.ts +21 -0
- package/dist/pii-scrubber.js +75 -0
- package/dist/pii-scrubber.js.map +7 -0
- package/dist/playwright-recorder.d.ts +39 -0
- package/dist/playwright-recorder.js +123 -0
- package/dist/playwright-recorder.js.map +7 -0
- package/dist/policies/crawl-safety.d.ts +9 -0
- package/dist/policies/crawl-safety.js +41 -0
- package/dist/policies/crawl-safety.js.map +7 -0
- package/dist/profiles/domain-profile.d.ts +19 -0
- package/dist/profiles/domain-profile.js +160 -0
- package/dist/profiles/domain-profile.js.map +7 -0
- package/dist/proxy.d.ts +50 -0
- package/dist/proxy.js +165 -0
- package/dist/proxy.js.map +7 -0
- package/dist/rate-limiter.d.ts +31 -0
- package/dist/rate-limiter.js +64 -0
- package/dist/rate-limiter.js.map +7 -0
- package/dist/redirect-tracker.d.ts +37 -0
- package/dist/redirect-tracker.js +65 -0
- package/dist/redirect-tracker.js.map +7 -0
- package/dist/redis-frontier.d.ts +48 -0
- package/dist/redis-frontier.js +126 -0
- package/dist/redis-frontier.js.map +7 -0
- package/dist/redis-queue.d.ts +48 -0
- package/dist/redis-queue.js +165 -0
- package/dist/redis-queue.js.map +7 -0
- package/dist/response-cache.d.ts +38 -0
- package/dist/response-cache.js +140 -0
- package/dist/response-cache.js.map +7 -0
- package/dist/responsive-detector.d.ts +29 -0
- package/dist/responsive-detector.js +46 -0
- package/dist/responsive-detector.js.map +7 -0
- package/dist/retry-policy.d.ts +20 -0
- package/dist/retry-policy.js +54 -0
- package/dist/retry-policy.js.map +7 -0
- package/dist/robots-txt.d.ts +9 -0
- package/dist/robots-txt.js +77 -0
- package/dist/robots-txt.js.map +7 -0
- package/dist/screen-reader.d.ts +15 -0
- package/dist/screen-reader.js +48 -0
- package/dist/screen-reader.js.map +7 -0
- package/dist/screenshot-masker.d.ts +28 -0
- package/dist/screenshot-masker.js +62 -0
- package/dist/screenshot-masker.js.map +7 -0
- package/dist/seo-checker.d.ts +33 -0
- package/dist/seo-checker.js +91 -0
- package/dist/seo-checker.js.map +7 -0
- package/dist/service-worker-interceptor.d.ts +36 -0
- package/dist/service-worker-interceptor.js +136 -0
- package/dist/service-worker-interceptor.js.map +7 -0
- package/dist/session-capture.d.ts +39 -0
- package/dist/session-capture.js +274 -0
- package/dist/session-capture.js.map +7 -0
- package/dist/session-export.d.ts +49 -0
- package/dist/session-export.js +117 -0
- package/dist/session-export.js.map +7 -0
- package/dist/shadow-dom-crawler.d.ts +35 -0
- package/dist/shadow-dom-crawler.js +132 -0
- package/dist/shadow-dom-crawler.js.map +7 -0
- package/dist/sitemap-writer.d.ts +21 -0
- package/dist/sitemap-writer.js +47 -0
- package/dist/sitemap-writer.js.map +7 -0
- package/dist/sitemap.d.ts +51 -0
- package/dist/sitemap.js +160 -0
- package/dist/sitemap.js.map +7 -0
- package/dist/skyvern.d.ts +56 -0
- package/dist/skyvern.js +129 -0
- package/dist/skyvern.js.map +7 -0
- package/dist/sqs-queue.d.ts +13 -0
- package/dist/sqs-queue.js +115 -0
- package/dist/sqs-queue.js.map +7 -0
- package/dist/sso-token.d.ts +29 -0
- package/dist/sso-token.js +76 -0
- package/dist/sso-token.js.map +7 -0
- package/dist/state-graph.d.ts +59 -0
- package/dist/state-graph.js +193 -0
- package/dist/state-graph.js.map +7 -0
- package/dist/stealth/human-behavior.d.ts +20 -0
- package/dist/stealth/human-behavior.js +202 -0
- package/dist/stealth/human-behavior.js.map +7 -0
- package/dist/telemetry.d.ts +34 -0
- package/dist/telemetry.js +77 -0
- package/dist/telemetry.js.map +7 -0
- package/dist/test-executor.d.ts +81 -0
- package/dist/test-executor.js +443 -0
- package/dist/test-executor.js.map +7 -0
- package/dist/tls-fingerprint.d.ts +50 -0
- package/dist/tls-fingerprint.js +81 -0
- package/dist/tls-fingerprint.js.map +7 -0
- package/dist/url-pattern-completer.d.ts +28 -0
- package/dist/url-pattern-completer.js +122 -0
- package/dist/url-pattern-completer.js.map +7 -0
- package/dist/visual-diff-verdict.d.ts +40 -0
- package/dist/visual-diff-verdict.js +27 -0
- package/dist/visual-diff-verdict.js.map +7 -0
- package/dist/webhook-notifier.d.ts +30 -0
- package/dist/webhook-notifier.js +76 -0
- package/dist/webhook-notifier.js.map +7 -0
- package/dist/websocket-detector.d.ts +24 -0
- package/dist/websocket-detector.js +60 -0
- package/dist/websocket-detector.js.map +7 -0
- package/dist/worker.d.ts +1 -0
- package/dist/worker.js +33 -0
- package/dist/worker.js.map +7 -0
- package/package.json +49 -4
- package/src/accessibility-auditor.ts +102 -0
- package/src/accessibility-tree-crawler.ts +246 -0
- package/src/advanced-ui-probes.ts +470 -0
- package/src/ai-login.ts +298 -0
- package/src/auth-token-rotator.ts +89 -0
- package/src/auth.ts +418 -0
- package/src/browser-extension.ts +186 -0
- package/src/captcha-solver.ts +143 -0
- package/src/color-contrast.ts +116 -0
- package/src/content-chunker.ts +170 -0
- package/src/content-dedup.ts +99 -0
- package/src/cookie-consent.ts +106 -0
- package/src/cookie-jar.ts +91 -0
- package/src/crawl-session-manager.ts +231 -0
- package/src/crawler.ts +7123 -0
- package/src/crawler.ts.rej +12 -0
- package/src/crud-interaction.ts +386 -0
- package/src/custom-js-injector.ts +93 -0
- package/src/dark-mode-capture.ts +110 -0
- package/src/dead-link-detector.ts +76 -0
- package/src/dropdown-enumerator.ts +312 -0
- package/src/element-hash.ts +16 -0
- package/src/email-otp.ts +252 -0
- package/src/engine-router.ts +141 -0
- package/src/export/bulk-export.ts +85 -0
- package/src/export/vector-export.ts +160 -0
- package/src/extractors/advanced.ts +309 -0
- package/src/extractors/artifact-envelope.ts +111 -0
- package/src/extractors/branding-extractor.ts +159 -0
- package/src/extractors/summary-extractor.ts +49 -0
- package/src/failure-analysis-agent.ts +51 -0
- package/src/feature-flag-detector.ts +97 -0
- package/src/filter-pagination-crawler.ts +500 -0
- package/src/form-detector.ts +103 -0
- package/src/frontier.test.ts +64 -0
- package/src/frontier.ts +144 -0
- package/src/gap-pass.ts +165 -0
- package/src/graphql-detector.ts +129 -0
- package/src/http-engine.ts +453 -0
- package/src/incremental-diff.ts +259 -0
- package/src/index.ts +52 -0
- package/src/input-fuzzer.ts +129 -0
- package/src/interactive-probe.ts +1136 -0
- package/src/job-queue.test.ts +187 -0
- package/src/job-queue.ts +4504 -0
- package/src/js-coverage.ts +87 -0
- package/src/js-router-detector.ts +112 -0
- package/src/language-detector.ts +82 -0
- package/src/link-classifier.ts +65 -0
- package/src/live-view.ts +146 -0
- package/src/locator-extractor.ts +309 -0
- package/src/logger.ts +9 -0
- package/src/markdown-extract.test.ts +73 -0
- package/src/markdown-extract.ts +76 -0
- package/src/multi-context-handler.ts +296 -0
- package/src/native-crawler.ts +1108 -0
- package/src/nested-window-crawler.ts +310 -0
- package/src/network-action-correlator.ts +320 -0
- package/src/network-observer.ts +204 -0
- package/src/network-throttle.ts +69 -0
- package/src/openapi-generator.ts +135 -0
- package/src/pdf-extractor.ts +123 -0
- package/src/performance-metrics-collector.ts +243 -0
- package/src/performance-metrics.ts +134 -0
- package/src/phash.ts +184 -0
- package/src/pii-scrubber.ts +99 -0
- package/src/playwright-recorder.ts +170 -0
- package/src/policies/crawl-safety.ts +52 -0
- package/src/profiles/domain-profile.ts +190 -0
- package/src/proxy.ts +238 -0
- package/src/rate-limiter.ts +87 -0
- package/src/redirect-tracker.ts +97 -0
- package/src/redis-frontier.ts +143 -0
- package/src/redis-queue.ts +208 -0
- package/src/response-cache.ts +150 -0
- package/src/responsive-detector.ts +92 -0
- package/src/retry-policy.ts +87 -0
- package/src/robots-txt.ts +96 -0
- package/src/screen-reader.ts +98 -0
- package/src/screenshot-masker.ts +98 -0
- package/src/seo-checker.ts +134 -0
- package/src/service-worker-interceptor.ts +185 -0
- package/src/session-capture.ts +335 -0
- package/src/session-export.test.ts +75 -0
- package/src/session-export.ts +172 -0
- package/src/shadow-dom-crawler.ts +198 -0
- package/src/sitemap-writer.ts +65 -0
- package/src/sitemap.test.ts +127 -0
- package/src/sitemap.ts +233 -0
- package/src/skyvern.ts +199 -0
- package/src/sqs-queue.ts +129 -0
- package/src/sso-token.ts +120 -0
- package/src/state-graph.ts +280 -0
- package/src/stealth/human-behavior.ts +227 -0
- package/src/telemetry.ts +122 -0
- package/src/test-executor.test.ts +72 -0
- package/src/test-executor.ts +578 -0
- package/src/tls-fingerprint.ts +111 -0
- package/src/url-pattern-completer.ts +161 -0
- package/src/vendor.d.ts +20 -0
- package/src/visual-diff-verdict.test.ts +80 -0
- package/src/visual-diff-verdict.ts +59 -0
- package/src/webhook-notifier.ts +109 -0
- package/src/websocket-detector.ts +83 -0
- package/src/worker.ts +50 -0
- package/tsconfig.json +13 -0
- package/types/accessibility-auditor.d.ts +29 -0
- package/types/accessibility-tree-crawler.d.ts +44 -0
- package/types/advanced-ui-probes.d.ts +47 -0
- package/types/ai-login.d.ts +32 -0
- package/types/auth-token-rotator.d.ts +47 -0
- package/types/auth.d.ts +20 -0
- package/types/browser-extension.d.ts +37 -0
- package/types/bull-worker.d.ts +6 -0
- package/types/captcha-solver.d.ts +16 -0
- package/types/color-contrast.d.ts +30 -0
- package/types/content-chunker.d.ts +18 -0
- package/types/content-dedup.d.ts +23 -0
- package/types/cookie-consent.d.ts +25 -0
- package/types/cookie-jar.d.ts +20 -0
- package/types/crawl-session-manager.d.ts +44 -0
- package/types/crawler.d.ts +233 -0
- package/types/crud-interaction.d.ts +31 -0
- package/types/custom-js-injector.d.ts +42 -0
- package/types/dark-mode-capture.d.ts +23 -0
- package/types/dead-link-detector.d.ts +37 -0
- package/types/dropdown-enumerator.d.ts +20 -0
- package/types/element-hash.d.ts +9 -0
- package/types/email-otp.d.ts +42 -0
- package/types/engine-router.d.ts +47 -0
- package/types/export/bulk-export.d.ts +26 -0
- package/types/export/vector-export.d.ts +76 -0
- package/types/extractors/advanced.d.ts +64 -0
- package/types/extractors/artifact-envelope.d.ts +63 -0
- package/types/extractors/branding-extractor.d.ts +46 -0
- package/types/extractors/summary-extractor.d.ts +10 -0
- package/types/failure-analysis-agent.d.ts +16 -0
- package/types/feature-flag-detector.d.ts +30 -0
- package/types/filter-pagination-crawler.d.ts +52 -0
- package/types/form-detector.d.ts +34 -0
- package/types/frontier.d.ts +40 -0
- package/types/gap-pass.d.ts +36 -0
- package/types/graphql-detector.d.ts +39 -0
- package/types/http-engine.d.ts +80 -0
- package/types/incremental-diff.d.ts +116 -0
- package/types/index.d.ts +52 -0
- package/types/input-fuzzer.d.ts +30 -0
- package/types/interactive-probe.d.ts +144 -0
- package/types/job-queue.d.ts +308 -0
- package/types/js-coverage.d.ts +29 -0
- package/types/js-router-detector.d.ts +17 -0
- package/types/language-detector.d.ts +22 -0
- package/types/link-classifier.d.ts +23 -0
- package/types/live-view.d.ts +36 -0
- package/types/locator-extractor.d.ts +46 -0
- package/types/logger.d.ts +2 -0
- package/types/markdown-extract.d.ts +8 -0
- package/types/multi-context-handler.d.ts +55 -0
- package/types/native-crawler.d.ts +69 -0
- package/types/nested-window-crawler.d.ts +41 -0
- package/types/network-action-correlator.d.ts +47 -0
- package/types/network-observer.d.ts +49 -0
- package/types/network-throttle.d.ts +25 -0
- package/types/openapi-generator.d.ts +49 -0
- package/types/pdf-extractor.d.ts +28 -0
- package/types/performance-metrics-collector.d.ts +50 -0
- package/types/performance-metrics.d.ts +52 -0
- package/types/phash.d.ts +50 -0
- package/types/pii-scrubber.d.ts +21 -0
- package/types/playwright-recorder.d.ts +39 -0
- package/types/policies/crawl-safety.d.ts +9 -0
- package/types/profiles/domain-profile.d.ts +19 -0
- package/types/proxy.d.ts +50 -0
- package/types/rate-limiter.d.ts +31 -0
- package/types/redirect-tracker.d.ts +37 -0
- package/types/redis-frontier.d.ts +48 -0
- package/types/redis-queue.d.ts +48 -0
- package/types/response-cache.d.ts +38 -0
- package/types/responsive-detector.d.ts +29 -0
- package/types/retry-policy.d.ts +20 -0
- package/types/robots-txt.d.ts +9 -0
- package/types/screen-reader.d.ts +15 -0
- package/types/screenshot-masker.d.ts +28 -0
- package/types/seo-checker.d.ts +33 -0
- package/types/service-worker-interceptor.d.ts +36 -0
- package/types/session-capture.d.ts +39 -0
- package/types/session-export.d.ts +49 -0
- package/types/shadow-dom-crawler.d.ts +35 -0
- package/types/sitemap-writer.d.ts +21 -0
- package/types/sitemap.d.ts +51 -0
- package/types/skyvern.d.ts +56 -0
- package/types/sqs-queue.d.ts +13 -0
- package/types/sso-token.d.ts +29 -0
- package/types/state-graph.d.ts +59 -0
- package/types/stealth/human-behavior.d.ts +20 -0
- package/types/telemetry.d.ts +34 -0
- package/types/test-executor.d.ts +84 -0
- package/types/tls-fingerprint.d.ts +50 -0
- package/types/url-pattern-completer.d.ts +28 -0
- package/types/visual-diff-verdict.d.ts +40 -0
- package/types/webhook-notifier.d.ts +30 -0
- package/types/websocket-detector.d.ts +24 -0
- package/types/worker.d.ts +1 -0
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Service Worker detection — inspects SW registrations, cache storage, background
|
|
3
|
+
* sync queues, and PWA installability for a crawled page.
|
|
4
|
+
* Surfaces offline-first contracts and push-capable surfaces that HTTP-only
|
|
5
|
+
* crawlers cannot observe.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { Page } from 'playwright';
|
|
9
|
+
|
|
10
|
+
export interface SwRegistration {
|
|
11
|
+
scope: string;
|
|
12
|
+
scriptURL: string | null;
|
|
13
|
+
state: 'active' | 'installing' | 'waiting' | 'none';
|
|
14
|
+
updateViaCache: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface SwCacheEntry {
|
|
18
|
+
cacheName: string;
|
|
19
|
+
entryCount: number;
|
|
20
|
+
sampleUrls: string[];
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface BackgroundSyncInfo {
|
|
24
|
+
scope: string;
|
|
25
|
+
pendingTags: string[];
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface PwaInfo {
|
|
29
|
+
isInstallable: boolean;
|
|
30
|
+
manifestUrl: string | null;
|
|
31
|
+
hasPushPermission: boolean;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface ServiceWorkerResult {
|
|
35
|
+
supported: boolean;
|
|
36
|
+
active: boolean;
|
|
37
|
+
registrations: SwRegistration[];
|
|
38
|
+
cacheStorages: SwCacheEntry[];
|
|
39
|
+
backgroundSync: BackgroundSyncInfo[];
|
|
40
|
+
pwa: PwaInfo;
|
|
41
|
+
totalCachedEntries: number;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const EMPTY_RESULT: ServiceWorkerResult = {
|
|
45
|
+
supported: false,
|
|
46
|
+
active: false,
|
|
47
|
+
registrations: [],
|
|
48
|
+
cacheStorages: [],
|
|
49
|
+
backgroundSync: [],
|
|
50
|
+
pwa: { isInstallable: false, manifestUrl: null, hasPushPermission: false },
|
|
51
|
+
totalCachedEntries: 0,
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
async function detectServiceWorkers(
|
|
55
|
+
page: any,
|
|
56
|
+
): Promise<{ supported: boolean; active: boolean; registrations: SwRegistration[] }> {
|
|
57
|
+
const result = await page.evaluate(async () => {
|
|
58
|
+
if (!('serviceWorker' in navigator)) return { supported: false, active: false, registrations: [] };
|
|
59
|
+
try {
|
|
60
|
+
const controller = navigator.serviceWorker.controller;
|
|
61
|
+
const regs = await navigator.serviceWorker.getRegistrations();
|
|
62
|
+
return {
|
|
63
|
+
supported: true,
|
|
64
|
+
active: !!controller,
|
|
65
|
+
registrations: regs.map(r => ({
|
|
66
|
+
scope: r.scope,
|
|
67
|
+
scriptURL: r.active?.scriptURL || r.installing?.scriptURL || r.waiting?.scriptURL || null,
|
|
68
|
+
state: r.active ? 'active' : r.installing ? 'installing' : r.waiting ? 'waiting' : 'none',
|
|
69
|
+
updateViaCache: r.updateViaCache,
|
|
70
|
+
})),
|
|
71
|
+
};
|
|
72
|
+
} catch {
|
|
73
|
+
return { supported: true, active: false, registrations: [] };
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
return result as { supported: boolean; active: boolean; registrations: SwRegistration[] };
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async function enumerateCacheStorage(page: any): Promise<SwCacheEntry[]> {
|
|
80
|
+
const caches = await page.evaluate(async () => {
|
|
81
|
+
if (!('caches' in window)) return [];
|
|
82
|
+
try {
|
|
83
|
+
const names = await window.caches.keys();
|
|
84
|
+
const result = [];
|
|
85
|
+
for (const name of names) {
|
|
86
|
+
try {
|
|
87
|
+
const cache = await window.caches.open(name);
|
|
88
|
+
const keys = await cache.keys();
|
|
89
|
+
result.push({
|
|
90
|
+
cacheName: name,
|
|
91
|
+
entryCount: keys.length,
|
|
92
|
+
sampleUrls: keys.slice(0, 15).map(k => {
|
|
93
|
+
try { return new URL(k.url).pathname; } catch { return k.url.slice(0, 80); }
|
|
94
|
+
}),
|
|
95
|
+
});
|
|
96
|
+
} catch { /* skip inaccessible cache */ }
|
|
97
|
+
}
|
|
98
|
+
return result;
|
|
99
|
+
} catch {
|
|
100
|
+
return [];
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
return caches as SwCacheEntry[];
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async function detectBackgroundSync(page: any): Promise<BackgroundSyncInfo[]> {
|
|
107
|
+
const syncInfo = await page.evaluate(async () => {
|
|
108
|
+
if (!('serviceWorker' in navigator)) return [];
|
|
109
|
+
try {
|
|
110
|
+
const regs = await navigator.serviceWorker.getRegistrations();
|
|
111
|
+
const result = [];
|
|
112
|
+
for (const reg of regs) {
|
|
113
|
+
if ('sync' in reg) {
|
|
114
|
+
try {
|
|
115
|
+
const tags = await (reg as any).sync.getTags();
|
|
116
|
+
result.push({ scope: reg.scope, pendingTags: tags || [] });
|
|
117
|
+
} catch {
|
|
118
|
+
result.push({ scope: reg.scope, pendingTags: [] });
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return result;
|
|
123
|
+
} catch {
|
|
124
|
+
return [];
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
return syncInfo as BackgroundSyncInfo[];
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
async function detectPwa(page: any): Promise<PwaInfo> {
|
|
131
|
+
const pwa = await page.evaluate(() => {
|
|
132
|
+
const manifestEl = document.querySelector('link[rel="manifest"]');
|
|
133
|
+
const manifestUrl = manifestEl?.getAttribute('href') || null;
|
|
134
|
+
const hasSW = 'serviceWorker' in navigator && !!navigator.serviceWorker.controller;
|
|
135
|
+
let hasPushPermission = false;
|
|
136
|
+
try { hasPushPermission = Notification.permission === 'granted'; } catch { /* ignore */ }
|
|
137
|
+
return {
|
|
138
|
+
isInstallable: !!(manifestUrl && hasSW),
|
|
139
|
+
manifestUrl,
|
|
140
|
+
hasPushPermission,
|
|
141
|
+
};
|
|
142
|
+
});
|
|
143
|
+
return pwa as PwaInfo;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export async function inspectServiceWorker(page: any): Promise<ServiceWorkerResult> {
|
|
147
|
+
try {
|
|
148
|
+
// Allow SW on HTTPS or localhost (http://localhost and http://127.0.0.1 support SW)
|
|
149
|
+
const url = page.url();
|
|
150
|
+
const isSecureContext = url.startsWith('https://') ||
|
|
151
|
+
/^http:\/\/(localhost|127\.0\.0\.1)(:\d+)?/.test(url);
|
|
152
|
+
if (!isSecureContext) return { ...EMPTY_RESULT };
|
|
153
|
+
|
|
154
|
+
const [swResult, cacheResult, syncResult, pwaResult] = await Promise.allSettled([
|
|
155
|
+
detectServiceWorkers(page),
|
|
156
|
+
enumerateCacheStorage(page),
|
|
157
|
+
detectBackgroundSync(page),
|
|
158
|
+
detectPwa(page),
|
|
159
|
+
]);
|
|
160
|
+
|
|
161
|
+
const sw = swResult.status === 'fulfilled'
|
|
162
|
+
? swResult.value
|
|
163
|
+
: { supported: false, active: false, registrations: [] };
|
|
164
|
+
|
|
165
|
+
const cacheStorages = cacheResult.status === 'fulfilled' ? cacheResult.value : [];
|
|
166
|
+
const backgroundSync = syncResult.status === 'fulfilled' ? syncResult.value : [];
|
|
167
|
+
const pwa = pwaResult.status === 'fulfilled'
|
|
168
|
+
? pwaResult.value
|
|
169
|
+
: EMPTY_RESULT.pwa;
|
|
170
|
+
|
|
171
|
+
const totalCachedEntries = cacheStorages.reduce((sum, c) => sum + c.entryCount, 0);
|
|
172
|
+
|
|
173
|
+
return {
|
|
174
|
+
supported: sw.supported,
|
|
175
|
+
active: sw.active,
|
|
176
|
+
registrations: sw.registrations,
|
|
177
|
+
cacheStorages,
|
|
178
|
+
backgroundSync,
|
|
179
|
+
pwa,
|
|
180
|
+
totalCachedEntries,
|
|
181
|
+
};
|
|
182
|
+
} catch {
|
|
183
|
+
return { ...EMPTY_RESULT };
|
|
184
|
+
}
|
|
185
|
+
}
|
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
import { chromium } from 'playwright';
|
|
2
|
+
import { randomBytes } from 'node:crypto';
|
|
3
|
+
import { login } from './auth.js';
|
|
4
|
+
import { generateTOTP } from './job-queue.js';
|
|
5
|
+
import { getNonHeadlessUserAgent } from './crawler.js';
|
|
6
|
+
|
|
7
|
+
interface CaptureSession {
|
|
8
|
+
browser: any;
|
|
9
|
+
context: any;
|
|
10
|
+
timer: ReturnType<typeof setTimeout>;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const activeSessions = new Map<string, CaptureSession>();
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Opens a headed (visible) browser at appUrl and returns a captureId.
|
|
17
|
+
* The user logs in manually, then calls completeSessionCapture() to harvest storageState.
|
|
18
|
+
* Sessions expire after 10 minutes if not completed.
|
|
19
|
+
*
|
|
20
|
+
* Throws 'HEADLESS_ENV' if running in Docker/Linux without a display server.
|
|
21
|
+
* The API route surfaces this as a 503 so the frontend can switch to manual JSON paste.
|
|
22
|
+
*/
|
|
23
|
+
export async function startSessionCapture(appUrl: string): Promise<string> {
|
|
24
|
+
const inDocker = process.env.PLAYWRIGHT_IN_DOCKER === 'true';
|
|
25
|
+
const noDisplay = process.platform === 'linux' && !process.env.DISPLAY;
|
|
26
|
+
if (inDocker || noDisplay) {
|
|
27
|
+
throw new Error('HEADLESS_ENV');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const id = randomBytes(16).toString('hex');
|
|
31
|
+
const browser = await chromium.launch({ headless: false });
|
|
32
|
+
const userAgent = await getNonHeadlessUserAgent(chromium.executablePath());
|
|
33
|
+
const context = await browser.newContext({ userAgent });
|
|
34
|
+
const page = await context.newPage();
|
|
35
|
+
await page.goto(appUrl, { waitUntil: 'domcontentloaded' }).catch(() => {});
|
|
36
|
+
const timer = setTimeout(() => cancelSessionCapture(id).catch(() => {}), 10 * 60 * 1000);
|
|
37
|
+
activeSessions.set(id, { browser, context, timer });
|
|
38
|
+
return id;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Captures storageState from the open browser, closes it, and returns the JSON string. */
|
|
42
|
+
export async function completeSessionCapture(captureId: string): Promise<string> {
|
|
43
|
+
const session = activeSessions.get(captureId);
|
|
44
|
+
if (!session) throw new Error('Capture session not found or expired.');
|
|
45
|
+
clearTimeout(session.timer);
|
|
46
|
+
const state = await session.context.storageState();
|
|
47
|
+
await session.browser.close().catch(() => {});
|
|
48
|
+
activeSessions.delete(captureId);
|
|
49
|
+
return JSON.stringify(state);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Closes the browser without capturing — called on Cancel. */
|
|
53
|
+
export async function cancelSessionCapture(captureId: string): Promise<void> {
|
|
54
|
+
const session = activeSessions.get(captureId);
|
|
55
|
+
if (!session) return;
|
|
56
|
+
clearTimeout(session.timer);
|
|
57
|
+
await session.browser.close().catch(() => {});
|
|
58
|
+
activeSessions.delete(captureId);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Post-login exploration: navigates nav links, cycles tabs, advances carousels,
|
|
63
|
+
* opens accordions, scrolls — so the browser accumulates full session state before capture.
|
|
64
|
+
* All best-effort: never throws, never fails the capture.
|
|
65
|
+
*/
|
|
66
|
+
async function exploreAppAfterLogin(page: any): Promise<void> {
|
|
67
|
+
const sleep = (ms: number) => page.waitForTimeout(ms).catch(() => {});
|
|
68
|
+
|
|
69
|
+
const isVisible = `
|
|
70
|
+
function isVisible(el) {
|
|
71
|
+
const s = window.getComputedStyle(el);
|
|
72
|
+
if (s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) < 0.1) return false;
|
|
73
|
+
const r = el.getBoundingClientRect();
|
|
74
|
+
return r.width > 4 && r.height > 4;
|
|
75
|
+
}
|
|
76
|
+
`;
|
|
77
|
+
|
|
78
|
+
try {
|
|
79
|
+
const origin = await page.evaluate(() => location.origin).catch(() => '');
|
|
80
|
+
const startUrl = await page.evaluate(() => location.href).catch(() => '');
|
|
81
|
+
|
|
82
|
+
// ── 1. Expand collapsed nav menus / hamburgers ──────────────────────────
|
|
83
|
+
await page.evaluate(new Function(`
|
|
84
|
+
${isVisible}
|
|
85
|
+
const inBadCtx = el => {
|
|
86
|
+
let p = el.parentElement;
|
|
87
|
+
while (p) {
|
|
88
|
+
const role = p.getAttribute('role') ?? '';
|
|
89
|
+
if (['dialog','alertdialog','combobox','listbox'].includes(role)) return true;
|
|
90
|
+
p = p.parentElement;
|
|
91
|
+
}
|
|
92
|
+
return false;
|
|
93
|
+
};
|
|
94
|
+
const targets = new Set();
|
|
95
|
+
document.querySelectorAll('[aria-haspopup]:not([aria-expanded="true"])').forEach(el => {
|
|
96
|
+
if (isVisible(el) && !inBadCtx(el)) targets.add(el);
|
|
97
|
+
});
|
|
98
|
+
document.querySelectorAll('[aria-expanded="false"]').forEach(el => {
|
|
99
|
+
if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadCtx(el)) targets.add(el);
|
|
100
|
+
});
|
|
101
|
+
document.querySelectorAll('[data-state="closed"],[data-headlessui-state="closed"]').forEach(el => {
|
|
102
|
+
if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadCtx(el)) targets.add(el);
|
|
103
|
+
});
|
|
104
|
+
targets.forEach(el => { try { el.click(); } catch {} });
|
|
105
|
+
`) as () => void).catch(() => {});
|
|
106
|
+
await sleep(600);
|
|
107
|
+
|
|
108
|
+
// ── 2. Collect nav links from the expanded DOM ──────────────────────────
|
|
109
|
+
const navLinks: string[] = await page.evaluate(new Function(`
|
|
110
|
+
${isVisible}
|
|
111
|
+
const links = new Set();
|
|
112
|
+
const navEls = document.querySelectorAll('nav a, [role="navigation"] a, aside a, header a');
|
|
113
|
+
navEls.forEach(a => {
|
|
114
|
+
const href = a.getAttribute('href');
|
|
115
|
+
if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:')) return;
|
|
116
|
+
if (!isVisible(a)) return;
|
|
117
|
+
try {
|
|
118
|
+
const url = new URL(href, location.href);
|
|
119
|
+
if (url.origin === location.origin) links.add(url.href);
|
|
120
|
+
} catch {}
|
|
121
|
+
});
|
|
122
|
+
return [...links].slice(0, 10); // cap at 10 nav pages
|
|
123
|
+
`) as () => string[]).catch(() => []);
|
|
124
|
+
|
|
125
|
+
// Dismiss opened menus before navigating
|
|
126
|
+
await page.keyboard?.press('Escape').catch(() => {});
|
|
127
|
+
await sleep(200);
|
|
128
|
+
|
|
129
|
+
// ── 3. Visit each nav link, interact, return ────────────────────────────
|
|
130
|
+
for (const link of navLinks) {
|
|
131
|
+
try {
|
|
132
|
+
if (link === startUrl) continue;
|
|
133
|
+
await page.goto(link, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => {});
|
|
134
|
+
await sleep(400);
|
|
135
|
+
await interactWithCurrentPage(page, isVisible, sleep);
|
|
136
|
+
} catch { /* skip broken page */ }
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Return to start page
|
|
140
|
+
if (navLinks.length > 0) {
|
|
141
|
+
await page.goto(startUrl, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => {});
|
|
142
|
+
await sleep(400);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// ── 4. Final interaction pass on landing page ───────────────────────────
|
|
146
|
+
await interactWithCurrentPage(page, isVisible, sleep);
|
|
147
|
+
|
|
148
|
+
await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => {});
|
|
149
|
+
} catch {
|
|
150
|
+
// Exploration is best-effort
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Interacts with tabs, carousels, accordions, and scrolls on the current page. */
|
|
155
|
+
async function interactWithCurrentPage(page: any, isVisible: string, sleep: (ms: number) => Promise<void>): Promise<void> {
|
|
156
|
+
// Cycle through tabs
|
|
157
|
+
await page.evaluate(new Function(`
|
|
158
|
+
${isVisible}
|
|
159
|
+
const tabs = [...document.querySelectorAll('[role="tab"]')].filter(isVisible);
|
|
160
|
+
tabs.forEach(tab => { try { tab.click(); } catch {} });
|
|
161
|
+
`) as () => void).catch(() => {});
|
|
162
|
+
await sleep(400);
|
|
163
|
+
|
|
164
|
+
// Advance carousels (next buttons)
|
|
165
|
+
await page.evaluate(new Function(`
|
|
166
|
+
${isVisible}
|
|
167
|
+
const nextBtns = [...document.querySelectorAll(
|
|
168
|
+
'button[aria-label*="next" i], button[aria-label*="forward" i], button[aria-label*="right" i], ' +
|
|
169
|
+
'[class*="carousel" i] button, [class*="slider" i] button, [data-direction="next"]'
|
|
170
|
+
)].filter(isVisible);
|
|
171
|
+
// Click next up to 3 times per carousel
|
|
172
|
+
for (let i = 0; i < 3; i++) {
|
|
173
|
+
nextBtns.forEach(btn => { try { btn.click(); } catch {} });
|
|
174
|
+
}
|
|
175
|
+
`) as () => void).catch(() => {});
|
|
176
|
+
await sleep(400);
|
|
177
|
+
|
|
178
|
+
// Open closed accordions / disclosure panels
|
|
179
|
+
await page.evaluate(new Function(`
|
|
180
|
+
${isVisible}
|
|
181
|
+
document.querySelectorAll(
|
|
182
|
+
'[data-state="closed"] button, [aria-expanded="false"], ' +
|
|
183
|
+
'details:not([open]) > summary, [class*="accordion" i] button'
|
|
184
|
+
).forEach(el => {
|
|
185
|
+
if (isVisible(el)) { try { el.click(); } catch {} }
|
|
186
|
+
});
|
|
187
|
+
`) as () => void).catch(() => {});
|
|
188
|
+
await sleep(300);
|
|
189
|
+
|
|
190
|
+
// Scroll to bottom then back to top
|
|
191
|
+
await page.evaluate(() => { window.scrollTo({ top: document.body.scrollHeight, behavior: 'instant' }); }).catch(() => {});
|
|
192
|
+
await sleep(300);
|
|
193
|
+
await page.evaluate(() => { window.scrollTo({ top: 0, behavior: 'instant' }); }).catch(() => {});
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Headless login + session capture — works on Docker/cloud (no display needed).
|
|
198
|
+
* Tries DOM-based login first (fast, free). If it fails and llmConfig is provided,
|
|
199
|
+
* falls back to Stagehand AI-guided login (handles modals, multi-step, marketing pages).
|
|
200
|
+
*
|
|
201
|
+
* Returns { storageState, cookies, localStorage } on success.
|
|
202
|
+
* Throws a human-readable error on failure.
|
|
203
|
+
*/
|
|
204
|
+
export async function validateAndCaptureSession(
|
|
205
|
+
appUrl: string,
|
|
206
|
+
creds: {
|
|
207
|
+
username: string; password: string; totpSecret?: string;
|
|
208
|
+
httpBasicUsername?: string; httpBasicPassword?: string;
|
|
209
|
+
},
|
|
210
|
+
llmConfig?: { modelName: string; apiKey: string },
|
|
211
|
+
captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },
|
|
212
|
+
loginInstructions?: string,
|
|
213
|
+
): Promise<{ storageState: string; cookies: number; localStorage: number; finalUrl?: string }> {
|
|
214
|
+
const browser = await chromium.launch({ headless: true });
|
|
215
|
+
try {
|
|
216
|
+
const userAgent = await getNonHeadlessUserAgent(chromium.executablePath());
|
|
217
|
+
const contextOpts: any = { userAgent };
|
|
218
|
+
if (creds.httpBasicUsername && creds.httpBasicPassword) {
|
|
219
|
+
contextOpts.httpCredentials = { username: creds.httpBasicUsername, password: creds.httpBasicPassword };
|
|
220
|
+
}
|
|
221
|
+
const context = await browser.newContext(contextOpts);
|
|
222
|
+
const page = await context.newPage();
|
|
223
|
+
|
|
224
|
+
const getOneTimeCode = creds.totpSecret
|
|
225
|
+
? async () => generateTOTP(creds.totpSecret!)
|
|
226
|
+
: undefined;
|
|
227
|
+
|
|
228
|
+
let loggedIn: boolean;
|
|
229
|
+
try {
|
|
230
|
+
loggedIn = await login(page, appUrl, {
|
|
231
|
+
username: creds.username,
|
|
232
|
+
password: creds.password,
|
|
233
|
+
getOneTimeCode,
|
|
234
|
+
});
|
|
235
|
+
} catch (loginErr: any) {
|
|
236
|
+
// login() throws when it can read a real error message from the page
|
|
237
|
+
const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);
|
|
238
|
+
throw Object.assign(loginErr, {
|
|
239
|
+
failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// Stagehand AI fallback when DOM login fails and LLM config is available
|
|
244
|
+
if (!loggedIn && llmConfig) {
|
|
245
|
+
let sh: any = null;
|
|
246
|
+
try {
|
|
247
|
+
const { Stagehand } = await import('@browserbasehq/stagehand');
|
|
248
|
+
const { loginWithStagehand } = await import('./ai-login.js');
|
|
249
|
+
sh = new Stagehand({
|
|
250
|
+
env: 'LOCAL',
|
|
251
|
+
model: { modelName: llmConfig.modelName as any, apiKey: llmConfig.apiKey },
|
|
252
|
+
serverCache: false,
|
|
253
|
+
verbose: 0,
|
|
254
|
+
localBrowserLaunchOptions: { headless: true, args: [`--user-agent=${userAgent}`] },
|
|
255
|
+
});
|
|
256
|
+
await sh.init();
|
|
257
|
+
const shPage = await (sh as any).resolvePage();
|
|
258
|
+
await shPage.goto(appUrl, { waitUntil: 'networkidle', timeout: 30_000 }).catch(() => {});
|
|
259
|
+
const shLoggedIn = await loginWithStagehand(
|
|
260
|
+
sh,
|
|
261
|
+
{ username: creds.username, password: creds.password, getOneTimeCode },
|
|
262
|
+
appUrl,
|
|
263
|
+
captchaOpts,
|
|
264
|
+
undefined,
|
|
265
|
+
loginInstructions,
|
|
266
|
+
);
|
|
267
|
+
if (shLoggedIn) {
|
|
268
|
+
const shCtx = (sh as any).context ?? (sh as any).browserContext;
|
|
269
|
+
if (shCtx) {
|
|
270
|
+
const shState = await shCtx.storageState();
|
|
271
|
+
const shCookies = shState.cookies?.length ?? 0;
|
|
272
|
+
const shLocalStorage = shState.origins?.reduce((s: number, o: any) => s + (o.localStorage?.length ?? 0), 0) ?? 0;
|
|
273
|
+
if (shCookies > 0 || shLocalStorage > 0) {
|
|
274
|
+
await sh.close().catch(() => {});
|
|
275
|
+
sh = null;
|
|
276
|
+
return { storageState: JSON.stringify(shState), cookies: shCookies, localStorage: shLocalStorage };
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
loggedIn = true; // Stagehand succeeded but we'll capture from the plain browser context below
|
|
280
|
+
}
|
|
281
|
+
} catch (shErr: any) {
|
|
282
|
+
const msg = String(shErr?.message ?? shErr);
|
|
283
|
+
if (msg.includes('CAPTCHA') || shErr?.isSSOError) throw shErr;
|
|
284
|
+
// Stagehand failed silently — fall through to "credentials rejected"
|
|
285
|
+
} finally {
|
|
286
|
+
if (sh) await sh.close().catch(() => {});
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
if (!loggedIn) {
|
|
291
|
+
const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);
|
|
292
|
+
const challenge = await page.evaluate(() => {
|
|
293
|
+
const text = (document.body?.innerText ?? '').toLowerCase();
|
|
294
|
+
const hasCloudflare = /verify you are human|checking your browser|cloudflare|cf-challenge|turnstile/.test(text) ||
|
|
295
|
+
!!document.querySelector('iframe[src*="challenges.cloudflare.com"], .cf-turnstile, [data-sitekey]');
|
|
296
|
+
const hasCaptcha = hasCloudflare || /captcha|recaptcha|hcaptcha/.test(text) ||
|
|
297
|
+
!!document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], #captcha, .captcha, [id*="captcha"], [class*="captcha"]');
|
|
298
|
+
return { hasCloudflare, hasCaptcha };
|
|
299
|
+
}).catch(() => ({ hasCloudflare: false, hasCaptcha: false }));
|
|
300
|
+
const message = challenge.hasCloudflare
|
|
301
|
+
? 'Cloudflare human verification blocked headless validation. This cannot be bypassed reliably in code; use Live Browser or manual session capture, configure a CAPTCHA solver, or allowlist the crawler IP.'
|
|
302
|
+
: challenge.hasCaptcha
|
|
303
|
+
? 'A CAPTCHA blocked headless validation. Configure CAPTCHA Auto-Solve or use Live Browser/manual session capture.'
|
|
304
|
+
: 'Login failed — credentials rejected or the login flow requires a manual/SSO step. Check credentials, saved login instructions, or use Live Browser session capture.';
|
|
305
|
+
const err = Object.assign(new Error(message), {
|
|
306
|
+
failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,
|
|
307
|
+
reason: challenge.hasCloudflare ? 'CLOUDFLARE_CHALLENGE' : challenge.hasCaptcha ? 'CAPTCHA_CHALLENGE' : 'LOGIN_NOT_CONFIRMED',
|
|
308
|
+
});
|
|
309
|
+
throw err;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// Wait for post-login navigation to settle before capturing session
|
|
313
|
+
await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});
|
|
314
|
+
|
|
315
|
+
// Capture post-login landing URL before exploration navigates away
|
|
316
|
+
const finalUrl = page.url();
|
|
317
|
+
|
|
318
|
+
// Post-login exploration: navigate tabs, expand menus, scroll to trigger lazy-loaded state
|
|
319
|
+
await exploreAppAfterLogin(page);
|
|
320
|
+
|
|
321
|
+
const state = await context.storageState();
|
|
322
|
+
const cookies = state.cookies?.length ?? 0;
|
|
323
|
+
const localStorage = state.origins?.reduce(
|
|
324
|
+
(sum, o) => sum + (o.localStorage?.length ?? 0), 0
|
|
325
|
+
) ?? 0;
|
|
326
|
+
|
|
327
|
+
if (cookies === 0 && localStorage === 0) {
|
|
328
|
+
throw new Error('Login appeared to succeed but no session data was captured — try manual session capture instead');
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
return { storageState: JSON.stringify(state), cookies, localStorage, finalUrl };
|
|
332
|
+
} finally {
|
|
333
|
+
await browser.close().catch(() => {});
|
|
334
|
+
}
|
|
335
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import test from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { toCypressSnippet, toSeleniumPythonDict, toWebdriverIOSnippet, exportSessionAs, type StorageState } from './session-export.js';
|
|
4
|
+
|
|
5
|
+
const sampleState: StorageState = {
|
|
6
|
+
cookies: [
|
|
7
|
+
{ name: 'session_id', value: 'abc123', domain: '.example.com', path: '/', secure: true, httpOnly: true, sameSite: 'Lax', expires: 1893456000 },
|
|
8
|
+
],
|
|
9
|
+
origins: [
|
|
10
|
+
{ origin: 'https://example.com', localStorage: [{ name: 'authToken', value: 'xyz789' }] },
|
|
11
|
+
],
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
test('toCypressSnippet: emits cy.setCookie for each cookie and cy.window().localStorage for each origin', () => {
|
|
15
|
+
const out = toCypressSnippet(sampleState);
|
|
16
|
+
assert.match(out, /cy\.setCookie\("session_id", "abc123"/);
|
|
17
|
+
assert.match(out, /domain: "\.example\.com"/);
|
|
18
|
+
assert.match(out, /secure: true/);
|
|
19
|
+
assert.match(out, /httpOnly: true/);
|
|
20
|
+
assert.match(out, /cy\.visit\("https:\/\/example\.com"\)/);
|
|
21
|
+
assert.match(out, /localStorage\.setItem\("authToken", "xyz789"\)/);
|
|
22
|
+
assert.match(out, /Cypress\.Commands\.add\('restoreZetaSession'/);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test('toSeleniumPythonDict: navigates to each cookie domain before add_cookie, and to each origin for localStorage', () => {
|
|
26
|
+
const out = toSeleniumPythonDict(sampleState);
|
|
27
|
+
assert.match(out, /driver\.get\("https:\/\/example\.com"\)/);
|
|
28
|
+
assert.match(out, /driver\.add_cookie\(/);
|
|
29
|
+
assert.match(out, /"name": "session_id"/);
|
|
30
|
+
assert.match(out, /"value": "abc123"/);
|
|
31
|
+
assert.match(out, /driver\.execute_script\(/);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test('toSeleniumPythonDict: emits Python True/False, not JSON true/false, for boolean fields', () => {
|
|
35
|
+
const out = toSeleniumPythonDict(sampleState);
|
|
36
|
+
// sampleState's cookie has secure: true — must render as Python's True.
|
|
37
|
+
assert.match(out, /"secure": True/);
|
|
38
|
+
assert.doesNotMatch(out, /:\s*true\b/);
|
|
39
|
+
assert.doesNotMatch(out, /:\s*false\b/);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test('toWebdriverIOSnippet: browser.setCookies() array shape close to Playwright cookies, plus localStorage via execute()', () => {
|
|
43
|
+
const out = toWebdriverIOSnippet(sampleState);
|
|
44
|
+
assert.match(out, /browser\.setCookies\(/);
|
|
45
|
+
assert.match(out, /"name": "session_id"/);
|
|
46
|
+
assert.match(out, /"domain": "\.example\.com"/);
|
|
47
|
+
assert.match(out, /browser\.url\("https:\/\/example\.com"\)/);
|
|
48
|
+
assert.match(out, /localStorage\.setItem\("authToken", "xyz789"\)/);
|
|
49
|
+
assert.match(out, /module\.exports = \{ restoreZetaSession \}/);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test('exportSessionAs: returns correct filename/mimeType per format', () => {
|
|
53
|
+
const cy = exportSessionAs('cypress', sampleState);
|
|
54
|
+
assert.equal(cy.filename, 'zeta-session.cy-commands.js');
|
|
55
|
+
assert.equal(cy.mimeType, 'application/javascript');
|
|
56
|
+
|
|
57
|
+
const py = exportSessionAs('selenium-python', sampleState);
|
|
58
|
+
assert.equal(py.filename, 'zeta_session.py');
|
|
59
|
+
assert.equal(py.mimeType, 'text/x-python');
|
|
60
|
+
|
|
61
|
+
const wdio = exportSessionAs('webdriverio', sampleState);
|
|
62
|
+
assert.equal(wdio.filename, 'zeta-session.wdio.js');
|
|
63
|
+
assert.equal(wdio.mimeType, 'application/javascript');
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test('exportSessionAs: throws on an unknown format', () => {
|
|
67
|
+
assert.throws(() => exportSessionAs('unknown' as any, sampleState), /Unknown session export format/);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
test('handles empty cookies/origins gracefully (no crash, minimal output)', () => {
|
|
71
|
+
const empty: StorageState = { cookies: [], origins: [] };
|
|
72
|
+
assert.doesNotThrow(() => toCypressSnippet(empty));
|
|
73
|
+
assert.doesNotThrow(() => toSeleniumPythonDict(empty));
|
|
74
|
+
assert.doesNotThrow(() => toWebdriverIOSnippet(empty));
|
|
75
|
+
});
|