@taodigital/crawler 0.0.0-stage → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +82 -0
- package/README.md +10 -2
- package/dist/accessibility-auditor.d.ts +29 -0
- package/dist/accessibility-auditor.js +57 -0
- package/dist/accessibility-auditor.js.map +7 -0
- package/dist/accessibility-tree-crawler.d.ts +44 -0
- package/dist/accessibility-tree-crawler.js +156 -0
- package/dist/accessibility-tree-crawler.js.map +7 -0
- package/dist/advanced-ui-probes.d.ts +47 -0
- package/dist/advanced-ui-probes.js +310 -0
- package/dist/advanced-ui-probes.js.map +7 -0
- package/dist/ai-login.d.ts +32 -0
- package/dist/ai-login.js +250 -0
- package/dist/ai-login.js.map +7 -0
- package/dist/auth-token-rotator.d.ts +47 -0
- package/dist/auth-token-rotator.js +58 -0
- package/dist/auth-token-rotator.js.map +7 -0
- package/dist/auth.d.ts +20 -0
- package/dist/auth.js +348 -0
- package/dist/auth.js.map +7 -0
- package/dist/browser-extension.d.ts +37 -0
- package/dist/browser-extension.js +113 -0
- package/dist/browser-extension.js.map +7 -0
- package/dist/captcha-solver.d.ts +16 -0
- package/dist/captcha-solver.js +105 -0
- package/dist/captcha-solver.js.map +7 -0
- package/dist/color-contrast.d.ts +30 -0
- package/dist/color-contrast.js +78 -0
- package/dist/color-contrast.js.map +7 -0
- package/dist/content-chunker.d.ts +18 -0
- package/dist/content-chunker.js +152 -0
- package/dist/content-chunker.js.map +7 -0
- package/dist/content-dedup.d.ts +23 -0
- package/dist/content-dedup.js +60 -0
- package/dist/content-dedup.js.map +7 -0
- package/dist/cookie-consent.d.ts +25 -0
- package/dist/cookie-consent.js +78 -0
- package/dist/cookie-consent.js.map +7 -0
- package/dist/cookie-jar.d.ts +20 -0
- package/dist/cookie-jar.js +61 -0
- package/dist/cookie-jar.js.map +7 -0
- package/dist/crawl-session-manager.d.ts +44 -0
- package/dist/crawl-session-manager.js +145 -0
- package/dist/crawl-session-manager.js.map +7 -0
- package/dist/crawler.d.ts +233 -0
- package/dist/crawler.js +6239 -0
- package/dist/crawler.js.map +7 -0
- package/dist/crud-interaction.d.ts +31 -0
- package/dist/crud-interaction.js +264 -0
- package/dist/crud-interaction.js.map +7 -0
- package/dist/custom-js-injector.d.ts +42 -0
- package/dist/custom-js-injector.js +50 -0
- package/dist/custom-js-injector.js.map +7 -0
- package/dist/dark-mode-capture.d.ts +23 -0
- package/dist/dark-mode-capture.js +75 -0
- package/dist/dark-mode-capture.js.map +7 -0
- package/dist/dead-link-detector.d.ts +37 -0
- package/dist/dead-link-detector.js +50 -0
- package/dist/dead-link-detector.js.map +7 -0
- package/dist/dropdown-enumerator.d.ts +20 -0
- package/dist/dropdown-enumerator.js +230 -0
- package/dist/dropdown-enumerator.js.map +7 -0
- package/dist/element-hash.d.ts +9 -0
- package/dist/element-hash.js +9 -0
- package/dist/element-hash.js.map +7 -0
- package/dist/email-otp.d.ts +42 -0
- package/dist/email-otp.js +177 -0
- package/dist/email-otp.js.map +7 -0
- package/dist/engine-router.d.ts +47 -0
- package/dist/engine-router.js +94 -0
- package/dist/engine-router.js.map +7 -0
- package/dist/export/bulk-export.d.ts +26 -0
- package/dist/export/bulk-export.js +76 -0
- package/dist/export/bulk-export.js.map +7 -0
- package/dist/export/vector-export.d.ts +76 -0
- package/dist/export/vector-export.js +85 -0
- package/dist/export/vector-export.js.map +7 -0
- package/dist/extractors/advanced.d.ts +64 -0
- package/dist/extractors/advanced.js +265 -0
- package/dist/extractors/advanced.js.map +7 -0
- package/dist/extractors/artifact-envelope.d.ts +63 -0
- package/dist/extractors/artifact-envelope.js +63 -0
- package/dist/extractors/artifact-envelope.js.map +7 -0
- package/dist/extractors/branding-extractor.d.ts +46 -0
- package/dist/extractors/branding-extractor.js +104 -0
- package/dist/extractors/branding-extractor.js.map +7 -0
- package/dist/extractors/summary-extractor.d.ts +10 -0
- package/dist/extractors/summary-extractor.js +36 -0
- package/dist/extractors/summary-extractor.js.map +7 -0
- package/dist/failure-analysis-agent.d.ts +16 -0
- package/dist/failure-analysis-agent.js +42 -0
- package/dist/failure-analysis-agent.js.map +7 -0
- package/dist/feature-flag-detector.d.ts +30 -0
- package/dist/feature-flag-detector.js +55 -0
- package/dist/feature-flag-detector.js.map +7 -0
- package/dist/filter-pagination-crawler.d.ts +52 -0
- package/dist/filter-pagination-crawler.js +299 -0
- package/dist/filter-pagination-crawler.js.map +7 -0
- package/dist/form-detector.d.ts +34 -0
- package/dist/form-detector.js +58 -0
- package/dist/form-detector.js.map +7 -0
- package/dist/frontier.d.ts +40 -0
- package/dist/frontier.js +103 -0
- package/dist/frontier.js.map +7 -0
- package/dist/gap-pass.d.ts +36 -0
- package/dist/gap-pass.js +115 -0
- package/dist/gap-pass.js.map +7 -0
- package/dist/graphql-detector.d.ts +39 -0
- package/dist/graphql-detector.js +86 -0
- package/dist/graphql-detector.js.map +7 -0
- package/dist/http-engine.d.ts +80 -0
- package/dist/http-engine.js +318 -0
- package/dist/http-engine.js.map +7 -0
- package/dist/incremental-diff.d.ts +116 -0
- package/dist/incremental-diff.js +137 -0
- package/dist/incremental-diff.js.map +7 -0
- package/dist/index.d.ts +52 -0
- package/dist/index.js +83 -0
- package/dist/index.js.map +7 -0
- package/dist/input-fuzzer.d.ts +30 -0
- package/dist/input-fuzzer.js +105 -0
- package/dist/input-fuzzer.js.map +7 -0
- package/dist/interactive-probe.d.ts +144 -0
- package/dist/interactive-probe.js +865 -0
- package/dist/interactive-probe.js.map +7 -0
- package/dist/job-queue.d.ts +297 -0
- package/dist/job-queue.js +3812 -0
- package/dist/job-queue.js.map +7 -0
- package/dist/js-coverage.d.ts +29 -0
- package/dist/js-coverage.js +56 -0
- package/dist/js-coverage.js.map +7 -0
- package/dist/js-router-detector.d.ts +17 -0
- package/dist/js-router-detector.js +90 -0
- package/dist/js-router-detector.js.map +7 -0
- package/dist/language-detector.d.ts +22 -0
- package/dist/language-detector.js +57 -0
- package/dist/language-detector.js.map +7 -0
- package/dist/link-classifier.d.ts +23 -0
- package/dist/link-classifier.js +40 -0
- package/dist/link-classifier.js.map +7 -0
- package/dist/live-view.d.ts +36 -0
- package/dist/live-view.js +105 -0
- package/dist/live-view.js.map +7 -0
- package/dist/locator-extractor.d.ts +46 -0
- package/dist/locator-extractor.js +220 -0
- package/dist/locator-extractor.js.map +7 -0
- package/dist/logger.d.ts +2 -0
- package/dist/logger.js +10 -0
- package/dist/logger.js.map +7 -0
- package/dist/markdown-extract.d.ts +8 -0
- package/dist/markdown-extract.js +54 -0
- package/dist/markdown-extract.js.map +7 -0
- package/dist/multi-context-handler.d.ts +55 -0
- package/dist/multi-context-handler.js +182 -0
- package/dist/multi-context-handler.js.map +7 -0
- package/dist/native-crawler.d.ts +69 -0
- package/dist/native-crawler.js +839 -0
- package/dist/native-crawler.js.map +7 -0
- package/dist/nested-window-crawler.d.ts +41 -0
- package/dist/nested-window-crawler.js +172 -0
- package/dist/nested-window-crawler.js.map +7 -0
- package/dist/network-action-correlator.d.ts +47 -0
- package/dist/network-action-correlator.js +209 -0
- package/dist/network-action-correlator.js.map +7 -0
- package/dist/network-observer.d.ts +49 -0
- package/dist/network-observer.js +149 -0
- package/dist/network-observer.js.map +7 -0
- package/dist/network-throttle.d.ts +25 -0
- package/dist/network-throttle.js +34 -0
- package/dist/network-throttle.js.map +7 -0
- package/dist/openapi-generator.d.ts +49 -0
- package/dist/openapi-generator.js +89 -0
- package/dist/openapi-generator.js.map +7 -0
- package/dist/pdf-extractor.d.ts +28 -0
- package/dist/pdf-extractor.js +81 -0
- package/dist/pdf-extractor.js.map +7 -0
- package/dist/performance-metrics-collector.d.ts +50 -0
- package/dist/performance-metrics-collector.js +155 -0
- package/dist/performance-metrics-collector.js.map +7 -0
- package/dist/performance-metrics.d.ts +52 -0
- package/dist/performance-metrics.js +83 -0
- package/dist/performance-metrics.js.map +7 -0
- package/dist/phash.d.ts +50 -0
- package/dist/phash.js +136 -0
- package/dist/phash.js.map +7 -0
- package/dist/pii-scrubber.d.ts +21 -0
- package/dist/pii-scrubber.js +75 -0
- package/dist/pii-scrubber.js.map +7 -0
- package/dist/playwright-recorder.d.ts +39 -0
- package/dist/playwright-recorder.js +123 -0
- package/dist/playwright-recorder.js.map +7 -0
- package/dist/policies/crawl-safety.d.ts +9 -0
- package/dist/policies/crawl-safety.js +41 -0
- package/dist/policies/crawl-safety.js.map +7 -0
- package/dist/profiles/domain-profile.d.ts +19 -0
- package/dist/profiles/domain-profile.js +160 -0
- package/dist/profiles/domain-profile.js.map +7 -0
- package/dist/proxy.d.ts +50 -0
- package/dist/proxy.js +165 -0
- package/dist/proxy.js.map +7 -0
- package/dist/rate-limiter.d.ts +31 -0
- package/dist/rate-limiter.js +64 -0
- package/dist/rate-limiter.js.map +7 -0
- package/dist/redirect-tracker.d.ts +37 -0
- package/dist/redirect-tracker.js +65 -0
- package/dist/redirect-tracker.js.map +7 -0
- package/dist/redis-frontier.d.ts +48 -0
- package/dist/redis-frontier.js +126 -0
- package/dist/redis-frontier.js.map +7 -0
- package/dist/redis-queue.d.ts +48 -0
- package/dist/redis-queue.js +165 -0
- package/dist/redis-queue.js.map +7 -0
- package/dist/response-cache.d.ts +38 -0
- package/dist/response-cache.js +140 -0
- package/dist/response-cache.js.map +7 -0
- package/dist/responsive-detector.d.ts +29 -0
- package/dist/responsive-detector.js +46 -0
- package/dist/responsive-detector.js.map +7 -0
- package/dist/retry-policy.d.ts +20 -0
- package/dist/retry-policy.js +54 -0
- package/dist/retry-policy.js.map +7 -0
- package/dist/robots-txt.d.ts +9 -0
- package/dist/robots-txt.js +77 -0
- package/dist/robots-txt.js.map +7 -0
- package/dist/screen-reader.d.ts +15 -0
- package/dist/screen-reader.js +48 -0
- package/dist/screen-reader.js.map +7 -0
- package/dist/screenshot-masker.d.ts +28 -0
- package/dist/screenshot-masker.js +62 -0
- package/dist/screenshot-masker.js.map +7 -0
- package/dist/seo-checker.d.ts +33 -0
- package/dist/seo-checker.js +91 -0
- package/dist/seo-checker.js.map +7 -0
- package/dist/service-worker-interceptor.d.ts +36 -0
- package/dist/service-worker-interceptor.js +136 -0
- package/dist/service-worker-interceptor.js.map +7 -0
- package/dist/session-capture.d.ts +39 -0
- package/dist/session-capture.js +274 -0
- package/dist/session-capture.js.map +7 -0
- package/dist/session-export.d.ts +49 -0
- package/dist/session-export.js +117 -0
- package/dist/session-export.js.map +7 -0
- package/dist/shadow-dom-crawler.d.ts +35 -0
- package/dist/shadow-dom-crawler.js +132 -0
- package/dist/shadow-dom-crawler.js.map +7 -0
- package/dist/sitemap-writer.d.ts +21 -0
- package/dist/sitemap-writer.js +47 -0
- package/dist/sitemap-writer.js.map +7 -0
- package/dist/sitemap.d.ts +51 -0
- package/dist/sitemap.js +160 -0
- package/dist/sitemap.js.map +7 -0
- package/dist/skyvern.d.ts +56 -0
- package/dist/skyvern.js +129 -0
- package/dist/skyvern.js.map +7 -0
- package/dist/sqs-queue.d.ts +13 -0
- package/dist/sqs-queue.js +115 -0
- package/dist/sqs-queue.js.map +7 -0
- package/dist/sso-token.d.ts +29 -0
- package/dist/sso-token.js +76 -0
- package/dist/sso-token.js.map +7 -0
- package/dist/state-graph.d.ts +59 -0
- package/dist/state-graph.js +193 -0
- package/dist/state-graph.js.map +7 -0
- package/dist/stealth/human-behavior.d.ts +20 -0
- package/dist/stealth/human-behavior.js +202 -0
- package/dist/stealth/human-behavior.js.map +7 -0
- package/dist/telemetry.d.ts +34 -0
- package/dist/telemetry.js +77 -0
- package/dist/telemetry.js.map +7 -0
- package/dist/test-executor.d.ts +81 -0
- package/dist/test-executor.js +443 -0
- package/dist/test-executor.js.map +7 -0
- package/dist/tls-fingerprint.d.ts +50 -0
- package/dist/tls-fingerprint.js +81 -0
- package/dist/tls-fingerprint.js.map +7 -0
- package/dist/url-pattern-completer.d.ts +28 -0
- package/dist/url-pattern-completer.js +122 -0
- package/dist/url-pattern-completer.js.map +7 -0
- package/dist/visual-diff-verdict.d.ts +40 -0
- package/dist/visual-diff-verdict.js +27 -0
- package/dist/visual-diff-verdict.js.map +7 -0
- package/dist/webhook-notifier.d.ts +30 -0
- package/dist/webhook-notifier.js +76 -0
- package/dist/webhook-notifier.js.map +7 -0
- package/dist/websocket-detector.d.ts +24 -0
- package/dist/websocket-detector.js +60 -0
- package/dist/websocket-detector.js.map +7 -0
- package/dist/worker.d.ts +1 -0
- package/dist/worker.js +33 -0
- package/dist/worker.js.map +7 -0
- package/package.json +49 -4
- package/src/accessibility-auditor.ts +102 -0
- package/src/accessibility-tree-crawler.ts +246 -0
- package/src/advanced-ui-probes.ts +470 -0
- package/src/ai-login.ts +298 -0
- package/src/auth-token-rotator.ts +89 -0
- package/src/auth.ts +418 -0
- package/src/browser-extension.ts +186 -0
- package/src/captcha-solver.ts +143 -0
- package/src/color-contrast.ts +116 -0
- package/src/content-chunker.ts +170 -0
- package/src/content-dedup.ts +99 -0
- package/src/cookie-consent.ts +106 -0
- package/src/cookie-jar.ts +91 -0
- package/src/crawl-session-manager.ts +231 -0
- package/src/crawler.ts +7123 -0
- package/src/crawler.ts.rej +12 -0
- package/src/crud-interaction.ts +386 -0
- package/src/custom-js-injector.ts +93 -0
- package/src/dark-mode-capture.ts +110 -0
- package/src/dead-link-detector.ts +76 -0
- package/src/dropdown-enumerator.ts +312 -0
- package/src/element-hash.ts +16 -0
- package/src/email-otp.ts +252 -0
- package/src/engine-router.ts +141 -0
- package/src/export/bulk-export.ts +85 -0
- package/src/export/vector-export.ts +160 -0
- package/src/extractors/advanced.ts +309 -0
- package/src/extractors/artifact-envelope.ts +111 -0
- package/src/extractors/branding-extractor.ts +159 -0
- package/src/extractors/summary-extractor.ts +49 -0
- package/src/failure-analysis-agent.ts +51 -0
- package/src/feature-flag-detector.ts +97 -0
- package/src/filter-pagination-crawler.ts +500 -0
- package/src/form-detector.ts +103 -0
- package/src/frontier.test.ts +64 -0
- package/src/frontier.ts +144 -0
- package/src/gap-pass.ts +165 -0
- package/src/graphql-detector.ts +129 -0
- package/src/http-engine.ts +453 -0
- package/src/incremental-diff.ts +259 -0
- package/src/index.ts +52 -0
- package/src/input-fuzzer.ts +129 -0
- package/src/interactive-probe.ts +1136 -0
- package/src/job-queue.test.ts +187 -0
- package/src/job-queue.ts +4504 -0
- package/src/js-coverage.ts +87 -0
- package/src/js-router-detector.ts +112 -0
- package/src/language-detector.ts +82 -0
- package/src/link-classifier.ts +65 -0
- package/src/live-view.ts +146 -0
- package/src/locator-extractor.ts +309 -0
- package/src/logger.ts +9 -0
- package/src/markdown-extract.test.ts +73 -0
- package/src/markdown-extract.ts +76 -0
- package/src/multi-context-handler.ts +296 -0
- package/src/native-crawler.ts +1108 -0
- package/src/nested-window-crawler.ts +310 -0
- package/src/network-action-correlator.ts +320 -0
- package/src/network-observer.ts +204 -0
- package/src/network-throttle.ts +69 -0
- package/src/openapi-generator.ts +135 -0
- package/src/pdf-extractor.ts +123 -0
- package/src/performance-metrics-collector.ts +243 -0
- package/src/performance-metrics.ts +134 -0
- package/src/phash.ts +184 -0
- package/src/pii-scrubber.ts +99 -0
- package/src/playwright-recorder.ts +170 -0
- package/src/policies/crawl-safety.ts +52 -0
- package/src/profiles/domain-profile.ts +190 -0
- package/src/proxy.ts +238 -0
- package/src/rate-limiter.ts +87 -0
- package/src/redirect-tracker.ts +97 -0
- package/src/redis-frontier.ts +143 -0
- package/src/redis-queue.ts +208 -0
- package/src/response-cache.ts +150 -0
- package/src/responsive-detector.ts +92 -0
- package/src/retry-policy.ts +87 -0
- package/src/robots-txt.ts +96 -0
- package/src/screen-reader.ts +98 -0
- package/src/screenshot-masker.ts +98 -0
- package/src/seo-checker.ts +134 -0
- package/src/service-worker-interceptor.ts +185 -0
- package/src/session-capture.ts +335 -0
- package/src/session-export.test.ts +75 -0
- package/src/session-export.ts +172 -0
- package/src/shadow-dom-crawler.ts +198 -0
- package/src/sitemap-writer.ts +65 -0
- package/src/sitemap.test.ts +127 -0
- package/src/sitemap.ts +233 -0
- package/src/skyvern.ts +199 -0
- package/src/sqs-queue.ts +129 -0
- package/src/sso-token.ts +120 -0
- package/src/state-graph.ts +280 -0
- package/src/stealth/human-behavior.ts +227 -0
- package/src/telemetry.ts +122 -0
- package/src/test-executor.test.ts +72 -0
- package/src/test-executor.ts +578 -0
- package/src/tls-fingerprint.ts +111 -0
- package/src/url-pattern-completer.ts +161 -0
- package/src/vendor.d.ts +20 -0
- package/src/visual-diff-verdict.test.ts +80 -0
- package/src/visual-diff-verdict.ts +59 -0
- package/src/webhook-notifier.ts +109 -0
- package/src/websocket-detector.ts +83 -0
- package/src/worker.ts +50 -0
- package/tsconfig.json +13 -0
- package/types/accessibility-auditor.d.ts +29 -0
- package/types/accessibility-tree-crawler.d.ts +44 -0
- package/types/advanced-ui-probes.d.ts +47 -0
- package/types/ai-login.d.ts +32 -0
- package/types/auth-token-rotator.d.ts +47 -0
- package/types/auth.d.ts +20 -0
- package/types/browser-extension.d.ts +37 -0
- package/types/bull-worker.d.ts +6 -0
- package/types/captcha-solver.d.ts +16 -0
- package/types/color-contrast.d.ts +30 -0
- package/types/content-chunker.d.ts +18 -0
- package/types/content-dedup.d.ts +23 -0
- package/types/cookie-consent.d.ts +25 -0
- package/types/cookie-jar.d.ts +20 -0
- package/types/crawl-session-manager.d.ts +44 -0
- package/types/crawler.d.ts +233 -0
- package/types/crud-interaction.d.ts +31 -0
- package/types/custom-js-injector.d.ts +42 -0
- package/types/dark-mode-capture.d.ts +23 -0
- package/types/dead-link-detector.d.ts +37 -0
- package/types/dropdown-enumerator.d.ts +20 -0
- package/types/element-hash.d.ts +9 -0
- package/types/email-otp.d.ts +42 -0
- package/types/engine-router.d.ts +47 -0
- package/types/export/bulk-export.d.ts +26 -0
- package/types/export/vector-export.d.ts +76 -0
- package/types/extractors/advanced.d.ts +64 -0
- package/types/extractors/artifact-envelope.d.ts +63 -0
- package/types/extractors/branding-extractor.d.ts +46 -0
- package/types/extractors/summary-extractor.d.ts +10 -0
- package/types/failure-analysis-agent.d.ts +16 -0
- package/types/feature-flag-detector.d.ts +30 -0
- package/types/filter-pagination-crawler.d.ts +52 -0
- package/types/form-detector.d.ts +34 -0
- package/types/frontier.d.ts +40 -0
- package/types/gap-pass.d.ts +36 -0
- package/types/graphql-detector.d.ts +39 -0
- package/types/http-engine.d.ts +80 -0
- package/types/incremental-diff.d.ts +116 -0
- package/types/index.d.ts +52 -0
- package/types/input-fuzzer.d.ts +30 -0
- package/types/interactive-probe.d.ts +144 -0
- package/types/job-queue.d.ts +308 -0
- package/types/js-coverage.d.ts +29 -0
- package/types/js-router-detector.d.ts +17 -0
- package/types/language-detector.d.ts +22 -0
- package/types/link-classifier.d.ts +23 -0
- package/types/live-view.d.ts +36 -0
- package/types/locator-extractor.d.ts +46 -0
- package/types/logger.d.ts +2 -0
- package/types/markdown-extract.d.ts +8 -0
- package/types/multi-context-handler.d.ts +55 -0
- package/types/native-crawler.d.ts +69 -0
- package/types/nested-window-crawler.d.ts +41 -0
- package/types/network-action-correlator.d.ts +47 -0
- package/types/network-observer.d.ts +49 -0
- package/types/network-throttle.d.ts +25 -0
- package/types/openapi-generator.d.ts +49 -0
- package/types/pdf-extractor.d.ts +28 -0
- package/types/performance-metrics-collector.d.ts +50 -0
- package/types/performance-metrics.d.ts +52 -0
- package/types/phash.d.ts +50 -0
- package/types/pii-scrubber.d.ts +21 -0
- package/types/playwright-recorder.d.ts +39 -0
- package/types/policies/crawl-safety.d.ts +9 -0
- package/types/profiles/domain-profile.d.ts +19 -0
- package/types/proxy.d.ts +50 -0
- package/types/rate-limiter.d.ts +31 -0
- package/types/redirect-tracker.d.ts +37 -0
- package/types/redis-frontier.d.ts +48 -0
- package/types/redis-queue.d.ts +48 -0
- package/types/response-cache.d.ts +38 -0
- package/types/responsive-detector.d.ts +29 -0
- package/types/retry-policy.d.ts +20 -0
- package/types/robots-txt.d.ts +9 -0
- package/types/screen-reader.d.ts +15 -0
- package/types/screenshot-masker.d.ts +28 -0
- package/types/seo-checker.d.ts +33 -0
- package/types/service-worker-interceptor.d.ts +36 -0
- package/types/session-capture.d.ts +39 -0
- package/types/session-export.d.ts +49 -0
- package/types/shadow-dom-crawler.d.ts +35 -0
- package/types/sitemap-writer.d.ts +21 -0
- package/types/sitemap.d.ts +51 -0
- package/types/skyvern.d.ts +56 -0
- package/types/sqs-queue.d.ts +13 -0
- package/types/sso-token.d.ts +29 -0
- package/types/state-graph.d.ts +59 -0
- package/types/stealth/human-behavior.d.ts +20 -0
- package/types/telemetry.d.ts +34 -0
- package/types/test-executor.d.ts +84 -0
- package/types/tls-fingerprint.d.ts +50 -0
- package/types/url-pattern-completer.d.ts +28 -0
- package/types/visual-diff-verdict.d.ts +40 -0
- package/types/webhook-notifier.d.ts +30 -0
- package/types/websocket-detector.d.ts +24 -0
- package/types/worker.d.ts +1 -0
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Responsive design detector — captures page at multiple breakpoints
|
|
3
|
+
* and detects layout differences between desktop and mobile viewports.
|
|
4
|
+
*
|
|
5
|
+
* Breakpoints: mobile(375), tablet(768), desktop(1280), wide(1920)
|
|
6
|
+
* Detects: viewport-specific content visibility, menu collapse state,
|
|
7
|
+
* font size changes, hidden/shown elements.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import type { Page } from 'playwright';
|
|
11
|
+
|
|
12
|
+
export interface ViewportCapture {
|
|
13
|
+
width: number;
|
|
14
|
+
height: number;
|
|
15
|
+
label: 'mobile' | 'tablet' | 'desktop' | 'wide';
|
|
16
|
+
screenshotBuffer?: Buffer;
|
|
17
|
+
visibleTextLength: number;
|
|
18
|
+
hasHamburgerMenu: boolean;
|
|
19
|
+
hiddenElements: number; // display:none element count
|
|
20
|
+
scrollHeight: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface ResponsiveReport {
|
|
24
|
+
url: string;
|
|
25
|
+
captures: ViewportCapture[];
|
|
26
|
+
isResponsive: boolean; // true if layout changes between viewports
|
|
27
|
+
hasMobileMenu: boolean;
|
|
28
|
+
issues: string[];
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const BREAKPOINTS: Array<{ width: number; height: number; label: ViewportCapture['label'] }> = [
|
|
32
|
+
{ width: 375, height: 812, label: 'mobile' },
|
|
33
|
+
{ width: 768, height: 1024, label: 'tablet' },
|
|
34
|
+
{ width: 1280, height: 800, label: 'desktop' },
|
|
35
|
+
];
|
|
36
|
+
|
|
37
|
+
export async function detectResponsive(
|
|
38
|
+
page: Page,
|
|
39
|
+
url: string,
|
|
40
|
+
opts: { captureScreenshots?: boolean } = {},
|
|
41
|
+
): Promise<ResponsiveReport> {
|
|
42
|
+
const captures: ViewportCapture[] = [];
|
|
43
|
+
const issues: string[] = [];
|
|
44
|
+
|
|
45
|
+
for (const bp of BREAKPOINTS) {
|
|
46
|
+
try {
|
|
47
|
+
await page.setViewportSize({ width: bp.width, height: bp.height });
|
|
48
|
+
await page.waitForTimeout(500); // allow CSS reflow
|
|
49
|
+
|
|
50
|
+
const data = await page.evaluate(() => ({
|
|
51
|
+
visibleTextLength: (document.body.innerText ?? '').length,
|
|
52
|
+
hasHamburgerMenu: !!(
|
|
53
|
+
document.querySelector('[class*="hamburger"],[class*="menu-toggle"],[class*="nav-toggle"],[aria-label*="menu"],[aria-label*="Menu"]')
|
|
54
|
+
),
|
|
55
|
+
hiddenElements: Array.from(document.querySelectorAll('*')).filter(
|
|
56
|
+
(el: any) => window.getComputedStyle(el).display === 'none'
|
|
57
|
+
).length,
|
|
58
|
+
scrollHeight: document.body.scrollHeight,
|
|
59
|
+
})).catch(() => ({ visibleTextLength: 0, hasHamburgerMenu: false, hiddenElements: 0, scrollHeight: 0 }));
|
|
60
|
+
|
|
61
|
+
let screenshotBuffer: Buffer | undefined;
|
|
62
|
+
if (opts.captureScreenshots) {
|
|
63
|
+
screenshotBuffer = await page.screenshot({ type: 'jpeg', quality: 60 }).catch(() => undefined);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
captures.push({ ...bp, ...data, screenshotBuffer });
|
|
67
|
+
} catch { /* viewport failed — skip */ }
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Restore to desktop
|
|
71
|
+
await page.setViewportSize({ width: 1280, height: 800 }).catch(() => {});
|
|
72
|
+
|
|
73
|
+
const mobile = captures.find(c => c.label === 'mobile');
|
|
74
|
+
const desktop = captures.find(c => c.label === 'desktop');
|
|
75
|
+
|
|
76
|
+
const isResponsive = !!(mobile && desktop && (
|
|
77
|
+
mobile.hiddenElements !== desktop.hiddenElements ||
|
|
78
|
+
mobile.scrollHeight !== desktop.scrollHeight ||
|
|
79
|
+
mobile.hasHamburgerMenu
|
|
80
|
+
));
|
|
81
|
+
|
|
82
|
+
const hasMobileMenu = captures.some(c => c.label === 'mobile' && c.hasHamburgerMenu);
|
|
83
|
+
|
|
84
|
+
if (mobile && desktop && mobile.visibleTextLength > desktop.visibleTextLength * 1.3) {
|
|
85
|
+
issues.push('Mobile shows significantly more text than desktop — possible responsive issue');
|
|
86
|
+
}
|
|
87
|
+
if (!isResponsive && captures.length > 1) {
|
|
88
|
+
issues.push('Layout appears identical across breakpoints — may not be responsive');
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
return { url, captures, isResponsive, hasMobileMenu, issues };
|
|
92
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retry policy — configurable exponential backoff with jitter.
|
|
3
|
+
* Covers: 5xx server errors, network timeouts, ECONNREFUSED, ECONNRESET.
|
|
4
|
+
* NOT retried: 4xx client errors (except 429 — handled separately), redirects.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
export interface RetryPolicyOpts {
|
|
8
|
+
maxRetries?: number; // default 3
|
|
9
|
+
baseDelayMs?: number; // default 1000
|
|
10
|
+
maxDelayMs?: number; // default 30000
|
|
11
|
+
jitterMs?: number; // default 500
|
|
12
|
+
retryOn?: number[]; // HTTP status codes to retry (default: [429, 500, 502, 503, 504])
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface RetryResult<T> {
|
|
16
|
+
value: T;
|
|
17
|
+
attempts: number;
|
|
18
|
+
totalDelayMs: number;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const DEFAULT_RETRY_STATUSES = [429, 500, 502, 503, 504];
|
|
22
|
+
|
|
23
|
+
function shouldRetry(error: any, retryOn: number[]): boolean {
|
|
24
|
+
// Network-level errors
|
|
25
|
+
if (error?.code && ['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'ENOTFOUND', 'EPIPE'].includes(error.code)) return true;
|
|
26
|
+
// Playwright navigation timeout
|
|
27
|
+
if (error?.message && /timeout|net::ERR_/i.test(error.message)) return true;
|
|
28
|
+
// HTTP status errors
|
|
29
|
+
if (typeof error?.status === 'number') return retryOn.includes(error.status);
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function delay(ms: number): Promise<void> {
|
|
34
|
+
return new Promise(r => setTimeout(r, ms));
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export async function withRetry<T>(
|
|
38
|
+
fn: (attempt: number) => Promise<T>,
|
|
39
|
+
opts: RetryPolicyOpts = {},
|
|
40
|
+
): Promise<RetryResult<T>> {
|
|
41
|
+
const {
|
|
42
|
+
maxRetries = 3,
|
|
43
|
+
baseDelayMs = 1_000,
|
|
44
|
+
maxDelayMs = 30_000,
|
|
45
|
+
jitterMs = 500,
|
|
46
|
+
retryOn = DEFAULT_RETRY_STATUSES,
|
|
47
|
+
} = opts;
|
|
48
|
+
|
|
49
|
+
let lastError: any;
|
|
50
|
+
let totalDelayMs = 0;
|
|
51
|
+
|
|
52
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
53
|
+
try {
|
|
54
|
+
const value = await fn(attempt);
|
|
55
|
+
return { value, attempts: attempt + 1, totalDelayMs };
|
|
56
|
+
} catch (err) {
|
|
57
|
+
lastError = err;
|
|
58
|
+
if (attempt === maxRetries) break;
|
|
59
|
+
if (!shouldRetry(err, retryOn)) throw err; // non-retryable — fail fast
|
|
60
|
+
|
|
61
|
+
const backoff = Math.min(maxDelayMs, baseDelayMs * Math.pow(2, attempt));
|
|
62
|
+
const jitter = Math.random() * jitterMs;
|
|
63
|
+
const waitMs = Math.round(backoff + jitter);
|
|
64
|
+
totalDelayMs += waitMs;
|
|
65
|
+
await delay(waitMs);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
throw lastError;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Wraps fetch() with retry policy */
|
|
72
|
+
export async function fetchWithRetry(
|
|
73
|
+
url: string,
|
|
74
|
+
init: RequestInit = {},
|
|
75
|
+
retryOpts: RetryPolicyOpts = {},
|
|
76
|
+
): Promise<Response> {
|
|
77
|
+
const result = await withRetry(async () => {
|
|
78
|
+
const res = await fetch(url, init);
|
|
79
|
+
if (retryOpts.retryOn?.includes(res.status) ?? DEFAULT_RETRY_STATUSES.includes(res.status)) {
|
|
80
|
+
const err: any = new Error(`HTTP ${res.status}`);
|
|
81
|
+
err.status = res.status;
|
|
82
|
+
throw err;
|
|
83
|
+
}
|
|
84
|
+
return res;
|
|
85
|
+
}, retryOpts);
|
|
86
|
+
return result.value;
|
|
87
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal robots.txt fetcher + parser. No external deps. Caches per origin.
|
|
3
|
+
* Implements the subset of robots.txt spec used in practice:
|
|
4
|
+
* User-agent, Disallow, Allow — longest match wins (RFC 9309).
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
interface RobotsRule {
|
|
8
|
+
userAgent: string;
|
|
9
|
+
disallow: string[];
|
|
10
|
+
allow: string[];
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function parseRobots(text: string): RobotsRule[] {
|
|
14
|
+
const rules: RobotsRule[] = [];
|
|
15
|
+
let current: RobotsRule | null = null;
|
|
16
|
+
|
|
17
|
+
for (const rawLine of text.split('\n')) {
|
|
18
|
+
const line = rawLine.split('#')[0].trim();
|
|
19
|
+
if (!line) { current = null; continue; }
|
|
20
|
+
const colonIdx = line.indexOf(':');
|
|
21
|
+
if (colonIdx === -1) continue;
|
|
22
|
+
const key = line.slice(0, colonIdx).trim().toLowerCase();
|
|
23
|
+
const value = line.slice(colonIdx + 1).trim();
|
|
24
|
+
|
|
25
|
+
if (key === 'user-agent') {
|
|
26
|
+
// Start new block only when current block already has rules, otherwise
|
|
27
|
+
// accumulate multiple User-agent lines into one block.
|
|
28
|
+
if (!current || current.disallow.length > 0 || current.allow.length > 0) {
|
|
29
|
+
current = { userAgent: value.toLowerCase(), disallow: [], allow: [] };
|
|
30
|
+
rules.push(current);
|
|
31
|
+
} else {
|
|
32
|
+
current.userAgent = value.toLowerCase();
|
|
33
|
+
}
|
|
34
|
+
} else if (key === 'disallow' && current && value) {
|
|
35
|
+
current.disallow.push(value);
|
|
36
|
+
} else if (key === 'allow' && current && value) {
|
|
37
|
+
current.allow.push(value);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
return rules;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function isAllowedByRules(rules: RobotsRule[], userAgent: string, pathname: string): boolean {
|
|
45
|
+
const ua = userAgent.toLowerCase();
|
|
46
|
+
const exactRules = rules.filter(r => r.userAgent === ua);
|
|
47
|
+
const wildcardRules = rules.filter(r => r.userAgent === '*');
|
|
48
|
+
const applicable = exactRules.length > 0 ? exactRules : wildcardRules;
|
|
49
|
+
|
|
50
|
+
if (applicable.length === 0) return true;
|
|
51
|
+
|
|
52
|
+
// Collect all matched allow/disallow directives; longest path wins.
|
|
53
|
+
const matched: Array<{ path: string; allow: boolean }> = [];
|
|
54
|
+
for (const rule of applicable) {
|
|
55
|
+
for (const p of rule.allow) {
|
|
56
|
+
if (pathname.startsWith(p)) matched.push({ path: p, allow: true });
|
|
57
|
+
}
|
|
58
|
+
for (const p of rule.disallow) {
|
|
59
|
+
if (pathname.startsWith(p)) matched.push({ path: p, allow: false });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (matched.length === 0) return true;
|
|
64
|
+
matched.sort((a, b) => b.path.length - a.path.length);
|
|
65
|
+
return matched[0].allow;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export class RobotsCache {
|
|
69
|
+
private readonly cache = new Map<string, RobotsRule[]>();
|
|
70
|
+
|
|
71
|
+
async isAllowed(url: string, userAgent = 'ZeTa-Crawler'): Promise<boolean> {
|
|
72
|
+
let origin: string;
|
|
73
|
+
let pathname: string;
|
|
74
|
+
try {
|
|
75
|
+
const u = new URL(url);
|
|
76
|
+
origin = u.origin;
|
|
77
|
+
pathname = u.pathname || '/';
|
|
78
|
+
} catch {
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if (!this.cache.has(origin)) {
|
|
83
|
+
try {
|
|
84
|
+
const res = await fetch(`${origin}/robots.txt`, {
|
|
85
|
+
headers: { 'User-Agent': userAgent },
|
|
86
|
+
signal: AbortSignal.timeout(5_000),
|
|
87
|
+
});
|
|
88
|
+
this.cache.set(origin, res.ok ? parseRobots(await res.text()) : []);
|
|
89
|
+
} catch {
|
|
90
|
+
this.cache.set(origin, []); // network error or timeout = no restrictions
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
return isAllowedByRules(this.cache.get(origin)!, userAgent, pathname);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import type { Page } from 'playwright';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Reads a page as a structured outline (cheap + reliable), the way
|
|
5
|
+
* Playwright MCP / the accessibility tree works — NOT by sending pixels to a
|
|
6
|
+
* vision model. Vision is a fallback for image-only pages (added later).
|
|
7
|
+
*
|
|
8
|
+
* Returns the interactive elements found, which the Screen Reader Agent then
|
|
9
|
+
* enriches with MEANING and saves to the Zeta Core.
|
|
10
|
+
*/
|
|
11
|
+
export interface RawElement {
|
|
12
|
+
role: string;
|
|
13
|
+
label: string;
|
|
14
|
+
selectorHint: string;
|
|
15
|
+
expectedData?: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export async function readScreen(page: any): Promise<RawElement[]> {
|
|
19
|
+
return page.evaluate(() => {
|
|
20
|
+
// esbuild injects __name() for named arrow-function constants (e.g. `const priority = ...`).
|
|
21
|
+
// Polyfill prevents ReferenceError when the serialized callback runs in the browser.
|
|
22
|
+
if (typeof (globalThis as any).__name === 'undefined') (globalThis as any).__name = (fn: any) => fn;
|
|
23
|
+
const out: { role: string; label: string; selectorHint: string; expectedData?: string }[] = [];
|
|
24
|
+
const seen = new Set<string>();
|
|
25
|
+
|
|
26
|
+
// Noise patterns to skip — cookie banners, footer chrome, version strings
|
|
27
|
+
// Intentionally excludes standalone "close" and "copy" — too many false positives
|
|
28
|
+
// (Close account, Close preview, Copy link, Copy to clipboard are all valid test targets)
|
|
29
|
+
const noiseLabel =
|
|
30
|
+
/^(cookie|consent|privacy policy|terms of use|copyright|©|\bv\d+\.\d+|got it|dismiss|skip to|powered by|copied|\s*$)/i;
|
|
31
|
+
|
|
32
|
+
// Include ARIA-role elements — Headless UI, Radix, Ant Design, Shadcn all use these instead of native tags
|
|
33
|
+
const nodes = document.querySelectorAll(
|
|
34
|
+
'input:not([type=hidden]), select, textarea, button, a[href], ' +
|
|
35
|
+
'[role="button"], [role="combobox"], [role="slider"], [role="switch"], ' +
|
|
36
|
+
'[role="tab"], [role="menuitem"], [role="searchbox"], [role="spinbutton"]'
|
|
37
|
+
);
|
|
38
|
+
|
|
39
|
+
nodes.forEach((el) => {
|
|
40
|
+
const htmlEl = el as HTMLElement;
|
|
41
|
+
|
|
42
|
+
// Skip invisible elements
|
|
43
|
+
const style = window.getComputedStyle(htmlEl);
|
|
44
|
+
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return;
|
|
45
|
+
const rect = el.getBoundingClientRect();
|
|
46
|
+
// Allow zero-size elements that carry an accessible label (sr-only icon buttons, skip-nav links)
|
|
47
|
+
const hasAccessibleLabel = !!(el.getAttribute('aria-label') || el.getAttribute('aria-labelledby'));
|
|
48
|
+
if (rect.width === 0 && rect.height === 0 && !hasAccessibleLabel) return;
|
|
49
|
+
|
|
50
|
+
// Derive role
|
|
51
|
+
const tag = el.tagName.toLowerCase();
|
|
52
|
+
const role =
|
|
53
|
+
el.getAttribute('role') ||
|
|
54
|
+
(tag === 'input' ? ((el as HTMLInputElement).type || 'text') : tag);
|
|
55
|
+
|
|
56
|
+
// Derive the best human-readable label (prefer aria > placeholder > inner text)
|
|
57
|
+
const raw = (
|
|
58
|
+
el.getAttribute('aria-label') ||
|
|
59
|
+
el.getAttribute('placeholder') ||
|
|
60
|
+
(el.getAttribute('title') ?? '') ||
|
|
61
|
+
htmlEl.innerText?.replace(/\s+/g, ' ').trim() ||
|
|
62
|
+
el.getAttribute('name') ||
|
|
63
|
+
''
|
|
64
|
+
).substring(0, 80);
|
|
65
|
+
|
|
66
|
+
// Skip empty, noise patterns, or very long labels (likely container text)
|
|
67
|
+
if (!raw || noiseLabel.test(raw) || raw.length > 70) return;
|
|
68
|
+
|
|
69
|
+
// Deduplicate by role+normalised label
|
|
70
|
+
const key = `${role}:${raw.toLowerCase()}`;
|
|
71
|
+
if (seen.has(key)) return;
|
|
72
|
+
seen.add(key);
|
|
73
|
+
|
|
74
|
+
let expectedData: string | undefined;
|
|
75
|
+
if (tag === 'input') {
|
|
76
|
+
const input = el as HTMLInputElement;
|
|
77
|
+
if (input.type === 'email' || /email/i.test(raw)) expectedData = 'valid_email';
|
|
78
|
+
else if (input.type === 'password' || /password/i.test(raw)) expectedData = 'password';
|
|
79
|
+
else if (input.type === 'tel' || /phone|mobile/i.test(raw)) expectedData = 'phone';
|
|
80
|
+
else if (input.type === 'date' || /\bdate\b/i.test(raw)) expectedData = 'date';
|
|
81
|
+
else if (/\b(code|otp|token)\b/i.test(raw)) expectedData = 'one_time_code';
|
|
82
|
+
else if (/\b(search|query)\b/i.test(raw)) expectedData = 'search_term';
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
out.push({ role, label: raw, selectorHint: tag, expectedData });
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
// Prioritise: form fields > buttons > links; within each group keep order
|
|
89
|
+
const priority = (e: typeof out[0]) => {
|
|
90
|
+
if (['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea'].includes(e.role)) return 0;
|
|
91
|
+
if (e.role === 'button' || e.role === 'submit') return 1;
|
|
92
|
+
return 2;
|
|
93
|
+
};
|
|
94
|
+
out.sort((a, b) => priority(a) - priority(b));
|
|
95
|
+
|
|
96
|
+
return out.slice(0, 25); // cap at 25 — enough context, not overwhelming
|
|
97
|
+
});
|
|
98
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Screenshot masker — blur or black-box PII regions in captured screenshots.
|
|
3
|
+
*
|
|
4
|
+
* Strategy: inject a CSS overlay into the page before screenshot capture that
|
|
5
|
+
* covers detected PII-likely elements (input[type=email], input[type=password],
|
|
6
|
+
* elements with PII-containing text) with a blur or solid black mask.
|
|
7
|
+
*
|
|
8
|
+
* This is a CSS-based approach (no image processing library needed):
|
|
9
|
+
* 1. Identify likely-PII DOM elements via selectors
|
|
10
|
+
* 2. Add an overlaid <div> with blur/black filter at each element's bounding box
|
|
11
|
+
* 3. Take screenshot
|
|
12
|
+
* 4. Remove overlays
|
|
13
|
+
*
|
|
14
|
+
* Falls back gracefully if elements are not found or overlay injection fails.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import type { Page } from 'playwright';
|
|
18
|
+
|
|
19
|
+
export interface MaskingOpts {
|
|
20
|
+
/** Additional CSS selectors to mask (in addition to defaults) */
|
|
21
|
+
extraSelectors?: string[];
|
|
22
|
+
/** Masking style: 'blur' or 'blackout' */
|
|
23
|
+
style?: 'blur' | 'blackout';
|
|
24
|
+
/** Blur strength in px (default 8) */
|
|
25
|
+
blurPx?: number;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// Default PII-sensitive selectors
|
|
29
|
+
const DEFAULT_PII_SELECTORS = [
|
|
30
|
+
'input[type="email"]',
|
|
31
|
+
'input[type="password"]',
|
|
32
|
+
'input[type="tel"]',
|
|
33
|
+
'input[autocomplete="cc-number"]',
|
|
34
|
+
'input[autocomplete="cc-csc"]',
|
|
35
|
+
'input[autocomplete="ssn"]',
|
|
36
|
+
'[data-pii="true"]',
|
|
37
|
+
'[data-sensitive="true"]',
|
|
38
|
+
];
|
|
39
|
+
|
|
40
|
+
export async function maskScreenshot(
|
|
41
|
+
page: Page,
|
|
42
|
+
opts: MaskingOpts = {},
|
|
43
|
+
): Promise<{ maskedCount: number; screenshot: Buffer }> {
|
|
44
|
+
const { extraSelectors = [], style = 'blur', blurPx = 8 } = opts;
|
|
45
|
+
const selectors = [...DEFAULT_PII_SELECTORS, ...extraSelectors];
|
|
46
|
+
|
|
47
|
+
let maskedCount = 0;
|
|
48
|
+
const overlayIds: string[] = [];
|
|
49
|
+
|
|
50
|
+
// Inject CSS overlays
|
|
51
|
+
try {
|
|
52
|
+
const result = await page.evaluate(({ selectors, style, blurPx }) => {
|
|
53
|
+
const ids: string[] = [];
|
|
54
|
+
for (const sel of selectors) {
|
|
55
|
+
try {
|
|
56
|
+
const elements = Array.from(document.querySelectorAll(sel));
|
|
57
|
+
for (const el of elements) {
|
|
58
|
+
const rect = el.getBoundingClientRect();
|
|
59
|
+
if (rect.width === 0 || rect.height === 0) continue;
|
|
60
|
+
const overlay = document.createElement('div');
|
|
61
|
+
const id = `__zeta_mask_${Math.random().toString(36).slice(2)}`;
|
|
62
|
+
overlay.id = id;
|
|
63
|
+
overlay.style.cssText = [
|
|
64
|
+
'position: fixed',
|
|
65
|
+
`top: ${rect.top + window.scrollY}px`,
|
|
66
|
+
`left: ${rect.left + window.scrollX}px`,
|
|
67
|
+
`width: ${rect.width}px`,
|
|
68
|
+
`height: ${rect.height}px`,
|
|
69
|
+
'z-index: 2147483647',
|
|
70
|
+
style === 'blur'
|
|
71
|
+
? `backdrop-filter: blur(${blurPx}px); -webkit-backdrop-filter: blur(${blurPx}px); background: rgba(0,0,0,0.1);`
|
|
72
|
+
: 'background: #000000;',
|
|
73
|
+
'pointer-events: none',
|
|
74
|
+
].join('; ');
|
|
75
|
+
document.body.appendChild(overlay);
|
|
76
|
+
ids.push(id);
|
|
77
|
+
}
|
|
78
|
+
} catch { /* selector error */ }
|
|
79
|
+
}
|
|
80
|
+
return ids;
|
|
81
|
+
}, { selectors, style, blurPx });
|
|
82
|
+
|
|
83
|
+
overlayIds.push(...result);
|
|
84
|
+
maskedCount = result.length;
|
|
85
|
+
} catch { /* non-fatal */ }
|
|
86
|
+
|
|
87
|
+
// Take screenshot with overlays active
|
|
88
|
+
const screenshot = await page.screenshot({ type: 'png', fullPage: false });
|
|
89
|
+
|
|
90
|
+
// Remove overlays
|
|
91
|
+
if (overlayIds.length > 0) {
|
|
92
|
+
await page.evaluate((ids) => {
|
|
93
|
+
for (const id of ids) document.getElementById(id)?.remove();
|
|
94
|
+
}, overlayIds).catch(() => {});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
return { maskedCount, screenshot };
|
|
98
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SEO indexability checker — determines if a page is crawlable and indexable.
|
|
3
|
+
*
|
|
4
|
+
* Checks:
|
|
5
|
+
* 1. X-Robots-Tag response header (noindex, nofollow, noarchive)
|
|
6
|
+
* 2. <meta name="robots"> tag (noindex, nofollow)
|
|
7
|
+
* 3. Canonical URL (self-canonical vs cross-canonical)
|
|
8
|
+
* 4. rel="nofollow" on outbound links
|
|
9
|
+
* 5. hreflang tags (multilingual)
|
|
10
|
+
* 6. Open Graph completeness (og:title, og:description, og:image)
|
|
11
|
+
* 7. Structured data presence (JSON-LD)
|
|
12
|
+
* 8. Meta description
|
|
13
|
+
* 9. Title tag length (< 60 chars recommended)
|
|
14
|
+
* 10. URL structure (query params, fragments, excessive depth)
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import type { Page } from 'playwright';
|
|
18
|
+
|
|
19
|
+
export interface SeoIssue {
|
|
20
|
+
rule: string;
|
|
21
|
+
severity: 'critical' | 'warning' | 'info';
|
|
22
|
+
message: string;
|
|
23
|
+
value?: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface SeoCheckResult {
|
|
27
|
+
url: string;
|
|
28
|
+
isIndexable: boolean;
|
|
29
|
+
canonicalUrl?: string;
|
|
30
|
+
isSelfCanonical?: boolean;
|
|
31
|
+
hreflang: Record<string, string>; // lang → url
|
|
32
|
+
issues: SeoIssue[];
|
|
33
|
+
score: number; // 0-100
|
|
34
|
+
checkedAt: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export async function checkSeo(page: Page, url: string, responseHeaders: Record<string, string> = {}): Promise<SeoCheckResult> {
|
|
38
|
+
const issues: SeoIssue[] = [];
|
|
39
|
+
const checkedAt = new Date().toISOString();
|
|
40
|
+
let isIndexable = true;
|
|
41
|
+
|
|
42
|
+
// 1. X-Robots-Tag
|
|
43
|
+
const robotsHeader = responseHeaders['x-robots-tag'] ?? '';
|
|
44
|
+
if (/noindex/i.test(robotsHeader)) {
|
|
45
|
+
isIndexable = false;
|
|
46
|
+
issues.push({ rule: 'x-robots-noindex', severity: 'critical', message: 'X-Robots-Tag: noindex found in response header', value: robotsHeader });
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// 2-9. Page-level checks
|
|
50
|
+
const pageData = await page.evaluate(() => {
|
|
51
|
+
const meta = (name: string) => document.querySelector(`meta[name="${name}"]`)?.getAttribute('content') ?? '';
|
|
52
|
+
const og = (prop: string) => document.querySelector(`meta[property="og:${prop}"]`)?.getAttribute('content') ?? '';
|
|
53
|
+
const canonical = document.querySelector('link[rel="canonical"]')?.getAttribute('href') ?? '';
|
|
54
|
+
const hreflang: Record<string, string> = {};
|
|
55
|
+
document.querySelectorAll('link[rel="alternate"][hreflang]').forEach((el: any) => {
|
|
56
|
+
hreflang[el.getAttribute('hreflang')] = el.getAttribute('href') ?? '';
|
|
57
|
+
});
|
|
58
|
+
const jsonLd = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).map((s: any) => s.textContent ?? '');
|
|
59
|
+
return {
|
|
60
|
+
robotsMeta: meta('robots'),
|
|
61
|
+
description: meta('description'),
|
|
62
|
+
title: document.title ?? '',
|
|
63
|
+
canonical,
|
|
64
|
+
hreflang,
|
|
65
|
+
ogTitle: og('title'),
|
|
66
|
+
ogDescription: og('description'),
|
|
67
|
+
ogImage: og('image'),
|
|
68
|
+
hasJsonLd: jsonLd.length > 0,
|
|
69
|
+
jsonLd,
|
|
70
|
+
};
|
|
71
|
+
}).catch(() => ({
|
|
72
|
+
robotsMeta: '', description: '', title: '', canonical: '', hreflang: {},
|
|
73
|
+
ogTitle: '', ogDescription: '', ogImage: '', hasJsonLd: false, jsonLd: [],
|
|
74
|
+
}));
|
|
75
|
+
|
|
76
|
+
// 2. Meta robots
|
|
77
|
+
if (/noindex/i.test(pageData.robotsMeta)) {
|
|
78
|
+
isIndexable = false;
|
|
79
|
+
issues.push({ rule: 'meta-robots-noindex', severity: 'critical', message: 'Meta robots noindex found', value: pageData.robotsMeta });
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// 3. Canonical
|
|
83
|
+
let isSelfCanonical: boolean | undefined;
|
|
84
|
+
if (pageData.canonical) {
|
|
85
|
+
isSelfCanonical = pageData.canonical === url || pageData.canonical.replace(/\/$/, '') === url.replace(/\/$/, '');
|
|
86
|
+
if (!isSelfCanonical) {
|
|
87
|
+
issues.push({ rule: 'cross-canonical', severity: 'warning', message: 'Cross-canonical points elsewhere', value: pageData.canonical });
|
|
88
|
+
}
|
|
89
|
+
} else {
|
|
90
|
+
issues.push({ rule: 'missing-canonical', severity: 'warning', message: 'No canonical link tag' });
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// 4. Title
|
|
94
|
+
if (!pageData.title) {
|
|
95
|
+
issues.push({ rule: 'missing-title', severity: 'critical', message: 'Page has no title tag' });
|
|
96
|
+
} else if (pageData.title.length > 60) {
|
|
97
|
+
issues.push({ rule: 'title-too-long', severity: 'warning', message: `Title ${pageData.title.length} chars (>60)`, value: pageData.title });
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// 5. Meta description
|
|
101
|
+
if (!pageData.description) {
|
|
102
|
+
issues.push({ rule: 'missing-description', severity: 'warning', message: 'Missing meta description' });
|
|
103
|
+
} else if (pageData.description.length > 160) {
|
|
104
|
+
issues.push({ rule: 'description-too-long', severity: 'info', message: `Description ${pageData.description.length} chars (>160)` });
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// 6. OG tags
|
|
108
|
+
if (!pageData.ogTitle) issues.push({ rule: 'missing-og-title', severity: 'info', message: 'Missing og:title' });
|
|
109
|
+
if (!pageData.ogImage) issues.push({ rule: 'missing-og-image', severity: 'info', message: 'Missing og:image' });
|
|
110
|
+
|
|
111
|
+
// 7. Structured data
|
|
112
|
+
if (!pageData.hasJsonLd) issues.push({ rule: 'no-structured-data', severity: 'info', message: 'No JSON-LD structured data found' });
|
|
113
|
+
|
|
114
|
+
// 8. URL depth
|
|
115
|
+
try {
|
|
116
|
+
const depth = new URL(url).pathname.split('/').filter(Boolean).length;
|
|
117
|
+
if (depth > 5) issues.push({ rule: 'deep-url', severity: 'info', message: `URL depth ${depth} (>5 levels)`, value: url });
|
|
118
|
+
} catch {}
|
|
119
|
+
|
|
120
|
+
const criticals = issues.filter(i => i.severity === 'critical').length;
|
|
121
|
+
const warnings = issues.filter(i => i.severity === 'warning').length;
|
|
122
|
+
const score = Math.max(0, 100 - criticals * 20 - warnings * 5 - issues.filter(i => i.severity === 'info').length * 2);
|
|
123
|
+
|
|
124
|
+
return {
|
|
125
|
+
url,
|
|
126
|
+
isIndexable,
|
|
127
|
+
canonicalUrl: pageData.canonical || undefined,
|
|
128
|
+
isSelfCanonical,
|
|
129
|
+
hreflang: pageData.hreflang,
|
|
130
|
+
issues,
|
|
131
|
+
score,
|
|
132
|
+
checkedAt,
|
|
133
|
+
};
|
|
134
|
+
}
|