@detiq/crawler 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +49 -0
- package/README.md +11 -0
- package/dist/accessibility-auditor.d.ts +29 -0
- package/dist/accessibility-auditor.js +57 -0
- package/dist/accessibility-auditor.js.map +7 -0
- package/dist/ai-login.d.ts +32 -0
- package/dist/ai-login.js +177 -0
- package/dist/ai-login.js.map +7 -0
- package/dist/auth-token-rotator.d.ts +47 -0
- package/dist/auth-token-rotator.js +58 -0
- package/dist/auth-token-rotator.js.map +7 -0
- package/dist/auth.d.ts +20 -0
- package/dist/auth.js +329 -0
- package/dist/auth.js.map +7 -0
- package/dist/browser-extension.d.ts +37 -0
- package/dist/browser-extension.js +113 -0
- package/dist/browser-extension.js.map +7 -0
- package/dist/captcha-solver.d.ts +16 -0
- package/dist/captcha-solver.js +105 -0
- package/dist/captcha-solver.js.map +7 -0
- package/dist/color-contrast.d.ts +30 -0
- package/dist/color-contrast.js +78 -0
- package/dist/color-contrast.js.map +7 -0
- package/dist/content-chunker.d.ts +18 -0
- package/dist/content-chunker.js +152 -0
- package/dist/content-chunker.js.map +7 -0
- package/dist/content-dedup.d.ts +23 -0
- package/dist/content-dedup.js +60 -0
- package/dist/content-dedup.js.map +7 -0
- package/dist/cookie-consent.d.ts +25 -0
- package/dist/cookie-consent.js +78 -0
- package/dist/cookie-consent.js.map +7 -0
- package/dist/cookie-jar.d.ts +20 -0
- package/dist/cookie-jar.js +61 -0
- package/dist/cookie-jar.js.map +7 -0
- package/dist/crawler.d.ts +179 -0
- package/dist/crawler.js +3672 -0
- package/dist/crawler.js.map +7 -0
- package/dist/custom-js-injector.d.ts +42 -0
- package/dist/custom-js-injector.js +50 -0
- package/dist/custom-js-injector.js.map +7 -0
- package/dist/dark-mode-capture.d.ts +23 -0
- package/dist/dark-mode-capture.js +75 -0
- package/dist/dark-mode-capture.js.map +7 -0
- package/dist/dead-link-detector.d.ts +37 -0
- package/dist/dead-link-detector.js +50 -0
- package/dist/dead-link-detector.js.map +7 -0
- package/dist/element-hash.d.ts +9 -0
- package/dist/element-hash.js +9 -0
- package/dist/element-hash.js.map +7 -0
- package/dist/email-otp.d.ts +42 -0
- package/dist/email-otp.js +177 -0
- package/dist/email-otp.js.map +7 -0
- package/dist/engine-router.d.ts +47 -0
- package/dist/engine-router.js +94 -0
- package/dist/engine-router.js.map +7 -0
- package/dist/export/bulk-export.d.ts +26 -0
- package/dist/export/bulk-export.js +76 -0
- package/dist/export/bulk-export.js.map +7 -0
- package/dist/export/vector-export.d.ts +76 -0
- package/dist/export/vector-export.js +85 -0
- package/dist/export/vector-export.js.map +7 -0
- package/dist/extractors/advanced.d.ts +54 -0
- package/dist/extractors/advanced.js +245 -0
- package/dist/extractors/advanced.js.map +7 -0
- package/dist/extractors/artifact-envelope.d.ts +63 -0
- package/dist/extractors/artifact-envelope.js +63 -0
- package/dist/extractors/artifact-envelope.js.map +7 -0
- package/dist/failure-analysis-agent.d.ts +16 -0
- package/dist/failure-analysis-agent.js +40 -0
- package/dist/failure-analysis-agent.js.map +7 -0
- package/dist/feature-flag-detector.d.ts +30 -0
- package/dist/feature-flag-detector.js +55 -0
- package/dist/feature-flag-detector.js.map +7 -0
- package/dist/form-detector.d.ts +34 -0
- package/dist/form-detector.js +58 -0
- package/dist/form-detector.js.map +7 -0
- package/dist/frontier.d.ts +32 -0
- package/dist/frontier.js +83 -0
- package/dist/frontier.js.map +7 -0
- package/dist/gap-pass.d.ts +35 -0
- package/dist/gap-pass.js +81 -0
- package/dist/gap-pass.js.map +7 -0
- package/dist/graphql-detector.d.ts +39 -0
- package/dist/graphql-detector.js +86 -0
- package/dist/graphql-detector.js.map +7 -0
- package/dist/http-engine.d.ts +80 -0
- package/dist/http-engine.js +304 -0
- package/dist/http-engine.js.map +7 -0
- package/dist/incremental-diff.d.ts +59 -0
- package/dist/incremental-diff.js +61 -0
- package/dist/incremental-diff.js.map +7 -0
- package/dist/index.d.ts +35 -0
- package/dist/index.js +61 -0
- package/dist/index.js.map +7 -0
- package/dist/input-fuzzer.d.ts +30 -0
- package/dist/input-fuzzer.js +105 -0
- package/dist/input-fuzzer.js.map +7 -0
- package/dist/interactive-probe.d.ts +104 -0
- package/dist/interactive-probe.js +442 -0
- package/dist/interactive-probe.js.map +7 -0
- package/dist/job-queue.d.ts +171 -0
- package/dist/job-queue.js +2256 -0
- package/dist/job-queue.js.map +7 -0
- package/dist/js-coverage.d.ts +29 -0
- package/dist/js-coverage.js +56 -0
- package/dist/js-coverage.js.map +7 -0
- package/dist/js-router-detector.d.ts +17 -0
- package/dist/js-router-detector.js +90 -0
- package/dist/js-router-detector.js.map +7 -0
- package/dist/language-detector.d.ts +22 -0
- package/dist/language-detector.js +57 -0
- package/dist/language-detector.js.map +7 -0
- package/dist/link-classifier.d.ts +23 -0
- package/dist/link-classifier.js +40 -0
- package/dist/link-classifier.js.map +7 -0
- package/dist/live-view.d.ts +36 -0
- package/dist/live-view.js +100 -0
- package/dist/live-view.js.map +7 -0
- package/dist/logger.d.ts +2 -0
- package/dist/logger.js +10 -0
- package/dist/logger.js.map +7 -0
- package/dist/markdown-extract.d.ts +8 -0
- package/dist/markdown-extract.js +54 -0
- package/dist/markdown-extract.js.map +7 -0
- package/dist/native-crawler.d.ts +69 -0
- package/dist/native-crawler.js +839 -0
- package/dist/native-crawler.js.map +7 -0
- package/dist/network-observer.d.ts +49 -0
- package/dist/network-observer.js +149 -0
- package/dist/network-observer.js.map +7 -0
- package/dist/network-throttle.d.ts +25 -0
- package/dist/network-throttle.js +34 -0
- package/dist/network-throttle.js.map +7 -0
- package/dist/openapi-generator.d.ts +49 -0
- package/dist/openapi-generator.js +89 -0
- package/dist/openapi-generator.js.map +7 -0
- package/dist/pdf-extractor.d.ts +28 -0
- package/dist/pdf-extractor.js +81 -0
- package/dist/pdf-extractor.js.map +7 -0
- package/dist/performance-metrics.d.ts +53 -0
- package/dist/performance-metrics.js +83 -0
- package/dist/performance-metrics.js.map +7 -0
- package/dist/phash.d.ts +38 -0
- package/dist/phash.js +115 -0
- package/dist/phash.js.map +7 -0
- package/dist/pii-scrubber.d.ts +21 -0
- package/dist/pii-scrubber.js +75 -0
- package/dist/pii-scrubber.js.map +7 -0
- package/dist/playwright-recorder.d.ts +39 -0
- package/dist/playwright-recorder.js +123 -0
- package/dist/playwright-recorder.js.map +7 -0
- package/dist/policies/crawl-safety.d.ts +9 -0
- package/dist/policies/crawl-safety.js +37 -0
- package/dist/policies/crawl-safety.js.map +7 -0
- package/dist/profiles/domain-profile.d.ts +19 -0
- package/dist/profiles/domain-profile.js +140 -0
- package/dist/profiles/domain-profile.js.map +7 -0
- package/dist/proxy.d.ts +50 -0
- package/dist/proxy.js +165 -0
- package/dist/proxy.js.map +7 -0
- package/dist/rate-limiter.d.ts +31 -0
- package/dist/rate-limiter.js +64 -0
- package/dist/rate-limiter.js.map +7 -0
- package/dist/redirect-tracker.d.ts +37 -0
- package/dist/redirect-tracker.js +65 -0
- package/dist/redirect-tracker.js.map +7 -0
- package/dist/redis-frontier.d.ts +48 -0
- package/dist/redis-frontier.js +126 -0
- package/dist/redis-frontier.js.map +7 -0
- package/dist/redis-queue.d.ts +48 -0
- package/dist/redis-queue.js +165 -0
- package/dist/redis-queue.js.map +7 -0
- package/dist/response-cache.d.ts +38 -0
- package/dist/response-cache.js +140 -0
- package/dist/response-cache.js.map +7 -0
- package/dist/responsive-detector.d.ts +29 -0
- package/dist/responsive-detector.js +46 -0
- package/dist/responsive-detector.js.map +7 -0
- package/dist/retry-policy.d.ts +20 -0
- package/dist/retry-policy.js +54 -0
- package/dist/retry-policy.js.map +7 -0
- package/dist/robots-txt.d.ts +9 -0
- package/dist/robots-txt.js +77 -0
- package/dist/robots-txt.js.map +7 -0
- package/dist/screen-reader.d.ts +16 -0
- package/dist/screen-reader.js +46 -0
- package/dist/screen-reader.js.map +7 -0
- package/dist/screenshot-masker.d.ts +28 -0
- package/dist/screenshot-masker.js +62 -0
- package/dist/screenshot-masker.js.map +7 -0
- package/dist/seo-checker.d.ts +33 -0
- package/dist/seo-checker.js +91 -0
- package/dist/seo-checker.js.map +7 -0
- package/dist/session-capture.d.ts +35 -0
- package/dist/session-capture.js +267 -0
- package/dist/session-capture.js.map +7 -0
- package/dist/session-export.d.ts +49 -0
- package/dist/session-export.js +117 -0
- package/dist/session-export.js.map +7 -0
- package/dist/sitemap-writer.d.ts +21 -0
- package/dist/sitemap-writer.js +47 -0
- package/dist/sitemap-writer.js.map +7 -0
- package/dist/sitemap.d.ts +51 -0
- package/dist/sitemap.js +152 -0
- package/dist/sitemap.js.map +7 -0
- package/dist/skyvern.d.ts +56 -0
- package/dist/skyvern.js +129 -0
- package/dist/skyvern.js.map +7 -0
- package/dist/sqs-queue.d.ts +2 -0
- package/dist/sqs-queue.js +47 -0
- package/dist/sqs-queue.js.map +7 -0
- package/dist/sso-token.d.ts +29 -0
- package/dist/sso-token.js +76 -0
- package/dist/sso-token.js.map +7 -0
- package/dist/stealth/human-behavior.d.ts +20 -0
- package/dist/stealth/human-behavior.js +168 -0
- package/dist/stealth/human-behavior.js.map +7 -0
- package/dist/telemetry.d.ts +34 -0
- package/dist/telemetry.js +77 -0
- package/dist/telemetry.js.map +7 -0
- package/dist/test-executor.d.ts +67 -0
- package/dist/test-executor.js +311 -0
- package/dist/test-executor.js.map +7 -0
- package/dist/tls-fingerprint.d.ts +50 -0
- package/dist/tls-fingerprint.js +81 -0
- package/dist/tls-fingerprint.js.map +7 -0
- package/dist/url-pattern-completer.d.ts +28 -0
- package/dist/url-pattern-completer.js +87 -0
- package/dist/url-pattern-completer.js.map +7 -0
- package/dist/visual-diff-verdict.d.ts +40 -0
- package/dist/visual-diff-verdict.js +25 -0
- package/dist/visual-diff-verdict.js.map +7 -0
- package/dist/webhook-notifier.d.ts +30 -0
- package/dist/webhook-notifier.js +76 -0
- package/dist/webhook-notifier.js.map +7 -0
- package/dist/websocket-detector.d.ts +24 -0
- package/dist/websocket-detector.js +60 -0
- package/dist/websocket-detector.js.map +7 -0
- package/dist/worker.d.ts +1 -0
- package/dist/worker.js +33 -0
- package/dist/worker.js.map +7 -0
- package/package.json +45 -0
- package/src/accessibility-auditor.ts +102 -0
- package/src/ai-login.ts +218 -0
- package/src/auth-token-rotator.ts +89 -0
- package/src/auth.ts +396 -0
- package/src/browser-extension.ts +186 -0
- package/src/captcha-solver.ts +143 -0
- package/src/color-contrast.ts +116 -0
- package/src/content-chunker.ts +170 -0
- package/src/content-dedup.ts +99 -0
- package/src/cookie-consent.ts +106 -0
- package/src/cookie-jar.ts +91 -0
- package/src/crawler.ts +4305 -0
- package/src/custom-js-injector.ts +93 -0
- package/src/dark-mode-capture.ts +110 -0
- package/src/dead-link-detector.ts +76 -0
- package/src/element-hash.ts +16 -0
- package/src/email-otp.ts +252 -0
- package/src/engine-router.ts +141 -0
- package/src/export/bulk-export.ts +85 -0
- package/src/export/vector-export.ts +160 -0
- package/src/extractors/advanced.ts +283 -0
- package/src/extractors/artifact-envelope.ts +111 -0
- package/src/failure-analysis-agent.ts +49 -0
- package/src/feature-flag-detector.ts +97 -0
- package/src/form-detector.ts +103 -0
- package/src/frontier.test.ts +64 -0
- package/src/frontier.ts +112 -0
- package/src/gap-pass.ts +122 -0
- package/src/graphql-detector.ts +129 -0
- package/src/http-engine.ts +433 -0
- package/src/incremental-diff.ts +121 -0
- package/src/index.ts +35 -0
- package/src/input-fuzzer.ts +129 -0
- package/src/interactive-probe.ts +623 -0
- package/src/job-queue.test.ts +120 -0
- package/src/job-queue.ts +2701 -0
- package/src/js-coverage.ts +87 -0
- package/src/js-router-detector.ts +112 -0
- package/src/language-detector.ts +82 -0
- package/src/link-classifier.ts +65 -0
- package/src/live-view.ts +141 -0
- package/src/logger.ts +9 -0
- package/src/markdown-extract.test.ts +73 -0
- package/src/markdown-extract.ts +76 -0
- package/src/native-crawler.ts +1106 -0
- package/src/network-observer.ts +204 -0
- package/src/network-throttle.ts +69 -0
- package/src/openapi-generator.ts +135 -0
- package/src/pdf-extractor.ts +123 -0
- package/src/performance-metrics.ts +134 -0
- package/src/phash.ts +151 -0
- package/src/pii-scrubber.ts +99 -0
- package/src/playwright-recorder.ts +170 -0
- package/src/policies/crawl-safety.ts +46 -0
- package/src/profiles/domain-profile.ts +167 -0
- package/src/proxy.ts +238 -0
- package/src/rate-limiter.ts +87 -0
- package/src/redirect-tracker.ts +97 -0
- package/src/redis-frontier.ts +143 -0
- package/src/redis-queue.ts +208 -0
- package/src/response-cache.ts +150 -0
- package/src/responsive-detector.ts +92 -0
- package/src/retry-policy.ts +87 -0
- package/src/robots-txt.ts +96 -0
- package/src/screen-reader.ts +89 -0
- package/src/screenshot-masker.ts +98 -0
- package/src/seo-checker.ts +134 -0
- package/src/session-capture.ts +325 -0
- package/src/session-export.test.ts +75 -0
- package/src/session-export.ts +172 -0
- package/src/sitemap-writer.ts +65 -0
- package/src/sitemap.test.ts +131 -0
- package/src/sitemap.ts +224 -0
- package/src/skyvern.ts +199 -0
- package/src/sqs-queue.ts +49 -0
- package/src/sso-token.ts +120 -0
- package/src/stealth/human-behavior.ts +191 -0
- package/src/telemetry.ts +122 -0
- package/src/test-executor.test.ts +72 -0
- package/src/test-executor.ts +424 -0
- package/src/tls-fingerprint.ts +111 -0
- package/src/url-pattern-completer.ts +114 -0
- package/src/visual-diff-verdict.test.ts +80 -0
- package/src/visual-diff-verdict.ts +57 -0
- package/src/webhook-notifier.ts +109 -0
- package/src/websocket-detector.ts +83 -0
- package/src/worker.ts +48 -0
- package/tsconfig.json +13 -0
- package/types/accessibility-auditor.d.ts +29 -0
- package/types/ai-login.d.ts +32 -0
- package/types/auth-token-rotator.d.ts +47 -0
- package/types/auth.d.ts +20 -0
- package/types/browser-extension.d.ts +37 -0
- package/types/bull-worker.d.ts +6 -0
- package/types/captcha-solver.d.ts +16 -0
- package/types/color-contrast.d.ts +30 -0
- package/types/content-chunker.d.ts +18 -0
- package/types/content-dedup.d.ts +23 -0
- package/types/cookie-consent.d.ts +25 -0
- package/types/cookie-jar.d.ts +20 -0
- package/types/crawler.d.ts +179 -0
- package/types/custom-js-injector.d.ts +42 -0
- package/types/dark-mode-capture.d.ts +23 -0
- package/types/dead-link-detector.d.ts +37 -0
- package/types/element-hash.d.ts +9 -0
- package/types/email-otp.d.ts +42 -0
- package/types/engine-router.d.ts +47 -0
- package/types/export/bulk-export.d.ts +26 -0
- package/types/export/vector-export.d.ts +76 -0
- package/types/extractors/advanced.d.ts +54 -0
- package/types/extractors/artifact-envelope.d.ts +63 -0
- package/types/failure-analysis-agent.d.ts +16 -0
- package/types/feature-flag-detector.d.ts +30 -0
- package/types/form-detector.d.ts +34 -0
- package/types/frontier.d.ts +32 -0
- package/types/gap-pass.d.ts +35 -0
- package/types/graphql-detector.d.ts +39 -0
- package/types/http-engine.d.ts +80 -0
- package/types/incremental-diff.d.ts +59 -0
- package/types/index.d.ts +35 -0
- package/types/input-fuzzer.d.ts +30 -0
- package/types/interactive-probe.d.ts +104 -0
- package/types/job-queue.d.ts +171 -0
- package/types/js-coverage.d.ts +29 -0
- package/types/js-router-detector.d.ts +17 -0
- package/types/language-detector.d.ts +22 -0
- package/types/link-classifier.d.ts +23 -0
- package/types/live-view.d.ts +36 -0
- package/types/logger.d.ts +2 -0
- package/types/markdown-extract.d.ts +8 -0
- package/types/native-crawler.d.ts +69 -0
- package/types/network-observer.d.ts +49 -0
- package/types/network-throttle.d.ts +25 -0
- package/types/openapi-generator.d.ts +49 -0
- package/types/pdf-extractor.d.ts +28 -0
- package/types/performance-metrics.d.ts +53 -0
- package/types/phash.d.ts +38 -0
- package/types/pii-scrubber.d.ts +21 -0
- package/types/playwright-recorder.d.ts +39 -0
- package/types/policies/crawl-safety.d.ts +9 -0
- package/types/profiles/domain-profile.d.ts +19 -0
- package/types/proxy.d.ts +50 -0
- package/types/rate-limiter.d.ts +31 -0
- package/types/redirect-tracker.d.ts +37 -0
- package/types/redis-frontier.d.ts +48 -0
- package/types/redis-queue.d.ts +48 -0
- package/types/response-cache.d.ts +38 -0
- package/types/responsive-detector.d.ts +29 -0
- package/types/retry-policy.d.ts +20 -0
- package/types/robots-txt.d.ts +9 -0
- package/types/screen-reader.d.ts +16 -0
- package/types/screenshot-masker.d.ts +28 -0
- package/types/seo-checker.d.ts +33 -0
- package/types/session-capture.d.ts +38 -0
- package/types/session-export.d.ts +49 -0
- package/types/sitemap-writer.d.ts +21 -0
- package/types/sitemap.d.ts +51 -0
- package/types/skyvern.d.ts +56 -0
- package/types/sqs-queue.d.ts +2 -0
- package/types/sso-token.d.ts +29 -0
- package/types/stealth/human-behavior.d.ts +20 -0
- package/types/telemetry.d.ts +34 -0
- package/types/test-executor.d.ts +67 -0
- package/types/tls-fingerprint.d.ts +50 -0
- package/types/url-pattern-completer.d.ts +28 -0
- package/types/visual-diff-verdict.d.ts +40 -0
- package/types/webhook-notifier.d.ts +30 -0
- package/types/websocket-detector.d.ts +24 -0
- package/types/worker.d.ts +1 -0
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/openapi-generator.ts"],
|
|
4
|
+
"sourcesContent": ["/**\n * OpenAPI 3.0 spec generator from observed network traffic.\n *\n * Takes the ApiEndpointObservationInput[] collected by the network observer\n * during crawl and generates a minimal but valid OpenAPI 3.0 spec.\n *\n * The spec captures: endpoints, methods, observed response shapes, status codes.\n * It does NOT capture request bodies (we'd need to intercept requests too) \u2014 that's\n * marked as a TODO. But it's enough for generating smoke-test suites.\n */\n\nimport type { ApiEndpointObservationInput } from './network-observer.js';\n\nexport interface OpenApiSpec {\n openapi: '3.0.3';\n info: { title: string; version: string; description?: string };\n servers: Array<{ url: string }>;\n paths: Record<string, Record<string, OpenApiOperation>>;\n}\n\ninterface OpenApiOperation {\n summary: string;\n parameters?: OpenApiParam[];\n responses: Record<string, { description: string; content?: Record<string, { schema: unknown }> }>;\n tags?: string[];\n}\n\ninterface OpenApiParam {\n name: string;\n in: 'path' | 'query';\n required: boolean;\n schema: { type: string };\n}\n\nfunction extractPathParams(urlPattern: string): OpenApiParam[] {\n const params: OpenApiParam[] = [];\n const matches = urlPattern.matchAll(/\\{(\\w+)\\}/g);\n for (const m of matches) {\n params.push({ name: m[1], in: 'path', required: true, schema: { type: 'string' } });\n }\n return params;\n}\n\nfunction methodSummary(method: string, path: string): string {\n const resource = path.split('/').filter(s => !s.startsWith('{') && s.length > 0).pop() ?? 'resource';\n const summaries: Record<string, string> = {\n GET: `Get ${resource}`,\n POST: `Create ${resource}`,\n PUT: `Update ${resource}`,\n PATCH: `Partially update ${resource}`,\n DELETE: `Delete ${resource}`,\n };\n return summaries[method.toUpperCase()] ?? `${method} ${resource}`;\n}\n\nfunction inferTag(path: string): string {\n const parts = path.split('/').filter(s => s.length > 0 && !s.startsWith('{'));\n return parts[0] ?? 'default';\n}\n\nexport function generateOpenApiSpec(\n observations: ApiEndpointObservationInput[],\n opts: { title?: string; baseUrl?: string; version?: string } = {},\n): OpenApiSpec {\n const { title = 'ZeTa Discovered API', baseUrl = '/', version = '1.0.0' } = opts;\n\n const spec: OpenApiSpec = {\n openapi: '3.0.3',\n info: { title, version, description: 'Auto-generated by ZeTa crawler from observed network traffic.' },\n servers: [{ url: baseUrl }],\n paths: {},\n };\n\n for (const obs of observations) {\n const path = obs.urlPattern;\n const method = obs.method.toLowerCase();\n\n if (!spec.paths[path]) spec.paths[path] = {};\n\n const params = extractPathParams(path);\n const statusCode = obs.statusCode ?? 200;\n const responses: OpenApiOperation['responses'] = {\n [String(statusCode)]: {\n description: statusCode < 400 ? 'Successful response' : 'Error response',\n content: obs.responseShape\n ? { 'application/json': { schema: shapeToJsonSchema(obs.responseShape) } }\n : undefined,\n },\n };\n\n const op: OpenApiOperation = {\n summary: methodSummary(obs.method, path),\n parameters: params.length > 0 ? params : undefined,\n responses,\n tags: [inferTag(path)],\n };\n\n spec.paths[path][method] = op;\n }\n\n return spec;\n}\n\nfunction shapeToJsonSchema(shape: unknown): unknown {\n if (Array.isArray(shape)) return { type: 'array', items: shape.length > 0 ? shapeToJsonSchema(shape[0]) : {} };\n if (shape && typeof shape === 'object') {\n const props: Record<string, unknown> = {};\n for (const [k, v] of Object.entries(shape as object)) props[k] = shapeToJsonSchema(v);\n return { type: 'object', properties: props };\n }\n return { type: String(shape) };\n}\n\nexport function specToYaml(spec: OpenApiSpec): string {\n // Minimal YAML serializer \u2014 avoids yaml dep\n function toYaml(val: unknown, indent = 0): string {\n const pad = ' '.repeat(indent);\n if (val === null || val === undefined) return 'null';\n if (typeof val === 'string') return val.includes('\\n') || val.includes(':') ? `\"${val.replace(/\"/g, '\\\\\"')}\"` : val;\n if (typeof val === 'number' || typeof val === 'boolean') return String(val);\n if (Array.isArray(val)) return val.map(v => `\\n${pad}- ${toYaml(v, indent + 1).trimStart()}`).join('') || '[]';\n if (typeof val === 'object') {\n const entries = Object.entries(val as object);\n if (entries.length === 0) return '{}';\n return entries.map(([k, v]) => {\n const vStr = toYaml(v, indent + 1);\n return typeof v === 'object' && v !== null && !Array.isArray(v)\n ? `\\n${pad}${k}:${vStr}`\n : `\\n${pad}${k}: ${vStr.trimStart()}`;\n }).join('');\n }\n return String(val);\n }\n return `openapi: \"${spec.openapi}\"${toYaml(spec, 0)}`;\n}\n"],
|
|
5
|
+
"mappings": "AAkCA,SAAS,kBAAkB,YAAoC;AAC7D,QAAM,SAAyB,CAAC;AAChC,QAAM,UAAU,WAAW,SAAS,YAAY;AAChD,aAAW,KAAK,SAAS;AACvB,WAAO,KAAK,EAAE,MAAM,EAAE,CAAC,GAAG,IAAI,QAAQ,UAAU,MAAM,QAAQ,EAAE,MAAM,SAAS,EAAE,CAAC;AAAA,EACpF;AACA,SAAO;AACT;AAEA,SAAS,cAAc,QAAgB,MAAsB;AAC3D,QAAM,WAAW,KAAK,MAAM,GAAG,EAAE,OAAO,OAAK,CAAC,EAAE,WAAW,GAAG,KAAK,EAAE,SAAS,CAAC,EAAE,IAAI,KAAK;AAC1F,QAAM,YAAoC;AAAA,IACxC,KAAK,OAAO,QAAQ;AAAA,IACpB,MAAM,UAAU,QAAQ;AAAA,IACxB,KAAK,UAAU,QAAQ;AAAA,IACvB,OAAO,oBAAoB,QAAQ;AAAA,IACnC,QAAQ,UAAU,QAAQ;AAAA,EAC5B;AACA,SAAO,UAAU,OAAO,YAAY,CAAC,KAAK,GAAG,MAAM,IAAI,QAAQ;AACjE;AAEA,SAAS,SAAS,MAAsB;AACtC,QAAM,QAAQ,KAAK,MAAM,GAAG,EAAE,OAAO,OAAK,EAAE,SAAS,KAAK,CAAC,EAAE,WAAW,GAAG,CAAC;AAC5E,SAAO,MAAM,CAAC,KAAK;AACrB;AAEO,SAAS,oBACd,cACA,OAA+D,CAAC,GACnD;AACb,QAAM,EAAE,QAAQ,uBAAuB,UAAU,KAAK,UAAU,QAAQ,IAAI;AAE5E,QAAM,OAAoB;AAAA,IACxB,SAAS;AAAA,IACT,MAAM,EAAE,OAAO,SAAS,aAAa,gEAAgE;AAAA,IACrG,SAAS,CAAC,EAAE,KAAK,QAAQ,CAAC;AAAA,IAC1B,OAAO,CAAC;AAAA,EACV;AAEA,aAAW,OAAO,cAAc;AAC9B,UAAM,OAAO,IAAI;AACjB,UAAM,SAAS,IAAI,OAAO,YAAY;AAEtC,QAAI,CAAC,KAAK,MAAM,IAAI,EAAG,MAAK,MAAM,IAAI,IAAI,CAAC;AAE3C,UAAM,SAAS,kBAAkB,IAAI;AACrC,UAAM,aAAa,IAAI,cAAc;AACrC,UAAM,YAA2C;AAAA,MAC/C,CAAC,OAAO,UAAU,CAAC,GAAG;AAAA,QACpB,aAAa,aAAa,MAAM,wBAAwB;AAAA,QACxD,SAAS,IAAI,gBACT,EAAE,oBAAoB,EAAE,QAAQ,kBAAkB,IAAI,aAAa,EAAE,EAAE,IACvE;AAAA,MACN;AAAA,IACF;AAEA,UAAM,KAAuB;AAAA,MAC3B,SAAS,cAAc,IAAI,QAAQ,IAAI;AAAA,MACvC,YAAY,OAAO,SAAS,IAAI,SAAS;AAAA,MACzC;AAAA,MACA,MAAM,CAAC,SAAS,IAAI,CAAC;AAAA,IACvB;AAEA,SAAK,MAAM,IAAI,EAAE,MAAM,IAAI;AAAA,EAC7B;AAEA,SAAO;AACT;AAEA,SAAS,kBAAkB,OAAyB;AAClD,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,EAAE,MAAM,SAAS,OAAO,MAAM,SAAS,IAAI,kBAAkB,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE;AAC7G,MAAI,SAAS,OAAO,UAAU,UAAU;AACtC,UAAM,QAAiC,CAAC;AACxC,eAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,KAAe,EAAG,OAAM,CAAC,IAAI,kBAAkB,CAAC;AACpF,WAAO,EAAE,MAAM,UAAU,YAAY,MAAM;AAAA,EAC7C;AACA,SAAO,EAAE,MAAM,OAAO,KAAK,EAAE;AAC/B;AAEO,SAAS,WAAW,MAA2B;AAEpD,WAAS,OAAO,KAAc,SAAS,GAAW;AAChD,UAAM,MAAM,KAAK,OAAO,MAAM;AAC9B,QAAI,QAAQ,QAAQ,QAAQ,OAAW,QAAO;AAC9C,QAAI,OAAO,QAAQ,SAAU,QAAO,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS,GAAG,IAAI,IAAI,IAAI,QAAQ,MAAM,KAAK,CAAC,MAAM;AAChH,QAAI,OAAO,QAAQ,YAAY,OAAO,QAAQ,UAAW,QAAO,OAAO,GAAG;AAC1E,QAAI,MAAM,QAAQ,GAAG,EAAG,QAAO,IAAI,IAAI,OAAK;AAAA,EAAK,GAAG,KAAK,OAAO,GAAG,SAAS,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK;AAC1G,QAAI,OAAO,QAAQ,UAAU;AAC3B,YAAM,UAAU,OAAO,QAAQ,GAAa;AAC5C,UAAI,QAAQ,WAAW,EAAG,QAAO;AACjC,aAAO,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM;AAC7B,cAAM,OAAO,OAAO,GAAG,SAAS,CAAC;AACjC,eAAO,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC,IAC1D;AAAA,EAAK,GAAG,GAAG,CAAC,IAAI,IAAI,KACpB;AAAA,EAAK,GAAG,GAAG,CAAC,KAAK,KAAK,UAAU,CAAC;AAAA,MACvC,CAAC,EAAE,KAAK,EAAE;AAAA,IACZ;AACA,WAAO,OAAO,GAAG;AAAA,EACnB;AACA,SAAO,aAAa,KAAK,OAAO,IAAI,OAAO,MAAM,CAAC,CAAC;AACrD;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PDF extraction — discovers and extracts text from PDF URLs found during crawl.
|
|
3
|
+
*
|
|
4
|
+
* When the HTTP engine or browser engine discovers a link to a .pdf file,
|
|
5
|
+
* this module fetches the PDF, extracts its text content, and returns it
|
|
6
|
+
* as a screen-compatible markdown string for test generation.
|
|
7
|
+
*
|
|
8
|
+
* Uses pdf-parse (already installed in API package, accessible via dynamic import).
|
|
9
|
+
*/
|
|
10
|
+
export interface PdfScreen {
|
|
11
|
+
url: string;
|
|
12
|
+
title: string;
|
|
13
|
+
markdown: string;
|
|
14
|
+
pageCount: number;
|
|
15
|
+
info: Record<string, string>;
|
|
16
|
+
sizeBytes: number;
|
|
17
|
+
}
|
|
18
|
+
export declare function isPdfUrl(url: string, contentType?: string): boolean;
|
|
19
|
+
export declare function extractPdfFromUrl(url: string, opts?: {
|
|
20
|
+
userAgent?: string;
|
|
21
|
+
timeoutMs?: number;
|
|
22
|
+
maxSizeBytes?: number;
|
|
23
|
+
}): Promise<PdfScreen | null>;
|
|
24
|
+
/**
|
|
25
|
+
* Filter a list of discovered URLs to find PDF links.
|
|
26
|
+
* Used by http-engine and browser crawler to decide which links to pass to extractPdfFromUrl.
|
|
27
|
+
*/
|
|
28
|
+
export declare function filterPdfUrls(urls: string[]): string[];
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
const PDF_EXT = /\.pdf(\?.*)?$/i;
|
|
2
|
+
const PDF_MIME = "application/pdf";
|
|
3
|
+
function isPdfUrl(url, contentType) {
|
|
4
|
+
return PDF_EXT.test(url) || (contentType?.includes(PDF_MIME) ?? false);
|
|
5
|
+
}
|
|
6
|
+
async function extractPdfFromUrl(url, opts = {}) {
|
|
7
|
+
const { userAgent = "ZeTa-Crawler/1.0", timeoutMs = 3e4, maxSizeBytes = 50 * 1024 * 1024 } = opts;
|
|
8
|
+
let buffer;
|
|
9
|
+
let sizeBytes;
|
|
10
|
+
try {
|
|
11
|
+
const response = await fetch(url, {
|
|
12
|
+
headers: { "User-Agent": userAgent, "Accept": "application/pdf,*/*" },
|
|
13
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
14
|
+
});
|
|
15
|
+
if (!response.ok) return null;
|
|
16
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
17
|
+
if (!contentType.includes("pdf") && !PDF_EXT.test(url)) return null;
|
|
18
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
19
|
+
sizeBytes = arrayBuffer.byteLength;
|
|
20
|
+
if (sizeBytes > maxSizeBytes) {
|
|
21
|
+
console.warn(`[pdf-extractor] PDF too large (${sizeBytes} bytes), skipping: ${url}`);
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
buffer = Buffer.from(arrayBuffer);
|
|
25
|
+
} catch (e) {
|
|
26
|
+
console.error(`[pdf-extractor] Failed to fetch PDF ${url}: ${e.message}`);
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
try {
|
|
30
|
+
const pdfParse = await import("pdf-parse/lib/pdf-parse.js").catch(
|
|
31
|
+
() => import("pdf-parse")
|
|
32
|
+
);
|
|
33
|
+
const parse = pdfParse.default ?? pdfParse;
|
|
34
|
+
const data = await parse(buffer, { max: 0 });
|
|
35
|
+
const title = data.info?.Title || extractTitleFromUrl(url);
|
|
36
|
+
const markdown = pdfTextToMarkdown(data.text ?? "", title);
|
|
37
|
+
return {
|
|
38
|
+
url,
|
|
39
|
+
title,
|
|
40
|
+
markdown,
|
|
41
|
+
pageCount: data.numpages ?? 0,
|
|
42
|
+
info: {
|
|
43
|
+
author: data.info?.Author ?? "",
|
|
44
|
+
subject: data.info?.Subject ?? "",
|
|
45
|
+
keywords: data.info?.Keywords ?? "",
|
|
46
|
+
creator: data.info?.Creator ?? ""
|
|
47
|
+
},
|
|
48
|
+
sizeBytes
|
|
49
|
+
};
|
|
50
|
+
} catch (e) {
|
|
51
|
+
console.error(`[pdf-extractor] Failed to parse PDF ${url}: ${e.message}`);
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
function extractTitleFromUrl(url) {
|
|
56
|
+
try {
|
|
57
|
+
const { pathname } = new URL(url);
|
|
58
|
+
const filename = pathname.split("/").pop() ?? "document";
|
|
59
|
+
return filename.replace(/\.pdf$/i, "").replace(/[-_]/g, " ").trim();
|
|
60
|
+
} catch {
|
|
61
|
+
return "PDF Document";
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
function pdfTextToMarkdown(text, title) {
|
|
65
|
+
if (!text.trim()) return `# ${title}
|
|
66
|
+
|
|
67
|
+
*No extractable text content.*`;
|
|
68
|
+
const cleaned = text.replace(/\f/g, "\n\n---\n\n").replace(/\r\n/g, "\n").replace(/\r/g, "\n").replace(/\n{3,}/g, "\n\n").replace(/[ \t]{2,}/g, " ").trim();
|
|
69
|
+
return `# ${title}
|
|
70
|
+
|
|
71
|
+
${cleaned}`;
|
|
72
|
+
}
|
|
73
|
+
function filterPdfUrls(urls) {
|
|
74
|
+
return urls.filter((u) => PDF_EXT.test(u));
|
|
75
|
+
}
|
|
76
|
+
export {
|
|
77
|
+
extractPdfFromUrl,
|
|
78
|
+
filterPdfUrls,
|
|
79
|
+
isPdfUrl
|
|
80
|
+
};
|
|
81
|
+
//# sourceMappingURL=pdf-extractor.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/pdf-extractor.ts"],
|
|
4
|
+
"sourcesContent": ["/**\n * PDF extraction \u2014 discovers and extracts text from PDF URLs found during crawl.\n *\n * When the HTTP engine or browser engine discovers a link to a .pdf file,\n * this module fetches the PDF, extracts its text content, and returns it\n * as a screen-compatible markdown string for test generation.\n *\n * Uses pdf-parse (already installed in API package, accessible via dynamic import).\n */\n\nconst PDF_EXT = /\\.pdf(\\?.*)?$/i;\nconst PDF_MIME = 'application/pdf';\n\nexport interface PdfScreen {\n url: string;\n title: string;\n markdown: string;\n pageCount: number;\n info: Record<string, string>;\n sizeBytes: number;\n}\n\nexport function isPdfUrl(url: string, contentType?: string): boolean {\n return PDF_EXT.test(url) || (contentType?.includes(PDF_MIME) ?? false);\n}\n\nexport async function extractPdfFromUrl(url: string, opts: {\n userAgent?: string;\n timeoutMs?: number;\n maxSizeBytes?: number;\n} = {}): Promise<PdfScreen | null> {\n const { userAgent = 'ZeTa-Crawler/1.0', timeoutMs = 30_000, maxSizeBytes = 50 * 1024 * 1024 } = opts;\n\n let buffer: Buffer;\n let sizeBytes: number;\n\n try {\n const response = await fetch(url, {\n headers: { 'User-Agent': userAgent, 'Accept': 'application/pdf,*/*' },\n signal: AbortSignal.timeout(timeoutMs),\n });\n if (!response.ok) return null;\n\n const contentType = response.headers.get('content-type') ?? '';\n if (!contentType.includes('pdf') && !PDF_EXT.test(url)) return null;\n\n const arrayBuffer = await response.arrayBuffer();\n sizeBytes = arrayBuffer.byteLength;\n if (sizeBytes > maxSizeBytes) {\n console.warn(`[pdf-extractor] PDF too large (${sizeBytes} bytes), skipping: ${url}`);\n return null;\n }\n buffer = Buffer.from(arrayBuffer);\n } catch (e: any) {\n console.error(`[pdf-extractor] Failed to fetch PDF ${url}: ${e.message}`);\n return null;\n }\n\n try {\n // pdf-parse is installed in the API package \u2014 use dynamic import\n // so this file compiles without adding pdf-parse to crawler's package.json\n // eslint-disable-next-line @typescript-eslint/ban-ts-comment\n // @ts-ignore \u2013 no types for pdf-parse/lib/pdf-parse.js\n const pdfParse = await import('pdf-parse/lib/pdf-parse.js').catch(\n () => import('pdf-parse' as any)\n ) as any;\n const parse = pdfParse.default ?? pdfParse;\n const data = await parse(buffer, { max: 0 }); // max: 0 = all pages\n\n const title = (data.info?.Title as string) || extractTitleFromUrl(url);\n const markdown = pdfTextToMarkdown(data.text ?? '', title);\n\n return {\n url,\n title,\n markdown,\n pageCount: data.numpages ?? 0,\n info: {\n author: data.info?.Author ?? '',\n subject: data.info?.Subject ?? '',\n keywords: data.info?.Keywords ?? '',\n creator: data.info?.Creator ?? '',\n },\n sizeBytes,\n };\n } catch (e: any) {\n console.error(`[pdf-extractor] Failed to parse PDF ${url}: ${e.message}`);\n return null;\n }\n}\n\nfunction extractTitleFromUrl(url: string): string {\n try {\n const { pathname } = new URL(url);\n const filename = pathname.split('/').pop() ?? 'document';\n return filename.replace(/\\.pdf$/i, '').replace(/[-_]/g, ' ').trim();\n } catch {\n return 'PDF Document';\n }\n}\n\nfunction pdfTextToMarkdown(text: string, title: string): string {\n if (!text.trim()) return `# ${title}\\n\\n*No extractable text content.*`;\n\n // Clean up common PDF extraction artifacts\n const cleaned = text\n .replace(/\\f/g, '\\n\\n---\\n\\n') // form feed = page break\n .replace(/\\r\\n/g, '\\n')\n .replace(/\\r/g, '\\n')\n .replace(/\\n{3,}/g, '\\n\\n')\n .replace(/[ \\t]{2,}/g, ' ')\n .trim();\n\n return `# ${title}\\n\\n${cleaned}`;\n}\n\n/**\n * Filter a list of discovered URLs to find PDF links.\n * Used by http-engine and browser crawler to decide which links to pass to extractPdfFromUrl.\n */\nexport function filterPdfUrls(urls: string[]): string[] {\n return urls.filter(u => PDF_EXT.test(u));\n}\n"],
|
|
5
|
+
"mappings": "AAUA,MAAM,UAAU;AAChB,MAAM,WAAW;AAWV,SAAS,SAAS,KAAa,aAA+B;AACnE,SAAO,QAAQ,KAAK,GAAG,MAAM,aAAa,SAAS,QAAQ,KAAK;AAClE;AAEA,eAAsB,kBAAkB,KAAa,OAIjD,CAAC,GAA8B;AACjC,QAAM,EAAE,YAAY,oBAAoB,YAAY,KAAQ,eAAe,KAAK,OAAO,KAAK,IAAI;AAEhG,MAAI;AACJ,MAAI;AAEJ,MAAI;AACF,UAAM,WAAW,MAAM,MAAM,KAAK;AAAA,MAChC,SAAS,EAAE,cAAc,WAAW,UAAU,sBAAsB;AAAA,MACpE,QAAQ,YAAY,QAAQ,SAAS;AAAA,IACvC,CAAC;AACD,QAAI,CAAC,SAAS,GAAI,QAAO;AAEzB,UAAM,cAAc,SAAS,QAAQ,IAAI,cAAc,KAAK;AAC5D,QAAI,CAAC,YAAY,SAAS,KAAK,KAAK,CAAC,QAAQ,KAAK,GAAG,EAAG,QAAO;AAE/D,UAAM,cAAc,MAAM,SAAS,YAAY;AAC/C,gBAAY,YAAY;AACxB,QAAI,YAAY,cAAc;AAC5B,cAAQ,KAAK,kCAAkC,SAAS,sBAAsB,GAAG,EAAE;AACnF,aAAO;AAAA,IACT;AACA,aAAS,OAAO,KAAK,WAAW;AAAA,EAClC,SAAS,GAAQ;AACf,YAAQ,MAAM,uCAAuC,GAAG,KAAK,EAAE,OAAO,EAAE;AACxE,WAAO;AAAA,EACT;AAEA,MAAI;AAKF,UAAM,WAAW,MAAM,OAAO,4BAA4B,EAAE;AAAA,MAC1D,MAAM,OAAO,WAAkB;AAAA,IACjC;AACA,UAAM,QAAQ,SAAS,WAAW;AAClC,UAAM,OAAO,MAAM,MAAM,QAAQ,EAAE,KAAK,EAAE,CAAC;AAE3C,UAAM,QAAS,KAAK,MAAM,SAAoB,oBAAoB,GAAG;AACrE,UAAM,WAAW,kBAAkB,KAAK,QAAQ,IAAI,KAAK;AAEzD,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW,KAAK,YAAY;AAAA,MAC5B,MAAM;AAAA,QACJ,QAAQ,KAAK,MAAM,UAAU;AAAA,QAC7B,SAAS,KAAK,MAAM,WAAW;AAAA,QAC/B,UAAU,KAAK,MAAM,YAAY;AAAA,QACjC,SAAS,KAAK,MAAM,WAAW;AAAA,MACjC;AAAA,MACA;AAAA,IACF;AAAA,EACF,SAAS,GAAQ;AACf,YAAQ,MAAM,uCAAuC,GAAG,KAAK,EAAE,OAAO,EAAE;AACxE,WAAO;AAAA,EACT;AACF;AAEA,SAAS,oBAAoB,KAAqB;AAChD,MAAI;AACF,UAAM,EAAE,SAAS,IAAI,IAAI,IAAI,GAAG;AAChC,UAAM,WAAW,SAAS,MAAM,GAAG,EAAE,IAAI,KAAK;AAC9C,WAAO,SAAS,QAAQ,WAAW,EAAE,EAAE,QAAQ,SAAS,GAAG,EAAE,KAAK;AAAA,EACpE,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,kBAAkB,MAAc,OAAuB;AAC9D,MAAI,CAAC,KAAK,KAAK,EAAG,QAAO,KAAK,KAAK;AAAA;AAAA;AAGnC,QAAM,UAAU,KACb,QAAQ,OAAO,aAAa,EAC5B,QAAQ,SAAS,IAAI,EACrB,QAAQ,OAAO,IAAI,EACnB,QAAQ,WAAW,MAAM,EACzB,QAAQ,cAAc,GAAG,EACzB,KAAK;AAER,SAAO,KAAK,KAAK;AAAA;AAAA,EAAO,OAAO;AACjC;AAMO,SAAS,cAAc,MAA0B;AACtD,SAAO,KAAK,OAAO,OAAK,QAAQ,KAAK,CAAC,CAAC;AACzC;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Performance metrics collection via CDP + Playwright page.evaluate().
|
|
3
|
+
*
|
|
4
|
+
* Collects Core Web Vitals and resource timing after page navigation.
|
|
5
|
+
* Stored alongside screen data for test generation context and regression detection.
|
|
6
|
+
*
|
|
7
|
+
* Metrics:
|
|
8
|
+
* - LCP (Largest Contentful Paint) — via PerformanceObserver in page
|
|
9
|
+
* - CLS (Cumulative Layout Shift) — via PerformanceObserver in page
|
|
10
|
+
* - TTFB (Time to First Byte) — from navigation timing
|
|
11
|
+
* - DOMContentLoaded — from navigation timing
|
|
12
|
+
* - Resource count and size by type
|
|
13
|
+
*/
|
|
14
|
+
import type { Page } from 'playwright';
|
|
15
|
+
export interface CoreWebVitals {
|
|
16
|
+
lcp?: number;
|
|
17
|
+
cls?: number;
|
|
18
|
+
ttfb?: number;
|
|
19
|
+
fcp?: number;
|
|
20
|
+
domContentLoaded?: number;
|
|
21
|
+
loadEventEnd?: number;
|
|
22
|
+
}
|
|
23
|
+
export interface ResourceSummary {
|
|
24
|
+
scripts: {
|
|
25
|
+
count: number;
|
|
26
|
+
totalBytes: number;
|
|
27
|
+
};
|
|
28
|
+
styles: {
|
|
29
|
+
count: number;
|
|
30
|
+
totalBytes: number;
|
|
31
|
+
};
|
|
32
|
+
images: {
|
|
33
|
+
count: number;
|
|
34
|
+
totalBytes: number;
|
|
35
|
+
};
|
|
36
|
+
fonts: {
|
|
37
|
+
count: number;
|
|
38
|
+
totalBytes: number;
|
|
39
|
+
};
|
|
40
|
+
other: {
|
|
41
|
+
count: number;
|
|
42
|
+
totalBytes: number;
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
export interface PagePerfMetrics {
|
|
46
|
+
url: string;
|
|
47
|
+
vitals: CoreWebVitals;
|
|
48
|
+
resources: ResourceSummary;
|
|
49
|
+
totalTransferBytes: number;
|
|
50
|
+
requestCount: number;
|
|
51
|
+
measuredAt: string;
|
|
52
|
+
}
|
|
53
|
+
export declare function collectPerfMetrics(page: Page, url: string): Promise<PagePerfMetrics>;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
async function collectPerfMetrics(page, url) {
|
|
2
|
+
const measuredAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
3
|
+
let vitals = {};
|
|
4
|
+
try {
|
|
5
|
+
vitals = await page.evaluate(() => {
|
|
6
|
+
return new Promise((resolve) => {
|
|
7
|
+
const result = {};
|
|
8
|
+
const nav = performance.getEntriesByType("navigation")[0];
|
|
9
|
+
if (nav) {
|
|
10
|
+
result.ttfb = Math.round(nav.responseStart - nav.requestStart);
|
|
11
|
+
result.domContentLoaded = Math.round(nav.domContentLoadedEventEnd);
|
|
12
|
+
result.loadEventEnd = Math.round(nav.loadEventEnd);
|
|
13
|
+
}
|
|
14
|
+
let lcpDone = false;
|
|
15
|
+
let clsDone = false;
|
|
16
|
+
const tryResolve = () => {
|
|
17
|
+
if (lcpDone && clsDone) resolve(result);
|
|
18
|
+
};
|
|
19
|
+
try {
|
|
20
|
+
new PerformanceObserver((list) => {
|
|
21
|
+
const entries = list.getEntries();
|
|
22
|
+
if (entries.length > 0) result.lcp = Math.round(entries[entries.length - 1].startTime);
|
|
23
|
+
lcpDone = true;
|
|
24
|
+
tryResolve();
|
|
25
|
+
}).observe({ type: "largest-contentful-paint", buffered: true });
|
|
26
|
+
} catch {
|
|
27
|
+
lcpDone = true;
|
|
28
|
+
}
|
|
29
|
+
try {
|
|
30
|
+
let cls = 0;
|
|
31
|
+
new PerformanceObserver((list) => {
|
|
32
|
+
for (const e of list.getEntries()) {
|
|
33
|
+
if (!e.hadRecentInput) cls += e.value;
|
|
34
|
+
}
|
|
35
|
+
result.cls = Math.round(cls * 1e3) / 1e3;
|
|
36
|
+
clsDone = true;
|
|
37
|
+
tryResolve();
|
|
38
|
+
}).observe({ type: "layout-shift", buffered: true });
|
|
39
|
+
} catch {
|
|
40
|
+
clsDone = true;
|
|
41
|
+
}
|
|
42
|
+
try {
|
|
43
|
+
const fcp = performance.getEntriesByName("first-contentful-paint")[0];
|
|
44
|
+
if (fcp) result.fcp = Math.round(fcp.startTime);
|
|
45
|
+
} catch {
|
|
46
|
+
}
|
|
47
|
+
setTimeout(() => resolve(result), 3e3);
|
|
48
|
+
});
|
|
49
|
+
}).catch(() => ({}));
|
|
50
|
+
} catch {
|
|
51
|
+
}
|
|
52
|
+
let resources = {
|
|
53
|
+
scripts: { count: 0, totalBytes: 0 },
|
|
54
|
+
styles: { count: 0, totalBytes: 0 },
|
|
55
|
+
images: { count: 0, totalBytes: 0 },
|
|
56
|
+
fonts: { count: 0, totalBytes: 0 },
|
|
57
|
+
other: { count: 0, totalBytes: 0 }
|
|
58
|
+
};
|
|
59
|
+
let totalTransferBytes = 0;
|
|
60
|
+
let requestCount = 0;
|
|
61
|
+
try {
|
|
62
|
+
const entries = await page.evaluate(
|
|
63
|
+
() => performance.getEntriesByType("resource").map((e) => ({
|
|
64
|
+
name: e.name,
|
|
65
|
+
initiatorType: e.initiatorType,
|
|
66
|
+
transferSize: e.transferSize ?? 0
|
|
67
|
+
}))
|
|
68
|
+
).catch(() => []);
|
|
69
|
+
for (const e of entries) {
|
|
70
|
+
requestCount++;
|
|
71
|
+
totalTransferBytes += e.transferSize;
|
|
72
|
+
const cat = e.initiatorType === "script" ? "scripts" : e.initiatorType === "css" || e.name?.endsWith(".css") ? "styles" : e.initiatorType === "img" ? "images" : e.name?.match(/\.(woff|woff2|ttf|otf|eot)/) ? "fonts" : "other";
|
|
73
|
+
resources[cat].count++;
|
|
74
|
+
resources[cat].totalBytes += e.transferSize;
|
|
75
|
+
}
|
|
76
|
+
} catch {
|
|
77
|
+
}
|
|
78
|
+
return { url, vitals, resources, totalTransferBytes, requestCount, measuredAt };
|
|
79
|
+
}
|
|
80
|
+
export {
|
|
81
|
+
collectPerfMetrics
|
|
82
|
+
};
|
|
83
|
+
//# sourceMappingURL=performance-metrics.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/performance-metrics.ts"],
|
|
4
|
+
"sourcesContent": ["/**\n * Performance metrics collection via CDP + Playwright page.evaluate().\n *\n * Collects Core Web Vitals and resource timing after page navigation.\n * Stored alongside screen data for test generation context and regression detection.\n *\n * Metrics:\n * - LCP (Largest Contentful Paint) \u2014 via PerformanceObserver in page\n * - CLS (Cumulative Layout Shift) \u2014 via PerformanceObserver in page\n * - TTFB (Time to First Byte) \u2014 from navigation timing\n * - DOMContentLoaded \u2014 from navigation timing\n * - Resource count and size by type\n */\n\nimport type { Page } from 'playwright';\n\nexport interface CoreWebVitals {\n lcp?: number; // ms\n cls?: number; // unitless score\n ttfb?: number; // ms\n fcp?: number; // First Contentful Paint ms\n domContentLoaded?: number; // ms\n loadEventEnd?: number; // ms\n}\n\nexport interface ResourceSummary {\n scripts: { count: number; totalBytes: number };\n styles: { count: number; totalBytes: number };\n images: { count: number; totalBytes: number };\n fonts: { count: number; totalBytes: number };\n other: { count: number; totalBytes: number };\n}\n\nexport interface PagePerfMetrics {\n url: string;\n vitals: CoreWebVitals;\n resources: ResourceSummary;\n totalTransferBytes: number;\n requestCount: number;\n measuredAt: string;\n}\n\nexport async function collectPerfMetrics(page: Page, url: string): Promise<PagePerfMetrics> {\n const measuredAt = new Date().toISOString();\n\n // Inject PerformanceObserver for LCP + CLS, wait up to 3s\n let vitals: CoreWebVitals = {};\n try {\n vitals = await page.evaluate(() => {\n return new Promise<any>((resolve) => {\n const result: any = {};\n\n // Navigation timing\n const nav = performance.getEntriesByType('navigation')[0] as any;\n if (nav) {\n result.ttfb = Math.round(nav.responseStart - nav.requestStart);\n result.domContentLoaded = Math.round(nav.domContentLoadedEventEnd);\n result.loadEventEnd = Math.round(nav.loadEventEnd);\n }\n\n let lcpDone = false;\n let clsDone = false;\n const tryResolve = () => { if (lcpDone && clsDone) resolve(result); };\n\n // LCP\n try {\n new PerformanceObserver((list) => {\n const entries = list.getEntries();\n if (entries.length > 0) result.lcp = Math.round((entries[entries.length - 1] as any).startTime);\n lcpDone = true;\n tryResolve();\n }).observe({ type: 'largest-contentful-paint', buffered: true });\n } catch { lcpDone = true; }\n\n // CLS\n try {\n let cls = 0;\n new PerformanceObserver((list) => {\n for (const e of list.getEntries() as any[]) {\n if (!e.hadRecentInput) cls += e.value;\n }\n result.cls = Math.round(cls * 1000) / 1000;\n clsDone = true;\n tryResolve();\n }).observe({ type: 'layout-shift', buffered: true });\n } catch { clsDone = true; }\n\n // FCP\n try {\n const fcp = performance.getEntriesByName('first-contentful-paint')[0] as any;\n if (fcp) result.fcp = Math.round(fcp.startTime);\n } catch {}\n\n // Fallback resolve after 3s\n setTimeout(() => resolve(result), 3_000);\n });\n }).catch(() => ({}));\n } catch { /* page closed or navigate already */ }\n\n // Resource timing\n let resources: ResourceSummary = {\n scripts: { count: 0, totalBytes: 0 },\n styles: { count: 0, totalBytes: 0 },\n images: { count: 0, totalBytes: 0 },\n fonts: { count: 0, totalBytes: 0 },\n other: { count: 0, totalBytes: 0 },\n };\n let totalTransferBytes = 0;\n let requestCount = 0;\n\n try {\n const entries = await page.evaluate(() =>\n performance.getEntriesByType('resource').map((e: any) => ({\n name: e.name,\n initiatorType: e.initiatorType,\n transferSize: e.transferSize ?? 0,\n }))\n ).catch(() => [] as any[]);\n\n for (const e of entries) {\n requestCount++;\n totalTransferBytes += e.transferSize;\n const cat = e.initiatorType === 'script' ? 'scripts'\n : e.initiatorType === 'css' || e.name?.endsWith('.css') ? 'styles'\n : e.initiatorType === 'img' ? 'images'\n : e.name?.match(/\\.(woff|woff2|ttf|otf|eot)/) ? 'fonts'\n : 'other';\n resources[cat as keyof ResourceSummary].count++;\n resources[cat as keyof ResourceSummary].totalBytes += e.transferSize;\n }\n } catch {}\n\n return { url, vitals, resources, totalTransferBytes, requestCount, measuredAt };\n}\n"],
|
|
5
|
+
"mappings": "AA0CA,eAAsB,mBAAmB,MAAY,KAAuC;AAC1F,QAAM,cAAa,oBAAI,KAAK,GAAE,YAAY;AAG1C,MAAI,SAAwB,CAAC;AAC7B,MAAI;AACF,aAAS,MAAM,KAAK,SAAS,MAAM;AACjC,aAAO,IAAI,QAAa,CAAC,YAAY;AACnC,cAAM,SAAc,CAAC;AAGrB,cAAM,MAAM,YAAY,iBAAiB,YAAY,EAAE,CAAC;AACxD,YAAI,KAAK;AACP,iBAAO,OAAO,KAAK,MAAM,IAAI,gBAAgB,IAAI,YAAY;AAC7D,iBAAO,mBAAmB,KAAK,MAAM,IAAI,wBAAwB;AACjE,iBAAO,eAAe,KAAK,MAAM,IAAI,YAAY;AAAA,QACnD;AAEA,YAAI,UAAU;AACd,YAAI,UAAU;AACd,cAAM,aAAa,MAAM;AAAE,cAAI,WAAW,QAAS,SAAQ,MAAM;AAAA,QAAG;AAGpE,YAAI;AACF,cAAI,oBAAoB,CAAC,SAAS;AAChC,kBAAM,UAAU,KAAK,WAAW;AAChC,gBAAI,QAAQ,SAAS,EAAG,QAAO,MAAM,KAAK,MAAO,QAAQ,QAAQ,SAAS,CAAC,EAAU,SAAS;AAC9F,sBAAU;AACV,uBAAW;AAAA,UACb,CAAC,EAAE,QAAQ,EAAE,MAAM,4BAA4B,UAAU,KAAK,CAAC;AAAA,QACjE,QAAQ;AAAE,oBAAU;AAAA,QAAM;AAG1B,YAAI;AACF,cAAI,MAAM;AACV,cAAI,oBAAoB,CAAC,SAAS;AAChC,uBAAW,KAAK,KAAK,WAAW,GAAY;AAC1C,kBAAI,CAAC,EAAE,eAAgB,QAAO,EAAE;AAAA,YAClC;AACA,mBAAO,MAAM,KAAK,MAAM,MAAM,GAAI,IAAI;AACtC,sBAAU;AACV,uBAAW;AAAA,UACb,CAAC,EAAE,QAAQ,EAAE,MAAM,gBAAgB,UAAU,KAAK,CAAC;AAAA,QACrD,QAAQ;AAAE,oBAAU;AAAA,QAAM;AAG1B,YAAI;AACF,gBAAM,MAAM,YAAY,iBAAiB,wBAAwB,EAAE,CAAC;AACpE,cAAI,IAAK,QAAO,MAAM,KAAK,MAAM,IAAI,SAAS;AAAA,QAChD,QAAQ;AAAA,QAAC;AAGT,mBAAW,MAAM,QAAQ,MAAM,GAAG,GAAK;AAAA,MACzC,CAAC;AAAA,IACH,CAAC,EAAE,MAAM,OAAO,CAAC,EAAE;AAAA,EACrB,QAAQ;AAAA,EAAwC;AAGhD,MAAI,YAA6B;AAAA,IAC/B,SAAS,EAAE,OAAO,GAAG,YAAY,EAAE;AAAA,IACnC,QAAQ,EAAE,OAAO,GAAG,YAAY,EAAE;AAAA,IAClC,QAAQ,EAAE,OAAO,GAAG,YAAY,EAAE;AAAA,IAClC,OAAO,EAAE,OAAO,GAAG,YAAY,EAAE;AAAA,IACjC,OAAO,EAAE,OAAO,GAAG,YAAY,EAAE;AAAA,EACnC;AACA,MAAI,qBAAqB;AACzB,MAAI,eAAe;AAEnB,MAAI;AACF,UAAM,UAAU,MAAM,KAAK;AAAA,MAAS,MAClC,YAAY,iBAAiB,UAAU,EAAE,IAAI,CAAC,OAAY;AAAA,QACxD,MAAM,EAAE;AAAA,QACR,eAAe,EAAE;AAAA,QACjB,cAAc,EAAE,gBAAgB;AAAA,MAClC,EAAE;AAAA,IACJ,EAAE,MAAM,MAAM,CAAC,CAAU;AAEzB,eAAW,KAAK,SAAS;AACvB;AACA,4BAAsB,EAAE;AACxB,YAAM,MAAM,EAAE,kBAAkB,WAAW,YACvC,EAAE,kBAAkB,SAAS,EAAE,MAAM,SAAS,MAAM,IAAI,WACxD,EAAE,kBAAkB,QAAQ,WAC5B,EAAE,MAAM,MAAM,4BAA4B,IAAI,UAC9C;AACJ,gBAAU,GAA4B,EAAE;AACxC,gBAAU,GAA4B,EAAE,cAAc,EAAE;AAAA,IAC1D;AAAA,EACF,QAAQ;AAAA,EAAC;AAET,SAAO,EAAE,KAAK,QAAQ,WAAW,oBAAoB,cAAc,WAAW;AAChF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
package/dist/phash.d.ts
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Composites neutral grey (128,128,128) over each ignoredRegion (fractional
|
|
3
|
+
* 0-1 coordinates) so masked content — timestamps, live counters, ads —
|
|
4
|
+
* contributes nothing to any downstream pixel comparison. Shared by every
|
|
5
|
+
* comparison path (whole-image hash, per-cell grid hash, AI vision) so a
|
|
6
|
+
* region masked once is actually ignored everywhere, not just in the
|
|
7
|
+
* cheapest check.
|
|
8
|
+
*/
|
|
9
|
+
export declare function applyMask(imagePath: string, ignoredRegions: Array<{
|
|
10
|
+
x: number;
|
|
11
|
+
y: number;
|
|
12
|
+
width: number;
|
|
13
|
+
height: number;
|
|
14
|
+
}>): Promise<Buffer>;
|
|
15
|
+
export declare function screenshotDiffPercent(pathA: string, pathB: string, ignoredRegions?: Array<{
|
|
16
|
+
x: number;
|
|
17
|
+
y: number;
|
|
18
|
+
width: number;
|
|
19
|
+
height: number;
|
|
20
|
+
}>): Promise<{
|
|
21
|
+
pHashDistance: number;
|
|
22
|
+
changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT';
|
|
23
|
+
}>;
|
|
24
|
+
/**
|
|
25
|
+
* Grid-based layout analysis: divides screenshots into a 3×3 grid and computes
|
|
26
|
+
* pHash per cell. Returns which regions changed and where, for the VisualDiff.changes field.
|
|
27
|
+
* Uses sharp's `extract` to crop each cell before hashing.
|
|
28
|
+
*/
|
|
29
|
+
export declare function gridLayoutDiff(pathA: string, pathB: string, ignoredRegions?: Array<{
|
|
30
|
+
x: number;
|
|
31
|
+
y: number;
|
|
32
|
+
width: number;
|
|
33
|
+
height: number;
|
|
34
|
+
}>): Promise<Array<{
|
|
35
|
+
location: string;
|
|
36
|
+
severity: string;
|
|
37
|
+
description: string;
|
|
38
|
+
}>>;
|
package/dist/phash.js
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import sharp from "sharp";
|
|
2
|
+
async function applyMask(imagePath, ignoredRegions) {
|
|
3
|
+
const meta = await sharp(imagePath).metadata();
|
|
4
|
+
const W = meta.width ?? 1200;
|
|
5
|
+
const H = meta.height ?? 800;
|
|
6
|
+
const overlays = await Promise.all(
|
|
7
|
+
ignoredRegions.map(async (r) => {
|
|
8
|
+
const rW = Math.max(1, Math.round(r.width * W));
|
|
9
|
+
const rH = Math.max(1, Math.round(r.height * H));
|
|
10
|
+
const buf = await sharp({
|
|
11
|
+
create: { width: rW, height: rH, channels: 3, background: { r: 128, g: 128, b: 128 } }
|
|
12
|
+
}).png().toBuffer();
|
|
13
|
+
return { input: buf, top: Math.round(r.y * H), left: Math.round(r.x * W) };
|
|
14
|
+
})
|
|
15
|
+
);
|
|
16
|
+
return sharp(imagePath).composite(overlays).png().toBuffer();
|
|
17
|
+
}
|
|
18
|
+
async function hashWithMasks(imagePath, ignoredRegions = []) {
|
|
19
|
+
const pipeline = ignoredRegions.length > 0 ? sharp(await applyMask(imagePath, ignoredRegions)) : sharp(imagePath);
|
|
20
|
+
const buf = await pipeline.resize(8, 8, { fit: "fill" }).greyscale().raw().toBuffer();
|
|
21
|
+
const pixels = Array.from(buf);
|
|
22
|
+
const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
|
|
23
|
+
return pixels.reduce((h, p, i) => p >= mean ? h | 1n << BigInt(63 - i) : h, 0n);
|
|
24
|
+
}
|
|
25
|
+
function hammingDistance(a, b) {
|
|
26
|
+
let xor = a ^ b;
|
|
27
|
+
let count = 0;
|
|
28
|
+
while (xor !== 0n) {
|
|
29
|
+
count += Number(xor & 1n);
|
|
30
|
+
xor >>= 1n;
|
|
31
|
+
}
|
|
32
|
+
return count;
|
|
33
|
+
}
|
|
34
|
+
async function screenshotDiffPercent(pathA, pathB, ignoredRegions = []) {
|
|
35
|
+
const [hashA, hashB] = await Promise.all([
|
|
36
|
+
hashWithMasks(pathA, ignoredRegions),
|
|
37
|
+
hashWithMasks(pathB, ignoredRegions)
|
|
38
|
+
]);
|
|
39
|
+
let xor = hashA ^ hashB;
|
|
40
|
+
let dist = 0;
|
|
41
|
+
while (xor !== 0n) {
|
|
42
|
+
dist += Number(xor & 1n);
|
|
43
|
+
xor >>= 1n;
|
|
44
|
+
}
|
|
45
|
+
const pct = parseFloat((dist / 64 * 100).toFixed(1));
|
|
46
|
+
const changeLevel = pct === 0 ? "IDENTICAL" : pct < 10 ? "MINOR" : pct < 30 ? "MODERATE" : "SIGNIFICANT";
|
|
47
|
+
return { pHashDistance: pct, changeLevel };
|
|
48
|
+
}
|
|
49
|
+
async function gridLayoutDiff(pathA, pathB, ignoredRegions = []) {
|
|
50
|
+
const GRID_LABELS = [
|
|
51
|
+
"top-left",
|
|
52
|
+
"top-center",
|
|
53
|
+
"top-right",
|
|
54
|
+
"mid-left",
|
|
55
|
+
"mid-center",
|
|
56
|
+
"mid-right",
|
|
57
|
+
"bottom-left",
|
|
58
|
+
"bottom-center",
|
|
59
|
+
"bottom-right"
|
|
60
|
+
];
|
|
61
|
+
try {
|
|
62
|
+
const [sourceA, sourceB] = ignoredRegions.length > 0 ? await Promise.all([applyMask(pathA, ignoredRegions), applyMask(pathB, ignoredRegions)]) : [pathA, pathB];
|
|
63
|
+
const [metaA, metaB] = await Promise.all([
|
|
64
|
+
sharp(sourceA).metadata(),
|
|
65
|
+
sharp(sourceB).metadata()
|
|
66
|
+
]);
|
|
67
|
+
const W = Math.min(metaA.width ?? 1200, metaB.width ?? 1200);
|
|
68
|
+
const H = Math.min(metaA.height ?? 800, metaB.height ?? 800);
|
|
69
|
+
const cW = Math.floor(W / 3);
|
|
70
|
+
const cH = Math.floor(H / 3);
|
|
71
|
+
const changes = [];
|
|
72
|
+
for (let row = 0; row < 3; row++) {
|
|
73
|
+
for (let col = 0; col < 3; col++) {
|
|
74
|
+
const region = { left: col * cW, top: row * cH, width: cW, height: cH };
|
|
75
|
+
const label = GRID_LABELS[row * 3 + col];
|
|
76
|
+
const [hashA, hashB] = await Promise.all([
|
|
77
|
+
sharp(sourceA).extract(region).resize(8, 8, { fit: "fill" }).greyscale().raw().toBuffer().then((buf) => {
|
|
78
|
+
const pixels = Array.from(buf);
|
|
79
|
+
const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
|
|
80
|
+
return pixels.reduce((h, p, i) => p >= mean ? h | 1n << BigInt(63 - i) : h, 0n);
|
|
81
|
+
}),
|
|
82
|
+
sharp(sourceB).extract(region).resize(8, 8, { fit: "fill" }).greyscale().raw().toBuffer().then((buf) => {
|
|
83
|
+
const pixels = Array.from(buf);
|
|
84
|
+
const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
|
|
85
|
+
return pixels.reduce((h, p, i) => p >= mean ? h | 1n << BigInt(63 - i) : h, 0n);
|
|
86
|
+
})
|
|
87
|
+
]);
|
|
88
|
+
let xor = hashA ^ hashB;
|
|
89
|
+
let dist = 0;
|
|
90
|
+
while (xor !== 0n) {
|
|
91
|
+
dist += Number(xor & 1n);
|
|
92
|
+
xor >>= 1n;
|
|
93
|
+
}
|
|
94
|
+
const pct = dist / 64 * 100;
|
|
95
|
+
if (pct >= 5) {
|
|
96
|
+
const severity = pct >= 20 ? "HIGH" : pct >= 10 ? "MEDIUM" : "LOW";
|
|
97
|
+
changes.push({
|
|
98
|
+
location: label,
|
|
99
|
+
severity,
|
|
100
|
+
description: `${Math.round(pct)}% visual change in ${label.replace("-", " ")} region`
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return changes;
|
|
106
|
+
} catch {
|
|
107
|
+
return [];
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
export {
|
|
111
|
+
applyMask,
|
|
112
|
+
gridLayoutDiff,
|
|
113
|
+
screenshotDiffPercent
|
|
114
|
+
};
|
|
115
|
+
//# sourceMappingURL=phash.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/phash.ts"],
|
|
4
|
+
"sourcesContent": ["import sharp from 'sharp';\nimport type { OverlayOptions } from 'sharp';\n\n/**\n * Composites neutral grey (128,128,128) over each ignoredRegion (fractional\n * 0-1 coordinates) so masked content \u2014 timestamps, live counters, ads \u2014\n * contributes nothing to any downstream pixel comparison. Shared by every\n * comparison path (whole-image hash, per-cell grid hash, AI vision) so a\n * region masked once is actually ignored everywhere, not just in the\n * cheapest check.\n */\nexport async function applyMask(\n imagePath: string,\n ignoredRegions: Array<{ x: number; y: number; width: number; height: number }>,\n): Promise<Buffer> {\n const meta = await sharp(imagePath).metadata();\n const W = meta.width ?? 1200;\n const H = meta.height ?? 800;\n\n const overlays: OverlayOptions[] = await Promise.all(\n ignoredRegions.map(async (r) => {\n const rW = Math.max(1, Math.round(r.width * W));\n const rH = Math.max(1, Math.round(r.height * H));\n const buf = await sharp({\n create: { width: rW, height: rH, channels: 3, background: { r: 128, g: 128, b: 128 } },\n }).png().toBuffer();\n return { input: buf, top: Math.round(r.y * H), left: Math.round(r.x * W) } as OverlayOptions;\n }),\n );\n\n return sharp(imagePath).composite(overlays).png().toBuffer();\n}\n\n/**\n * Compute an 8x8 average hash (aHash) of an image file, with optional mask overlays.\n * ignoredRegions are fractional (0-1 range), converted to pixels here.\n * Masked regions are filled with neutral grey (128,128,128) before hashing.\n */\nasync function hashWithMasks(\n imagePath: string,\n ignoredRegions: Array<{ x: number; y: number; width: number; height: number }> = [],\n): Promise<bigint> {\n const pipeline = ignoredRegions.length > 0\n ? sharp(await applyMask(imagePath, ignoredRegions))\n : sharp(imagePath);\n\n const buf = await pipeline.resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer();\n const pixels = Array.from(buf);\n const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;\n return pixels.reduce((h, p, i) => (p >= mean ? h | (1n << BigInt(63 - i)) : h), 0n);\n}\n\nfunction hammingDistance(a: bigint, b: bigint): number {\n let xor = a ^ b;\n let count = 0;\n while (xor !== 0n) { count += Number(xor & 1n); xor >>= 1n; }\n return count;\n}\n\nexport async function screenshotDiffPercent(\n pathA: string,\n pathB: string,\n ignoredRegions: Array<{ x: number; y: number; width: number; height: number }> = [],\n): Promise<{ pHashDistance: number; changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT' }> {\n const [hashA, hashB] = await Promise.all([\n hashWithMasks(pathA, ignoredRegions),\n hashWithMasks(pathB, ignoredRegions),\n ]);\n let xor = hashA ^ hashB;\n let dist = 0;\n while (xor !== 0n) { dist += Number(xor & 1n); xor >>= 1n; }\n const pct = parseFloat(((dist / 64) * 100).toFixed(1));\n const changeLevel = pct === 0 ? 'IDENTICAL' : pct < 10 ? 'MINOR' : pct < 30 ? 'MODERATE' : 'SIGNIFICANT';\n return { pHashDistance: pct, changeLevel };\n}\n\n/**\n * Grid-based layout analysis: divides screenshots into a 3\u00D73 grid and computes\n * pHash per cell. Returns which regions changed and where, for the VisualDiff.changes field.\n * Uses sharp's `extract` to crop each cell before hashing.\n */\nexport async function gridLayoutDiff(\n pathA: string,\n pathB: string,\n ignoredRegions: Array<{ x: number; y: number; width: number; height: number }> = [],\n): Promise<Array<{ location: string; severity: string; description: string }>> {\n const GRID_LABELS = [\n 'top-left', 'top-center', 'top-right',\n 'mid-left', 'mid-center', 'mid-right',\n 'bottom-left', 'bottom-center', 'bottom-right',\n ];\n\n try {\n // Mask before cropping into cells \u2014 a masked region (e.g. a timestamp)\n // then contributes nothing to any cell's hash, instead of only being\n // ignored by the separate whole-image hash in screenshotDiffPercent().\n const [sourceA, sourceB] = ignoredRegions.length > 0\n ? await Promise.all([applyMask(pathA, ignoredRegions), applyMask(pathB, ignoredRegions)])\n : [pathA, pathB];\n\n const [metaA, metaB] = await Promise.all([\n sharp(sourceA).metadata(),\n sharp(sourceB).metadata(),\n ]);\n const W = Math.min(metaA.width ?? 1200, metaB.width ?? 1200);\n const H = Math.min(metaA.height ?? 800, metaB.height ?? 800);\n const cW = Math.floor(W / 3);\n const cH = Math.floor(H / 3);\n\n const changes: Array<{ location: string; severity: string; description: string }> = [];\n\n for (let row = 0; row < 3; row++) {\n for (let col = 0; col < 3; col++) {\n const region = { left: col * cW, top: row * cH, width: cW, height: cH };\n const label = GRID_LABELS[row * 3 + col];\n\n const [hashA, hashB] = await Promise.all([\n sharp(sourceA).extract(region).resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer()\n .then((buf) => {\n const pixels = Array.from(buf);\n const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;\n return pixels.reduce((h, p, i) => p >= mean ? h | (1n << BigInt(63 - i)) : h, 0n);\n }),\n sharp(sourceB).extract(region).resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer()\n .then((buf) => {\n const pixels = Array.from(buf);\n const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;\n return pixels.reduce((h, p, i) => p >= mean ? h | (1n << BigInt(63 - i)) : h, 0n);\n }),\n ]);\n\n let xor = hashA ^ hashB;\n let dist = 0;\n while (xor !== 0n) { dist += Number(xor & 1n); xor >>= 1n; }\n const pct = (dist / 64) * 100;\n\n if (pct >= 5) {\n const severity = pct >= 20 ? 'HIGH' : pct >= 10 ? 'MEDIUM' : 'LOW';\n changes.push({\n location: label,\n severity,\n description: `${Math.round(pct)}% visual change in ${label.replace('-', ' ')} region`,\n });\n }\n }\n }\n return changes;\n } catch {\n return [];\n }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,WAAW;AAWlB,eAAsB,UACpB,WACA,gBACiB;AACjB,QAAM,OAAO,MAAM,MAAM,SAAS,EAAE,SAAS;AAC7C,QAAM,IAAI,KAAK,SAAS;AACxB,QAAM,IAAI,KAAK,UAAU;AAEzB,QAAM,WAA6B,MAAM,QAAQ;AAAA,IAC/C,eAAe,IAAI,OAAO,MAAM;AAC9B,YAAM,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,EAAE,QAAQ,CAAC,CAAC;AAC9C,YAAM,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,EAAE,SAAS,CAAC,CAAC;AAC/C,YAAM,MAAM,MAAM,MAAM;AAAA,QACtB,QAAQ,EAAE,OAAO,IAAI,QAAQ,IAAI,UAAU,GAAG,YAAY,EAAE,GAAG,KAAK,GAAG,KAAK,GAAG,IAAI,EAAE;AAAA,MACvF,CAAC,EAAE,IAAI,EAAE,SAAS;AAClB,aAAO,EAAE,OAAO,KAAK,KAAK,KAAK,MAAM,EAAE,IAAI,CAAC,GAAG,MAAM,KAAK,MAAM,EAAE,IAAI,CAAC,EAAE;AAAA,IAC3E,CAAC;AAAA,EACH;AAEA,SAAO,MAAM,SAAS,EAAE,UAAU,QAAQ,EAAE,IAAI,EAAE,SAAS;AAC7D;AAOA,eAAe,cACb,WACA,iBAAiF,CAAC,GACjE;AACjB,QAAM,WAAW,eAAe,SAAS,IACrC,MAAM,MAAM,UAAU,WAAW,cAAc,CAAC,IAChD,MAAM,SAAS;AAEnB,QAAM,MAAM,MAAM,SAAS,OAAO,GAAG,GAAG,EAAE,KAAK,OAAO,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS;AACpF,QAAM,SAAS,MAAM,KAAK,GAAG;AAC7B,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,SAAO,OAAO,OAAO,CAAC,GAAG,GAAG,MAAO,KAAK,OAAO,IAAK,MAAM,OAAO,KAAK,CAAC,IAAK,GAAI,EAAE;AACpF;AAEA,SAAS,gBAAgB,GAAW,GAAmB;AACrD,MAAI,MAAM,IAAI;AACd,MAAI,QAAQ;AACZ,SAAO,QAAQ,IAAI;AAAE,aAAS,OAAO,MAAM,EAAE;AAAG,YAAQ;AAAA,EAAI;AAC5D,SAAO;AACT;AAEA,eAAsB,sBACpB,OACA,OACA,iBAAiF,CAAC,GACmB;AACrG,QAAM,CAAC,OAAO,KAAK,IAAI,MAAM,QAAQ,IAAI;AAAA,IACvC,cAAc,OAAO,cAAc;AAAA,IACnC,cAAc,OAAO,cAAc;AAAA,EACrC,CAAC;AACD,MAAI,MAAM,QAAQ;AAClB,MAAI,OAAO;AACX,SAAO,QAAQ,IAAI;AAAE,YAAQ,OAAO,MAAM,EAAE;AAAG,YAAQ;AAAA,EAAI;AAC3D,QAAM,MAAM,YAAa,OAAO,KAAM,KAAK,QAAQ,CAAC,CAAC;AACrD,QAAM,cAAc,QAAQ,IAAI,cAAc,MAAM,KAAK,UAAU,MAAM,KAAK,aAAa;AAC3F,SAAO,EAAE,eAAe,KAAK,YAAY;AAC3C;AAOA,eAAsB,eACpB,OACA,OACA,iBAAiF,CAAC,GACL;AAC7E,QAAM,cAAc;AAAA,IAClB;AAAA,IAAe;AAAA,IAAiB;AAAA,IAChC;AAAA,IAAe;AAAA,IAAiB;AAAA,IAChC;AAAA,IAAe;AAAA,IAAiB;AAAA,EAClC;AAEA,MAAI;AAIF,UAAM,CAAC,SAAS,OAAO,IAAI,eAAe,SAAS,IAC/C,MAAM,QAAQ,IAAI,CAAC,UAAU,OAAO,cAAc,GAAG,UAAU,OAAO,cAAc,CAAC,CAAC,IACtF,CAAC,OAAO,KAAK;AAEjB,UAAM,CAAC,OAAO,KAAK,IAAI,MAAM,QAAQ,IAAI;AAAA,MACvC,MAAM,OAAO,EAAE,SAAS;AAAA,MACxB,MAAM,OAAO,EAAE,SAAS;AAAA,IAC1B,CAAC;AACD,UAAM,IAAI,KAAK,IAAI,MAAM,SAAS,MAAM,MAAM,SAAS,IAAI;AAC3D,UAAM,IAAI,KAAK,IAAI,MAAM,UAAU,KAAK,MAAM,UAAU,GAAG;AAC3D,UAAM,KAAK,KAAK,MAAM,IAAI,CAAC;AAC3B,UAAM,KAAK,KAAK,MAAM,IAAI,CAAC;AAE3B,UAAM,UAA8E,CAAC;AAErF,aAAS,MAAM,GAAG,MAAM,GAAG,OAAO;AAChC,eAAS,MAAM,GAAG,MAAM,GAAG,OAAO;AAChC,cAAM,SAAS,EAAE,MAAM,MAAM,IAAI,KAAK,MAAM,IAAI,OAAO,IAAI,QAAQ,GAAG;AACtE,cAAM,QAAQ,YAAY,MAAM,IAAI,GAAG;AAEvC,cAAM,CAAC,OAAO,KAAK,IAAI,MAAM,QAAQ,IAAI;AAAA,UACvC,MAAM,OAAO,EAAE,QAAQ,MAAM,EAAE,OAAO,GAAG,GAAG,EAAE,KAAK,OAAO,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EACrF,KAAK,CAAC,QAAQ;AACb,kBAAM,SAAS,MAAM,KAAK,GAAG;AAC7B,kBAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,mBAAO,OAAO,OAAO,CAAC,GAAG,GAAG,MAAM,KAAK,OAAO,IAAK,MAAM,OAAO,KAAK,CAAC,IAAK,GAAG,EAAE;AAAA,UAClF,CAAC;AAAA,UACH,MAAM,OAAO,EAAE,QAAQ,MAAM,EAAE,OAAO,GAAG,GAAG,EAAE,KAAK,OAAO,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EACrF,KAAK,CAAC,QAAQ;AACb,kBAAM,SAAS,MAAM,KAAK,GAAG;AAC7B,kBAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,mBAAO,OAAO,OAAO,CAAC,GAAG,GAAG,MAAM,KAAK,OAAO,IAAK,MAAM,OAAO,KAAK,CAAC,IAAK,GAAG,EAAE;AAAA,UAClF,CAAC;AAAA,QACL,CAAC;AAED,YAAI,MAAM,QAAQ;AAClB,YAAI,OAAO;AACX,eAAO,QAAQ,IAAI;AAAE,kBAAQ,OAAO,MAAM,EAAE;AAAG,kBAAQ;AAAA,QAAI;AAC3D,cAAM,MAAO,OAAO,KAAM;AAE1B,YAAI,OAAO,GAAG;AACZ,gBAAM,WAAW,OAAO,KAAK,SAAS,OAAO,KAAK,WAAW;AAC7D,kBAAQ,KAAK;AAAA,YACX,UAAU;AAAA,YACV;AAAA,YACA,aAAa,GAAG,KAAK,MAAM,GAAG,CAAC,sBAAsB,MAAM,QAAQ,KAAK,GAAG,CAAC;AAAA,UAC9E,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PII scrubber — detects and redacts Personally Identifiable Information
|
|
3
|
+
* from crawled markdown, HTML, and extracted text before storage.
|
|
4
|
+
*
|
|
5
|
+
* GDPR Article 4(1): "any information relating to an identified or identifiable natural person"
|
|
6
|
+
*/
|
|
7
|
+
export interface PiiMatch {
|
|
8
|
+
type: 'email' | 'phone' | 'ssn' | 'credit-card' | 'ip-address' | 'passport' | 'iban' | 'dob';
|
|
9
|
+
value: string;
|
|
10
|
+
start: number;
|
|
11
|
+
end: number;
|
|
12
|
+
}
|
|
13
|
+
export interface ScrubResult {
|
|
14
|
+
text: string;
|
|
15
|
+
matches: PiiMatch[];
|
|
16
|
+
scrubbed: boolean;
|
|
17
|
+
}
|
|
18
|
+
export declare function detectPii(text: string): PiiMatch[];
|
|
19
|
+
export declare function scrubPii(text: string): ScrubResult;
|
|
20
|
+
export declare function hasPii(text: string): boolean;
|
|
21
|
+
export declare function scrubObject<T extends Record<string, unknown>>(obj: T, fields: (keyof T)[]): T;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
const PATTERNS = [
|
|
2
|
+
// Email — RFC 5321 simplified
|
|
3
|
+
{ type: "email", re: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g },
|
|
4
|
+
// US SSN — 9 digits with dashes or spaces
|
|
5
|
+
{ type: "ssn", re: /\b(?!000|666|9\d\d)\d{3}[-\s](?!00)\d{2}[-\s](?!0000)\d{4}\b/g },
|
|
6
|
+
// Credit card — Visa/MC/Amex/Discover with optional spaces/dashes
|
|
7
|
+
{ type: "credit-card", re: /\b(?:4\d{3}|5[1-5]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{3,4}\b/g },
|
|
8
|
+
// IBAN — international bank account (15-34 chars)
|
|
9
|
+
{ type: "iban", re: /\b[A-Z]{2}\d{2}[A-Z0-9]{1,30}\b/g },
|
|
10
|
+
// Phone — international or US formats
|
|
11
|
+
{ type: "phone", re: /(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}\b|\+\d{1,3}[-.\s]\d{2,4}[-.\s]\d{3,4}[-.\s]\d{3,4}/g },
|
|
12
|
+
// Date of birth — common formats
|
|
13
|
+
{ type: "dob", re: /\b(?:0?[1-9]|1[0-2])\/(?:0?[1-9]|[12]\d|3[01])\/(?:19|20)\d{2}\b/g },
|
|
14
|
+
// IPv4 private/internal addresses — only flag private-looking ones in content
|
|
15
|
+
{ type: "ip-address", re: /\b(?:192\.168\.\d{1,3}\.\d{1,3}|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|172\.(?:1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})\b/g }
|
|
16
|
+
];
|
|
17
|
+
const REDACT_MAP = {
|
|
18
|
+
email: "[EMAIL]",
|
|
19
|
+
phone: "[PHONE]",
|
|
20
|
+
ssn: "[SSN]",
|
|
21
|
+
"credit-card": "[CARD]",
|
|
22
|
+
"ip-address": "[IP]",
|
|
23
|
+
passport: "[PASSPORT]",
|
|
24
|
+
iban: "[IBAN]",
|
|
25
|
+
dob: "[DOB]"
|
|
26
|
+
};
|
|
27
|
+
function detectPii(text) {
|
|
28
|
+
const matches = [];
|
|
29
|
+
const seen = /* @__PURE__ */ new Set();
|
|
30
|
+
for (const { type, re } of PATTERNS) {
|
|
31
|
+
const r = new RegExp(re.source, re.flags.includes("g") ? re.flags : re.flags + "g");
|
|
32
|
+
let m;
|
|
33
|
+
while ((m = r.exec(text)) !== null) {
|
|
34
|
+
const key = `${type}:${m.index}`;
|
|
35
|
+
if (!seen.has(key)) {
|
|
36
|
+
seen.add(key);
|
|
37
|
+
matches.push({ type, value: m[0], start: m.index, end: m.index + m[0].length });
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return matches.sort((a, b) => a.start - b.start);
|
|
42
|
+
}
|
|
43
|
+
function scrubPii(text) {
|
|
44
|
+
const matches = detectPii(text);
|
|
45
|
+
if (matches.length === 0) return { text, matches: [], scrubbed: false };
|
|
46
|
+
let scrubbed = text;
|
|
47
|
+
for (let i = matches.length - 1; i >= 0; i--) {
|
|
48
|
+
const { type, start, end } = matches[i];
|
|
49
|
+
scrubbed = scrubbed.slice(0, start) + REDACT_MAP[type] + scrubbed.slice(end);
|
|
50
|
+
}
|
|
51
|
+
return { text: scrubbed, matches, scrubbed: true };
|
|
52
|
+
}
|
|
53
|
+
function hasPii(text) {
|
|
54
|
+
for (const { re } of PATTERNS) {
|
|
55
|
+
const r = new RegExp(re.source, re.flags);
|
|
56
|
+
if (r.test(text)) return true;
|
|
57
|
+
}
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
function scrubObject(obj, fields) {
|
|
61
|
+
const result = { ...obj };
|
|
62
|
+
for (const field of fields) {
|
|
63
|
+
if (typeof result[field] === "string") {
|
|
64
|
+
result[field] = scrubPii(result[field]).text;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return result;
|
|
68
|
+
}
|
|
69
|
+
export {
|
|
70
|
+
detectPii,
|
|
71
|
+
hasPii,
|
|
72
|
+
scrubObject,
|
|
73
|
+
scrubPii
|
|
74
|
+
};
|
|
75
|
+
//# sourceMappingURL=pii-scrubber.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/pii-scrubber.ts"],
|
|
4
|
+
"sourcesContent": ["/**\n * PII scrubber \u2014 detects and redacts Personally Identifiable Information\n * from crawled markdown, HTML, and extracted text before storage.\n *\n * GDPR Article 4(1): \"any information relating to an identified or identifiable natural person\"\n */\n\nexport interface PiiMatch {\n type: 'email' | 'phone' | 'ssn' | 'credit-card' | 'ip-address' | 'passport' | 'iban' | 'dob';\n value: string;\n start: number;\n end: number;\n}\n\nexport interface ScrubResult {\n text: string;\n matches: PiiMatch[];\n scrubbed: boolean;\n}\n\n// Patterns \u2014 ordered from most-specific to least-specific\nconst PATTERNS: Array<{ type: PiiMatch['type']; re: RegExp }> = [\n // Email \u2014 RFC 5321 simplified\n { type: 'email', re: /\\b[a-zA-Z0-9._%+\\-]+@[a-zA-Z0-9.\\-]+\\.[a-zA-Z]{2,}\\b/g },\n // US SSN \u2014 9 digits with dashes or spaces\n { type: 'ssn', re: /\\b(?!000|666|9\\d\\d)\\d{3}[-\\s](?!00)\\d{2}[-\\s](?!0000)\\d{4}\\b/g },\n // Credit card \u2014 Visa/MC/Amex/Discover with optional spaces/dashes\n { type: 'credit-card', re: /\\b(?:4\\d{3}|5[1-5]\\d{2}|3[47]\\d{2}|6(?:011|5\\d{2}))[-\\s]?\\d{4}[-\\s]?\\d{4}[-\\s]?\\d{3,4}\\b/g },\n // IBAN \u2014 international bank account (15-34 chars)\n { type: 'iban', re: /\\b[A-Z]{2}\\d{2}[A-Z0-9]{1,30}\\b/g },\n // Phone \u2014 international or US formats\n { type: 'phone', re: /(?:\\+?1[-.\\s]?)?\\(?\\d{3}\\)?[-.\\s]\\d{3}[-.\\s]\\d{4}\\b|\\+\\d{1,3}[-.\\s]\\d{2,4}[-.\\s]\\d{3,4}[-.\\s]\\d{3,4}/g },\n // Date of birth \u2014 common formats\n { type: 'dob', re: /\\b(?:0?[1-9]|1[0-2])\\/(?:0?[1-9]|[12]\\d|3[01])\\/(?:19|20)\\d{2}\\b/g },\n // IPv4 private/internal addresses \u2014 only flag private-looking ones in content\n { type: 'ip-address', re: /\\b(?:192\\.168\\.\\d{1,3}\\.\\d{1,3}|10\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|172\\.(?:1[6-9]|2\\d|3[01])\\.\\d{1,3}\\.\\d{1,3})\\b/g },\n];\n\nconst REDACT_MAP: Record<PiiMatch['type'], string> = {\n email: '[EMAIL]',\n phone: '[PHONE]',\n ssn: '[SSN]',\n 'credit-card': '[CARD]',\n 'ip-address': '[IP]',\n passport: '[PASSPORT]',\n iban: '[IBAN]',\n dob: '[DOB]',\n};\n\nexport function detectPii(text: string): PiiMatch[] {\n const matches: PiiMatch[] = [];\n const seen = new Set<string>();\n\n for (const { type, re } of PATTERNS) {\n const r = new RegExp(re.source, re.flags.includes('g') ? re.flags : re.flags + 'g');\n let m: RegExpExecArray | null;\n while ((m = r.exec(text)) !== null) {\n const key = `${type}:${m.index}`;\n if (!seen.has(key)) {\n seen.add(key);\n matches.push({ type, value: m[0], start: m.index, end: m.index + m[0].length });\n }\n }\n }\n\n return matches.sort((a, b) => a.start - b.start);\n}\n\nexport function scrubPii(text: string): ScrubResult {\n const matches = detectPii(text);\n if (matches.length === 0) return { text, matches: [], scrubbed: false };\n\n // Replace from end to start so indices stay valid\n let scrubbed = text;\n for (let i = matches.length - 1; i >= 0; i--) {\n const { type, start, end } = matches[i];\n scrubbed = scrubbed.slice(0, start) + REDACT_MAP[type] + scrubbed.slice(end);\n }\n\n return { text: scrubbed, matches, scrubbed: true };\n}\n\nexport function hasPii(text: string): boolean {\n for (const { re } of PATTERNS) {\n const r = new RegExp(re.source, re.flags);\n if (r.test(text)) return true;\n }\n return false;\n}\n\nexport function scrubObject<T extends Record<string, unknown>>(obj: T, fields: (keyof T)[]): T {\n const result = { ...obj };\n for (const field of fields) {\n if (typeof result[field] === 'string') {\n (result as any)[field] = scrubPii(result[field] as string).text;\n }\n }\n return result;\n}\n"],
|
|
5
|
+
"mappings": "AAqBA,MAAM,WAA0D;AAAA;AAAA,EAE9D,EAAE,MAAM,SAAS,IAAI,wDAAwD;AAAA;AAAA,EAE7E,EAAE,MAAM,OAAO,IAAI,gEAAgE;AAAA;AAAA,EAEnF,EAAE,MAAM,eAAe,IAAI,4FAA4F;AAAA;AAAA,EAEvH,EAAE,MAAM,QAAQ,IAAI,mCAAmC;AAAA;AAAA,EAEvD,EAAE,MAAM,SAAS,IAAI,wGAAwG;AAAA;AAAA,EAE7H,EAAE,MAAM,OAAO,IAAI,oEAAoE;AAAA;AAAA,EAEvF,EAAE,MAAM,cAAc,IAAI,gHAAgH;AAC5I;AAEA,MAAM,aAA+C;AAAA,EACnD,OAAO;AAAA,EACP,OAAO;AAAA,EACP,KAAK;AAAA,EACL,eAAe;AAAA,EACf,cAAc;AAAA,EACd,UAAU;AAAA,EACV,MAAM;AAAA,EACN,KAAK;AACP;AAEO,SAAS,UAAU,MAA0B;AAClD,QAAM,UAAsB,CAAC;AAC7B,QAAM,OAAO,oBAAI,IAAY;AAE7B,aAAW,EAAE,MAAM,GAAG,KAAK,UAAU;AACnC,UAAM,IAAI,IAAI,OAAO,GAAG,QAAQ,GAAG,MAAM,SAAS,GAAG,IAAI,GAAG,QAAQ,GAAG,QAAQ,GAAG;AAClF,QAAI;AACJ,YAAQ,IAAI,EAAE,KAAK,IAAI,OAAO,MAAM;AAClC,YAAM,MAAM,GAAG,IAAI,IAAI,EAAE,KAAK;AAC9B,UAAI,CAAC,KAAK,IAAI,GAAG,GAAG;AAClB,aAAK,IAAI,GAAG;AACZ,gBAAQ,KAAK,EAAE,MAAM,OAAO,EAAE,CAAC,GAAG,OAAO,EAAE,OAAO,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,CAAC;AAAA,MAChF;AAAA,IACF;AAAA,EACF;AAEA,SAAO,QAAQ,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACjD;AAEO,SAAS,SAAS,MAA2B;AAClD,QAAM,UAAU,UAAU,IAAI;AAC9B,MAAI,QAAQ,WAAW,EAAG,QAAO,EAAE,MAAM,SAAS,CAAC,GAAG,UAAU,MAAM;AAGtE,MAAI,WAAW;AACf,WAAS,IAAI,QAAQ,SAAS,GAAG,KAAK,GAAG,KAAK;AAC5C,UAAM,EAAE,MAAM,OAAO,IAAI,IAAI,QAAQ,CAAC;AACtC,eAAW,SAAS,MAAM,GAAG,KAAK,IAAI,WAAW,IAAI,IAAI,SAAS,MAAM,GAAG;AAAA,EAC7E;AAEA,SAAO,EAAE,MAAM,UAAU,SAAS,UAAU,KAAK;AACnD;AAEO,SAAS,OAAO,MAAuB;AAC5C,aAAW,EAAE,GAAG,KAAK,UAAU;AAC7B,UAAM,IAAI,IAAI,OAAO,GAAG,QAAQ,GAAG,KAAK;AACxC,QAAI,EAAE,KAAK,IAAI,EAAG,QAAO;AAAA,EAC3B;AACA,SAAO;AACT;AAEO,SAAS,YAA+C,KAAQ,QAAwB;AAC7F,QAAM,SAAS,EAAE,GAAG,IAAI;AACxB,aAAW,SAAS,QAAQ;AAC1B,QAAI,OAAO,OAAO,KAAK,MAAM,UAAU;AACrC,MAAC,OAAe,KAAK,IAAI,SAAS,OAAO,KAAK,CAAW,EAAE;AAAA,IAC7D;AAAA,EACF;AACA,SAAO;AACT;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Playwright test recorder — generate Playwright test code from crawl data.
|
|
3
|
+
*
|
|
4
|
+
* Given a list of crawled screens, generates a minimal Playwright test file that:
|
|
5
|
+
* 1. Navigates to each screen URL
|
|
6
|
+
* 2. Verifies the page title
|
|
7
|
+
* 3. Checks key elements visible on the page
|
|
8
|
+
* 4. Verifies no console errors (optional)
|
|
9
|
+
*
|
|
10
|
+
* This is a "smoke test" generator — not a full interaction recorder.
|
|
11
|
+
* Output: valid TypeScript Playwright test file ready to run with npx playwright test.
|
|
12
|
+
*/
|
|
13
|
+
export interface RecorderScreen {
|
|
14
|
+
id: string;
|
|
15
|
+
name: string;
|
|
16
|
+
url: string;
|
|
17
|
+
title?: string;
|
|
18
|
+
heading?: string;
|
|
19
|
+
markdownWordCount?: number;
|
|
20
|
+
smartLocator?: {
|
|
21
|
+
tag: string;
|
|
22
|
+
text?: string;
|
|
23
|
+
role?: string;
|
|
24
|
+
placeholder?: string;
|
|
25
|
+
nearText?: string;
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
export interface RecorderOpts {
|
|
29
|
+
baseUrl?: string;
|
|
30
|
+
includeConsoleCheck?: boolean;
|
|
31
|
+
testTimeout?: number;
|
|
32
|
+
projectName?: string;
|
|
33
|
+
useSmartLocators?: boolean;
|
|
34
|
+
}
|
|
35
|
+
export declare function generatePlaywrightTest(screens: RecorderScreen[], opts?: RecorderOpts): string;
|
|
36
|
+
export declare function generatePlaywrightConfig(opts?: {
|
|
37
|
+
baseUrl?: string;
|
|
38
|
+
reportDir?: string;
|
|
39
|
+
}): string;
|