@enricai/barnacle 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +958 -0
- package/dist/api/errors.d.ts +88 -0
- package/dist/api/errors.d.ts.map +1 -0
- package/dist/api/errors.js +164 -0
- package/dist/api/errors.js.map +1 -0
- package/dist/api/helpers/envelope.d.ts +13 -0
- package/dist/api/helpers/envelope.d.ts.map +1 -0
- package/dist/api/helpers/envelope.js +20 -0
- package/dist/api/helpers/envelope.js.map +1 -0
- package/dist/api/helpers/reply.d.ts +9 -0
- package/dist/api/helpers/reply.d.ts.map +1 -0
- package/dist/api/helpers/reply.js +14 -0
- package/dist/api/helpers/reply.js.map +1 -0
- package/dist/api/plugins/auth.d.ts +27 -0
- package/dist/api/plugins/auth.d.ts.map +1 -0
- package/dist/api/plugins/auth.js +111 -0
- package/dist/api/plugins/auth.js.map +1 -0
- package/dist/api/plugins/error-handler.d.ts +12 -0
- package/dist/api/plugins/error-handler.d.ts.map +1 -0
- package/dist/api/plugins/error-handler.js +72 -0
- package/dist/api/plugins/error-handler.js.map +1 -0
- package/dist/api/plugins/request-context.d.ts +17 -0
- package/dist/api/plugins/request-context.d.ts.map +1 -0
- package/dist/api/plugins/request-context.js +61 -0
- package/dist/api/plugins/request-context.js.map +1 -0
- package/dist/api/routes/health.d.ts +55 -0
- package/dist/api/routes/health.d.ts.map +1 -0
- package/dist/api/routes/health.js +137 -0
- package/dist/api/routes/health.js.map +1 -0
- package/dist/api/routes/plugins-introspection.d.ts +17 -0
- package/dist/api/routes/plugins-introspection.d.ts.map +1 -0
- package/dist/api/routes/plugins-introspection.js +13 -0
- package/dist/api/routes/plugins-introspection.js.map +1 -0
- package/dist/api/schemas/common.d.ts +114 -0
- package/dist/api/schemas/common.d.ts.map +1 -0
- package/dist/api/schemas/common.js +109 -0
- package/dist/api/schemas/common.js.map +1 -0
- package/dist/api/schemas/telemetry.d.ts +66 -0
- package/dist/api/schemas/telemetry.d.ts.map +1 -0
- package/dist/api/schemas/telemetry.js +52 -0
- package/dist/api/schemas/telemetry.js.map +1 -0
- package/dist/cache/keyed-ttl-cache.d.ts +45 -0
- package/dist/cache/keyed-ttl-cache.d.ts.map +1 -0
- package/dist/cache/keyed-ttl-cache.js +69 -0
- package/dist/cache/keyed-ttl-cache.js.map +1 -0
- package/dist/cache/response-cache.d.ts +37 -0
- package/dist/cache/response-cache.d.ts.map +1 -0
- package/dist/cache/response-cache.js +114 -0
- package/dist/cache/response-cache.js.map +1 -0
- package/dist/config.d.ts +258 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +138 -0
- package/dist/config.js.map +1 -0
- package/dist/lib/applicant-payload.d.ts +28 -0
- package/dist/lib/applicant-payload.d.ts.map +1 -0
- package/dist/lib/applicant-payload.js +20 -0
- package/dist/lib/applicant-payload.js.map +1 -0
- package/dist/lib/application-address.d.ts +16 -0
- package/dist/lib/application-address.d.ts.map +1 -0
- package/dist/lib/application-address.js +20 -0
- package/dist/lib/application-address.js.map +1 -0
- package/dist/lib/application-answers.d.ts +151 -0
- package/dist/lib/application-answers.d.ts.map +1 -0
- package/dist/lib/application-answers.js +95 -0
- package/dist/lib/application-answers.js.map +1 -0
- package/dist/lib/application-identity.d.ts +18 -0
- package/dist/lib/application-identity.d.ts.map +1 -0
- package/dist/lib/application-identity.js +20 -0
- package/dist/lib/application-identity.js.map +1 -0
- package/dist/lib/application-resume.d.ts +21 -0
- package/dist/lib/application-resume.d.ts.map +1 -0
- package/dist/lib/application-resume.js +23 -0
- package/dist/lib/application-resume.js.map +1 -0
- package/dist/lib/bedrock.d.ts +22 -0
- package/dist/lib/bedrock.d.ts.map +1 -0
- package/dist/lib/bedrock.js +35 -0
- package/dist/lib/bedrock.js.map +1 -0
- package/dist/lib/case-insensitive-headers.d.ts +15 -0
- package/dist/lib/case-insensitive-headers.d.ts.map +1 -0
- package/dist/lib/case-insensitive-headers.js +29 -0
- package/dist/lib/case-insensitive-headers.js.map +1 -0
- package/dist/lib/chromium-client-hints.d.ts +11 -0
- package/dist/lib/chromium-client-hints.d.ts.map +1 -0
- package/dist/lib/chromium-client-hints.js +17 -0
- package/dist/lib/chromium-client-hints.js.map +1 -0
- package/dist/lib/cookie-value.d.ts +8 -0
- package/dist/lib/cookie-value.d.ts.map +1 -0
- package/dist/lib/cookie-value.js +14 -0
- package/dist/lib/cookie-value.js.map +1 -0
- package/dist/lib/datadog.d.ts +8 -0
- package/dist/lib/datadog.d.ts.map +1 -0
- package/dist/lib/datadog.js +27 -0
- package/dist/lib/datadog.js.map +1 -0
- package/dist/lib/dd-metrics.d.ts +28 -0
- package/dist/lib/dd-metrics.d.ts.map +1 -0
- package/dist/lib/dd-metrics.js +62 -0
- package/dist/lib/dd-metrics.js.map +1 -0
- package/dist/lib/dispatch-metrics.d.ts +19 -0
- package/dist/lib/dispatch-metrics.d.ts.map +1 -0
- package/dist/lib/dispatch-metrics.js +65 -0
- package/dist/lib/dispatch-metrics.js.map +1 -0
- package/dist/lib/env.d.ts +37 -0
- package/dist/lib/env.d.ts.map +1 -0
- package/dist/lib/env.js +70 -0
- package/dist/lib/env.js.map +1 -0
- package/dist/lib/errors.d.ts +15 -0
- package/dist/lib/errors.d.ts.map +1 -0
- package/dist/lib/errors.js +20 -0
- package/dist/lib/errors.js.map +1 -0
- package/dist/lib/http.d.ts +7 -0
- package/dist/lib/http.d.ts.map +1 -0
- package/dist/lib/http.js +14 -0
- package/dist/lib/http.js.map +1 -0
- package/dist/lib/job-tracking.d.ts +17 -0
- package/dist/lib/job-tracking.d.ts.map +1 -0
- package/dist/lib/job-tracking.js +19 -0
- package/dist/lib/job-tracking.js.map +1 -0
- package/dist/lib/llm/anthropic-client.d.ts +12 -0
- package/dist/lib/llm/anthropic-client.d.ts.map +1 -0
- package/dist/lib/llm/anthropic-client.js +23 -0
- package/dist/lib/llm/anthropic-client.js.map +1 -0
- package/dist/lib/llm/judge.d.ts +57 -0
- package/dist/lib/llm/judge.d.ts.map +1 -0
- package/dist/lib/llm/judge.js +123 -0
- package/dist/lib/llm/judge.js.map +1 -0
- package/dist/lib/llm/judges/error-messages.d.ts +38 -0
- package/dist/lib/llm/judges/error-messages.d.ts.map +1 -0
- package/dist/lib/llm/judges/error-messages.js +72 -0
- package/dist/lib/llm/judges/error-messages.js.map +1 -0
- package/dist/lib/llm/judges/invalid-fields.d.ts +43 -0
- package/dist/lib/llm/judges/invalid-fields.d.ts.map +1 -0
- package/dist/lib/llm/judges/invalid-fields.js +80 -0
- package/dist/lib/llm/judges/invalid-fields.js.map +1 -0
- package/dist/lib/llm/judges/modal-priority.d.ts +41 -0
- package/dist/lib/llm/judges/modal-priority.d.ts.map +1 -0
- package/dist/lib/llm/judges/modal-priority.js +74 -0
- package/dist/lib/llm/judges/modal-priority.js.map +1 -0
- package/dist/lib/llm/judges/select-option.d.ts +49 -0
- package/dist/lib/llm/judges/select-option.d.ts.map +1 -0
- package/dist/lib/llm/judges/select-option.js +90 -0
- package/dist/lib/llm/judges/select-option.js.map +1 -0
- package/dist/lib/llm/judges/verify-submit.d.ts +68 -0
- package/dist/lib/llm/judges/verify-submit.d.ts.map +1 -0
- package/dist/lib/llm/judges/verify-submit.js +89 -0
- package/dist/lib/llm/judges/verify-submit.js.map +1 -0
- package/dist/lib/llm/schemas.d.ts +220 -0
- package/dist/lib/llm/schemas.d.ts.map +1 -0
- package/dist/lib/llm/schemas.js +239 -0
- package/dist/lib/llm/schemas.js.map +1 -0
- package/dist/lib/logging.d.ts +39 -0
- package/dist/lib/logging.d.ts.map +1 -0
- package/dist/lib/logging.js +219 -0
- package/dist/lib/logging.js.map +1 -0
- package/dist/lib/multipart.d.ts +34 -0
- package/dist/lib/multipart.d.ts.map +1 -0
- package/dist/lib/multipart.js +55 -0
- package/dist/lib/multipart.js.map +1 -0
- package/dist/lib/option-matcher.d.ts +15 -0
- package/dist/lib/option-matcher.d.ts.map +1 -0
- package/dist/lib/option-matcher.js +69 -0
- package/dist/lib/option-matcher.js.map +1 -0
- package/dist/lib/phone.d.ts +11 -0
- package/dist/lib/phone.d.ts.map +1 -0
- package/dist/lib/phone.js +17 -0
- package/dist/lib/phone.js.map +1 -0
- package/dist/lib/random.d.ts +22 -0
- package/dist/lib/random.d.ts.map +1 -0
- package/dist/lib/random.js +33 -0
- package/dist/lib/random.js.map +1 -0
- package/dist/lib/statsd.d.ts +11 -0
- package/dist/lib/statsd.d.ts.map +1 -0
- package/dist/lib/statsd.js +68 -0
- package/dist/lib/statsd.js.map +1 -0
- package/dist/lib/telemetry/call-capture.d.ts +89 -0
- package/dist/lib/telemetry/call-capture.d.ts.map +1 -0
- package/dist/lib/telemetry/call-capture.js +190 -0
- package/dist/lib/telemetry/call-capture.js.map +1 -0
- package/dist/lib/telemetry/call-types.d.ts +65 -0
- package/dist/lib/telemetry/call-types.d.ts.map +1 -0
- package/dist/lib/telemetry/call-types.js +68 -0
- package/dist/lib/telemetry/call-types.js.map +1 -0
- package/dist/lib/telemetry/run-state.d.ts +15 -0
- package/dist/lib/telemetry/run-state.d.ts.map +1 -0
- package/dist/lib/telemetry/run-state.js +23 -0
- package/dist/lib/telemetry/run-state.js.map +1 -0
- package/dist/lib/telemetry/s3-sink.d.ts +48 -0
- package/dist/lib/telemetry/s3-sink.d.ts.map +1 -0
- package/dist/lib/telemetry/s3-sink.js +229 -0
- package/dist/lib/telemetry/s3-sink.js.map +1 -0
- package/dist/lib/telemetry/submission-capture.d.ts +45 -0
- package/dist/lib/telemetry/submission-capture.d.ts.map +1 -0
- package/dist/lib/telemetry/submission-capture.js +57 -0
- package/dist/lib/telemetry/submission-capture.js.map +1 -0
- package/dist/lib/telemetry/telemetry-paths.d.ts +59 -0
- package/dist/lib/telemetry/telemetry-paths.d.ts.map +1 -0
- package/dist/lib/telemetry/telemetry-paths.js +103 -0
- package/dist/lib/telemetry/telemetry-paths.js.map +1 -0
- package/dist/lib/tracking-click.d.ts +20 -0
- package/dist/lib/tracking-click.d.ts.map +1 -0
- package/dist/lib/tracking-click.js +75 -0
- package/dist/lib/tracking-click.js.map +1 -0
- package/dist/lib/url-capture.d.ts +8 -0
- package/dist/lib/url-capture.d.ts.map +1 -0
- package/dist/lib/url-capture.js +17 -0
- package/dist/lib/url-capture.js.map +1 -0
- package/dist/lib/us-states.d.ts +20 -0
- package/dist/lib/us-states.d.ts.map +1 -0
- package/dist/lib/us-states.js +86 -0
- package/dist/lib/us-states.js.map +1 -0
- package/dist/lib/zod-multipart.d.ts +18 -0
- package/dist/lib/zod-multipart.d.ts.map +1 -0
- package/dist/lib/zod-multipart.js +35 -0
- package/dist/lib/zod-multipart.js.map +1 -0
- package/dist/plugins/config-plugin.d.ts +77 -0
- package/dist/plugins/config-plugin.d.ts.map +1 -0
- package/dist/plugins/config-plugin.js +216 -0
- package/dist/plugins/config-plugin.js.map +1 -0
- package/dist/plugins/discover.d.ts +72 -0
- package/dist/plugins/discover.d.ts.map +1 -0
- package/dist/plugins/discover.js +265 -0
- package/dist/plugins/discover.js.map +1 -0
- package/dist/plugins/json-schema-to-zod.d.ts +47 -0
- package/dist/plugins/json-schema-to-zod.d.ts.map +1 -0
- package/dist/plugins/json-schema-to-zod.js +85 -0
- package/dist/plugins/json-schema-to-zod.js.map +1 -0
- package/dist/plugins/loader.d.ts +34 -0
- package/dist/plugins/loader.d.ts.map +1 -0
- package/dist/plugins/loader.js +348 -0
- package/dist/plugins/loader.js.map +1 -0
- package/dist/plugins/plugin-api-version.d.ts +8 -0
- package/dist/plugins/plugin-api-version.d.ts.map +1 -0
- package/dist/plugins/plugin-api-version.js +11 -0
- package/dist/plugins/plugin-api-version.js.map +1 -0
- package/dist/plugins/plugin-manifest-envelope.d.ts +16 -0
- package/dist/plugins/plugin-manifest-envelope.d.ts.map +1 -0
- package/dist/plugins/plugin-manifest-envelope.js +19 -0
- package/dist/plugins/plugin-manifest-envelope.js.map +1 -0
- package/dist/scraper/behavioral-signals.d.ts +16 -0
- package/dist/scraper/behavioral-signals.d.ts.map +1 -0
- package/dist/scraper/behavioral-signals.js +30 -0
- package/dist/scraper/behavioral-signals.js.map +1 -0
- package/dist/scraper/errors.d.ts +193 -0
- package/dist/scraper/errors.d.ts.map +1 -0
- package/dist/scraper/errors.js +226 -0
- package/dist/scraper/errors.js.map +1 -0
- package/dist/scraper/fixtures.d.ts +23 -0
- package/dist/scraper/fixtures.d.ts.map +1 -0
- package/dist/scraper/fixtures.js +48 -0
- package/dist/scraper/fixtures.js.map +1 -0
- package/dist/scraper/flow-runner.d.ts +1319 -0
- package/dist/scraper/flow-runner.d.ts.map +1 -0
- package/dist/scraper/flow-runner.js +5637 -0
- package/dist/scraper/flow-runner.js.map +1 -0
- package/dist/scraper/graphql-client.d.ts +30 -0
- package/dist/scraper/graphql-client.d.ts.map +1 -0
- package/dist/scraper/graphql-client.js +14 -0
- package/dist/scraper/graphql-client.js.map +1 -0
- package/dist/scraper/http-client.d.ts +75 -0
- package/dist/scraper/http-client.d.ts.map +1 -0
- package/dist/scraper/http-client.js +189 -0
- package/dist/scraper/http-client.js.map +1 -0
- package/dist/scraper/http-status-classifier.d.ts +12 -0
- package/dist/scraper/http-status-classifier.d.ts.map +1 -0
- package/dist/scraper/http-status-classifier.js +29 -0
- package/dist/scraper/http-status-classifier.js.map +1 -0
- package/dist/scraper/metrics.d.ts +50 -0
- package/dist/scraper/metrics.d.ts.map +1 -0
- package/dist/scraper/metrics.js +85 -0
- package/dist/scraper/metrics.js.map +1 -0
- package/dist/scraper/navigate.d.ts +20 -0
- package/dist/scraper/navigate.d.ts.map +1 -0
- package/dist/scraper/navigate.js +30 -0
- package/dist/scraper/navigate.js.map +1 -0
- package/dist/scraper/oracle-sentinels.d.ts +22 -0
- package/dist/scraper/oracle-sentinels.d.ts.map +1 -0
- package/dist/scraper/oracle-sentinels.js +40 -0
- package/dist/scraper/oracle-sentinels.js.map +1 -0
- package/dist/scraper/parse-json-response.d.ts +16 -0
- package/dist/scraper/parse-json-response.d.ts.map +1 -0
- package/dist/scraper/parse-json-response.js +37 -0
- package/dist/scraper/parse-json-response.js.map +1 -0
- package/dist/scraper/pool.d.ts +37 -0
- package/dist/scraper/pool.d.ts.map +1 -0
- package/dist/scraper/pool.js +105 -0
- package/dist/scraper/pool.js.map +1 -0
- package/dist/scraper/rate-limited-json-client.d.ts +35 -0
- package/dist/scraper/rate-limited-json-client.d.ts.map +1 -0
- package/dist/scraper/rate-limited-json-client.js +29 -0
- package/dist/scraper/rate-limited-json-client.js.map +1 -0
- package/dist/scraper/raw-fetch.d.ts +49 -0
- package/dist/scraper/raw-fetch.d.ts.map +1 -0
- package/dist/scraper/raw-fetch.js +40 -0
- package/dist/scraper/raw-fetch.js.map +1 -0
- package/dist/scraper/require-response-field.d.ts +23 -0
- package/dist/scraper/require-response-field.d.ts.map +1 -0
- package/dist/scraper/require-response-field.js +42 -0
- package/dist/scraper/require-response-field.js.map +1 -0
- package/dist/scraper/retry.d.ts +33 -0
- package/dist/scraper/retry.d.ts.map +1 -0
- package/dist/scraper/retry.js +122 -0
- package/dist/scraper/retry.js.map +1 -0
- package/dist/scraper/session-browserbase.d.ts +35 -0
- package/dist/scraper/session-browserbase.d.ts.map +1 -0
- package/dist/scraper/session-browserbase.js +198 -0
- package/dist/scraper/session-browserbase.js.map +1 -0
- package/dist/scraper/session-shared.d.ts +29 -0
- package/dist/scraper/session-shared.d.ts.map +1 -0
- package/dist/scraper/session-shared.js +35 -0
- package/dist/scraper/session-shared.js.map +1 -0
- package/dist/scraper/session-steel.d.ts +31 -0
- package/dist/scraper/session-steel.d.ts.map +1 -0
- package/dist/scraper/session-steel.js +140 -0
- package/dist/scraper/session-steel.js.map +1 -0
- package/dist/scraper/session-warmup.d.ts +31 -0
- package/dist/scraper/session-warmup.d.ts.map +1 -0
- package/dist/scraper/session-warmup.js +46 -0
- package/dist/scraper/session-warmup.js.map +1 -0
- package/dist/scraper/session.d.ts +21 -0
- package/dist/scraper/session.d.ts.map +1 -0
- package/dist/scraper/session.js +28 -0
- package/dist/scraper/session.js.map +1 -0
- package/dist/scraper/stagehand-guard.d.ts +128 -0
- package/dist/scraper/stagehand-guard.d.ts.map +1 -0
- package/dist/scraper/stagehand-guard.js +337 -0
- package/dist/scraper/stagehand-guard.js.map +1 -0
- package/dist/scraper/throttle.d.ts +21 -0
- package/dist/scraper/throttle.d.ts.map +1 -0
- package/dist/scraper/throttle.js +45 -0
- package/dist/scraper/throttle.js.map +1 -0
- package/dist/scripts/hca-inbox-poll.d.ts +19 -0
- package/dist/scripts/hca-inbox-poll.d.ts.map +1 -0
- package/dist/scripts/hca-inbox-poll.js +151 -0
- package/dist/scripts/hca-inbox-poll.js.map +1 -0
- package/dist/scripts/judge-llm-batch.d.ts +98 -0
- package/dist/scripts/judge-llm-batch.d.ts.map +1 -0
- package/dist/scripts/judge-llm-batch.js +350 -0
- package/dist/scripts/judge-llm-batch.js.map +1 -0
- package/dist/scripts/llm-heal.d.ts +165 -0
- package/dist/scripts/llm-heal.d.ts.map +1 -0
- package/dist/scripts/llm-heal.js +595 -0
- package/dist/scripts/llm-heal.js.map +1 -0
- package/dist/scripts/migrate-telemetry-dir-names.d.ts +57 -0
- package/dist/scripts/migrate-telemetry-dir-names.d.ts.map +1 -0
- package/dist/scripts/migrate-telemetry-dir-names.js +156 -0
- package/dist/scripts/migrate-telemetry-dir-names.js.map +1 -0
- package/dist/scripts/probe-setinputfiles.d.ts +28 -0
- package/dist/scripts/probe-setinputfiles.d.ts.map +1 -0
- package/dist/scripts/probe-setinputfiles.js +95 -0
- package/dist/scripts/probe-setinputfiles.js.map +1 -0
- package/dist/scripts/recon-browser.d.ts +304 -0
- package/dist/scripts/recon-browser.d.ts.map +1 -0
- package/dist/scripts/recon-browser.js +1833 -0
- package/dist/scripts/recon-browser.js.map +1 -0
- package/dist/scripts/recon-generate.d.ts +182 -0
- package/dist/scripts/recon-generate.d.ts.map +1 -0
- package/dist/scripts/recon-generate.js +2572 -0
- package/dist/scripts/recon-generate.js.map +1 -0
- package/dist/scripts/recon-heal.d.ts +176 -0
- package/dist/scripts/recon-heal.d.ts.map +1 -0
- package/dist/scripts/recon-heal.js +584 -0
- package/dist/scripts/recon-heal.js.map +1 -0
- package/dist/scripts/recon-http.d.ts +19 -0
- package/dist/scripts/recon-http.d.ts.map +1 -0
- package/dist/scripts/recon-http.js +377 -0
- package/dist/scripts/recon-http.js.map +1 -0
- package/dist/scripts/recon-replay-jobs.d.ts +21 -0
- package/dist/scripts/recon-replay-jobs.d.ts.map +1 -0
- package/dist/scripts/recon-replay-jobs.js +221 -0
- package/dist/scripts/recon-replay-jobs.js.map +1 -0
- package/dist/scripts/recon-shared.d.ts +52 -0
- package/dist/scripts/recon-shared.d.ts.map +1 -0
- package/dist/scripts/recon-shared.js +68 -0
- package/dist/scripts/recon-shared.js.map +1 -0
- package/dist/scripts/recon-summarize.d.ts +15 -0
- package/dist/scripts/recon-summarize.d.ts.map +1 -0
- package/dist/scripts/recon-summarize.js +233 -0
- package/dist/scripts/recon-summarize.js.map +1 -0
- package/dist/scripts/smoke-test.d.ts +22 -0
- package/dist/scripts/smoke-test.d.ts.map +1 -0
- package/dist/scripts/smoke-test.js +236 -0
- package/dist/scripts/smoke-test.js.map +1 -0
- package/dist/server.d.ts +10 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +189 -0
- package/dist/server.js.map +1 -0
- package/dist/site-plugin.d.ts +226 -0
- package/dist/site-plugin.d.ts.map +1 -0
- package/dist/site-plugin.js +13 -0
- package/dist/site-plugin.js.map +1 -0
- package/dist/testing/answers-fixture.d.ts +13 -0
- package/dist/testing/answers-fixture.d.ts.map +1 -0
- package/dist/testing/answers-fixture.js +37 -0
- package/dist/testing/answers-fixture.js.map +1 -0
- package/dist/testing/batch-email-confirmation.d.ts +58 -0
- package/dist/testing/batch-email-confirmation.d.ts.map +1 -0
- package/dist/testing/batch-email-confirmation.js +62 -0
- package/dist/testing/batch-email-confirmation.js.map +1 -0
- package/dist/testing/batch-report.d.ts +33 -0
- package/dist/testing/batch-report.d.ts.map +1 -0
- package/dist/testing/batch-report.js +30 -0
- package/dist/testing/batch-report.js.map +1 -0
- package/dist/testing/contract-parity-suite.d.ts +55 -0
- package/dist/testing/contract-parity-suite.d.ts.map +1 -0
- package/dist/testing/contract-parity-suite.js +39 -0
- package/dist/testing/contract-parity-suite.js.map +1 -0
- package/dist/testing/coverage-guard-suite.d.ts +46 -0
- package/dist/testing/coverage-guard-suite.d.ts.map +1 -0
- package/dist/testing/coverage-guard-suite.js +39 -0
- package/dist/testing/coverage-guard-suite.js.map +1 -0
- package/dist/testing/fixtures/resume.pdf +54 -0
- package/dist/testing/integration-runner.d.ts +62 -0
- package/dist/testing/integration-runner.d.ts.map +1 -0
- package/dist/testing/integration-runner.js +46 -0
- package/dist/testing/integration-runner.js.map +1 -0
- package/dist/testing/mock-fetch-response.d.ts +7 -0
- package/dist/testing/mock-fetch-response.d.ts.map +1 -0
- package/dist/testing/mock-fetch-response.js +16 -0
- package/dist/testing/mock-fetch-response.js.map +1 -0
- package/dist/testing/persona-fixture.d.ts +77 -0
- package/dist/testing/persona-fixture.d.ts.map +1 -0
- package/dist/testing/persona-fixture.js +83 -0
- package/dist/testing/persona-fixture.js.map +1 -0
- package/dist/testing/replay-integration-suite.d.ts +50 -0
- package/dist/testing/replay-integration-suite.d.ts.map +1 -0
- package/dist/testing/replay-integration-suite.js +45 -0
- package/dist/testing/replay-integration-suite.js.map +1 -0
- package/dist/testing/resume-fixture.d.ts +39 -0
- package/dist/testing/resume-fixture.d.ts.map +1 -0
- package/dist/testing/resume-fixture.js +38 -0
- package/dist/testing/resume-fixture.js.map +1 -0
- package/dist/testmail/client.d.ts +61 -0
- package/dist/testmail/client.d.ts.map +1 -0
- package/dist/testmail/client.js +190 -0
- package/dist/testmail/client.js.map +1 -0
- package/dist/testmail/errors.d.ts +18 -0
- package/dist/testmail/errors.d.ts.map +1 -0
- package/dist/testmail/errors.js +29 -0
- package/dist/testmail/errors.js.map +1 -0
- package/dist/types/dispatch-metrics.d.ts +46 -0
- package/dist/types/dispatch-metrics.d.ts.map +1 -0
- package/dist/types/dispatch-metrics.js +25 -0
- package/dist/types/dispatch-metrics.js.map +1 -0
- package/dist/types/logging.d.ts +11 -0
- package/dist/types/logging.d.ts.map +1 -0
- package/dist/types/logging.js +3 -0
- package/dist/types/logging.js.map +1 -0
- package/package.json +283 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.classifyHttpStatus = classifyHttpStatus;
|
|
4
|
+
const errors_1 = require("../scraper/errors");
|
|
5
|
+
/**
|
|
6
|
+
* Pure status→error classifier shared by all raw-fetch callers that cannot
|
|
7
|
+
* use `createHttpClient` (e.g. multipart Buffer bodies, per-response header
|
|
8
|
+
* rotation). Throws the appropriate typed `ScraperError` for any non-2xx
|
|
9
|
+
* status; returns silently for 2xx so callers can proceed to body parsing.
|
|
10
|
+
*
|
|
11
|
+
* Deliberately does NOT wrap in `p-retry AbortError` — that is
|
|
12
|
+
* `createHttpClient`'s concern. Direct callers that don't use p-retry get
|
|
13
|
+
* plain throws that propagate naturally.
|
|
14
|
+
*/
|
|
15
|
+
function classifyHttpStatus(status, rawBodySnippet, contextLabel) {
|
|
16
|
+
if (status === 401 || status === 403) {
|
|
17
|
+
throw new errors_1.HttpBotChallengeError(`${contextLabel} returned ${status}: ${rawBodySnippet}`);
|
|
18
|
+
}
|
|
19
|
+
if (status === 429) {
|
|
20
|
+
throw new errors_1.HttpRateLimitError(`${contextLabel} returned ${status}: ${rawBodySnippet}`);
|
|
21
|
+
}
|
|
22
|
+
if (status >= 500) {
|
|
23
|
+
throw new errors_1.HttpServerError(`${contextLabel} returned ${status}: ${rawBodySnippet}`);
|
|
24
|
+
}
|
|
25
|
+
if (status >= 400) {
|
|
26
|
+
throw new errors_1.HttpSchemaError(`${contextLabel} returned ${status}: ${rawBodySnippet}`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=http-status-classifier.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http-status-classifier.js","sourceRoot":"","sources":["../../src/scraper/http-status-classifier.ts"],"names":[],"mappings":";;AAiBA,gDAiBC;AAlCD,6CAK0B;AAE1B;;;;;;;;;GASG;AACH,SAAgB,kBAAkB,CAChC,MAAc,EACd,cAAsB,EACtB,YAAoB;IAEpB,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACrC,MAAM,IAAI,8BAAqB,CAAC,GAAG,YAAY,aAAa,MAAM,KAAK,cAAc,EAAE,CAAC,CAAC;IAC3F,CAAC;IACD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACnB,MAAM,IAAI,2BAAkB,CAAC,GAAG,YAAY,aAAa,MAAM,KAAK,cAAc,EAAE,CAAC,CAAC;IACxF,CAAC;IACD,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QAClB,MAAM,IAAI,wBAAe,CAAC,GAAG,YAAY,aAAa,MAAM,KAAK,cAAc,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QAClB,MAAM,IAAI,wBAAe,CAAC,GAAG,YAAY,aAAa,MAAM,KAAK,cAAc,EAAE,CAAC,CAAC;IACrF,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* In-process counters for the spec's drift-detection signals (spec §6B).
|
|
3
|
+
* Exposed via /readyz so ops dashboards can alert when fallback rate rises —
|
|
4
|
+
* a rising fallback rate means the hot-path contract has drifted and recon
|
|
5
|
+
* needs re-running.
|
|
6
|
+
*/
|
|
7
|
+
/** Per-site drift-detection counters exposed on /readyz and used by ops dashboards. */
|
|
8
|
+
export interface SiteMetrics {
|
|
9
|
+
hotPathSuccess: number;
|
|
10
|
+
fallbackActivations: number;
|
|
11
|
+
rateLimitRejections: number;
|
|
12
|
+
/** p95 hot-path round-trip latency in ms over the last ~1000 samples. undefined until first sample. */
|
|
13
|
+
p95LatencyMs: number | undefined;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Records a successful direct-HTTP hot-path response for `siteId`. Called by
|
|
17
|
+
* dispatch() when `executeHttp()` resolves without error.
|
|
18
|
+
*/
|
|
19
|
+
export declare function recordHotPathSuccess(siteId: string): void;
|
|
20
|
+
/**
|
|
21
|
+
* Records a fallback activation for `siteId`. Called by dispatch() when the
|
|
22
|
+
* hot path throws `HttpSchemaError`, `HttpBotChallengeError`, or `HttpServerError`
|
|
23
|
+
* and the Stagehand browser path is engaged instead.
|
|
24
|
+
*/
|
|
25
|
+
export declare function recordFallbackActivation(siteId: string): void;
|
|
26
|
+
/**
|
|
27
|
+
* Records a rate-limit rejection for `siteId`. Called by dispatch() when the
|
|
28
|
+
* hot path throws `HttpRateLimitError` (429 from the target). A rising count
|
|
29
|
+
* signals the configured rps ceiling needs to be lowered.
|
|
30
|
+
*/
|
|
31
|
+
export declare function recordRateLimitRejection(siteId: string): void;
|
|
32
|
+
/**
|
|
33
|
+
* Records a hot-path round-trip latency sample and updates the p95 percentile.
|
|
34
|
+
* Reservoir is capped at RESERVOIR_SIZE to bound memory; once full, a random
|
|
35
|
+
* slot is replaced (Vitter's Algorithm R). Called by dispatch() on non-cached
|
|
36
|
+
* hot-path successes only — cache hits are memory reads and must not bias the
|
|
37
|
+
* upstream latency signal.
|
|
38
|
+
*/
|
|
39
|
+
export declare function recordHotPathLatency(siteId: string, latencyMs: number): void;
|
|
40
|
+
/**
|
|
41
|
+
* Returns a snapshot of all per-site counters. Safe to expose on /readyz —
|
|
42
|
+
* values are monotonically increasing integers since process start.
|
|
43
|
+
*/
|
|
44
|
+
export declare function allMetrics(): Record<string, SiteMetrics>;
|
|
45
|
+
/**
|
|
46
|
+
* Resets all counters. Intended for use in tests only — never call in
|
|
47
|
+
* production as it discards drift-detection history.
|
|
48
|
+
*/
|
|
49
|
+
export declare function resetMetrics(): void;
|
|
50
|
+
//# sourceMappingURL=metrics.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/scraper/metrics.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AAEH,uFAAuF;AACvF,MAAM,WAAW,WAAW;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uGAAuG;IACvG,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC;AAqBD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAEzD;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAE7D;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAE7D;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAW5E;AAED;;;GAGG;AACH,wBAAgB,UAAU,IAAI,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAExD;AAED;;;GAGG;AACH,wBAAgB,YAAY,IAAI,IAAI,CAGnC"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.recordHotPathSuccess = recordHotPathSuccess;
|
|
4
|
+
exports.recordFallbackActivation = recordFallbackActivation;
|
|
5
|
+
exports.recordRateLimitRejection = recordRateLimitRejection;
|
|
6
|
+
exports.recordHotPathLatency = recordHotPathLatency;
|
|
7
|
+
exports.allMetrics = allMetrics;
|
|
8
|
+
exports.resetMetrics = resetMetrics;
|
|
9
|
+
const random_1 = require("../lib/random");
|
|
10
|
+
const store = new Map();
|
|
11
|
+
const RESERVOIR_SIZE = 1000;
|
|
12
|
+
const latencyStore = new Map();
|
|
13
|
+
function ensureEntry(siteId) {
|
|
14
|
+
const existing = store.get(siteId);
|
|
15
|
+
if (existing)
|
|
16
|
+
return existing;
|
|
17
|
+
const entry = {
|
|
18
|
+
hotPathSuccess: 0,
|
|
19
|
+
fallbackActivations: 0,
|
|
20
|
+
rateLimitRejections: 0,
|
|
21
|
+
p95LatencyMs: undefined,
|
|
22
|
+
};
|
|
23
|
+
store.set(siteId, entry);
|
|
24
|
+
latencyStore.set(siteId, []);
|
|
25
|
+
return entry;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Records a successful direct-HTTP hot-path response for `siteId`. Called by
|
|
29
|
+
* dispatch() when `executeHttp()` resolves without error.
|
|
30
|
+
*/
|
|
31
|
+
function recordHotPathSuccess(siteId) {
|
|
32
|
+
ensureEntry(siteId).hotPathSuccess++;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Records a fallback activation for `siteId`. Called by dispatch() when the
|
|
36
|
+
* hot path throws `HttpSchemaError`, `HttpBotChallengeError`, or `HttpServerError`
|
|
37
|
+
* and the Stagehand browser path is engaged instead.
|
|
38
|
+
*/
|
|
39
|
+
function recordFallbackActivation(siteId) {
|
|
40
|
+
ensureEntry(siteId).fallbackActivations++;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Records a rate-limit rejection for `siteId`. Called by dispatch() when the
|
|
44
|
+
* hot path throws `HttpRateLimitError` (429 from the target). A rising count
|
|
45
|
+
* signals the configured rps ceiling needs to be lowered.
|
|
46
|
+
*/
|
|
47
|
+
function recordRateLimitRejection(siteId) {
|
|
48
|
+
ensureEntry(siteId).rateLimitRejections++;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Records a hot-path round-trip latency sample and updates the p95 percentile.
|
|
52
|
+
* Reservoir is capped at RESERVOIR_SIZE to bound memory; once full, a random
|
|
53
|
+
* slot is replaced (Vitter's Algorithm R). Called by dispatch() on non-cached
|
|
54
|
+
* hot-path successes only — cache hits are memory reads and must not bias the
|
|
55
|
+
* upstream latency signal.
|
|
56
|
+
*/
|
|
57
|
+
function recordHotPathLatency(siteId, latencyMs) {
|
|
58
|
+
const entry = ensureEntry(siteId);
|
|
59
|
+
// biome-ignore lint/style/noNonNullAssertion: ensureEntry() always sets this key
|
|
60
|
+
const reservoir = latencyStore.get(siteId);
|
|
61
|
+
if (reservoir.length < RESERVOIR_SIZE) {
|
|
62
|
+
reservoir.push(latencyMs);
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
reservoir[(0, random_1.randomIntInclusive)(0, RESERVOIR_SIZE - 1)] = latencyMs;
|
|
66
|
+
}
|
|
67
|
+
const sorted = [...reservoir].sort((a, b) => a - b);
|
|
68
|
+
entry.p95LatencyMs = sorted[Math.floor(sorted.length * 0.95)] ?? sorted[sorted.length - 1];
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Returns a snapshot of all per-site counters. Safe to expose on /readyz —
|
|
72
|
+
* values are monotonically increasing integers since process start.
|
|
73
|
+
*/
|
|
74
|
+
function allMetrics() {
|
|
75
|
+
return Object.fromEntries(store.entries());
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Resets all counters. Intended for use in tests only — never call in
|
|
79
|
+
* production as it discards drift-detection history.
|
|
80
|
+
*/
|
|
81
|
+
function resetMetrics() {
|
|
82
|
+
store.clear();
|
|
83
|
+
latencyStore.clear();
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=metrics.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/scraper/metrics.ts"],"names":[],"mappings":";;AAyCA,oDAEC;AAOD,4DAEC;AAOD,4DAEC;AASD,oDAWC;AAMD,gCAEC;AAMD,oCAGC;AAlGD,yCAAkD;AAkBlD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAC;AAE7C,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAC;AAEjD,SAAS,WAAW,CAAC,MAAc;IACjC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,MAAM,KAAK,GAAgB;QACzB,cAAc,EAAE,CAAC;QACjB,mBAAmB,EAAE,CAAC;QACtB,mBAAmB,EAAE,CAAC;QACtB,YAAY,EAAE,SAAS;KACxB,CAAC;IACF,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACzB,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC7B,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,MAAc;IACjD,WAAW,CAAC,MAAM,CAAC,CAAC,cAAc,EAAE,CAAC;AACvC,CAAC;AAED;;;;GAIG;AACH,SAAgB,wBAAwB,CAAC,MAAc;IACrD,WAAW,CAAC,MAAM,CAAC,CAAC,mBAAmB,EAAE,CAAC;AAC5C,CAAC;AAED;;;;GAIG;AACH,SAAgB,wBAAwB,CAAC,MAAc;IACrD,WAAW,CAAC,MAAM,CAAC,CAAC,mBAAmB,EAAE,CAAC;AAC5C,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,oBAAoB,CAAC,MAAc,EAAE,SAAiB;IACpE,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAClC,iFAAiF;IACjF,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC;IAC5C,IAAI,SAAS,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QACtC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5B,CAAC;SAAM,CAAC;QACN,SAAS,CAAC,IAAA,2BAAkB,EAAC,CAAC,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;IACnE,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACpD,KAAK,CAAC,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAC7F,CAAC;AAED;;;GAGG;AACH,SAAgB,UAAU;IACxB,OAAO,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED;;;GAGG;AACH,SAAgB,YAAY;IAC1B,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,YAAY,CAAC,KAAK,EAAE,CAAC;AACvB,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared apply-page navigation helper. Centralises the two-step pattern that
|
|
3
|
+
* every site browser-flow repeats: await the active Stagehand page, then
|
|
4
|
+
* navigate to the apply URL with networkidle so form scripts have fully
|
|
5
|
+
* settled before the caller starts filling fields.
|
|
6
|
+
*
|
|
7
|
+
* Scripts that interleave CDP listener wiring between awaitActivePage and goto
|
|
8
|
+
* (recon-browser.ts, recon-generate.ts, etc.) must NOT use this helper — they
|
|
9
|
+
* need the bare page reference before any navigation fires.
|
|
10
|
+
*/
|
|
11
|
+
import type { Page, Stagehand } from "@browserbasehq/stagehand";
|
|
12
|
+
import type { MetricsCollector } from "../lib/dispatch-metrics";
|
|
13
|
+
/**
|
|
14
|
+
* Awaits the active Stagehand page and navigates to `url` with
|
|
15
|
+
* `waitUntil: "networkidle"`. When a `MetricsCollector` is supplied, wraps
|
|
16
|
+
* the goto in a `"navigate"` step so the caller's metrics include navigation
|
|
17
|
+
* timing without each site having to repeat the bookkeeping.
|
|
18
|
+
*/
|
|
19
|
+
export declare function navigateActivePage(stagehand: Stagehand, url: string, collector?: MetricsCollector, timeoutMs?: number): Promise<Page>;
|
|
20
|
+
//# sourceMappingURL=navigate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"navigate.d.ts","sourceRoot":"","sources":["../../src/scraper/navigate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAEhE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE/D;;;;;GAKG;AACH,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,SAAS,EACpB,GAAG,EAAE,MAAM,EACX,SAAS,CAAC,EAAE,gBAAgB,EAC5B,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC,CASf"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Shared apply-page navigation helper. Centralises the two-step pattern that
|
|
4
|
+
* every site browser-flow repeats: await the active Stagehand page, then
|
|
5
|
+
* navigate to the apply URL with networkidle so form scripts have fully
|
|
6
|
+
* settled before the caller starts filling fields.
|
|
7
|
+
*
|
|
8
|
+
* Scripts that interleave CDP listener wiring between awaitActivePage and goto
|
|
9
|
+
* (recon-browser.ts, recon-generate.ts, etc.) must NOT use this helper — they
|
|
10
|
+
* need the bare page reference before any navigation fires.
|
|
11
|
+
*/
|
|
12
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.navigateActivePage = navigateActivePage;
|
|
14
|
+
/**
|
|
15
|
+
* Awaits the active Stagehand page and navigates to `url` with
|
|
16
|
+
* `waitUntil: "networkidle"`. When a `MetricsCollector` is supplied, wraps
|
|
17
|
+
* the goto in a `"navigate"` step so the caller's metrics include navigation
|
|
18
|
+
* timing without each site having to repeat the bookkeeping.
|
|
19
|
+
*/
|
|
20
|
+
async function navigateActivePage(stagehand, url, collector, timeoutMs) {
|
|
21
|
+
const page = await stagehand.context.awaitActivePage();
|
|
22
|
+
collector?.startStep("navigate");
|
|
23
|
+
await page.goto(url, {
|
|
24
|
+
waitUntil: "networkidle",
|
|
25
|
+
...(timeoutMs !== undefined && { timeoutMs }),
|
|
26
|
+
});
|
|
27
|
+
collector?.endStep("success");
|
|
28
|
+
return page;
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=navigate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"navigate.js","sourceRoot":"","sources":["../../src/scraper/navigate.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;AAYH,gDAcC;AApBD;;;;;GAKG;AACI,KAAK,UAAU,kBAAkB,CACtC,SAAoB,EACpB,GAAW,EACX,SAA4B,EAC5B,SAAkB;IAElB,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;IACvD,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;IACjC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;QACnB,SAAS,EAAE,aAAa;QACxB,GAAG,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAC9C,CAAC,CAAC;IACH,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC9B,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure classifier for Oracle HCM plain-text sentinel bodies. Oracle occasionally
|
|
3
|
+
* responds with a raw ASCII token instead of JSON; the token's prefix determines
|
|
4
|
+
* whether the failure is terminal (locked URL, must not retry) or transient
|
|
5
|
+
* (rate-limit window, may retry).
|
|
6
|
+
*
|
|
7
|
+
* Isolated here so the discriminator logic is exhaustively testable without
|
|
8
|
+
* coupling to the HTTP/p-retry machinery in http-client.ts.
|
|
9
|
+
*/
|
|
10
|
+
/** Classification of an Oracle plain-text sentinel body. */
|
|
11
|
+
export type OracleSentinelKind = "locked" | "transient" | "none";
|
|
12
|
+
/**
|
|
13
|
+
* Classifies an Oracle HCM plain-text response body.
|
|
14
|
+
*
|
|
15
|
+
* Returns `'locked'` for terminal sentinels (ORA_URL_LOCKED) that must not be
|
|
16
|
+
* retried. Returns `'transient'` for ORA_IRC_* tokens that represent a
|
|
17
|
+
* rate-limit window and should be retried. Returns `'none'` for anything else
|
|
18
|
+
* (HTML error pages, unexpected text) so the caller can fall through to its
|
|
19
|
+
* existing parse-and-retry logic.
|
|
20
|
+
*/
|
|
21
|
+
export declare function classifyOracleSentinel(rawText: string): OracleSentinelKind;
|
|
22
|
+
//# sourceMappingURL=oracle-sentinels.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"oracle-sentinels.d.ts","sourceRoot":"","sources":["../../src/scraper/oracle-sentinels.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,4DAA4D;AAC5D,MAAM,MAAM,kBAAkB,GAAG,QAAQ,GAAG,WAAW,GAAG,MAAM,CAAC;AASjE;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,MAAM,GAAG,kBAAkB,CAc1E"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Pure classifier for Oracle HCM plain-text sentinel bodies. Oracle occasionally
|
|
4
|
+
* responds with a raw ASCII token instead of JSON; the token's prefix determines
|
|
5
|
+
* whether the failure is terminal (locked URL, must not retry) or transient
|
|
6
|
+
* (rate-limit window, may retry).
|
|
7
|
+
*
|
|
8
|
+
* Isolated here so the discriminator logic is exhaustively testable without
|
|
9
|
+
* coupling to the HTTP/p-retry machinery in http-client.ts.
|
|
10
|
+
*/
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.classifyOracleSentinel = classifyOracleSentinel;
|
|
13
|
+
/**
|
|
14
|
+
* Tokens confirmed as terminal locked/rate-limit conditions. Oracle issues these
|
|
15
|
+
* after repeated attempts against the same requisition URL; retrying cannot
|
|
16
|
+
* succeed and may deepen the lock.
|
|
17
|
+
*/
|
|
18
|
+
const LOCKED_SENTINELS = ["ORA_URL_LOCKED"];
|
|
19
|
+
/**
|
|
20
|
+
* Classifies an Oracle HCM plain-text response body.
|
|
21
|
+
*
|
|
22
|
+
* Returns `'locked'` for terminal sentinels (ORA_URL_LOCKED) that must not be
|
|
23
|
+
* retried. Returns `'transient'` for ORA_IRC_* tokens that represent a
|
|
24
|
+
* rate-limit window and should be retried. Returns `'none'` for anything else
|
|
25
|
+
* (HTML error pages, unexpected text) so the caller can fall through to its
|
|
26
|
+
* existing parse-and-retry logic.
|
|
27
|
+
*/
|
|
28
|
+
function classifyOracleSentinel(rawText) {
|
|
29
|
+
const trimmed = rawText.trim();
|
|
30
|
+
for (const sentinel of LOCKED_SENTINELS) {
|
|
31
|
+
if (trimmed === sentinel) {
|
|
32
|
+
return "locked";
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
if (trimmed.startsWith("ORA_IRC_")) {
|
|
36
|
+
return "transient";
|
|
37
|
+
}
|
|
38
|
+
return "none";
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=oracle-sentinels.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"oracle-sentinels.js","sourceRoot":"","sources":["../../src/scraper/oracle-sentinels.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;AAqBH,wDAcC;AA9BD;;;;GAIG;AACH,MAAM,gBAAgB,GAAG,CAAC,gBAAgB,CAAU,CAAC;AAErD;;;;;;;;GAQG;AACH,SAAgB,sBAAsB,CAAC,OAAe;IACpD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAE/B,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;QACxC,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YACzB,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QACnC,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared JSON-parse + Zod-validate seam for rawFetch callers. Each plugin
|
|
3
|
+
* that builds on rawFetch needs this identical two-step (JSON.parse, then
|
|
4
|
+
* schema.safeParse) — extracting it here prevents subtle divergence in error
|
|
5
|
+
* message shapes and slice constants across plugins.
|
|
6
|
+
*/
|
|
7
|
+
import type { ZodType } from "zod/v4";
|
|
8
|
+
/**
|
|
9
|
+
* Parses `rawBody` as JSON, validates it against `schema`, and returns the
|
|
10
|
+
* narrowed result. Throws {@link HttpUrlLockedError} when the body is a locked
|
|
11
|
+
* Oracle sentinel (ORA_URL_LOCKED). Throws {@link HttpSchemaError} on other
|
|
12
|
+
* JSON parse failures or Zod schema mismatches, with `contextLabel` prefixed so
|
|
13
|
+
* error messages are immediately actionable in logs without additional wrapping.
|
|
14
|
+
*/
|
|
15
|
+
export declare function parseJsonResponse<T>(rawBody: string, schema: ZodType<T>, contextLabel: string): T;
|
|
16
|
+
//# sourceMappingURL=parse-json-response.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse-json-response.d.ts","sourceRoot":"","sources":["../../src/scraper/parse-json-response.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAMtC;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,CAAC,CAqBjG"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Shared JSON-parse + Zod-validate seam for rawFetch callers. Each plugin
|
|
4
|
+
* that builds on rawFetch needs this identical two-step (JSON.parse, then
|
|
5
|
+
* schema.safeParse) — extracting it here prevents subtle divergence in error
|
|
6
|
+
* message shapes and slice constants across plugins.
|
|
7
|
+
*/
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.parseJsonResponse = parseJsonResponse;
|
|
10
|
+
const errors_1 = require("../lib/errors");
|
|
11
|
+
const errors_2 = require("../scraper/errors");
|
|
12
|
+
const oracle_sentinels_1 = require("../scraper/oracle-sentinels");
|
|
13
|
+
/**
|
|
14
|
+
* Parses `rawBody` as JSON, validates it against `schema`, and returns the
|
|
15
|
+
* narrowed result. Throws {@link HttpUrlLockedError} when the body is a locked
|
|
16
|
+
* Oracle sentinel (ORA_URL_LOCKED). Throws {@link HttpSchemaError} on other
|
|
17
|
+
* JSON parse failures or Zod schema mismatches, with `contextLabel` prefixed so
|
|
18
|
+
* error messages are immediately actionable in logs without additional wrapping.
|
|
19
|
+
*/
|
|
20
|
+
function parseJsonResponse(rawBody, schema, contextLabel) {
|
|
21
|
+
let parsed;
|
|
22
|
+
try {
|
|
23
|
+
parsed = JSON.parse(rawBody);
|
|
24
|
+
}
|
|
25
|
+
catch (err) {
|
|
26
|
+
if ((0, oracle_sentinels_1.classifyOracleSentinel)(rawBody) === "locked") {
|
|
27
|
+
throw new errors_2.HttpUrlLockedError();
|
|
28
|
+
}
|
|
29
|
+
throw new errors_2.HttpSchemaError(`${contextLabel} non-JSON body: ${(0, errors_1.toErrorMessage)(err)} (first 200B: ${rawBody.slice(0, 200)})`);
|
|
30
|
+
}
|
|
31
|
+
const result = schema.safeParse(parsed);
|
|
32
|
+
if (!result.success) {
|
|
33
|
+
throw new errors_2.HttpSchemaError(`${contextLabel} body failed Zod parse: ${result.error.message.slice(0, 300)}`);
|
|
34
|
+
}
|
|
35
|
+
return result.data;
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=parse-json-response.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse-json-response.js","sourceRoot":"","sources":["../../src/scraper/parse-json-response.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;AAeH,8CAqBC;AAhCD,yCAA8C;AAC9C,6CAAuE;AACvE,iEAAoE;AAEpE;;;;;;GAMG;AACH,SAAgB,iBAAiB,CAAI,OAAe,EAAE,MAAkB,EAAE,YAAoB;IAC5F,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,IAAA,yCAAsB,EAAC,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YACjD,MAAM,IAAI,2BAAkB,EAAE,CAAC;QACjC,CAAC;QACD,MAAM,IAAI,wBAAe,CACvB,GAAG,YAAY,mBAAmB,IAAA,uBAAc,EAAC,GAAG,CAAC,iBAAiB,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAC/F,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACxC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,wBAAe,CACvB,GAAG,YAAY,2BAA2B,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC/E,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { type RetryOptions } from "../scraper/retry";
|
|
2
|
+
import { type BrowserSession } from "../scraper/session";
|
|
3
|
+
/**
|
|
4
|
+
* Runs `task` inside a freshly created browser session. The session is
|
|
5
|
+
* torn down in a `finally` block even if `task` throws. Retries are
|
|
6
|
+
* supplied by `withScraperRetry` — `SessionTimeoutError` causes a session
|
|
7
|
+
* tear-down and fresh re-creation for the next attempt.
|
|
8
|
+
*
|
|
9
|
+
* Each queued task is bounded by `TASK_TIMEOUT_MS`. A hung Stagehand
|
|
10
|
+
* operation would otherwise block the queue slot indefinitely; the timeout
|
|
11
|
+
* converts the hang into a `SessionTimeoutError` so the retry policy can
|
|
12
|
+
* restart the session and try again.
|
|
13
|
+
*/
|
|
14
|
+
export declare function runWithSession<T>(task: (session: BrowserSession) => Promise<T>, retryOptions?: Omit<RetryOptions, "onSessionRestart">, taskTimeoutMs?: number, sessionOpts?: {
|
|
15
|
+
advancedStealth?: boolean;
|
|
16
|
+
}): Promise<T>;
|
|
17
|
+
/**
|
|
18
|
+
* Exposed for tests and health probes. Returns the current number of
|
|
19
|
+
* queued + in-flight scraper tasks so /readyz can surface back-pressure.
|
|
20
|
+
*/
|
|
21
|
+
export declare function poolStats(): {
|
|
22
|
+
size: number;
|
|
23
|
+
pending: number;
|
|
24
|
+
concurrency: number;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* Drains the scraper queue on graceful shutdown — pauses new intake,
|
|
28
|
+
* waits for in-flight tasks' `finally` blocks to close their Steel
|
|
29
|
+
* sessions, then resolves. Leaving sessions open past process exit
|
|
30
|
+
* means Steel keeps billing until their own timeout kicks in.
|
|
31
|
+
*
|
|
32
|
+
* Bounded by `timeoutMs` so a hung scrape can't block SIGTERM forever;
|
|
33
|
+
* the orchestrator's grace period is usually 30s, so 20s is a safe
|
|
34
|
+
* default that leaves headroom for Fastify to flush.
|
|
35
|
+
*/
|
|
36
|
+
export declare function drainPool(timeoutMs?: number): Promise<void>;
|
|
37
|
+
//# sourceMappingURL=pool.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pool.d.ts","sourceRoot":"","sources":["../../src/scraper/pool.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,YAAY,EAAoB,MAAM,iBAAiB,CAAC;AACtE,OAAO,EAAE,KAAK,cAAc,EAAwB,MAAM,mBAAmB,CAAC;AAqB9E;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAAC,CAAC,EACpC,IAAI,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,OAAO,CAAC,CAAC,CAAC,EAC7C,YAAY,GAAE,IAAI,CAAC,YAAY,EAAE,kBAAkB,CAAM,EACzD,aAAa,SAAkB,EAC/B,WAAW,GAAE;IAAE,eAAe,CAAC,EAAE,OAAO,CAAA;CAAO,GAC9C,OAAO,CAAC,CAAC,CAAC,CA2CZ;AAED;;;GAGG;AACH,wBAAgB,SAAS,IAAI;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAMlF;AAED;;;;;;;;;GASG;AACH,wBAAsB,SAAS,CAAC,SAAS,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAKjE"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.runWithSession = runWithSession;
|
|
7
|
+
exports.poolStats = poolStats;
|
|
8
|
+
exports.drainPool = drainPool;
|
|
9
|
+
const p_queue_1 = __importDefault(require("p-queue"));
|
|
10
|
+
const config_1 = require("../config");
|
|
11
|
+
const logging_1 = require("../lib/logging");
|
|
12
|
+
const errors_1 = require("../scraper/errors");
|
|
13
|
+
const retry_1 = require("../scraper/retry");
|
|
14
|
+
const session_1 = require("../scraper/session");
|
|
15
|
+
const logger = (0, logging_1.getLogger)({ name: "scraper/pool" });
|
|
16
|
+
/**
|
|
17
|
+
* Per-task hard timeout default. A hung Stagehand operation (infinite network wait,
|
|
18
|
+
* frozen CDP connection) would otherwise block the queue slot indefinitely,
|
|
19
|
+
* draining pool capacity without recovery. This ceiling converts a silent
|
|
20
|
+
* hang into a SessionTimeoutError that the retry policy can act on.
|
|
21
|
+
* Individual plugins may override this via SitePluginMeta.taskTimeoutMs.
|
|
22
|
+
*/
|
|
23
|
+
const TASK_TIMEOUT_MS = 60 * 60 * 1_000; // 60 minutes
|
|
24
|
+
/**
|
|
25
|
+
* Shared queue that limits how many scraper tasks run concurrently across
|
|
26
|
+
* the whole process. Sessions are created on demand inside each queued
|
|
27
|
+
* task, not pre-warmed, so Steel billing stays proportional to actual
|
|
28
|
+
* traffic.
|
|
29
|
+
*/
|
|
30
|
+
const queue = new p_queue_1.default({ concurrency: config_1.config.scraper.poolSize });
|
|
31
|
+
/**
|
|
32
|
+
* Runs `task` inside a freshly created browser session. The session is
|
|
33
|
+
* torn down in a `finally` block even if `task` throws. Retries are
|
|
34
|
+
* supplied by `withScraperRetry` — `SessionTimeoutError` causes a session
|
|
35
|
+
* tear-down and fresh re-creation for the next attempt.
|
|
36
|
+
*
|
|
37
|
+
* Each queued task is bounded by `TASK_TIMEOUT_MS`. A hung Stagehand
|
|
38
|
+
* operation would otherwise block the queue slot indefinitely; the timeout
|
|
39
|
+
* converts the hang into a `SessionTimeoutError` so the retry policy can
|
|
40
|
+
* restart the session and try again.
|
|
41
|
+
*/
|
|
42
|
+
async function runWithSession(task, retryOptions = {}, taskTimeoutMs = TASK_TIMEOUT_MS, sessionOpts = {}) {
|
|
43
|
+
return queue.add(async () => {
|
|
44
|
+
const sessionRef = { session: null };
|
|
45
|
+
const ensureSession = async () => {
|
|
46
|
+
if (!sessionRef.session) {
|
|
47
|
+
sessionRef.session = await (0, session_1.createBrowserSession)(sessionOpts);
|
|
48
|
+
}
|
|
49
|
+
return sessionRef.session;
|
|
50
|
+
};
|
|
51
|
+
const closeSession = async () => {
|
|
52
|
+
if (sessionRef.session) {
|
|
53
|
+
await sessionRef.session.close();
|
|
54
|
+
sessionRef.session = null;
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
try {
|
|
58
|
+
return await (0, retry_1.withScraperRetry)(async () => {
|
|
59
|
+
const session = await ensureSession();
|
|
60
|
+
const timeout = new Promise((_, reject) => {
|
|
61
|
+
const t = setTimeout(() => reject(new errors_1.SessionTimeoutError(`task exceeded ${taskTimeoutMs}ms`)), taskTimeoutMs);
|
|
62
|
+
t.unref();
|
|
63
|
+
});
|
|
64
|
+
return Promise.race([task(session), timeout]);
|
|
65
|
+
}, {
|
|
66
|
+
...retryOptions,
|
|
67
|
+
onSessionRestart: async () => {
|
|
68
|
+
logger.info("restarting scraper session after timeout");
|
|
69
|
+
await closeSession();
|
|
70
|
+
},
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
finally {
|
|
74
|
+
await closeSession();
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Exposed for tests and health probes. Returns the current number of
|
|
80
|
+
* queued + in-flight scraper tasks so /readyz can surface back-pressure.
|
|
81
|
+
*/
|
|
82
|
+
function poolStats() {
|
|
83
|
+
return {
|
|
84
|
+
size: queue.size,
|
|
85
|
+
pending: queue.pending,
|
|
86
|
+
concurrency: queue.concurrency,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Drains the scraper queue on graceful shutdown — pauses new intake,
|
|
91
|
+
* waits for in-flight tasks' `finally` blocks to close their Steel
|
|
92
|
+
* sessions, then resolves. Leaving sessions open past process exit
|
|
93
|
+
* means Steel keeps billing until their own timeout kicks in.
|
|
94
|
+
*
|
|
95
|
+
* Bounded by `timeoutMs` so a hung scrape can't block SIGTERM forever;
|
|
96
|
+
* the orchestrator's grace period is usually 30s, so 20s is a safe
|
|
97
|
+
* default that leaves headroom for Fastify to flush.
|
|
98
|
+
*/
|
|
99
|
+
async function drainPool(timeoutMs = 20_000) {
|
|
100
|
+
queue.pause();
|
|
101
|
+
const onIdle = queue.onIdle();
|
|
102
|
+
await Promise.race([onIdle, new Promise((resolve) => setTimeout(resolve, timeoutMs))]);
|
|
103
|
+
queue.clear();
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=pool.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pool.js","sourceRoot":"","sources":["../../src/scraper/pool.ts"],"names":[],"mappings":";;;;;AAsCA,wCAgDC;AAMD,8BAMC;AAYD,8BAKC;AAnHD,sDAA6B;AAE7B,qCAAkC;AAClC,2CAA0C;AAC1C,6CAAuD;AACvD,2CAAsE;AACtE,+CAA8E;AAE9E,MAAM,MAAM,GAAG,IAAA,mBAAS,EAAC,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;AAEnD;;;;;;GAMG;AACH,MAAM,eAAe,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,aAAa;AAEtD;;;;;GAKG;AACH,MAAM,KAAK,GAAG,IAAI,iBAAM,CAAC,EAAE,WAAW,EAAE,eAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;AAEnE;;;;;;;;;;GAUG;AACI,KAAK,UAAU,cAAc,CAClC,IAA6C,EAC7C,eAAuD,EAAE,EACzD,aAAa,GAAG,eAAe,EAC/B,cAA6C,EAAE;IAE/C,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;QAC1B,MAAM,UAAU,GAAuC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAEzE,MAAM,aAAa,GAAG,KAAK,IAA6B,EAAE;YACxD,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;gBACxB,UAAU,CAAC,OAAO,GAAG,MAAM,IAAA,8BAAoB,EAAC,WAAW,CAAC,CAAC;YAC/D,CAAC;YACD,OAAO,UAAU,CAAC,OAAO,CAAC;QAC5B,CAAC,CAAC;QAEF,MAAM,YAAY,GAAG,KAAK,IAAmB,EAAE;YAC7C,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;gBACvB,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,UAAU,CAAC,OAAO,GAAG,IAAI,CAAC;YAC5B,CAAC;QACH,CAAC,CAAC;QAEF,IAAI,CAAC;YACH,OAAO,MAAM,IAAA,wBAAgB,EAC3B,KAAK,IAAI,EAAE;gBACT,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;gBACtC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;oBAC/C,MAAM,CAAC,GAAG,UAAU,CAClB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,4BAAmB,CAAC,iBAAiB,aAAa,IAAI,CAAC,CAAC,EACzE,aAAa,CACd,CAAC;oBACF,CAAC,CAAC,KAAK,EAAE,CAAC;gBACZ,CAAC,CAAC,CAAC;gBACH,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YAChD,CAAC,EACD;gBACE,GAAG,YAAY;gBACf,gBAAgB,EAAE,KAAK,IAAI,EAAE;oBAC3B,MAAM,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;oBACxD,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;aACF,CACF,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,YAAY,EAAE,CAAC;QACvB,CAAC;IACH,CAAC,CAAe,CAAC;AACnB,CAAC;AAED;;;GAGG;AACH,SAAgB,SAAS;IACvB,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,WAAW,EAAE,KAAK,CAAC,WAAW;KAC/B,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,SAAS,CAAC,SAAS,GAAG,MAAM;IAChD,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;IAC9B,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7F,KAAK,CAAC,KAAK,EAAE,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { ZodType } from "zod/v4";
|
|
2
|
+
import type { HttpRequestInit } from "../scraper/http-client";
|
|
3
|
+
/**
|
|
4
|
+
* Options for building a Chromium-flavoured rate-limited JSON client.
|
|
5
|
+
* Bundles the Bottleneck configuration, Chromium client-hint values, and any
|
|
6
|
+
* site-specific headers into one call so each plugin avoids hand-rolling the
|
|
7
|
+
* identical scaffold.
|
|
8
|
+
*/
|
|
9
|
+
export interface RateLimitedJsonClientOptions<TResponse> {
|
|
10
|
+
/** Bottleneck `minTime` in milliseconds (inverse of requests-per-second ceiling). */
|
|
11
|
+
minTimeMs: number;
|
|
12
|
+
/** Full User-Agent string captured during recon. */
|
|
13
|
+
userAgent: string;
|
|
14
|
+
/** `sec-ch-ua` header value captured during recon. */
|
|
15
|
+
secChUa: string;
|
|
16
|
+
/** Platform string for `sec-ch-ua-platform`, e.g. `"Linux"`. */
|
|
17
|
+
platform: string;
|
|
18
|
+
/**
|
|
19
|
+
* Site-specific headers layered on top of the Chromium client-hint quartet.
|
|
20
|
+
* These are merged last, so a plugin can override a hint value if recon
|
|
21
|
+
* demands it.
|
|
22
|
+
*/
|
|
23
|
+
extraHeaders?: Record<string, string>;
|
|
24
|
+
/** Zod schema that validates and narrows each response body. */
|
|
25
|
+
schema: ZodType<TResponse>;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Site-agnostic factory that wires together Bottleneck + chromiumClientHints +
|
|
29
|
+
* createHttpClient so individual plugins don't repeat the same three-step
|
|
30
|
+
* scaffold. Each plugin supplies its own `minTimeMs`, Chromium hint values,
|
|
31
|
+
* and site-specific `extraHeaders`; the factory owns the limiter lifecycle and
|
|
32
|
+
* the header merge order (hints first, extraHeaders on top).
|
|
33
|
+
*/
|
|
34
|
+
export declare function createRateLimitedJsonClient<TResponse>(opts: RateLimitedJsonClientOptions<TResponse>): (url: string, init?: HttpRequestInit) => Promise<TResponse>;
|
|
35
|
+
//# sourceMappingURL=rate-limited-json-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rate-limited-json-client.d.ts","sourceRoot":"","sources":["../../src/scraper/rate-limited-json-client.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAGtC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAG7D;;;;;GAKG;AACH,MAAM,WAAW,4BAA4B,CAAC,SAAS;IACrD,qFAAqF;IACrF,SAAS,EAAE,MAAM,CAAC;IAClB,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,sDAAsD;IACtD,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,gEAAgE;IAChE,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;CAC5B;AAED;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CAAC,SAAS,EACnD,IAAI,EAAE,4BAA4B,CAAC,SAAS,CAAC,GAC5C,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,SAAS,CAAC,CAW7D"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.createRateLimitedJsonClient = createRateLimitedJsonClient;
|
|
7
|
+
const bottleneck_1 = __importDefault(require("bottleneck"));
|
|
8
|
+
const chromium_client_hints_1 = require("../lib/chromium-client-hints");
|
|
9
|
+
const http_client_1 = require("../scraper/http-client");
|
|
10
|
+
/**
|
|
11
|
+
* Site-agnostic factory that wires together Bottleneck + chromiumClientHints +
|
|
12
|
+
* createHttpClient so individual plugins don't repeat the same three-step
|
|
13
|
+
* scaffold. Each plugin supplies its own `minTimeMs`, Chromium hint values,
|
|
14
|
+
* and site-specific `extraHeaders`; the factory owns the limiter lifecycle and
|
|
15
|
+
* the header merge order (hints first, extraHeaders on top).
|
|
16
|
+
*/
|
|
17
|
+
function createRateLimitedJsonClient(opts) {
|
|
18
|
+
const bottleneck = new bottleneck_1.default({ minTime: opts.minTimeMs });
|
|
19
|
+
const baseHeaders = {
|
|
20
|
+
...(0, chromium_client_hints_1.chromiumClientHints)({
|
|
21
|
+
userAgent: opts.userAgent,
|
|
22
|
+
secChUa: opts.secChUa,
|
|
23
|
+
platform: opts.platform,
|
|
24
|
+
}),
|
|
25
|
+
...(opts.extraHeaders ?? {}),
|
|
26
|
+
};
|
|
27
|
+
return (0, http_client_1.createHttpClient)({ schema: opts.schema, bottleneck, baseHeaders });
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=rate-limited-json-client.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rate-limited-json-client.js","sourceRoot":"","sources":["../../src/scraper/rate-limited-json-client.ts"],"names":[],"mappings":";;;;;AAuCA,kEAaC;AApDD,4DAAoC;AAGpC,uEAAkE;AAElE,uDAAyD;AA2BzD;;;;;;GAMG;AACH,SAAgB,2BAA2B,CACzC,IAA6C;IAE7C,MAAM,UAAU,GAAG,IAAI,oBAAU,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IAC/D,MAAM,WAAW,GAA2B;QAC1C,GAAG,IAAA,2CAAmB,EAAC;YACrB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,CAAC;QACF,GAAG,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;KAC7B,CAAC;IACF,OAAO,IAAA,8BAAgB,EAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC;AAC5E,CAAC"}
|