@apifuse/provider-sdk 2.2.0-beta.5 → 2.2.0-beta.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +537 -33
- package/CHANGELOG.md +205 -1
- package/README.md +83 -22
- package/SUBMISSION.md +2 -2
- package/bin/apifuse-check.ts +165 -11
- package/bin/apifuse-dev.ts +57 -13
- package/bin/apifuse-migrate-operation-declaration.ts +55 -0
- package/bin/apifuse-migrate-shape.ts +202 -0
- package/bin/apifuse-pack-check.ts +22 -2
- package/bin/apifuse-pack-smoke.ts +57 -2
- package/bin/apifuse-pack-types.ts +357 -38
- package/bin/apifuse-perf.ts +14 -13
- package/bin/apifuse-record.ts +709 -72
- package/bin/apifuse-submit-check.ts +2344 -324
- package/bin/apifuse-sync-assets.ts +117 -0
- package/bin/submit-check-delimited-text.ts +50 -0
- package/dist/auth-turn/index.d.ts +3 -3
- package/dist/auth-turn/index.js +1 -1
- package/dist/auth.d.ts +14 -0
- package/dist/auth.js +67 -0
- package/dist/ceremonies/index.d.ts +16 -0
- package/dist/ceremonies/index.js +141 -36
- package/dist/cli/commands.d.ts +1 -1
- package/dist/cli/commands.js +27 -0
- package/dist/cli/create.d.ts +4 -1
- package/dist/cli/create.js +42 -37
- package/dist/cli/migrate-operation-declaration.d.ts +59 -0
- package/dist/cli/migrate-operation-declaration.js +1249 -0
- package/dist/cli/migrate-operation-shape.d.ts +44 -0
- package/dist/cli/migrate-operation-shape.js +113 -0
- package/dist/cli/migrate-provider-shape.d.ts +52 -0
- package/dist/cli/migrate-provider-shape.js +578 -0
- package/dist/cli/prompt-assets.d.ts +80 -0
- package/dist/cli/prompt-assets.js +743 -0
- package/dist/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/dist/cli/templates/provider/Dockerfile.tpl +1 -1
- package/dist/cli/templates/provider/README.md.tpl +5 -5
- package/dist/cli/templates/provider/index.ts.tpl +6 -3
- package/dist/cli/templates/provider/operations/ping.ts.tpl +4 -1
- package/dist/cli/templates/provider/provider.json.tpl +6 -0
- package/dist/config/loader.d.ts +179 -16
- package/dist/config/loader.js +435 -127
- package/dist/contract-serialization.js +4 -8
- package/dist/contract-types.d.ts +12 -5
- package/dist/contract.js +24 -10
- package/dist/declaration-validation.d.ts +33 -0
- package/dist/declaration-validation.js +266 -0
- package/dist/define.d.ts +66 -29
- package/dist/define.js +778 -69
- package/dist/dev.d.ts +3 -0
- package/dist/dev.js +1 -1
- package/dist/engine.d.ts +78 -0
- package/dist/engine.js +133 -0
- package/dist/error-observability.d.ts +7 -0
- package/dist/error-observability.js +61 -0
- package/dist/error-resolution.d.ts +4 -0
- package/dist/error-resolution.js +122 -0
- package/dist/errors.d.ts +33 -0
- package/dist/errors.js +40 -0
- package/dist/fixture-sanitization.d.ts +28 -0
- package/dist/fixture-sanitization.js +227 -0
- package/dist/health-scenario.d.ts +1842 -0
- package/dist/health-scenario.js +624 -0
- package/dist/index.d.ts +21 -10
- package/dist/index.js +12 -7
- package/dist/lint.d.ts +23 -11
- package/dist/lint.js +572 -69
- package/dist/native-address.d.ts +43 -0
- package/dist/native-address.js +281 -0
- package/dist/native-egress-policy.d.ts +31 -0
- package/dist/native-egress-policy.js +288 -0
- package/dist/observability.d.ts +5 -2
- package/dist/observability.js +48 -1
- package/dist/provider.d.ts +10 -2
- package/dist/provider.js +4 -1
- package/dist/runtime/auth-flow.d.ts +5 -1
- package/dist/runtime/auth-flow.js +6 -0
- package/dist/runtime/browser.d.ts +1 -0
- package/dist/runtime/browser.js +492 -49
- package/dist/runtime/cache.d.ts +1 -0
- package/dist/runtime/cache.js +169 -15
- package/dist/runtime/choice-wordlist.d.ts +9 -0
- package/dist/runtime/choice-wordlist.js +138 -0
- package/dist/runtime/choice.d.ts +13 -1
- package/dist/runtime/choice.js +490 -102
- package/dist/runtime/chrome149-header-order.d.ts +58 -0
- package/dist/runtime/chrome149-header-order.js +289 -0
- package/dist/runtime/env.js +12 -0
- package/dist/runtime/executor.d.ts +3 -2
- package/dist/runtime/executor.js +26 -25
- package/dist/runtime/http.d.ts +1 -0
- package/dist/runtime/http.js +515 -53
- package/dist/runtime/insights.js +2 -2
- package/dist/runtime/instrumentation.d.ts +2 -2
- package/dist/runtime/instrumentation.js +366 -8
- package/dist/runtime/native-network-errors.d.ts +33 -0
- package/dist/runtime/native-network-errors.js +69 -0
- package/dist/runtime/native-network.d.ts +96 -0
- package/dist/runtime/native-network.js +1232 -0
- package/dist/runtime/ocr.d.ts +29 -0
- package/dist/runtime/ocr.js +440 -0
- package/dist/runtime/otlp.d.ts +71 -2
- package/dist/runtime/otlp.js +397 -16
- package/dist/runtime/proxy-errors.js +6 -2
- package/dist/runtime/proxy-nodemaven.d.ts +56 -0
- package/dist/runtime/proxy-nodemaven.js +146 -0
- package/dist/runtime/proxy-telemetry.d.ts +80 -1
- package/dist/runtime/proxy-telemetry.js +154 -47
- package/dist/runtime/redirects.d.ts +29 -0
- package/dist/runtime/redirects.js +36 -0
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +4 -2
- package/dist/runtime/request-options.d.ts +68 -1
- package/dist/runtime/request-options.js +548 -0
- package/dist/runtime/resolver-config.d.ts +6 -0
- package/dist/runtime/resolver-config.js +6 -0
- package/dist/runtime/resolver-public.d.ts +1 -0
- package/dist/runtime/resolver-public.js +1 -0
- package/dist/runtime/resolver-shared.d.ts +14 -0
- package/dist/runtime/resolver-shared.js +12 -0
- package/dist/runtime/resolver-vendors/bindings.d.ts +48 -0
- package/dist/runtime/resolver-vendors/bindings.js +40 -0
- package/dist/runtime/resolver-vendors/browser.d.ts +22 -0
- package/dist/runtime/resolver-vendors/browser.js +377 -0
- package/dist/runtime/resolver-vendors/capsolver.d.ts +22 -0
- package/dist/runtime/resolver-vendors/capsolver.js +531 -0
- package/dist/runtime/resolver-vendors/hosts.d.ts +2 -0
- package/dist/runtime/resolver-vendors/hosts.js +33 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +24 -0
- package/dist/runtime/resolver-vendors/twocaptcha.js +408 -0
- package/dist/runtime/resolver-vendors/types.d.ts +94 -0
- package/dist/runtime/resolver-vendors/types.js +96 -0
- package/dist/runtime/resolver.d.ts +72 -0
- package/dist/runtime/resolver.js +771 -0
- package/dist/runtime/secrets.d.ts +27 -0
- package/dist/runtime/secrets.js +51 -0
- package/dist/runtime/state.d.ts +3 -0
- package/dist/runtime/state.js +277 -71
- package/dist/runtime/stealth-cookies.d.ts +20 -0
- package/dist/runtime/stealth-cookies.js +111 -0
- package/dist/runtime/stealth.d.ts +41 -5
- package/dist/runtime/stealth.js +997 -291
- package/dist/runtime/stt.js +1 -12
- package/dist/runtime/timeout.d.ts +5 -0
- package/dist/runtime/timeout.js +12 -0
- package/dist/runtime/trace-config.d.ts +12 -0
- package/dist/runtime/trace-config.js +62 -0
- package/dist/runtime/trace.d.ts +5 -0
- package/dist/runtime/trace.js +43 -10
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/error-observability.d.ts +1 -0
- package/dist/server/error-observability.js +1 -0
- package/dist/server/index.d.ts +5 -2
- package/dist/server/index.js +2 -2
- package/dist/server/self-test-input-tokens.d.ts +2 -1
- package/dist/server/self-test-input-tokens.js +18 -14
- package/dist/server/self-test.d.ts +14 -3
- package/dist/server/self-test.js +129 -58
- package/dist/server/serve-implementation.d.ts +230 -0
- package/dist/server/serve-implementation.js +2265 -0
- package/dist/server/serve.d.ts +1 -70
- package/dist/server/serve.js +1 -1131
- package/dist/server/trace-output.d.ts +6 -0
- package/dist/server/trace-output.js +79 -0
- package/dist/server/types.d.ts +30 -5
- package/dist/server/types.js +13 -1
- package/dist/stateful/errors.d.ts +19 -0
- package/dist/stateful/errors.js +24 -0
- package/dist/stateful/http-provider-event-emitter.d.ts +40 -0
- package/dist/stateful/http-provider-event-emitter.js +237 -0
- package/dist/stateful/http-session-owner-registry.d.ts +44 -0
- package/dist/stateful/http-session-owner-registry.js +210 -0
- package/dist/stateful/index.d.ts +18 -0
- package/dist/stateful/index.js +18 -0
- package/dist/stateful/provider-event-delivery-failures.d.ts +32 -0
- package/dist/stateful/provider-event-delivery-failures.js +43 -0
- package/dist/stateful/provider-event-pipeline-metrics.d.ts +46 -0
- package/dist/stateful/provider-event-pipeline-metrics.js +48 -0
- package/dist/stateful/provider-event-pipeline.d.ts +50 -0
- package/dist/stateful/provider-event-pipeline.js +1 -0
- package/dist/stateful/provider-events.d.ts +101 -0
- package/dist/stateful/provider-events.js +289 -0
- package/dist/stateful/session-key.d.ts +15 -0
- package/dist/stateful/session-key.js +86 -0
- package/dist/stateful/stateful-provider-adapter-context.d.ts +5 -0
- package/dist/stateful/stateful-provider-adapter-context.js +42 -0
- package/dist/stateful/stateful-provider-adapter-metrics.d.ts +15 -0
- package/dist/stateful/stateful-provider-adapter-metrics.js +21 -0
- package/dist/stateful/stateful-provider-adapter.d.ts +98 -0
- package/dist/stateful/stateful-provider-adapter.js +287 -0
- package/dist/stateful/stateful-provider-observability.d.ts +62 -0
- package/dist/stateful/stateful-provider-observability.js +161 -0
- package/dist/stateful/stateful-provider-owner-forwarder.d.ts +41 -0
- package/dist/stateful/stateful-provider-owner-forwarder.js +215 -0
- package/dist/stateful/stateful-provider-runtime-context.d.ts +32 -0
- package/dist/stateful/stateful-provider-runtime-context.js +60 -0
- package/dist/stateful/stateful-provider-runtime-executor.d.ts +34 -0
- package/dist/stateful/stateful-provider-runtime-executor.js +52 -0
- package/dist/stateful/stateful-provider-session-routing.d.ts +67 -0
- package/dist/stateful/stateful-provider-session-routing.js +345 -0
- package/dist/stateful/stateful-provider-session-runtime.d.ts +98 -0
- package/dist/stateful/stateful-provider-session-runtime.js +245 -0
- package/dist/stateful-signing.d.ts +18 -0
- package/dist/stateful-signing.js +27 -0
- package/dist/stealth/profiles.d.ts +9 -4
- package/dist/stealth/profiles.js +160 -211
- package/dist/stream-evidence.d.ts +74 -0
- package/dist/stream-evidence.js +785 -0
- package/dist/stream.js +7 -1
- package/dist/testing/index.d.ts +2 -1
- package/dist/testing/index.js +2 -1
- package/dist/testing/run.d.ts +32 -2
- package/dist/testing/run.js +489 -21
- package/dist/trace-sanitization.d.ts +5 -0
- package/dist/trace-sanitization.js +45 -0
- package/dist/types.d.ts +673 -131
- package/dist/types.js +1 -0
- package/package.json +45 -5
- package/src/auth-turn/index.ts +1 -1
- package/src/auth.ts +118 -0
- package/src/ceremonies/index.ts +189 -46
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/approval-override.ts.txt +6 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/codemod-syntax.ts.txt +3 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/connection-precedence.ts.txt +10 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/docs-conflict.ts.txt +8 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/examples-hoisted-array-spread.ts.txt +15 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/examples-imported-array-spread.ts.txt +9 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/examples-map.ts.txt +5 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/examples-mixed-array-spread.ts.txt +24 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/examples-operation.ts.txt +16 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/factory-map.ts.txt +3 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/hoist-all.ts.txt +31 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/hoisted-const.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/imported-spread.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-map.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-spread-cast-tail.ts.txt +21 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-spread-ekitan.ts.txt +11 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/inline-spread-override.ts.txt +14 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/missing-english-locale.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/no-safety.ts.txt +6 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/non-literal.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/redundant-approval.ts.txt +6 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/safety-conflict.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/stream.ts.txt +7 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/tool-router-spread.ts.txt +15 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/unparseable.ts.txt +4 -0
- package/src/cli/__tests__/fixtures/migrate-operation-declaration/verbatim-template.ts.txt +12 -0
- package/src/cli/commands.ts +33 -0
- package/src/cli/create.ts +50 -37
- package/src/cli/migrate-operation-declaration.ts +1751 -0
- package/src/cli/migrate-operation-shape.ts +184 -0
- package/src/cli/migrate-provider-shape.ts +772 -0
- package/src/cli/prompt-assets.ts +865 -0
- package/src/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/src/cli/templates/provider/Dockerfile.tpl +1 -1
- package/src/cli/templates/provider/README.md.tpl +5 -5
- package/src/cli/templates/provider/index.ts.tpl +6 -3
- package/src/cli/templates/provider/operations/ping.ts.tpl +4 -1
- package/src/cli/templates/provider/provider.json.tpl +6 -0
- package/src/config/loader.ts +690 -163
- package/src/contract-serialization.ts +5 -7
- package/src/contract-types.ts +12 -5
- package/src/contract.ts +24 -10
- package/src/declaration-validation.ts +330 -0
- package/src/define.ts +1048 -131
- package/src/dev.ts +4 -1
- package/src/engine.ts +279 -0
- package/src/error-observability.ts +64 -0
- package/src/error-resolution.ts +127 -0
- package/src/errors.ts +68 -0
- package/src/fixture-sanitization.ts +264 -0
- package/src/health-scenario.ts +875 -0
- package/src/index.ts +220 -13
- package/src/lint.ts +698 -90
- package/src/native-address.ts +340 -0
- package/src/native-egress-policy.ts +358 -0
- package/src/observability.ts +51 -1
- package/src/provider.ts +161 -3
- package/src/runtime/auth-flow.ts +12 -0
- package/src/runtime/browser.ts +661 -63
- package/src/runtime/cache.ts +189 -14
- package/src/runtime/choice-wordlist.ts +145 -0
- package/src/runtime/choice.ts +631 -120
- package/src/runtime/chrome149-header-order.ts +330 -0
- package/src/runtime/env.ts +13 -0
- package/src/runtime/executor.ts +43 -31
- package/src/runtime/http.ts +641 -61
- package/src/runtime/insights.ts +2 -2
- package/src/runtime/instrumentation.ts +520 -15
- package/src/runtime/native-network-errors.ts +99 -0
- package/src/runtime/native-network.ts +1605 -0
- package/src/runtime/ocr.ts +523 -0
- package/src/runtime/otlp.ts +467 -21
- package/src/runtime/proxy-errors.ts +12 -4
- package/src/runtime/proxy-nodemaven.ts +221 -0
- package/src/runtime/proxy-telemetry.ts +244 -75
- package/src/runtime/redirects.ts +66 -0
- package/src/runtime/redis.ts +7 -2
- package/src/runtime/request-options.ts +680 -1
- package/src/runtime/resolver-config.ts +6 -0
- package/src/runtime/resolver-public.ts +23 -0
- package/src/runtime/resolver-shared.ts +33 -0
- package/src/runtime/resolver-vendors/bindings.ts +56 -0
- package/src/runtime/resolver-vendors/browser.ts +533 -0
- package/src/runtime/resolver-vendors/capsolver.ts +707 -0
- package/src/runtime/resolver-vendors/hosts.ts +38 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +540 -0
- package/src/runtime/resolver-vendors/types.ts +212 -0
- package/src/runtime/resolver.ts +1157 -0
- package/src/runtime/secrets.ts +64 -0
- package/src/runtime/state.ts +394 -77
- package/src/runtime/stealth-cookies.ts +132 -0
- package/src/runtime/stealth.ts +1280 -336
- package/src/runtime/stt.ts +1 -19
- package/src/runtime/timeout.ts +18 -0
- package/src/runtime/trace-config.ts +78 -0
- package/src/runtime/trace.ts +57 -17
- package/src/serve.ts +6 -1
- package/src/server/error-observability.ts +1 -0
- package/src/server/index.ts +34 -2
- package/src/server/self-test-input-tokens.ts +29 -14
- package/src/server/self-test.ts +191 -68
- package/src/server/serve-implementation.ts +3378 -0
- package/src/server/serve.ts +1 -1632
- package/src/server/trace-output.ts +129 -0
- package/src/server/types.ts +13 -1
- package/src/stateful/README.md +146 -0
- package/src/stateful/errors.ts +35 -0
- package/src/stateful/http-provider-event-emitter.ts +314 -0
- package/src/stateful/http-session-owner-registry.ts +306 -0
- package/src/stateful/index.ts +18 -0
- package/src/stateful/provider-event-delivery-failures.ts +80 -0
- package/src/stateful/provider-event-pipeline-metrics.ts +95 -0
- package/src/stateful/provider-event-pipeline.ts +61 -0
- package/src/stateful/provider-events.ts +462 -0
- package/src/stateful/session-key.ts +111 -0
- package/src/stateful/stateful-provider-adapter-context.ts +59 -0
- package/src/stateful/stateful-provider-adapter-metrics.ts +48 -0
- package/src/stateful/stateful-provider-adapter.ts +562 -0
- package/src/stateful/stateful-provider-observability.ts +261 -0
- package/src/stateful/stateful-provider-owner-forwarder.ts +287 -0
- package/src/stateful/stateful-provider-runtime-context.ts +92 -0
- package/src/stateful/stateful-provider-runtime-executor.ts +96 -0
- package/src/stateful/stateful-provider-session-routing.ts +546 -0
- package/src/stateful/stateful-provider-session-runtime.ts +403 -0
- package/src/stateful-signing.ts +46 -0
- package/src/stealth/profiles.ts +202 -231
- package/src/stream-evidence.ts +988 -0
- package/src/stream.ts +8 -1
- package/src/testing/index.ts +10 -1
- package/src/testing/run.ts +658 -15
- package/src/trace-sanitization.ts +63 -0
- package/src/types.ts +788 -174
- package/dist/cli/templates/provider/CLAUDE.md.tpl +0 -1
- package/src/cli/templates/provider/CLAUDE.md.tpl +0 -1
- /package/dist/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
package/src/config/loader.ts
CHANGED
|
@@ -1,11 +1,31 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { existsSync } from "node:fs";
|
|
3
|
+
import { createRequire } from "node:module";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
|
|
5
|
-
import { Redis } from "ioredis";
|
|
6
|
-
|
|
7
|
-
import type { ProviderProxyPolicy, TraceConfig } from "../types.js";
|
|
8
|
-
|
|
6
|
+
import type { Redis } from "ioredis";
|
|
7
|
+
|
|
8
|
+
import type { ProviderProxyPolicy, ProviderProxyProvider, TraceConfig } from "../types.js";
|
|
9
|
+
import {
|
|
10
|
+
NODEMAVEN_DEFAULT_PROTOCOL,
|
|
11
|
+
NODEMAVEN_FILTER_ENV,
|
|
12
|
+
NODEMAVEN_MAX_POOL_SIZE,
|
|
13
|
+
NODEMAVEN_PASSWORD_ENV,
|
|
14
|
+
NODEMAVEN_USERNAME_ENV,
|
|
15
|
+
type ProxyProtocol,
|
|
16
|
+
nodemavenPoolSize,
|
|
17
|
+
synthesizeNodemavenProxy,
|
|
18
|
+
} from "../runtime/proxy-nodemaven.js";
|
|
19
|
+
|
|
20
|
+
export type { ProxyProtocol } from "../runtime/proxy-nodemaven.js";
|
|
21
|
+
|
|
22
|
+
/** Proxy vendors with SDK-managed resolution. */
|
|
23
|
+
export type ProxyVendorName = "smartproxy" | "nodemaven";
|
|
24
|
+
|
|
25
|
+
// "smartproxy" here is api.smartproxy.org — a residential proxy with an IP
|
|
26
|
+
// extraction API (app_key → raw ip:port pool). It is NOT the company formerly
|
|
27
|
+
// named Smartproxy (smartproxy.com), which rebranded to Decodo in 2025 and is
|
|
28
|
+
// modelled separately as the `decodo` gateway vendor. Do not conflate them.
|
|
9
29
|
export const SMARTPROXY_APP_KEY_ENV = "APIFUSE__PROXY__SMARTPROXY_APP_KEY";
|
|
10
30
|
export const SMARTPROXY_MAX_LIFETIME_MINUTES = 2000;
|
|
11
31
|
export const DEFAULT_SMARTPROXY_POOL_SIZE = 20;
|
|
@@ -17,15 +37,6 @@ export const PROVIDER_CACHE_REDIS_URL_ENV = "APIFUSE__PROVIDER__CACHE_REDIS_URL"
|
|
|
17
37
|
export const PROVIDER_STATE_REDIS_URL_ENV = "APIFUSE__PROVIDER__STATE_REDIS_URL";
|
|
18
38
|
export const REDIS_URL_ENV = "APIFUSE__REDIS__URL";
|
|
19
39
|
|
|
20
|
-
export type ProxyOptions = {
|
|
21
|
-
url: string;
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
export type ProxyConfig = Partial<ProxyOptions> & {
|
|
25
|
-
provider?: string;
|
|
26
|
-
apiKey?: string;
|
|
27
|
-
};
|
|
28
|
-
|
|
29
40
|
export type BrowserConfig = {
|
|
30
41
|
executablePath?: string;
|
|
31
42
|
headless?: boolean;
|
|
@@ -37,7 +48,6 @@ export type SessionConfig = {
|
|
|
37
48
|
};
|
|
38
49
|
|
|
39
50
|
export type ApiFuseConfig = {
|
|
40
|
-
proxy?: ProxyConfig;
|
|
41
51
|
browser?: BrowserConfig;
|
|
42
52
|
session?: SessionConfig;
|
|
43
53
|
trace?: TraceConfig;
|
|
@@ -47,12 +57,33 @@ export type ApiFuseConfig = {
|
|
|
47
57
|
export type ProxyResolutionOptions = {
|
|
48
58
|
proxy?: string;
|
|
49
59
|
upstream?: { proxy?: boolean | ProviderProxyPolicy };
|
|
50
|
-
apifuseConfig?: Pick<ApiFuseConfig, "proxy">;
|
|
51
60
|
proxyPolicy?: ProviderProxyPolicy;
|
|
52
61
|
affinityKey?: string;
|
|
53
62
|
/** Zero-based proxy-pool attempt index used by SDK transports for failover. */
|
|
54
63
|
proxyAttempt?: number;
|
|
64
|
+
/**
|
|
65
|
+
* Tunnelling protocols the calling transport can use. When a resolved
|
|
66
|
+
* protocol is not in this set the resolver fails with
|
|
67
|
+
* `PROXY_PROTOCOL_UNSUPPORTED` instead of silently downgrading. Unset means
|
|
68
|
+
* permissive (both protocols allowed).
|
|
69
|
+
*/
|
|
70
|
+
transportProtocols?: readonly ProxyProtocol[];
|
|
71
|
+
/**
|
|
72
|
+
* Explicit protocol override. Internal — for the verification harness and
|
|
73
|
+
* tests, or an advanced caller. Normal callers omit it and each vendor uses
|
|
74
|
+
* its own benchmarked default protocol (see VENDOR_DEFAULT_PROTOCOL). Not an
|
|
75
|
+
* env var and not a provider-policy field.
|
|
76
|
+
*/
|
|
77
|
+
protocol?: ProxyProtocol;
|
|
78
|
+
/**
|
|
79
|
+
* Gateway pool "refresh" generation. Bumped by transports on pool refresh to
|
|
80
|
+
* derive a fresh gateway session set (ignored by allocation-style vendors,
|
|
81
|
+
* whose refresh is driven by cache invalidation).
|
|
82
|
+
*/
|
|
83
|
+
proxyRefreshEpoch?: number;
|
|
55
84
|
telemetry?: ProxyTelemetrySink;
|
|
85
|
+
/** Engine-owned credential snapshot. Provider code must never populate this field. */
|
|
86
|
+
engineCredentials?: Readonly<Record<string, string>>;
|
|
56
87
|
};
|
|
57
88
|
|
|
58
89
|
export type ProxyCacheStatus =
|
|
@@ -73,8 +104,14 @@ export type SmartproxyAllocatorBodyClass =
|
|
|
73
104
|
| "text_without_proxies"
|
|
74
105
|
| "usable_proxy_endpoints";
|
|
75
106
|
|
|
107
|
+
export type ProxyUserAgentSource = "declared" | "defaulted";
|
|
108
|
+
|
|
76
109
|
export type ProxyResolutionTelemetryEvent = {
|
|
77
|
-
provider:
|
|
110
|
+
provider: ProxyVendorName;
|
|
111
|
+
/** Defaults to `"ok"` when omitted. */
|
|
112
|
+
outcome?: "ok" | "error";
|
|
113
|
+
userAgentSource?: ProxyUserAgentSource;
|
|
114
|
+
protocol?: ProxyProtocol;
|
|
78
115
|
cacheStatus: ProxyCacheStatus;
|
|
79
116
|
cacheHit: boolean;
|
|
80
117
|
resolutionMs: number;
|
|
@@ -92,7 +129,7 @@ export type ProxyResolutionTelemetryEvent = {
|
|
|
92
129
|
};
|
|
93
130
|
|
|
94
131
|
export type ProxyAttemptTelemetryEvent = {
|
|
95
|
-
provider:
|
|
132
|
+
provider: ProxyVendorName;
|
|
96
133
|
attempt: number;
|
|
97
134
|
poolIndex?: number;
|
|
98
135
|
proxyHash?: string;
|
|
@@ -102,31 +139,69 @@ export type ProxyAttemptTelemetryEvent = {
|
|
|
102
139
|
durationMs?: number;
|
|
103
140
|
};
|
|
104
141
|
|
|
142
|
+
export type ProxyVendorFailoverTelemetryEvent = {
|
|
143
|
+
/** Vendor that failed or was skipped. */
|
|
144
|
+
vendor: ProxyVendorName;
|
|
145
|
+
/** Vendor tried next, or undefined when the chain is exhausted. */
|
|
146
|
+
nextVendor?: ProxyVendorName;
|
|
147
|
+
phase: "resolution" | "transport";
|
|
148
|
+
reason: "no_credentials" | "allocation_failed" | "pool_exhausted" | "protocol_unsupported";
|
|
149
|
+
attempt?: number;
|
|
150
|
+
};
|
|
151
|
+
|
|
105
152
|
export type ProxyTelemetrySink = {
|
|
106
153
|
recordProxyResolution(event: ProxyResolutionTelemetryEvent): void;
|
|
107
154
|
recordProxyAttempt?(event: ProxyAttemptTelemetryEvent): void;
|
|
155
|
+
recordProxyVendorFailover?(event: ProxyVendorFailoverTelemetryEvent): void;
|
|
108
156
|
};
|
|
109
157
|
|
|
158
|
+
export type ProxyResolutionSource =
|
|
159
|
+
| "explicit"
|
|
160
|
+
| "env"
|
|
161
|
+
| "config"
|
|
162
|
+
| "smartproxy-allocator"
|
|
163
|
+
| "nodemaven-gateway";
|
|
164
|
+
|
|
110
165
|
export type ResolvedProxyConfig = {
|
|
111
166
|
shouldWarn: boolean;
|
|
112
167
|
url?: string;
|
|
113
|
-
|
|
168
|
+
/** SDK-native vendor that supplied the URL, when applicable. */
|
|
169
|
+
vendor?: ProxyVendorName;
|
|
170
|
+
source?: ProxyResolutionSource;
|
|
171
|
+
protocol?: ProxyProtocol;
|
|
114
172
|
diagnostics?: Record<string, string | number | boolean>;
|
|
115
173
|
};
|
|
116
174
|
|
|
175
|
+
export type ProxyResolutionErrorCode =
|
|
176
|
+
| "PROXY_REQUIRED"
|
|
177
|
+
| "PROXY_ALLOCATION_FAILED"
|
|
178
|
+
| "PROXY_PROTOCOL_UNSUPPORTED";
|
|
179
|
+
|
|
117
180
|
export class ProxyResolutionError extends Error {
|
|
118
|
-
readonly code:
|
|
181
|
+
readonly code: ProxyResolutionErrorCode;
|
|
119
182
|
readonly telemetry?: ProxyResolutionTelemetryEvent;
|
|
183
|
+
readonly vendor?: ProxyVendorName;
|
|
184
|
+
readonly vendorChain?: ProxyVendorName[];
|
|
185
|
+
readonly protocol?: ProxyProtocol;
|
|
120
186
|
|
|
121
187
|
constructor(
|
|
122
|
-
code:
|
|
188
|
+
code: ProxyResolutionErrorCode,
|
|
123
189
|
message: string,
|
|
124
|
-
options?: {
|
|
190
|
+
options?: {
|
|
191
|
+
cause?: unknown;
|
|
192
|
+
telemetry?: ProxyResolutionTelemetryEvent;
|
|
193
|
+
vendor?: ProxyVendorName;
|
|
194
|
+
vendorChain?: ProxyVendorName[];
|
|
195
|
+
protocol?: ProxyProtocol;
|
|
196
|
+
},
|
|
125
197
|
) {
|
|
126
198
|
super(message, options);
|
|
127
199
|
this.name = "ProxyResolutionError";
|
|
128
200
|
this.code = code;
|
|
129
201
|
this.telemetry = options?.telemetry;
|
|
202
|
+
this.vendor = options?.vendor;
|
|
203
|
+
this.vendorChain = options?.vendorChain;
|
|
204
|
+
this.protocol = options?.protocol;
|
|
130
205
|
}
|
|
131
206
|
}
|
|
132
207
|
|
|
@@ -149,6 +224,7 @@ type ProxyRedisClient = Pick<
|
|
|
149
224
|
>;
|
|
150
225
|
|
|
151
226
|
const proxyCache = new Map<string, CachedProxyPool>();
|
|
227
|
+
const require = createRequire(import.meta.url);
|
|
152
228
|
const proxyInflight = new Map<string, Promise<SmartproxyAllocationResult>>();
|
|
153
229
|
const invalidatedProxyKeys = new Map<string, number>();
|
|
154
230
|
const redisClients = new Map<string, ProxyRedisClient>();
|
|
@@ -208,7 +284,8 @@ function getProxyRedis(): ProxyRedisClient | undefined {
|
|
|
208
284
|
const existing = redisClients.get(redisUrl);
|
|
209
285
|
if (existing) return existing;
|
|
210
286
|
|
|
211
|
-
const
|
|
287
|
+
const { Redis: RedisClient } = require("ioredis") as typeof import("ioredis");
|
|
288
|
+
const redis = new RedisClient(redisUrl, {
|
|
212
289
|
connectTimeout: REDIS_TIMEOUT_MS,
|
|
213
290
|
enableOfflineQueue: false,
|
|
214
291
|
lazyConnect: true,
|
|
@@ -296,6 +373,7 @@ function telemetryForFailure(
|
|
|
296
373
|
): ProxyResolutionTelemetryEvent {
|
|
297
374
|
return {
|
|
298
375
|
provider: "smartproxy",
|
|
376
|
+
outcome: "error",
|
|
299
377
|
cacheStatus,
|
|
300
378
|
cacheHit: false,
|
|
301
379
|
resolutionMs: Math.max(0, Date.now() - startedAt),
|
|
@@ -365,70 +443,7 @@ function serializeSmartproxyPool(pool: CachedProxyPool): string {
|
|
|
365
443
|
|
|
366
444
|
function normalizeProxyUrl(url?: string): string | undefined {
|
|
367
445
|
const normalized = url?.trim();
|
|
368
|
-
return normalized
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
function readPositiveIntegerEnv(name: string): string | undefined {
|
|
372
|
-
const raw = process.env[name]?.trim();
|
|
373
|
-
if (!raw) return undefined;
|
|
374
|
-
if (!/^[1-9]\d*$/.test(raw)) {
|
|
375
|
-
throw new Error(`${name} must be a positive integer`);
|
|
376
|
-
}
|
|
377
|
-
return raw;
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
function applyStickyProxySession(proxyUrl: string): string {
|
|
381
|
-
let parsed: URL;
|
|
382
|
-
try {
|
|
383
|
-
parsed = new URL(proxyUrl);
|
|
384
|
-
} catch {
|
|
385
|
-
return proxyUrl;
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
if (!parsed.hostname || !parsed.username || !parsed.password) {
|
|
389
|
-
return proxyUrl;
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
const host = parsed.hostname.toLowerCase();
|
|
393
|
-
if (!host.includes("smartproxy") && !host.includes("decodo")) {
|
|
394
|
-
return proxyUrl;
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
const username = decodeURIComponent(parsed.username);
|
|
398
|
-
const sessionId = process.env.APIFUSE__PROXY__SESSION_ID?.trim() || "apifuse-shared";
|
|
399
|
-
const sessionDuration = readPositiveIntegerEnv("APIFUSE__PROXY__SESSION_DURATION");
|
|
400
|
-
const stickyUsername = host.includes("smartproxy")
|
|
401
|
-
? buildSmartproxyUsername(username, sessionId, sessionDuration)
|
|
402
|
-
: buildDecodoUsername(username, sessionId, sessionDuration ?? "60");
|
|
403
|
-
|
|
404
|
-
parsed.username = stickyUsername;
|
|
405
|
-
return parsed.toString();
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
function buildSmartproxyUsername(
|
|
409
|
-
username: string,
|
|
410
|
-
sessionId: string,
|
|
411
|
-
sessionDuration?: string,
|
|
412
|
-
): string {
|
|
413
|
-
const parts = username.split("_");
|
|
414
|
-
const configuredLife = parts.find((part) => part.startsWith("life-"))?.slice("life-".length);
|
|
415
|
-
const baseUsername = parts
|
|
416
|
-
.filter((part) => !part.startsWith("session-") && !part.startsWith("life-"))
|
|
417
|
-
.join("_");
|
|
418
|
-
return `${baseUsername}_session-${sessionId}_life-${sessionDuration ?? configuredLife ?? "60"}`;
|
|
419
|
-
}
|
|
420
|
-
|
|
421
|
-
function buildDecodoUsername(username: string, sessionId: string, sessionDuration: string): string {
|
|
422
|
-
const withoutSticky = username.replace(/-session-.+-sessionduration-\d+$/, "");
|
|
423
|
-
const baseUsername = withoutSticky.startsWith("user-") ? withoutSticky : `user-${withoutSticky}`;
|
|
424
|
-
return `${baseUsername}-session-${sessionId}-sessionduration-${sessionDuration}`;
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
function syncProxyEnv(config: ApiFuseConfig): void {
|
|
428
|
-
const configProxyUrl = normalizeProxyUrl(config.proxy?.url);
|
|
429
|
-
if (!process.env.APIFUSE__PROXY__URL && configProxyUrl) {
|
|
430
|
-
process.env.APIFUSE__PROXY__URL = configProxyUrl;
|
|
431
|
-
}
|
|
446
|
+
return normalized || undefined;
|
|
432
447
|
}
|
|
433
448
|
|
|
434
449
|
export function resolveProxyConfig(options: ProxyResolutionOptions = {}): ResolvedProxyConfig {
|
|
@@ -448,16 +463,6 @@ export function resolveProxyConfig(options: ProxyResolutionOptions = {}): Resolv
|
|
|
448
463
|
return { shouldWarn: false };
|
|
449
464
|
}
|
|
450
465
|
|
|
451
|
-
const envProxyUrl = normalizeProxyUrl(process.env.APIFUSE__PROXY__URL);
|
|
452
|
-
if (envProxyUrl) {
|
|
453
|
-
return { shouldWarn: false, url: envProxyUrl };
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
const configuredProxyUrl = normalizeProxyUrl(options.apifuseConfig?.proxy?.url);
|
|
457
|
-
if (configuredProxyUrl) {
|
|
458
|
-
return { shouldWarn: false, url: configuredProxyUrl };
|
|
459
|
-
}
|
|
460
|
-
|
|
461
466
|
return { shouldWarn: true };
|
|
462
467
|
}
|
|
463
468
|
|
|
@@ -477,60 +482,286 @@ export async function resolveProxyConfigAsync(
|
|
|
477
482
|
return { shouldWarn: false };
|
|
478
483
|
}
|
|
479
484
|
|
|
480
|
-
const
|
|
481
|
-
if (
|
|
485
|
+
const chain = resolveVendorChain(policy);
|
|
486
|
+
if (chain.length === 0) {
|
|
487
|
+
const declared = declaredVendorChain(policy);
|
|
488
|
+
const deprecated = declared.filter((vendor) => vendor === "decodo" || vendor === "custom");
|
|
489
|
+
if (policy.mode === "required") {
|
|
490
|
+
const providerIds =
|
|
491
|
+
declared.length > 0 ? declared.map((vendor) => `"${vendor}"`).join(", ") : "none";
|
|
492
|
+
const deprecatedDetail =
|
|
493
|
+
deprecated.length > 0
|
|
494
|
+
? ` Deprecated vendor(s): ${deprecated.map((vendor) => `"${vendor}"`).join(", ")}.`
|
|
495
|
+
: "";
|
|
496
|
+
throw new ProxyResolutionError(
|
|
497
|
+
"PROXY_REQUIRED",
|
|
498
|
+
`Required proxy policy has no SDK-managed adapter for provider id(s): ${providerIds}.${deprecatedDetail} Use "smartproxy" or "nodemaven".`,
|
|
499
|
+
);
|
|
500
|
+
}
|
|
501
|
+
// Deprecated decodo/custom providers have no SDK-managed adapter. Optional
|
|
502
|
+
// policies preserve the warning-only behavior and may continue directly.
|
|
482
503
|
return resolveProxyConfig({
|
|
483
504
|
...options,
|
|
484
505
|
upstream: { proxy: true },
|
|
485
506
|
});
|
|
486
507
|
}
|
|
487
508
|
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
509
|
+
// Protocol is chosen per vendor (each vendor's benchmarked-best), with an
|
|
510
|
+
// optional explicit override for the harness/tests. Both are tunnelling
|
|
511
|
+
// schemes. transportProtocols is what the calling transport can actually use.
|
|
512
|
+
const transportProtocols = options.transportProtocols ?? (["http", "socks5"] as const);
|
|
513
|
+
|
|
514
|
+
const sizes = chain.map((vendor) => vendorPoolSize(vendor, policy));
|
|
515
|
+
const total = sizes.reduce((sum, size) => sum + size, 0);
|
|
516
|
+
const normalizedAttempt = normalizeAttemptIndex(options.proxyAttempt);
|
|
517
|
+
const { vendorIndex: startVendorIndex, poolIndex: startPoolIndex } = mapFlatAttempt(
|
|
518
|
+
total > 0 ? normalizedAttempt % total : 0,
|
|
519
|
+
sizes,
|
|
520
|
+
);
|
|
521
|
+
const refreshEpoch = normalizeAttemptIndex(options.proxyRefreshEpoch);
|
|
522
|
+
|
|
523
|
+
let lastError: unknown;
|
|
524
|
+
let blockedProtocol: ProxyProtocol | undefined;
|
|
525
|
+
for (let vendorIndex = startVendorIndex; vendorIndex < chain.length; vendorIndex++) {
|
|
526
|
+
const vendor = chain[vendorIndex] as ProxyVendorName;
|
|
527
|
+
const nextVendor = chain[vendorIndex + 1];
|
|
528
|
+
const poolIndex = vendorIndex === startVendorIndex ? startPoolIndex : 0;
|
|
529
|
+
const protocol = options.protocol ?? VENDOR_DEFAULT_PROTOCOL[vendor];
|
|
530
|
+
|
|
531
|
+
if (!vendorHasCredentials(vendor, options.engineCredentials)) {
|
|
532
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
533
|
+
vendor,
|
|
534
|
+
nextVendor,
|
|
535
|
+
phase: "resolution",
|
|
536
|
+
reason: "no_credentials",
|
|
537
|
+
});
|
|
538
|
+
continue;
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
// The calling transport must be able to use this vendor's protocol; if not,
|
|
542
|
+
// fail over to the next vendor rather than silently downgrading.
|
|
543
|
+
if (!transportProtocols.includes(protocol)) {
|
|
544
|
+
blockedProtocol = protocol;
|
|
545
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
546
|
+
vendor,
|
|
547
|
+
nextVendor,
|
|
548
|
+
phase: "resolution",
|
|
549
|
+
reason: "protocol_unsupported",
|
|
550
|
+
});
|
|
551
|
+
continue;
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
try {
|
|
555
|
+
return await resolveWithVendor(vendor, policy, options, {
|
|
556
|
+
protocol,
|
|
557
|
+
poolIndex,
|
|
558
|
+
refreshEpoch,
|
|
559
|
+
credentials: options.engineCredentials,
|
|
560
|
+
});
|
|
561
|
+
} catch (error) {
|
|
562
|
+
// Config/programming errors (invalid filter, etc.) are not vendor
|
|
563
|
+
// outages — propagate them rather than failing over.
|
|
564
|
+
if (!(error instanceof ProxyResolutionError)) {
|
|
565
|
+
throw error;
|
|
566
|
+
}
|
|
567
|
+
if (error.telemetry) {
|
|
568
|
+
options.telemetry?.recordProxyResolution(error.telemetry);
|
|
569
|
+
}
|
|
570
|
+
lastError = error;
|
|
571
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
572
|
+
vendor,
|
|
573
|
+
nextVendor,
|
|
574
|
+
phase: "resolution",
|
|
575
|
+
reason: "allocation_failed",
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
if (policy.mode === "required") {
|
|
581
|
+
if (lastError) {
|
|
582
|
+
throw lastError instanceof ProxyResolutionError
|
|
583
|
+
? lastError
|
|
584
|
+
: new ProxyResolutionError(
|
|
585
|
+
"PROXY_ALLOCATION_FAILED",
|
|
586
|
+
`All proxy vendors [${chain.join(", ")}] failed for required proxy egress.`,
|
|
587
|
+
{ cause: lastError, vendorChain: chain },
|
|
588
|
+
);
|
|
589
|
+
}
|
|
590
|
+
if (blockedProtocol) {
|
|
491
591
|
throw new ProxyResolutionError(
|
|
492
|
-
"
|
|
493
|
-
`
|
|
592
|
+
"PROXY_PROTOCOL_UNSUPPORTED",
|
|
593
|
+
`No proxy vendor in [${chain.join(", ")}] could serve a protocol supported by this transport (supports: ${transportProtocols.join(", ")}; vendor wanted "${blockedProtocol}"). Route this provider through the stealth transport.`,
|
|
594
|
+
{ protocol: blockedProtocol, vendorChain: chain },
|
|
494
595
|
);
|
|
495
596
|
}
|
|
496
|
-
|
|
597
|
+
throw new ProxyResolutionError(
|
|
598
|
+
"PROXY_REQUIRED",
|
|
599
|
+
`Proxy egress is required but no vendor credentials are configured. Missing: ${chain
|
|
600
|
+
.map((vendor) => `${missingCredentialEnv(vendor)} (${vendor})`)
|
|
601
|
+
.join(", ")}.`,
|
|
602
|
+
{ vendorChain: chain },
|
|
603
|
+
);
|
|
497
604
|
}
|
|
498
|
-
|
|
605
|
+
return { shouldWarn: true };
|
|
606
|
+
}
|
|
499
607
|
|
|
608
|
+
/**
|
|
609
|
+
* Resolve the proxy URL for a provider-owned consumer such as a CAPTCHA solver.
|
|
610
|
+
* Vendor allocation and failover remain owned by the SDK.
|
|
611
|
+
*/
|
|
612
|
+
export async function resolveProxy(
|
|
613
|
+
options: ProxyResolutionOptions = {},
|
|
614
|
+
): Promise<ResolvedProxyConfig> {
|
|
615
|
+
const resolved = await resolveProxyConfigAsync(options);
|
|
616
|
+
const vendor = vendorFromResolvedSource(resolved.source);
|
|
617
|
+
return vendor ? { ...resolved, vendor } : resolved;
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
/**
|
|
621
|
+
* Each vendor's default egress protocol, chosen from live KR benchmarks. HTTP
|
|
622
|
+
* CONNECT wins for nodemaven (socks5 adds ~500ms through the gateway) and ties
|
|
623
|
+
* for smartproxy, and is the only protocol ctx.http (Bun native fetch) supports.
|
|
624
|
+
* Override per call via ProxyResolutionOptions.protocol (harness/tests).
|
|
625
|
+
*/
|
|
626
|
+
export const VENDOR_DEFAULT_PROTOCOL: Readonly<Record<ProxyVendorName, ProxyProtocol>> = {
|
|
627
|
+
smartproxy: "http",
|
|
628
|
+
nodemaven: NODEMAVEN_DEFAULT_PROTOCOL,
|
|
629
|
+
};
|
|
630
|
+
|
|
631
|
+
/**
|
|
632
|
+
* Guard the No-MITM invariant: a resolved proxy URL must use a tunnelling scheme
|
|
633
|
+
* (http CONNECT or socks5) so the client TLS handshake reaches the origin
|
|
634
|
+
* end-to-end. Anything else would intercept TLS and break fingerprinting.
|
|
635
|
+
*/
|
|
636
|
+
export function assertTunnelingScheme(url: string): void {
|
|
637
|
+
let scheme: string;
|
|
500
638
|
try {
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
639
|
+
scheme = new URL(url).protocol.replace(/:$/, "").toLowerCase();
|
|
640
|
+
} catch {
|
|
641
|
+
throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `Malformed proxy URL: ${url}`);
|
|
642
|
+
}
|
|
643
|
+
if (scheme !== "http" && scheme !== "socks5") {
|
|
644
|
+
throw new ProxyResolutionError(
|
|
645
|
+
"PROXY_ALLOCATION_FAILED",
|
|
646
|
+
`Resolved proxy scheme "${scheme}" is not a tunnelling scheme (expected http or socks5). Refusing to route TLS through a non-tunnelling proxy.`,
|
|
506
647
|
);
|
|
507
|
-
|
|
508
|
-
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
export type ProxyVendorResolutionContext = {
|
|
652
|
+
readonly protocol: ProxyProtocol;
|
|
653
|
+
readonly poolIndex: number;
|
|
654
|
+
readonly refreshEpoch: number;
|
|
655
|
+
/** Explicit vendor credentials. Omit only on the legacy ambient-env path. */
|
|
656
|
+
readonly credentials?: Readonly<Record<string, string>>;
|
|
657
|
+
/** Disable non-policy env defaults for deterministic injected adapters. */
|
|
658
|
+
readonly ambientDefaults?: boolean;
|
|
659
|
+
/** Disable env-discovered Redis sharing for deterministic injected adapters. */
|
|
660
|
+
readonly sharedCache?: boolean;
|
|
661
|
+
};
|
|
662
|
+
|
|
663
|
+
export async function resolveWithVendor(
|
|
664
|
+
vendor: ProxyVendorName,
|
|
665
|
+
policy: ProviderProxyPolicy,
|
|
666
|
+
options: ProxyResolutionOptions,
|
|
667
|
+
context: ProxyVendorResolutionContext,
|
|
668
|
+
): Promise<ResolvedProxyConfig> {
|
|
669
|
+
if (vendor === "nodemaven") {
|
|
670
|
+
const startedAt = Date.now();
|
|
671
|
+
const username = (
|
|
672
|
+
context.credentials === undefined
|
|
673
|
+
? process.env[NODEMAVEN_USERNAME_ENV]
|
|
674
|
+
: context.credentials[NODEMAVEN_USERNAME_ENV]
|
|
675
|
+
)?.trim();
|
|
676
|
+
const password = (
|
|
677
|
+
context.credentials === undefined
|
|
678
|
+
? process.env[NODEMAVEN_PASSWORD_ENV]
|
|
679
|
+
: context.credentials[NODEMAVEN_PASSWORD_ENV]
|
|
680
|
+
)?.trim();
|
|
681
|
+
const filter =
|
|
682
|
+
context.credentials === undefined
|
|
683
|
+
? process.env[NODEMAVEN_FILTER_ENV]
|
|
684
|
+
: context.credentials[NODEMAVEN_FILTER_ENV];
|
|
685
|
+
if (!username || !password) {
|
|
686
|
+
throw new ProxyResolutionError(
|
|
687
|
+
"PROXY_ALLOCATION_FAILED",
|
|
688
|
+
`NodeMaven credentials missing: set ${NODEMAVEN_USERNAME_ENV} and ${NODEMAVEN_PASSWORD_ENV}.`,
|
|
689
|
+
{ vendor: "nodemaven" },
|
|
690
|
+
);
|
|
691
|
+
}
|
|
692
|
+
const synthesized = synthesizeNodemavenProxy({
|
|
693
|
+
policy,
|
|
694
|
+
credentials: {
|
|
695
|
+
username,
|
|
696
|
+
password,
|
|
697
|
+
...(filter ? { filter } : {}),
|
|
698
|
+
},
|
|
699
|
+
affinityKey: options.affinityKey,
|
|
700
|
+
protocol: context.protocol,
|
|
701
|
+
poolIndex: context.poolIndex,
|
|
702
|
+
refreshEpoch: context.refreshEpoch,
|
|
703
|
+
country: resolveSmartproxyCountry(policy, context.ambientDefaults !== false),
|
|
704
|
+
});
|
|
705
|
+
options.telemetry?.recordProxyResolution({
|
|
706
|
+
provider: "nodemaven",
|
|
707
|
+
protocol: synthesized.protocol,
|
|
708
|
+
cacheStatus: "disabled",
|
|
709
|
+
cacheHit: false,
|
|
710
|
+
resolutionMs: Math.max(0, Date.now() - startedAt),
|
|
711
|
+
attempts: 1,
|
|
712
|
+
});
|
|
713
|
+
assertTunnelingScheme(synthesized.url);
|
|
509
714
|
return {
|
|
510
715
|
shouldWarn: false,
|
|
511
|
-
url:
|
|
512
|
-
source: "
|
|
716
|
+
url: synthesized.url,
|
|
717
|
+
source: "nodemaven-gateway",
|
|
718
|
+
protocol: synthesized.protocol,
|
|
513
719
|
diagnostics: {
|
|
514
|
-
...
|
|
515
|
-
|
|
516
|
-
poolIndex,
|
|
720
|
+
...synthesized.diagnostics,
|
|
721
|
+
poolIndex: context.poolIndex,
|
|
517
722
|
},
|
|
518
723
|
};
|
|
519
|
-
} catch (error) {
|
|
520
|
-
if (error instanceof ProxyResolutionError && error.telemetry) {
|
|
521
|
-
options.telemetry?.recordProxyResolution(error.telemetry);
|
|
522
|
-
}
|
|
523
|
-
if (policy.mode === "required") {
|
|
524
|
-
throw error instanceof ProxyResolutionError
|
|
525
|
-
? error
|
|
526
|
-
: new ProxyResolutionError(
|
|
527
|
-
"PROXY_ALLOCATION_FAILED",
|
|
528
|
-
"Smartproxy allocator failed for required proxy egress.",
|
|
529
|
-
{ cause: error },
|
|
530
|
-
);
|
|
531
|
-
}
|
|
532
|
-
return { shouldWarn: true };
|
|
533
724
|
}
|
|
725
|
+
|
|
726
|
+
// smartproxy allocation-style vendor.
|
|
727
|
+
const appKey = (
|
|
728
|
+
context.credentials === undefined
|
|
729
|
+
? process.env[SMARTPROXY_APP_KEY_ENV]
|
|
730
|
+
: context.credentials[SMARTPROXY_APP_KEY_ENV]
|
|
731
|
+
)?.trim();
|
|
732
|
+
if (!appKey) {
|
|
733
|
+
// Guarded by vendorHasCredentials; treated as a vendor-internal failure.
|
|
734
|
+
throw new ProxyResolutionError(
|
|
735
|
+
"PROXY_ALLOCATION_FAILED",
|
|
736
|
+
`${SMARTPROXY_APP_KEY_ENV} is not configured.`,
|
|
737
|
+
{ vendor: "smartproxy" },
|
|
738
|
+
);
|
|
739
|
+
}
|
|
740
|
+
const lifetimeMinutes = resolveSmartproxyLifetime(policy, context.ambientDefaults !== false);
|
|
741
|
+
const allocated = await allocateSmartproxy(
|
|
742
|
+
policy,
|
|
743
|
+
appKey,
|
|
744
|
+
lifetimeMinutes,
|
|
745
|
+
options.affinityKey,
|
|
746
|
+
context.protocol,
|
|
747
|
+
context.ambientDefaults !== false,
|
|
748
|
+
context.sharedCache !== false,
|
|
749
|
+
);
|
|
750
|
+
options.telemetry?.recordProxyResolution({ ...allocated.telemetry, protocol: context.protocol });
|
|
751
|
+
const poolIndex = selectProxyPoolIndex(allocated.pool.urls.length, context.poolIndex);
|
|
752
|
+
const url = allocated.pool.urls[poolIndex];
|
|
753
|
+
if (url) assertTunnelingScheme(url);
|
|
754
|
+
return {
|
|
755
|
+
shouldWarn: false,
|
|
756
|
+
url,
|
|
757
|
+
source: "smartproxy-allocator",
|
|
758
|
+
protocol: context.protocol,
|
|
759
|
+
diagnostics: {
|
|
760
|
+
...allocated.pool.diagnostics,
|
|
761
|
+
poolSize: allocated.pool.urls.length,
|
|
762
|
+
poolIndex,
|
|
763
|
+
},
|
|
764
|
+
};
|
|
534
765
|
}
|
|
535
766
|
|
|
536
767
|
function resolvePolicy(options: ProxyResolutionOptions): ProviderProxyPolicy | undefined {
|
|
@@ -544,21 +775,240 @@ function resolvePolicy(options: ProxyResolutionOptions): ProviderProxyPolicy | u
|
|
|
544
775
|
return undefined;
|
|
545
776
|
}
|
|
546
777
|
|
|
547
|
-
function
|
|
548
|
-
return
|
|
549
|
-
|
|
778
|
+
function isRegistryVendor(name: string | undefined): name is ProxyVendorName {
|
|
779
|
+
return name === "smartproxy" || name === "nodemaven";
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
function declaredVendorChain(policy: ProviderProxyPolicy): ProviderProxyProvider[] {
|
|
783
|
+
const declared = policy.providers?.length
|
|
784
|
+
? policy.providers
|
|
785
|
+
: [policy.provider ?? envDefaultProvider()];
|
|
786
|
+
return declared.filter((vendor): vendor is ProviderProxyProvider => vendor !== undefined);
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
/**
|
|
790
|
+
* Ordered list of SDK-native proxy vendors declared by the policy. `providers`
|
|
791
|
+
* takes precedence over the legacy singular `provider`; the platform default
|
|
792
|
+
* env is the final fallback. Non-registry names (decodo/custom) are dropped so
|
|
793
|
+
* an all-deprecated chain has no managed adapter.
|
|
794
|
+
*/
|
|
795
|
+
export function resolveVendorChain(policy: ProviderProxyPolicy): ProxyVendorName[] {
|
|
796
|
+
const chain: ProxyVendorName[] = [];
|
|
797
|
+
for (const name of declaredVendorChain(policy)) {
|
|
798
|
+
if (isRegistryVendor(name) && !chain.includes(name)) {
|
|
799
|
+
chain.push(name);
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
return chain;
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
function envDefaultProvider(): ProviderProxyProvider | undefined {
|
|
806
|
+
const raw = process.env[DEFAULT_PROXY_PROVIDER_ENV]?.trim().toLowerCase();
|
|
807
|
+
return (raw as ProviderProxyProvider | undefined) ?? undefined;
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
function vendorHasCredentials(
|
|
811
|
+
vendor: ProxyVendorName,
|
|
812
|
+
credentials?: Readonly<Record<string, string>>,
|
|
813
|
+
): boolean {
|
|
814
|
+
if (vendor === "nodemaven") {
|
|
815
|
+
return Boolean(
|
|
816
|
+
(credentials === undefined
|
|
817
|
+
? process.env[NODEMAVEN_USERNAME_ENV]
|
|
818
|
+
: credentials[NODEMAVEN_USERNAME_ENV]
|
|
819
|
+
)?.trim() &&
|
|
820
|
+
(credentials === undefined
|
|
821
|
+
? process.env[NODEMAVEN_PASSWORD_ENV]
|
|
822
|
+
: credentials[NODEMAVEN_PASSWORD_ENV]
|
|
823
|
+
)?.trim(),
|
|
824
|
+
);
|
|
825
|
+
}
|
|
826
|
+
return Boolean(
|
|
827
|
+
(credentials === undefined
|
|
828
|
+
? process.env[SMARTPROXY_APP_KEY_ENV]
|
|
829
|
+
: credentials[SMARTPROXY_APP_KEY_ENV]
|
|
830
|
+
)?.trim(),
|
|
550
831
|
);
|
|
551
832
|
}
|
|
552
833
|
|
|
553
|
-
function
|
|
834
|
+
function missingCredentialEnv(vendor: ProxyVendorName): string {
|
|
835
|
+
return vendor === "nodemaven" ? "APIFUSE__PROXY__NODEMAVEN_USERNAME" : SMARTPROXY_APP_KEY_ENV;
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
function vendorPoolSize(vendor: ProxyVendorName, policy: ProviderProxyPolicy): number {
|
|
839
|
+
return vendor === "nodemaven" ? nodemavenPoolSize(policy) : resolveSmartproxyPoolSize(policy);
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
/**
|
|
843
|
+
* Total attempt span across a policy's vendor chain — the sum of each vendor's
|
|
844
|
+
* pool size. Transports use this so successive attempts rotate a vendor's pool
|
|
845
|
+
* and then fail over to the next vendor via the flat attempt index. With one
|
|
846
|
+
* vendor this equals that vendor's pool size (today's behaviour).
|
|
847
|
+
*/
|
|
848
|
+
export function resolvePolicyProxyPoolSpan(policy: ProviderProxyPolicy): number {
|
|
849
|
+
const chain = resolveVendorChain(policy);
|
|
850
|
+
if (chain.length === 0) return resolveSmartproxyPoolSize(policy);
|
|
851
|
+
return chain.reduce((sum, vendor) => sum + vendorPoolSize(vendor, policy), 0);
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
function vendorMaxPoolSize(vendor: ProxyVendorName): number {
|
|
855
|
+
return vendor === "nodemaven" ? NODEMAVEN_MAX_POOL_SIZE : SMARTPROXY_MAX_POOL_SIZE;
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
/**
|
|
859
|
+
* Absolute upper bound on a chain's attempt span — the sum of each vendor's
|
|
860
|
+
* *maximum* pool size. Unlike `resolvePolicyProxyPoolSpan` (the configured
|
|
861
|
+
* span), this backstop is independent of `session.poolSize`, so it never
|
|
862
|
+
* truncates a legitimately large pool below the point where the flat attempt
|
|
863
|
+
* index would cross into the next vendor (e.g. a 50-slot NodeMaven pool).
|
|
864
|
+
*/
|
|
865
|
+
export function maxPolicyProxyPoolSpan(policy: ProviderProxyPolicy): number {
|
|
866
|
+
const chain = resolveVendorChain(policy);
|
|
867
|
+
if (chain.length === 0) return SMARTPROXY_MAX_POOL_SIZE;
|
|
868
|
+
return chain.reduce((sum, vendor) => sum + vendorMaxPoolSize(vendor), 0);
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
const UNSAFE_TRANSPORT_RETRY_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE", "TRACE"]);
|
|
872
|
+
|
|
873
|
+
/**
|
|
874
|
+
* Transport-retry attempt cap for a policy-managed request. A transport failure
|
|
875
|
+
* rotates the flat attempt index onto the *next* endpoint (and, once the index
|
|
876
|
+
* passes the primary vendor's pool span, the *next vendor*), so the cap must be
|
|
877
|
+
* the chain's full pool span for failover to reach the fallback vendor — the
|
|
878
|
+
* per-endpoint retry budget (default 3) never gets there.
|
|
879
|
+
*
|
|
880
|
+
* The span only widens beyond the caller's retry budget when ALL hold:
|
|
881
|
+
* - the request is policy-allocator managed (not a caller-supplied proxy URL);
|
|
882
|
+
* - the caller did NOT pin an explicit retry policy — `HttpRetryOptions.attempts`
|
|
883
|
+
* is the documented total-attempt ceiling and must be honoured verbatim;
|
|
884
|
+
* - the method is safe/idempotent — an unsafe request must never be duplicated
|
|
885
|
+
* across the pool even if some framework default would allow it;
|
|
886
|
+
* - the policy resolves a non-empty *registry* vendor chain (smartproxy /
|
|
887
|
+
* nodemaven). Deprecated vendors (custom / decodo) resolve no managed pool,
|
|
888
|
+
* so there is no possible endpoint crossover — they keep the retry budget.
|
|
889
|
+
*
|
|
890
|
+
* The widened cap is bounded by the chain's true maximum span (sum of each
|
|
891
|
+
* vendor's max pool size), so a large NodeMaven pool (≤50) stays reachable and
|
|
892
|
+
* a pathological chain can never spin unbounded.
|
|
893
|
+
*/
|
|
894
|
+
/**
|
|
895
|
+
* True when a policy request is in *implicit chain-rotation* mode: successive
|
|
896
|
+
* transport attempts rotate the flat index across the concatenated vendor pool
|
|
897
|
+
* spans (and, past the primary vendor's span, into the fallback vendor). This is
|
|
898
|
+
* the ONLY mode in which the transport loop widens its attempt cap AND
|
|
899
|
+
* de-duplicates repeated endpoints — the two behaviours must share one predicate
|
|
900
|
+
* so they never diverge. It holds when ALL of the widening conditions hold:
|
|
901
|
+
* - the request is policy-allocator managed (not a caller-supplied proxy URL);
|
|
902
|
+
* - the caller did NOT pin an explicit retry policy — its `attempts` ceiling is
|
|
903
|
+
* the documented contract and must be honoured verbatim against whatever
|
|
904
|
+
* endpoint each attempt resolves (even a repeated one), so no de-duplication;
|
|
905
|
+
* - the method is safe/idempotent — an unsafe request is never duplicated;
|
|
906
|
+
* - the policy resolves a non-empty registry vendor chain (smartproxy /
|
|
907
|
+
* nodemaven). Deprecated vendors (custom / decodo) resolve no managed
|
|
908
|
+
* endpoint, so there is nothing to rotate or de-duplicate.
|
|
909
|
+
*/
|
|
910
|
+
export function policyRotatesTransportVendorChain(input: {
|
|
911
|
+
policy: ProviderProxyPolicy | undefined;
|
|
912
|
+
usesPolicyAllocator: boolean;
|
|
913
|
+
explicitRetry: boolean;
|
|
914
|
+
method: string;
|
|
915
|
+
}): boolean {
|
|
916
|
+
if (!input.usesPolicyAllocator || !input.policy || input.explicitRetry) {
|
|
917
|
+
return false;
|
|
918
|
+
}
|
|
919
|
+
if (UNSAFE_TRANSPORT_RETRY_METHODS.has(input.method.toUpperCase())) {
|
|
920
|
+
return false;
|
|
921
|
+
}
|
|
922
|
+
return resolveVendorChain(input.policy).length > 0;
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
export function resolvePolicyTransportAttemptCap(input: {
|
|
926
|
+
policy: ProviderProxyPolicy | undefined;
|
|
927
|
+
usesPolicyAllocator: boolean;
|
|
928
|
+
retryAttempts: number;
|
|
929
|
+
explicitRetry: boolean;
|
|
930
|
+
method: string;
|
|
931
|
+
}): number {
|
|
932
|
+
const budget = Math.max(1, Math.floor(input.retryAttempts));
|
|
933
|
+
if (
|
|
934
|
+
!policyRotatesTransportVendorChain({
|
|
935
|
+
policy: input.policy,
|
|
936
|
+
usesPolicyAllocator: input.usesPolicyAllocator,
|
|
937
|
+
explicitRetry: input.explicitRetry,
|
|
938
|
+
method: input.method,
|
|
939
|
+
})
|
|
940
|
+
) {
|
|
941
|
+
return budget;
|
|
942
|
+
}
|
|
943
|
+
const span = Math.min(
|
|
944
|
+
maxPolicyProxyPoolSpan(input.policy as ProviderProxyPolicy),
|
|
945
|
+
resolvePolicyProxyPoolSpan(input.policy as ProviderProxyPolicy),
|
|
946
|
+
);
|
|
947
|
+
return Math.max(budget, span);
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
/**
|
|
951
|
+
* A registry vendor chain (smartproxy/nodemaven) resolves a potentially
|
|
952
|
+
* *different* endpoint per flat attempt index, so a transport retry should
|
|
953
|
+
* advance across endpoints and de-duplicate once the chain stops yielding new
|
|
954
|
+
* ones. Deprecated custom/decodo policies have an empty registry chain and no
|
|
955
|
+
* managed endpoint, so the transport loop has nothing to rotate or de-duplicate.
|
|
956
|
+
*/
|
|
957
|
+
export function policyResolvesRegistryVendorChain(
|
|
958
|
+
policy: ProviderProxyPolicy | undefined,
|
|
959
|
+
): boolean {
|
|
960
|
+
return Boolean(policy) && resolveVendorChain(policy as ProviderProxyPolicy).length > 0;
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
/** Map a resolved proxy source label to the vendor that served it. */
|
|
964
|
+
export function vendorFromResolvedSource(
|
|
965
|
+
source: ResolvedProxyConfig["source"],
|
|
966
|
+
): ProxyVendorName | undefined {
|
|
967
|
+
if (source === "nodemaven-gateway") return "nodemaven";
|
|
968
|
+
if (source === "smartproxy-allocator") return "smartproxy";
|
|
969
|
+
return undefined;
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
function normalizeAttemptIndex(attempt: number | undefined): number {
|
|
973
|
+
return Number.isFinite(attempt) ? Math.max(0, Math.floor(attempt as number)) : 0;
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
/**
|
|
977
|
+
* Map a flat attempt index into (vendorIndex, poolIndex) by concatenating each
|
|
978
|
+
* vendor's pool space in chain order. With a single vendor this reduces to
|
|
979
|
+
* `attempt % poolSize`, preserving today's behaviour exactly.
|
|
980
|
+
*/
|
|
981
|
+
export function mapFlatAttempt(
|
|
982
|
+
flat: number,
|
|
983
|
+
sizes: readonly number[],
|
|
984
|
+
): { vendorIndex: number; poolIndex: number } {
|
|
985
|
+
let cursor = flat;
|
|
986
|
+
for (let vendorIndex = 0; vendorIndex < sizes.length; vendorIndex++) {
|
|
987
|
+
const size = Math.max(1, sizes[vendorIndex] ?? 1);
|
|
988
|
+
if (cursor < size) {
|
|
989
|
+
return { vendorIndex, poolIndex: cursor };
|
|
990
|
+
}
|
|
991
|
+
cursor -= size;
|
|
992
|
+
}
|
|
993
|
+
return { vendorIndex: 0, poolIndex: 0 };
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
function resolveSmartproxyCountry(
|
|
997
|
+
policy: ProviderProxyPolicy,
|
|
998
|
+
ambientDefaults = true,
|
|
999
|
+
): string | undefined {
|
|
554
1000
|
return (
|
|
555
|
-
policy.geo?.country ??
|
|
1001
|
+
policy.geo?.country ??
|
|
1002
|
+
(ambientDefaults
|
|
1003
|
+
? process.env[DEFAULT_PROXY_COUNTRY_ENV]?.trim().toUpperCase() || undefined
|
|
1004
|
+
: undefined)
|
|
556
1005
|
);
|
|
557
1006
|
}
|
|
558
1007
|
|
|
559
|
-
function resolveSmartproxyLifetime(policy: ProviderProxyPolicy): number {
|
|
1008
|
+
function resolveSmartproxyLifetime(policy: ProviderProxyPolicy, ambientDefaults = true): number {
|
|
560
1009
|
const configuredLifetime =
|
|
561
|
-
policy.session?.lifetimeMinutes ??
|
|
1010
|
+
policy.session?.lifetimeMinutes ??
|
|
1011
|
+
(ambientDefaults ? readPositiveNumberEnv(DEFAULT_PROXY_LIFETIME_ENV, 30) : 30);
|
|
562
1012
|
return Math.min(SMARTPROXY_MAX_LIFETIME_MINUTES, Math.max(1, Math.floor(configuredLifetime)));
|
|
563
1013
|
}
|
|
564
1014
|
|
|
@@ -589,13 +1039,21 @@ function selectProxyPoolIndex(poolSize: number, attempt = 0): number {
|
|
|
589
1039
|
|
|
590
1040
|
function buildSmartproxyCacheKey(
|
|
591
1041
|
policy: ProviderProxyPolicy,
|
|
1042
|
+
appKey: string,
|
|
592
1043
|
affinityKey: string | undefined,
|
|
593
1044
|
lifetimeMinutes: number,
|
|
1045
|
+
protocol: ProxyProtocol,
|
|
1046
|
+
ambientDefaults = true,
|
|
594
1047
|
): string {
|
|
595
1048
|
const poolSize = resolveSmartproxyPoolSize(policy);
|
|
596
1049
|
return JSON.stringify({
|
|
597
1050
|
provider: "smartproxy",
|
|
598
|
-
|
|
1051
|
+
credentialHash: createHash("sha256")
|
|
1052
|
+
.update("apifuse-smartproxy-credential:v1\0")
|
|
1053
|
+
.update(appKey)
|
|
1054
|
+
.digest("hex"),
|
|
1055
|
+
protocol,
|
|
1056
|
+
country: resolveSmartproxyCountry(policy, ambientDefaults),
|
|
599
1057
|
affinity: policy.session?.affinity ?? "request",
|
|
600
1058
|
affinityKey: (policy.session?.affinity ?? "request") === "request" ? undefined : affinityKey,
|
|
601
1059
|
lifetimeMinutes,
|
|
@@ -608,8 +1066,18 @@ async function allocateSmartproxy(
|
|
|
608
1066
|
appKey: string,
|
|
609
1067
|
lifetimeMinutes: number,
|
|
610
1068
|
affinityKey: string | undefined,
|
|
1069
|
+
protocol: ProxyProtocol,
|
|
1070
|
+
ambientDefaults = true,
|
|
1071
|
+
sharedCache = true,
|
|
611
1072
|
): Promise<SmartproxyAllocationResult> {
|
|
612
|
-
const cacheKey = buildSmartproxyCacheKey(
|
|
1073
|
+
const cacheKey = buildSmartproxyCacheKey(
|
|
1074
|
+
policy,
|
|
1075
|
+
appKey,
|
|
1076
|
+
affinityKey,
|
|
1077
|
+
lifetimeMinutes,
|
|
1078
|
+
protocol,
|
|
1079
|
+
ambientDefaults,
|
|
1080
|
+
);
|
|
613
1081
|
const startedAt = Date.now();
|
|
614
1082
|
const now = startedAt;
|
|
615
1083
|
const invalidatedUntil = invalidatedProxyKeys.get(cacheKey) ?? 0;
|
|
@@ -617,7 +1085,15 @@ async function allocateSmartproxy(
|
|
|
617
1085
|
const cached = proxyCache.get(cacheKey);
|
|
618
1086
|
if (!skipCached && cached && isFresh(cached, now)) {
|
|
619
1087
|
if (shouldSoftRefresh(cached, now)) {
|
|
620
|
-
void refreshSmartproxyPool(
|
|
1088
|
+
void refreshSmartproxyPool(
|
|
1089
|
+
cacheKey,
|
|
1090
|
+
policy,
|
|
1091
|
+
appKey,
|
|
1092
|
+
lifetimeMinutes,
|
|
1093
|
+
protocol,
|
|
1094
|
+
ambientDefaults,
|
|
1095
|
+
sharedCache,
|
|
1096
|
+
);
|
|
621
1097
|
return {
|
|
622
1098
|
pool: cached,
|
|
623
1099
|
telemetry: telemetryForPool(cached, "soft_stale_refresh", startedAt, {
|
|
@@ -631,7 +1107,7 @@ async function allocateSmartproxy(
|
|
|
631
1107
|
};
|
|
632
1108
|
}
|
|
633
1109
|
|
|
634
|
-
if (!skipCached) {
|
|
1110
|
+
if (!skipCached && sharedCache) {
|
|
635
1111
|
const redisResult = await readSmartproxyRedisPool(cacheKey, startedAt);
|
|
636
1112
|
if (redisResult) return redisResult;
|
|
637
1113
|
}
|
|
@@ -653,6 +1129,9 @@ async function allocateSmartproxy(
|
|
|
653
1129
|
appKey,
|
|
654
1130
|
lifetimeMinutes,
|
|
655
1131
|
startedAt,
|
|
1132
|
+
protocol,
|
|
1133
|
+
ambientDefaults,
|
|
1134
|
+
sharedCache,
|
|
656
1135
|
).finally(() => {
|
|
657
1136
|
proxyInflight.delete(cacheKey);
|
|
658
1137
|
});
|
|
@@ -691,11 +1170,24 @@ async function refreshSmartproxyPool(
|
|
|
691
1170
|
policy: ProviderProxyPolicy,
|
|
692
1171
|
appKey: string,
|
|
693
1172
|
lifetimeMinutes: number,
|
|
1173
|
+
protocol: ProxyProtocol,
|
|
1174
|
+
ambientDefaults: boolean,
|
|
1175
|
+
sharedCache: boolean,
|
|
694
1176
|
): Promise<void> {
|
|
695
1177
|
try {
|
|
696
|
-
await allocateSmartproxyShared(
|
|
697
|
-
|
|
698
|
-
|
|
1178
|
+
await allocateSmartproxyShared(
|
|
1179
|
+
cacheKey,
|
|
1180
|
+
policy,
|
|
1181
|
+
appKey,
|
|
1182
|
+
lifetimeMinutes,
|
|
1183
|
+
Date.now(),
|
|
1184
|
+
protocol,
|
|
1185
|
+
ambientDefaults,
|
|
1186
|
+
sharedCache,
|
|
1187
|
+
{
|
|
1188
|
+
background: true,
|
|
1189
|
+
},
|
|
1190
|
+
);
|
|
699
1191
|
} catch {
|
|
700
1192
|
// Soft refresh is opportunistic; current fresh pool remains usable.
|
|
701
1193
|
}
|
|
@@ -707,9 +1199,12 @@ async function allocateSmartproxyShared(
|
|
|
707
1199
|
appKey: string,
|
|
708
1200
|
lifetimeMinutes: number,
|
|
709
1201
|
startedAt: number,
|
|
1202
|
+
protocol: ProxyProtocol,
|
|
1203
|
+
ambientDefaults: boolean,
|
|
1204
|
+
sharedCache: boolean,
|
|
710
1205
|
options: { background?: boolean } = {},
|
|
711
1206
|
): Promise<SmartproxyAllocationResult> {
|
|
712
|
-
const redis = getProxyRedis();
|
|
1207
|
+
const redis = sharedCache ? getProxyRedis() : undefined;
|
|
713
1208
|
if (!redis || !(await ensureRedisReady(redis))) {
|
|
714
1209
|
return await allocateAndStoreSmartproxyPool(
|
|
715
1210
|
cacheKey,
|
|
@@ -717,7 +1212,7 @@ async function allocateSmartproxyShared(
|
|
|
717
1212
|
appKey,
|
|
718
1213
|
lifetimeMinutes,
|
|
719
1214
|
startedAt,
|
|
720
|
-
{ cacheStatus: "allocator" },
|
|
1215
|
+
{ cacheStatus: "allocator", protocol, ambientDefaults },
|
|
721
1216
|
);
|
|
722
1217
|
}
|
|
723
1218
|
|
|
@@ -742,6 +1237,8 @@ async function allocateSmartproxyShared(
|
|
|
742
1237
|
cacheStatus: options.background ? "soft_stale_refresh" : "allocator",
|
|
743
1238
|
redis,
|
|
744
1239
|
poolKey,
|
|
1240
|
+
protocol,
|
|
1241
|
+
ambientDefaults,
|
|
745
1242
|
},
|
|
746
1243
|
);
|
|
747
1244
|
} finally {
|
|
@@ -897,10 +1394,19 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
897
1394
|
cacheStatus: ProxyCacheStatus;
|
|
898
1395
|
redis?: ProxyRedisClient;
|
|
899
1396
|
poolKey?: string;
|
|
1397
|
+
protocol: ProxyProtocol;
|
|
1398
|
+
ambientDefaults: boolean;
|
|
900
1399
|
},
|
|
901
1400
|
): Promise<SmartproxyAllocationResult> {
|
|
902
1401
|
const poolSize = resolveSmartproxyPoolSize(policy);
|
|
903
|
-
const allocatorUrl = buildSmartproxyAllocatorUrl(
|
|
1402
|
+
const allocatorUrl = buildSmartproxyAllocatorUrl(
|
|
1403
|
+
policy,
|
|
1404
|
+
appKey,
|
|
1405
|
+
lifetimeMinutes,
|
|
1406
|
+
poolSize,
|
|
1407
|
+
options.protocol,
|
|
1408
|
+
options.ambientDefaults,
|
|
1409
|
+
);
|
|
904
1410
|
const allocatorStartedAt = Date.now();
|
|
905
1411
|
const allocatorDeadlineAt = allocatorStartedAt + smartproxyAllocatorDeadlineMs();
|
|
906
1412
|
let allocation: SmartproxyAllocatorSuccess | undefined;
|
|
@@ -914,6 +1420,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
914
1420
|
allocatorUrl,
|
|
915
1421
|
attempt,
|
|
916
1422
|
allocatorDeadlineAt,
|
|
1423
|
+
options.protocol,
|
|
917
1424
|
);
|
|
918
1425
|
if (attemptResult.ok) {
|
|
919
1426
|
allocation = attemptResult;
|
|
@@ -969,7 +1476,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
969
1476
|
expiresAt: allocatedAt + ttlMs,
|
|
970
1477
|
diagnostics: {
|
|
971
1478
|
provider: "smartproxy",
|
|
972
|
-
country: resolveSmartproxyCountry(policy) ?? "default",
|
|
1479
|
+
country: resolveSmartproxyCountry(policy, options.ambientDefaults) ?? "default",
|
|
973
1480
|
lifetimeMinutes,
|
|
974
1481
|
affinity: policy.session?.affinity ?? "request",
|
|
975
1482
|
rawConnect: true,
|
|
@@ -1034,6 +1541,7 @@ async function fetchSmartproxyAllocatorAttempt(
|
|
|
1034
1541
|
allocatorUrl: string,
|
|
1035
1542
|
attempt: number,
|
|
1036
1543
|
deadlineAt: number,
|
|
1544
|
+
protocol: ProxyProtocol,
|
|
1037
1545
|
): Promise<SmartproxyAllocatorAttemptResult> {
|
|
1038
1546
|
const { controller, dispose } = createDeadlineAbortController(deadlineAt);
|
|
1039
1547
|
let response: Response;
|
|
@@ -1077,7 +1585,7 @@ async function fetchSmartproxyAllocatorAttempt(
|
|
|
1077
1585
|
};
|
|
1078
1586
|
}
|
|
1079
1587
|
|
|
1080
|
-
const urls = parseSmartproxyAllocatorProxies(body);
|
|
1588
|
+
const urls = parseSmartproxyAllocatorProxies(body, protocol);
|
|
1081
1589
|
const bodyClass = classifySmartproxyAllocatorBody(body, urls);
|
|
1082
1590
|
if (urls.length === 0) {
|
|
1083
1591
|
return {
|
|
@@ -1117,22 +1625,32 @@ function smartproxyAllocatorFailureMessage(
|
|
|
1117
1625
|
return "Smartproxy allocator response did not contain a usable proxy endpoint.";
|
|
1118
1626
|
}
|
|
1119
1627
|
|
|
1628
|
+
// Smartproxy get-ip-v3 `protocol` param: 1 = HTTP. The SOCKS5 value ("2") is a
|
|
1629
|
+
// best-effort mapping pending live vendor confirmation; http is the default and
|
|
1630
|
+
// the only value exercised in production today.
|
|
1631
|
+
const SMARTPROXY_PROTOCOL_PARAM: Record<ProxyProtocol, string> = {
|
|
1632
|
+
http: "1",
|
|
1633
|
+
socks5: "2",
|
|
1634
|
+
};
|
|
1635
|
+
|
|
1120
1636
|
function buildSmartproxyAllocatorUrl(
|
|
1121
1637
|
policy: ProviderProxyPolicy,
|
|
1122
1638
|
appKey: string,
|
|
1123
1639
|
lifetimeMinutes: number,
|
|
1124
1640
|
poolSize: number,
|
|
1641
|
+
protocol: ProxyProtocol,
|
|
1642
|
+
ambientDefaults = true,
|
|
1125
1643
|
): string {
|
|
1126
1644
|
const params = new URLSearchParams({
|
|
1127
1645
|
app_key: appKey,
|
|
1128
1646
|
pt: "9",
|
|
1129
1647
|
num: String(poolSize),
|
|
1130
1648
|
life: String(lifetimeMinutes),
|
|
1131
|
-
protocol:
|
|
1649
|
+
protocol: SMARTPROXY_PROTOCOL_PARAM[protocol],
|
|
1132
1650
|
format: "txt",
|
|
1133
1651
|
lb: "\\n",
|
|
1134
1652
|
});
|
|
1135
|
-
const country = resolveSmartproxyCountry(policy);
|
|
1653
|
+
const country = resolveSmartproxyCountry(policy, ambientDefaults);
|
|
1136
1654
|
if (country) {
|
|
1137
1655
|
params.set("cc", country);
|
|
1138
1656
|
}
|
|
@@ -1141,7 +1659,8 @@ function buildSmartproxyAllocatorUrl(
|
|
|
1141
1659
|
return `https://api.smartproxy.org/web_v1/ip/get-ip-v3?${params.toString()}`;
|
|
1142
1660
|
}
|
|
1143
1661
|
|
|
1144
|
-
function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
1662
|
+
function parseSmartproxyAllocatorProxies(body: string, protocol: ProxyProtocol): string[] {
|
|
1663
|
+
const scheme = protocol === "socks5" ? "socks5" : "http";
|
|
1145
1664
|
const trimmed = body.trim();
|
|
1146
1665
|
if (!trimmed) {
|
|
1147
1666
|
return [];
|
|
@@ -1162,7 +1681,7 @@ function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
|
1162
1681
|
"port" in item && (typeof item.port === "string" || typeof item.port === "number")
|
|
1163
1682
|
? item.port
|
|
1164
1683
|
: "";
|
|
1165
|
-
return ip && port ?
|
|
1684
|
+
return ip && port ? `${scheme}://${ip}:${port}` : null;
|
|
1166
1685
|
})
|
|
1167
1686
|
.filter((url): url is string => url !== null);
|
|
1168
1687
|
}
|
|
@@ -1174,7 +1693,7 @@ function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
|
1174
1693
|
.split(/\r?\n/)
|
|
1175
1694
|
.map((item) => item.trim())
|
|
1176
1695
|
.filter((item) => /^\d{1,3}(?:\.\d{1,3}){3}:\d{2,5}$/.test(item))
|
|
1177
|
-
.map((line) =>
|
|
1696
|
+
.map((line) => `${scheme}://${line}`);
|
|
1178
1697
|
}
|
|
1179
1698
|
|
|
1180
1699
|
function classifySmartproxyAllocatorBody(
|
|
@@ -1207,12 +1726,24 @@ function markSmartproxyCacheInvalidated(options: ProxyResolutionOptions = {}): s
|
|
|
1207
1726
|
if (!policy || policy.mode === "disabled") {
|
|
1208
1727
|
return undefined;
|
|
1209
1728
|
}
|
|
1210
|
-
if (
|
|
1729
|
+
if (!resolveVendorChain(policy).includes("smartproxy")) {
|
|
1211
1730
|
return undefined;
|
|
1212
1731
|
}
|
|
1213
1732
|
|
|
1214
1733
|
const lifetimeMinutes = resolveSmartproxyLifetime(policy);
|
|
1215
|
-
const
|
|
1734
|
+
const appKey = (
|
|
1735
|
+
options.engineCredentials === undefined
|
|
1736
|
+
? process.env[SMARTPROXY_APP_KEY_ENV]
|
|
1737
|
+
: options.engineCredentials[SMARTPROXY_APP_KEY_ENV]
|
|
1738
|
+
)?.trim();
|
|
1739
|
+
if (!appKey) return undefined;
|
|
1740
|
+
const cacheKey = buildSmartproxyCacheKey(
|
|
1741
|
+
policy,
|
|
1742
|
+
appKey,
|
|
1743
|
+
options.affinityKey,
|
|
1744
|
+
lifetimeMinutes,
|
|
1745
|
+
options.protocol ?? VENDOR_DEFAULT_PROTOCOL.smartproxy,
|
|
1746
|
+
);
|
|
1216
1747
|
invalidatedProxyKeys.set(cacheKey, Date.now() + SMARTPROXY_INVALIDATION_SKIP_REDIS_MS);
|
|
1217
1748
|
proxyCache.delete(cacheKey);
|
|
1218
1749
|
proxyInflight.delete(cacheKey);
|
|
@@ -1262,17 +1793,13 @@ export async function loadApiFuseConfig(dir: string = process.cwd()): Promise<Ap
|
|
|
1262
1793
|
const tsPath = path.resolve(dir, "apifuse.config.ts");
|
|
1263
1794
|
if (existsSync(tsPath)) {
|
|
1264
1795
|
const config = await importConfig(tsPath);
|
|
1265
|
-
|
|
1266
|
-
syncProxyEnv(resolvedConfig);
|
|
1267
|
-
return resolvedConfig;
|
|
1796
|
+
return config ?? {};
|
|
1268
1797
|
}
|
|
1269
1798
|
|
|
1270
1799
|
const jsPath = path.resolve(dir, "apifuse.config.js");
|
|
1271
1800
|
if (existsSync(jsPath)) {
|
|
1272
1801
|
const config = await importConfig(jsPath);
|
|
1273
|
-
|
|
1274
|
-
syncProxyEnv(resolvedConfig);
|
|
1275
|
-
return resolvedConfig;
|
|
1802
|
+
return config ?? {};
|
|
1276
1803
|
}
|
|
1277
1804
|
|
|
1278
1805
|
return {};
|