@apifuse/provider-sdk 2.2.0-beta.3 → 2.2.0-beta.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +493 -5
- package/CHANGELOG.md +131 -1
- package/README.md +52 -6
- package/SUBMISSION.md +1 -1
- package/bin/apifuse-check.ts +106 -62
- package/bin/apifuse-create.ts +1 -1
- package/bin/apifuse-dev.ts +63 -55
- package/bin/apifuse-pack-check.ts +22 -2
- package/bin/apifuse-pack-smoke.ts +78 -82
- package/bin/apifuse-pack-types.ts +583 -0
- package/bin/apifuse-perf.ts +59 -140
- package/bin/apifuse-record.ts +698 -113
- package/bin/apifuse-submit-check.ts +517 -44
- package/bin/apifuse-sync-assets.ts +117 -0
- package/bin/apifuse.ts +1 -1
- package/bin/submit-check-delimited-text.ts +50 -0
- package/bin/submit-check-xml.ts +1 -1
- package/dist/auth-turn/index.d.ts +4 -4
- package/dist/auth-turn/index.js +1 -1
- package/dist/auth.d.ts +16 -2
- package/dist/auth.js +76 -18
- package/dist/ceremonies/index.d.ts +9 -1
- package/dist/ceremonies/index.js +117 -29
- package/dist/cli/commands.d.ts +1 -1
- package/dist/cli/commands.js +8 -0
- package/dist/cli/create.d.ts +3 -0
- package/dist/cli/create.js +34 -35
- package/dist/cli/prompt-assets.d.ts +80 -0
- package/dist/cli/prompt-assets.js +743 -0
- package/dist/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/dist/cli/templates/provider/README.md.tpl +4 -4
- package/dist/config/loader.d.ts +176 -17
- package/dist/config/loader.js +434 -161
- package/dist/contract-serialization.d.ts +2 -2
- package/dist/contract-serialization.js +7 -14
- package/dist/contract-types.d.ts +3 -2
- package/dist/contract.d.ts +3 -3
- package/dist/contract.js +6 -6
- package/dist/declaration-validation.d.ts +23 -0
- package/dist/declaration-validation.js +159 -0
- package/dist/define.d.ts +13 -1
- package/dist/define.js +391 -122
- package/dist/dev.d.ts +1 -1
- package/dist/dev.js +1 -1
- package/dist/error-resolution.d.ts +4 -0
- package/dist/error-resolution.js +123 -0
- package/dist/errors.d.ts +19 -1
- package/dist/errors.js +41 -3
- package/dist/fixture-sanitization.d.ts +26 -0
- package/dist/fixture-sanitization.js +216 -0
- package/dist/i18n/catalog.d.ts +2 -2
- package/dist/i18n/catalog.js +4 -10
- package/dist/i18n/index.d.ts +2 -2
- package/dist/i18n/index.js +2 -2
- package/dist/i18n/keys.d.ts +2 -2
- package/dist/index.d.ts +50 -42
- package/dist/index.js +41 -37
- package/dist/lint.d.ts +6 -1
- package/dist/lint.js +370 -18
- package/dist/native-address.d.ts +43 -0
- package/dist/native-address.js +281 -0
- package/dist/native-egress-policy.d.ts +31 -0
- package/dist/native-egress-policy.js +288 -0
- package/dist/observability.d.ts +5 -2
- package/dist/observability.js +48 -1
- package/dist/provider.d.ts +13 -11
- package/dist/provider.js +10 -9
- package/dist/public-schema-field-lint.d.ts +1 -1
- package/dist/recipes/gov-api.js +1 -1
- package/dist/runtime/auth-flow.d.ts +3 -1
- package/dist/runtime/auth-flow.js +8 -3
- package/dist/runtime/browser.d.ts +1 -1
- package/dist/runtime/browser.js +138 -40
- package/dist/runtime/cache.d.ts +2 -1
- package/dist/runtime/cache.js +173 -23
- package/dist/runtime/choice-wordlist.d.ts +9 -0
- package/dist/runtime/choice-wordlist.js +138 -0
- package/dist/runtime/choice.d.ts +14 -1
- package/dist/runtime/choice.js +566 -101
- package/dist/runtime/credential.d.ts +1 -1
- package/dist/runtime/credential.js +1 -1
- package/dist/runtime/env.d.ts +1 -1
- package/dist/runtime/executor.d.ts +1 -1
- package/dist/runtime/executor.js +25 -3
- package/dist/runtime/http.d.ts +3 -2
- package/dist/runtime/http.js +517 -55
- package/dist/runtime/insights.d.ts +1 -1
- package/dist/runtime/insights.js +6 -13
- package/dist/runtime/instrumentation.d.ts +2 -2
- package/dist/runtime/instrumentation.js +371 -23
- package/dist/runtime/keyring.js +1 -1
- package/dist/runtime/namespace.js +1 -1
- package/dist/runtime/native-network-errors.d.ts +33 -0
- package/dist/runtime/native-network-errors.js +69 -0
- package/dist/runtime/native-network.d.ts +96 -0
- package/dist/runtime/native-network.js +1232 -0
- package/dist/runtime/ocr.d.ts +29 -0
- package/dist/runtime/ocr.js +440 -0
- package/dist/runtime/otlp.d.ts +1 -1
- package/dist/runtime/perf.d.ts +1 -1
- package/dist/runtime/provider.d.ts +1 -1
- package/dist/runtime/provider.js +1 -2
- package/dist/runtime/proxy-errors.d.ts +1 -1
- package/dist/runtime/proxy-errors.js +9 -7
- package/dist/runtime/proxy-nodemaven.d.ts +56 -0
- package/dist/runtime/proxy-nodemaven.js +146 -0
- package/dist/runtime/proxy-retry-policy.d.ts +2 -2
- package/dist/runtime/proxy-retry-policy.js +2 -2
- package/dist/runtime/proxy-telemetry.d.ts +2 -1
- package/dist/runtime/proxy-telemetry.js +58 -52
- package/dist/runtime/redirects.d.ts +29 -0
- package/dist/runtime/redirects.js +36 -0
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +5 -5
- package/dist/runtime/request-options.d.ts +68 -1
- package/dist/runtime/request-options.js +548 -0
- package/dist/runtime/resolver-config.d.ts +6 -0
- package/dist/runtime/resolver-config.js +6 -0
- package/dist/runtime/resolver-public.d.ts +1 -0
- package/dist/runtime/resolver-public.js +1 -0
- package/dist/runtime/resolver-shared.d.ts +3 -0
- package/dist/runtime/resolver-shared.js +12 -0
- package/dist/runtime/resolver-vendors/bindings.d.ts +48 -0
- package/dist/runtime/resolver-vendors/bindings.js +40 -0
- package/dist/runtime/resolver-vendors/browser.d.ts +20 -0
- package/dist/runtime/resolver-vendors/browser.js +282 -0
- package/dist/runtime/resolver-vendors/hosts.d.ts +2 -0
- package/dist/runtime/resolver-vendors/hosts.js +33 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +24 -0
- package/dist/runtime/resolver-vendors/twocaptcha.js +368 -0
- package/dist/runtime/resolver-vendors/types.d.ts +83 -0
- package/dist/runtime/resolver-vendors/types.js +69 -0
- package/dist/runtime/resolver.d.ts +59 -0
- package/dist/runtime/resolver.js +705 -0
- package/dist/runtime/secrets.d.ts +27 -0
- package/dist/runtime/secrets.js +51 -0
- package/dist/runtime/state.d.ts +5 -2
- package/dist/runtime/state.js +280 -74
- package/dist/runtime/stealth-cookies.d.ts +20 -0
- package/dist/runtime/stealth-cookies.js +111 -0
- package/dist/runtime/stealth.d.ts +30 -5
- package/dist/runtime/stealth.js +523 -259
- package/dist/runtime/stt.d.ts +1 -1
- package/dist/runtime/stt.js +12 -27
- package/dist/runtime/timeout.d.ts +5 -0
- package/dist/runtime/timeout.js +12 -0
- package/dist/runtime/trace.d.ts +2 -2
- package/dist/runtime/trace.js +2 -4
- package/dist/runtime/waterfall.d.ts +1 -1
- package/dist/schema.d.ts +1 -1
- package/dist/schema.js +7 -15
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/index.d.ts +7 -7
- package/dist/server/index.js +6 -6
- package/dist/server/self-test-input-tokens.d.ts +2 -1
- package/dist/server/self-test-input-tokens.js +18 -14
- package/dist/server/self-test-redaction.d.ts +1 -1
- package/dist/server/self-test-redaction.js +1 -1
- package/dist/server/self-test.d.ts +117 -3
- package/dist/server/self-test.js +787 -151
- package/dist/server/serve-implementation.d.ts +210 -0
- package/dist/server/serve-implementation.js +2078 -0
- package/dist/server/serve.d.ts +1 -70
- package/dist/server/serve.js +1 -1143
- package/dist/server/types.d.ts +34 -9
- package/dist/server/types.js +8 -1
- package/dist/stateful/errors.d.ts +19 -0
- package/dist/stateful/errors.js +24 -0
- package/dist/stateful/http-provider-event-emitter.d.ts +40 -0
- package/dist/stateful/http-provider-event-emitter.js +237 -0
- package/dist/stateful/http-session-owner-registry.d.ts +44 -0
- package/dist/stateful/http-session-owner-registry.js +210 -0
- package/dist/stateful/index.d.ts +18 -0
- package/dist/stateful/index.js +18 -0
- package/dist/stateful/provider-event-delivery-failures.d.ts +32 -0
- package/dist/stateful/provider-event-delivery-failures.js +43 -0
- package/dist/stateful/provider-event-pipeline-metrics.d.ts +46 -0
- package/dist/stateful/provider-event-pipeline-metrics.js +48 -0
- package/dist/stateful/provider-event-pipeline.d.ts +50 -0
- package/dist/stateful/provider-event-pipeline.js +1 -0
- package/dist/stateful/provider-events.d.ts +101 -0
- package/dist/stateful/provider-events.js +289 -0
- package/dist/stateful/session-key.d.ts +15 -0
- package/dist/stateful/session-key.js +86 -0
- package/dist/stateful/stateful-provider-adapter-context.d.ts +5 -0
- package/dist/stateful/stateful-provider-adapter-context.js +42 -0
- package/dist/stateful/stateful-provider-adapter-metrics.d.ts +15 -0
- package/dist/stateful/stateful-provider-adapter-metrics.js +21 -0
- package/dist/stateful/stateful-provider-adapter.d.ts +98 -0
- package/dist/stateful/stateful-provider-adapter.js +287 -0
- package/dist/stateful/stateful-provider-observability.d.ts +62 -0
- package/dist/stateful/stateful-provider-observability.js +161 -0
- package/dist/stateful/stateful-provider-owner-forwarder.d.ts +41 -0
- package/dist/stateful/stateful-provider-owner-forwarder.js +215 -0
- package/dist/stateful/stateful-provider-runtime-context.d.ts +32 -0
- package/dist/stateful/stateful-provider-runtime-context.js +60 -0
- package/dist/stateful/stateful-provider-runtime-executor.d.ts +34 -0
- package/dist/stateful/stateful-provider-runtime-executor.js +52 -0
- package/dist/stateful/stateful-provider-session-routing.d.ts +67 -0
- package/dist/stateful/stateful-provider-session-routing.js +345 -0
- package/dist/stateful/stateful-provider-session-runtime.d.ts +98 -0
- package/dist/stateful/stateful-provider-session-runtime.js +245 -0
- package/dist/stateful-signing.d.ts +18 -0
- package/dist/stateful-signing.js +27 -0
- package/dist/stealth/profiles.d.ts +1 -1
- package/dist/stealth/profiles.js +21 -21
- package/dist/stream-evidence.d.ts +74 -0
- package/dist/stream-evidence.js +785 -0
- package/dist/stream.d.ts +1 -1
- package/dist/stream.js +7 -1
- package/dist/testing/index.d.ts +3 -2
- package/dist/testing/index.js +3 -2
- package/dist/testing/run.d.ts +32 -2
- package/dist/testing/run.js +488 -28
- package/dist/types.d.ts +566 -19
- package/dist/types.js +1 -0
- package/dist/user-input.d.ts +30 -0
- package/dist/user-input.js +66 -0
- package/package.json +42 -7
- package/src/auth-turn/index.ts +2 -2
- package/src/auth.ts +146 -86
- package/src/ceremonies/index.ts +167 -92
- package/src/cli/commands.ts +10 -0
- package/src/cli/create.ts +42 -35
- package/src/cli/prompt-assets.ts +865 -0
- package/src/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/src/cli/templates/provider/README.md.tpl +4 -4
- package/src/config/loader.ts +667 -289
- package/src/contract-serialization.ts +10 -18
- package/src/contract-types.ts +3 -2
- package/src/contract.ts +14 -28
- package/src/declaration-validation.ts +202 -0
- package/src/define.ts +631 -495
- package/src/dev.ts +4 -9
- package/src/error-resolution.ts +128 -0
- package/src/errors.ts +56 -11
- package/src/fixture-sanitization.ts +247 -0
- package/src/i18n/catalog.ts +10 -32
- package/src/i18n/index.ts +2 -2
- package/src/i18n/keys.ts +5 -11
- package/src/index.ts +158 -44
- package/src/lint.ts +488 -154
- package/src/native-address.ts +340 -0
- package/src/native-egress-policy.ts +358 -0
- package/src/observability.ts +51 -1
- package/src/provider.ts +66 -11
- package/src/public-schema-field-lint.ts +7 -33
- package/src/recipes/gov-api.ts +2 -5
- package/src/runtime/auth-flow.ts +13 -7
- package/src/runtime/browser.ts +252 -207
- package/src/runtime/cache.ts +209 -81
- package/src/runtime/choice-wordlist.ts +145 -0
- package/src/runtime/choice.ts +758 -197
- package/src/runtime/credential.ts +2 -2
- package/src/runtime/env.ts +1 -1
- package/src/runtime/executor.ts +37 -19
- package/src/runtime/http.ts +645 -65
- package/src/runtime/insights.ts +15 -53
- package/src/runtime/instrumentation.ts +530 -67
- package/src/runtime/keyring.ts +7 -19
- package/src/runtime/namespace.ts +2 -7
- package/src/runtime/native-network-errors.ts +99 -0
- package/src/runtime/native-network.ts +1605 -0
- package/src/runtime/ocr.ts +523 -0
- package/src/runtime/otlp.ts +12 -23
- package/src/runtime/perf.ts +1 -1
- package/src/runtime/provider.ts +4 -9
- package/src/runtime/proxy-errors.ts +29 -42
- package/src/runtime/proxy-nodemaven.ts +221 -0
- package/src/runtime/proxy-retry-policy.ts +3 -3
- package/src/runtime/proxy-telemetry.ts +84 -77
- package/src/runtime/redirects.ts +66 -0
- package/src/runtime/redis.ts +10 -13
- package/src/runtime/request-options.ts +679 -9
- package/src/runtime/resolver-config.ts +6 -0
- package/src/runtime/resolver-public.ts +18 -0
- package/src/runtime/resolver-shared.ts +17 -0
- package/src/runtime/resolver-vendors/bindings.ts +56 -0
- package/src/runtime/resolver-vendors/browser.ts +408 -0
- package/src/runtime/resolver-vendors/hosts.ts +38 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +500 -0
- package/src/runtime/resolver-vendors/types.ts +173 -0
- package/src/runtime/resolver.ts +1060 -0
- package/src/runtime/secrets.ts +64 -0
- package/src/runtime/state.ts +399 -161
- package/src/runtime/stealth-cookies.ts +132 -0
- package/src/runtime/stealth.ts +681 -295
- package/src/runtime/stt.ts +39 -113
- package/src/runtime/timeout.ts +18 -0
- package/src/runtime/trace.ts +14 -44
- package/src/runtime/waterfall.ts +5 -18
- package/src/schema.ts +23 -84
- package/src/serve.ts +6 -1
- package/src/server/index.ts +30 -7
- package/src/server/self-test-input-tokens.ts +29 -14
- package/src/server/self-test-redaction.ts +2 -2
- package/src/server/self-test.ts +1030 -180
- package/src/server/serve-implementation.ts +3062 -0
- package/src/server/serve.ts +1 -1781
- package/src/server/types.ts +12 -13
- package/src/stateful/README.md +146 -0
- package/src/stateful/errors.ts +35 -0
- package/src/stateful/http-provider-event-emitter.ts +314 -0
- package/src/stateful/http-session-owner-registry.ts +306 -0
- package/src/stateful/index.ts +18 -0
- package/src/stateful/provider-event-delivery-failures.ts +80 -0
- package/src/stateful/provider-event-pipeline-metrics.ts +95 -0
- package/src/stateful/provider-event-pipeline.ts +61 -0
- package/src/stateful/provider-events.ts +462 -0
- package/src/stateful/session-key.ts +111 -0
- package/src/stateful/stateful-provider-adapter-context.ts +59 -0
- package/src/stateful/stateful-provider-adapter-metrics.ts +48 -0
- package/src/stateful/stateful-provider-adapter.ts +562 -0
- package/src/stateful/stateful-provider-observability.ts +261 -0
- package/src/stateful/stateful-provider-owner-forwarder.ts +287 -0
- package/src/stateful/stateful-provider-runtime-context.ts +92 -0
- package/src/stateful/stateful-provider-runtime-executor.ts +96 -0
- package/src/stateful/stateful-provider-session-routing.ts +546 -0
- package/src/stateful/stateful-provider-session-runtime.ts +403 -0
- package/src/stateful-signing.ts +46 -0
- package/src/stealth/profiles.ts +27 -33
- package/src/stream-evidence.ts +988 -0
- package/src/stream.ts +16 -20
- package/src/testing/index.ts +11 -2
- package/src/testing/run.ts +668 -74
- package/src/types.ts +665 -35
- package/src/user-input.ts +118 -0
- package/dist/cli/templates/provider/CLAUDE.md.tpl +0 -1
- package/src/cli/templates/provider/CLAUDE.md.tpl +0 -1
- /package/dist/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
package/src/config/loader.ts
CHANGED
|
@@ -1,34 +1,43 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { existsSync } from "node:fs";
|
|
3
|
+
import { createRequire } from "node:module";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
|
|
5
|
-
import Redis from "ioredis";
|
|
6
|
-
|
|
7
|
-
import type { ProviderProxyPolicy, TraceConfig } from "../types";
|
|
8
|
-
|
|
6
|
+
import type { Redis } from "ioredis";
|
|
7
|
+
|
|
8
|
+
import type { ProviderProxyPolicy, ProviderProxyProvider, TraceConfig } from "../types.js";
|
|
9
|
+
import {
|
|
10
|
+
NODEMAVEN_DEFAULT_PROTOCOL,
|
|
11
|
+
NODEMAVEN_FILTER_ENV,
|
|
12
|
+
NODEMAVEN_MAX_POOL_SIZE,
|
|
13
|
+
NODEMAVEN_PASSWORD_ENV,
|
|
14
|
+
NODEMAVEN_USERNAME_ENV,
|
|
15
|
+
type ProxyProtocol,
|
|
16
|
+
hasNodemavenCredentials,
|
|
17
|
+
nodemavenPoolSize,
|
|
18
|
+
synthesizeNodemavenProxy,
|
|
19
|
+
} from "../runtime/proxy-nodemaven.js";
|
|
20
|
+
|
|
21
|
+
export type { ProxyProtocol } from "../runtime/proxy-nodemaven.js";
|
|
22
|
+
|
|
23
|
+
/** Proxy vendors with SDK-managed resolution. */
|
|
24
|
+
export type ProxyVendorName = "smartproxy" | "nodemaven";
|
|
25
|
+
|
|
26
|
+
// "smartproxy" here is api.smartproxy.org — a residential proxy with an IP
|
|
27
|
+
// extraction API (app_key → raw ip:port pool). It is NOT the company formerly
|
|
28
|
+
// named Smartproxy (smartproxy.com), which rebranded to Decodo in 2025 and is
|
|
29
|
+
// modelled separately as the `decodo` gateway vendor. Do not conflate them.
|
|
9
30
|
export const SMARTPROXY_APP_KEY_ENV = "APIFUSE__PROXY__SMARTPROXY_APP_KEY";
|
|
10
31
|
export const SMARTPROXY_MAX_LIFETIME_MINUTES = 2000;
|
|
11
32
|
export const DEFAULT_SMARTPROXY_POOL_SIZE = 20;
|
|
12
33
|
export const SMARTPROXY_MAX_POOL_SIZE = 20;
|
|
13
34
|
export const DEFAULT_PROXY_PROVIDER_ENV = "APIFUSE__PROXY__PROVIDER";
|
|
14
35
|
export const DEFAULT_PROXY_COUNTRY_ENV = "APIFUSE__PROXY__DEFAULT_COUNTRY";
|
|
15
|
-
export const DEFAULT_PROXY_LIFETIME_ENV =
|
|
16
|
-
|
|
17
|
-
export const
|
|
18
|
-
"APIFUSE__PROVIDER__CACHE_REDIS_URL";
|
|
19
|
-
export const PROVIDER_STATE_REDIS_URL_ENV =
|
|
20
|
-
"APIFUSE__PROVIDER__STATE_REDIS_URL";
|
|
36
|
+
export const DEFAULT_PROXY_LIFETIME_ENV = "APIFUSE__PROXY__DEFAULT_LIFETIME_MINUTES";
|
|
37
|
+
export const PROVIDER_CACHE_REDIS_URL_ENV = "APIFUSE__PROVIDER__CACHE_REDIS_URL";
|
|
38
|
+
export const PROVIDER_STATE_REDIS_URL_ENV = "APIFUSE__PROVIDER__STATE_REDIS_URL";
|
|
21
39
|
export const REDIS_URL_ENV = "APIFUSE__REDIS__URL";
|
|
22
40
|
|
|
23
|
-
export type ProxyOptions = {
|
|
24
|
-
url: string;
|
|
25
|
-
};
|
|
26
|
-
|
|
27
|
-
export type ProxyConfig = Partial<ProxyOptions> & {
|
|
28
|
-
provider?: string;
|
|
29
|
-
apiKey?: string;
|
|
30
|
-
};
|
|
31
|
-
|
|
32
41
|
export type BrowserConfig = {
|
|
33
42
|
executablePath?: string;
|
|
34
43
|
headless?: boolean;
|
|
@@ -40,7 +49,6 @@ export type SessionConfig = {
|
|
|
40
49
|
};
|
|
41
50
|
|
|
42
51
|
export type ApiFuseConfig = {
|
|
43
|
-
proxy?: ProxyConfig;
|
|
44
52
|
browser?: BrowserConfig;
|
|
45
53
|
session?: SessionConfig;
|
|
46
54
|
trace?: TraceConfig;
|
|
@@ -50,11 +58,30 @@ export type ApiFuseConfig = {
|
|
|
50
58
|
export type ProxyResolutionOptions = {
|
|
51
59
|
proxy?: string;
|
|
52
60
|
upstream?: { proxy?: boolean | ProviderProxyPolicy };
|
|
53
|
-
apifuseConfig?: Pick<ApiFuseConfig, "proxy">;
|
|
54
61
|
proxyPolicy?: ProviderProxyPolicy;
|
|
55
62
|
affinityKey?: string;
|
|
56
63
|
/** Zero-based proxy-pool attempt index used by SDK transports for failover. */
|
|
57
64
|
proxyAttempt?: number;
|
|
65
|
+
/**
|
|
66
|
+
* Tunnelling protocols the calling transport can use. When a resolved
|
|
67
|
+
* protocol is not in this set the resolver fails with
|
|
68
|
+
* `PROXY_PROTOCOL_UNSUPPORTED` instead of silently downgrading. Unset means
|
|
69
|
+
* permissive (both protocols allowed).
|
|
70
|
+
*/
|
|
71
|
+
transportProtocols?: readonly ProxyProtocol[];
|
|
72
|
+
/**
|
|
73
|
+
* Explicit protocol override. Internal — for the verification harness and
|
|
74
|
+
* tests, or an advanced caller. Normal callers omit it and each vendor uses
|
|
75
|
+
* its own benchmarked default protocol (see VENDOR_DEFAULT_PROTOCOL). Not an
|
|
76
|
+
* env var and not a provider-policy field.
|
|
77
|
+
*/
|
|
78
|
+
protocol?: ProxyProtocol;
|
|
79
|
+
/**
|
|
80
|
+
* Gateway pool "refresh" generation. Bumped by transports on pool refresh to
|
|
81
|
+
* derive a fresh gateway session set (ignored by allocation-style vendors,
|
|
82
|
+
* whose refresh is driven by cache invalidation).
|
|
83
|
+
*/
|
|
84
|
+
proxyRefreshEpoch?: number;
|
|
58
85
|
telemetry?: ProxyTelemetrySink;
|
|
59
86
|
};
|
|
60
87
|
|
|
@@ -76,8 +103,12 @@ export type SmartproxyAllocatorBodyClass =
|
|
|
76
103
|
| "text_without_proxies"
|
|
77
104
|
| "usable_proxy_endpoints";
|
|
78
105
|
|
|
106
|
+
export type ProxyUserAgentSource = "declared" | "defaulted";
|
|
107
|
+
|
|
79
108
|
export type ProxyResolutionTelemetryEvent = {
|
|
80
|
-
provider:
|
|
109
|
+
provider: ProxyVendorName;
|
|
110
|
+
userAgentSource?: ProxyUserAgentSource;
|
|
111
|
+
protocol?: ProxyProtocol;
|
|
81
112
|
cacheStatus: ProxyCacheStatus;
|
|
82
113
|
cacheHit: boolean;
|
|
83
114
|
resolutionMs: number;
|
|
@@ -95,7 +126,7 @@ export type ProxyResolutionTelemetryEvent = {
|
|
|
95
126
|
};
|
|
96
127
|
|
|
97
128
|
export type ProxyAttemptTelemetryEvent = {
|
|
98
|
-
provider:
|
|
129
|
+
provider: ProxyVendorName;
|
|
99
130
|
attempt: number;
|
|
100
131
|
poolIndex?: number;
|
|
101
132
|
proxyHash?: string;
|
|
@@ -105,31 +136,69 @@ export type ProxyAttemptTelemetryEvent = {
|
|
|
105
136
|
durationMs?: number;
|
|
106
137
|
};
|
|
107
138
|
|
|
139
|
+
export type ProxyVendorFailoverTelemetryEvent = {
|
|
140
|
+
/** Vendor that failed or was skipped. */
|
|
141
|
+
vendor: ProxyVendorName;
|
|
142
|
+
/** Vendor tried next, or undefined when the chain is exhausted. */
|
|
143
|
+
nextVendor?: ProxyVendorName;
|
|
144
|
+
phase: "resolution" | "transport";
|
|
145
|
+
reason: "no_credentials" | "allocation_failed" | "pool_exhausted" | "protocol_unsupported";
|
|
146
|
+
attempt?: number;
|
|
147
|
+
};
|
|
148
|
+
|
|
108
149
|
export type ProxyTelemetrySink = {
|
|
109
150
|
recordProxyResolution(event: ProxyResolutionTelemetryEvent): void;
|
|
110
151
|
recordProxyAttempt?(event: ProxyAttemptTelemetryEvent): void;
|
|
152
|
+
recordProxyVendorFailover?(event: ProxyVendorFailoverTelemetryEvent): void;
|
|
111
153
|
};
|
|
112
154
|
|
|
155
|
+
export type ProxyResolutionSource =
|
|
156
|
+
| "explicit"
|
|
157
|
+
| "env"
|
|
158
|
+
| "config"
|
|
159
|
+
| "smartproxy-allocator"
|
|
160
|
+
| "nodemaven-gateway";
|
|
161
|
+
|
|
113
162
|
export type ResolvedProxyConfig = {
|
|
114
163
|
shouldWarn: boolean;
|
|
115
164
|
url?: string;
|
|
116
|
-
|
|
165
|
+
/** SDK-native vendor that supplied the URL, when applicable. */
|
|
166
|
+
vendor?: ProxyVendorName;
|
|
167
|
+
source?: ProxyResolutionSource;
|
|
168
|
+
protocol?: ProxyProtocol;
|
|
117
169
|
diagnostics?: Record<string, string | number | boolean>;
|
|
118
170
|
};
|
|
119
171
|
|
|
172
|
+
export type ProxyResolutionErrorCode =
|
|
173
|
+
| "PROXY_REQUIRED"
|
|
174
|
+
| "PROXY_ALLOCATION_FAILED"
|
|
175
|
+
| "PROXY_PROTOCOL_UNSUPPORTED";
|
|
176
|
+
|
|
120
177
|
export class ProxyResolutionError extends Error {
|
|
121
|
-
readonly code:
|
|
178
|
+
readonly code: ProxyResolutionErrorCode;
|
|
122
179
|
readonly telemetry?: ProxyResolutionTelemetryEvent;
|
|
180
|
+
readonly vendor?: ProxyVendorName;
|
|
181
|
+
readonly vendorChain?: ProxyVendorName[];
|
|
182
|
+
readonly protocol?: ProxyProtocol;
|
|
123
183
|
|
|
124
184
|
constructor(
|
|
125
|
-
code:
|
|
185
|
+
code: ProxyResolutionErrorCode,
|
|
126
186
|
message: string,
|
|
127
|
-
options?: {
|
|
187
|
+
options?: {
|
|
188
|
+
cause?: unknown;
|
|
189
|
+
telemetry?: ProxyResolutionTelemetryEvent;
|
|
190
|
+
vendor?: ProxyVendorName;
|
|
191
|
+
vendorChain?: ProxyVendorName[];
|
|
192
|
+
protocol?: ProxyProtocol;
|
|
193
|
+
},
|
|
128
194
|
) {
|
|
129
195
|
super(message, options);
|
|
130
196
|
this.name = "ProxyResolutionError";
|
|
131
197
|
this.code = code;
|
|
132
198
|
this.telemetry = options?.telemetry;
|
|
199
|
+
this.vendor = options?.vendor;
|
|
200
|
+
this.vendorChain = options?.vendorChain;
|
|
201
|
+
this.protocol = options?.protocol;
|
|
133
202
|
}
|
|
134
203
|
}
|
|
135
204
|
|
|
@@ -152,6 +221,7 @@ type ProxyRedisClient = Pick<
|
|
|
152
221
|
>;
|
|
153
222
|
|
|
154
223
|
const proxyCache = new Map<string, CachedProxyPool>();
|
|
224
|
+
const require = createRequire(import.meta.url);
|
|
155
225
|
const proxyInflight = new Map<string, Promise<SmartproxyAllocationResult>>();
|
|
156
226
|
const invalidatedProxyKeys = new Map<string, number>();
|
|
157
227
|
const redisClients = new Map<string, ProxyRedisClient>();
|
|
@@ -165,8 +235,7 @@ const SMARTPROXY_LOCK_POLL_MAX_MS = 9_000;
|
|
|
165
235
|
const SMARTPROXY_DEADLINE_MARGIN_MS = 1_000;
|
|
166
236
|
const SMARTPROXY_INVALIDATION_SKIP_REDIS_MS = 30_000;
|
|
167
237
|
const SMARTPROXY_ALLOCATOR_MAX_ATTEMPTS = 3;
|
|
168
|
-
const SMARTPROXY_ALLOCATOR_DEADLINE_MS =
|
|
169
|
-
SMARTPROXY_LOCK_TTL_MS - SMARTPROXY_DEADLINE_MARGIN_MS;
|
|
238
|
+
const SMARTPROXY_ALLOCATOR_DEADLINE_MS = SMARTPROXY_LOCK_TTL_MS - SMARTPROXY_DEADLINE_MARGIN_MS;
|
|
170
239
|
const SMARTPROXY_ALLOCATOR_RETRY_BASE_MS = 25;
|
|
171
240
|
// Smartproxy API extraction returns fresh IP:port candidates; the `life`
|
|
172
241
|
// parameter controls session duration intent, not a hard endpoint lease. Keep
|
|
@@ -197,15 +266,11 @@ export function providerStateRedisUrlFromEnv(): string | undefined {
|
|
|
197
266
|
}
|
|
198
267
|
|
|
199
268
|
/** @internal Test-only hook for exercising shared proxy-cache behavior. */
|
|
200
|
-
export function __setProxyRedisForTests(
|
|
201
|
-
redis: ProxyRedisClient | undefined,
|
|
202
|
-
): void {
|
|
269
|
+
export function __setProxyRedisForTests(redis: ProxyRedisClient | undefined): void {
|
|
203
270
|
proxyRedisForTests = redis;
|
|
204
271
|
}
|
|
205
272
|
|
|
206
|
-
export function __setSmartproxyAllocatorDeadlineMsForTests(
|
|
207
|
-
deadlineMs: number | undefined,
|
|
208
|
-
): void {
|
|
273
|
+
export function __setSmartproxyAllocatorDeadlineMsForTests(deadlineMs: number | undefined): void {
|
|
209
274
|
smartproxyAllocatorDeadlineMsForTests = deadlineMs;
|
|
210
275
|
}
|
|
211
276
|
|
|
@@ -216,7 +281,8 @@ function getProxyRedis(): ProxyRedisClient | undefined {
|
|
|
216
281
|
const existing = redisClients.get(redisUrl);
|
|
217
282
|
if (existing) return existing;
|
|
218
283
|
|
|
219
|
-
const
|
|
284
|
+
const { Redis: RedisClient } = require("ioredis") as typeof import("ioredis");
|
|
285
|
+
const redis = new RedisClient(redisUrl, {
|
|
220
286
|
connectTimeout: REDIS_TIMEOUT_MS,
|
|
221
287
|
enableOfflineQueue: false,
|
|
222
288
|
lazyConnect: true,
|
|
@@ -230,9 +296,7 @@ function getProxyRedis(): ProxyRedisClient | undefined {
|
|
|
230
296
|
return redis;
|
|
231
297
|
}
|
|
232
298
|
|
|
233
|
-
async function withRedisTimeout<T>(
|
|
234
|
-
operation: () => Promise<T>,
|
|
235
|
-
): Promise<T | undefined> {
|
|
299
|
+
async function withRedisTimeout<T>(operation: () => Promise<T>): Promise<T | undefined> {
|
|
236
300
|
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
|
237
301
|
try {
|
|
238
302
|
const timeout = new Promise<undefined>((resolve) => {
|
|
@@ -318,17 +382,11 @@ function isRecord(value: unknown): value is Record<string, unknown> {
|
|
|
318
382
|
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
319
383
|
}
|
|
320
384
|
|
|
321
|
-
function toProxyDiagnostics(
|
|
322
|
-
value: unknown,
|
|
323
|
-
): Record<string, string | number | boolean> | undefined {
|
|
385
|
+
function toProxyDiagnostics(value: unknown): Record<string, string | number | boolean> | undefined {
|
|
324
386
|
if (!isRecord(value)) return undefined;
|
|
325
387
|
const diagnostics: Record<string, string | number | boolean> = {};
|
|
326
388
|
for (const [key, item] of Object.entries(value)) {
|
|
327
|
-
if (
|
|
328
|
-
typeof item === "string" ||
|
|
329
|
-
typeof item === "number" ||
|
|
330
|
-
typeof item === "boolean"
|
|
331
|
-
) {
|
|
389
|
+
if (typeof item === "string" || typeof item === "number" || typeof item === "boolean") {
|
|
332
390
|
diagnostics[key] = item;
|
|
333
391
|
}
|
|
334
392
|
}
|
|
@@ -381,89 +439,10 @@ function serializeSmartproxyPool(pool: CachedProxyPool): string {
|
|
|
381
439
|
|
|
382
440
|
function normalizeProxyUrl(url?: string): string | undefined {
|
|
383
441
|
const normalized = url?.trim();
|
|
384
|
-
return normalized
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
function readPositiveIntegerEnv(name: string): string | undefined {
|
|
388
|
-
const raw = process.env[name]?.trim();
|
|
389
|
-
if (!raw) return undefined;
|
|
390
|
-
if (!/^[1-9]\d*$/.test(raw)) {
|
|
391
|
-
throw new Error(`${name} must be a positive integer`);
|
|
392
|
-
}
|
|
393
|
-
return raw;
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
function applyStickyProxySession(proxyUrl: string): string {
|
|
397
|
-
let parsed: URL;
|
|
398
|
-
try {
|
|
399
|
-
parsed = new URL(proxyUrl);
|
|
400
|
-
} catch {
|
|
401
|
-
return proxyUrl;
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
if (!parsed.hostname || !parsed.username || !parsed.password) {
|
|
405
|
-
return proxyUrl;
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
const host = parsed.hostname.toLowerCase();
|
|
409
|
-
if (!host.includes("smartproxy") && !host.includes("decodo")) {
|
|
410
|
-
return proxyUrl;
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
const username = decodeURIComponent(parsed.username);
|
|
414
|
-
const sessionId =
|
|
415
|
-
process.env.APIFUSE__PROXY__SESSION_ID?.trim() || "apifuse-shared";
|
|
416
|
-
const sessionDuration = readPositiveIntegerEnv(
|
|
417
|
-
"APIFUSE__PROXY__SESSION_DURATION",
|
|
418
|
-
);
|
|
419
|
-
const stickyUsername = host.includes("smartproxy")
|
|
420
|
-
? buildSmartproxyUsername(username, sessionId, sessionDuration)
|
|
421
|
-
: buildDecodoUsername(username, sessionId, sessionDuration ?? "60");
|
|
422
|
-
|
|
423
|
-
parsed.username = stickyUsername;
|
|
424
|
-
return parsed.toString();
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
function buildSmartproxyUsername(
|
|
428
|
-
username: string,
|
|
429
|
-
sessionId: string,
|
|
430
|
-
sessionDuration?: string,
|
|
431
|
-
): string {
|
|
432
|
-
const parts = username.split("_");
|
|
433
|
-
const configuredLife = parts
|
|
434
|
-
.find((part) => part.startsWith("life-"))
|
|
435
|
-
?.slice("life-".length);
|
|
436
|
-
const baseUsername = parts
|
|
437
|
-
.filter((part) => !part.startsWith("session-") && !part.startsWith("life-"))
|
|
438
|
-
.join("_");
|
|
439
|
-
return `${baseUsername}_session-${sessionId}_life-${sessionDuration ?? configuredLife ?? "60"}`;
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
function buildDecodoUsername(
|
|
443
|
-
username: string,
|
|
444
|
-
sessionId: string,
|
|
445
|
-
sessionDuration: string,
|
|
446
|
-
): string {
|
|
447
|
-
const withoutSticky = username.replace(
|
|
448
|
-
/-session-.+-sessionduration-\d+$/,
|
|
449
|
-
"",
|
|
450
|
-
);
|
|
451
|
-
const baseUsername = withoutSticky.startsWith("user-")
|
|
452
|
-
? withoutSticky
|
|
453
|
-
: `user-${withoutSticky}`;
|
|
454
|
-
return `${baseUsername}-session-${sessionId}-sessionduration-${sessionDuration}`;
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
function syncProxyEnv(config: ApiFuseConfig): void {
|
|
458
|
-
const configProxyUrl = normalizeProxyUrl(config.proxy?.url);
|
|
459
|
-
if (!process.env.APIFUSE__PROXY__URL && configProxyUrl) {
|
|
460
|
-
process.env.APIFUSE__PROXY__URL = configProxyUrl;
|
|
461
|
-
}
|
|
442
|
+
return normalized || undefined;
|
|
462
443
|
}
|
|
463
444
|
|
|
464
|
-
export function resolveProxyConfig(
|
|
465
|
-
options: ProxyResolutionOptions = {},
|
|
466
|
-
): ResolvedProxyConfig {
|
|
445
|
+
export function resolveProxyConfig(options: ProxyResolutionOptions = {}): ResolvedProxyConfig {
|
|
467
446
|
const explicitProxyUrl = normalizeProxyUrl(options.proxy);
|
|
468
447
|
if (explicitProxyUrl) {
|
|
469
448
|
return { shouldWarn: false, url: explicitProxyUrl };
|
|
@@ -480,18 +459,6 @@ export function resolveProxyConfig(
|
|
|
480
459
|
return { shouldWarn: false };
|
|
481
460
|
}
|
|
482
461
|
|
|
483
|
-
const envProxyUrl = normalizeProxyUrl(process.env.APIFUSE__PROXY__URL);
|
|
484
|
-
if (envProxyUrl) {
|
|
485
|
-
return { shouldWarn: false, url: envProxyUrl };
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
const configuredProxyUrl = normalizeProxyUrl(
|
|
489
|
-
options.apifuseConfig?.proxy?.url,
|
|
490
|
-
);
|
|
491
|
-
if (configuredProxyUrl) {
|
|
492
|
-
return { shouldWarn: false, url: configuredProxyUrl };
|
|
493
|
-
}
|
|
494
|
-
|
|
495
462
|
return { shouldWarn: true };
|
|
496
463
|
}
|
|
497
464
|
|
|
@@ -511,68 +478,288 @@ export async function resolveProxyConfigAsync(
|
|
|
511
478
|
return { shouldWarn: false };
|
|
512
479
|
}
|
|
513
480
|
|
|
514
|
-
const
|
|
515
|
-
if (
|
|
481
|
+
const chain = resolveVendorChain(policy);
|
|
482
|
+
if (chain.length === 0) {
|
|
483
|
+
const declared = declaredVendorChain(policy);
|
|
484
|
+
const deprecated = declared.filter((vendor) => vendor === "decodo" || vendor === "custom");
|
|
485
|
+
if (policy.mode === "required") {
|
|
486
|
+
const providerIds =
|
|
487
|
+
declared.length > 0 ? declared.map((vendor) => `"${vendor}"`).join(", ") : "none";
|
|
488
|
+
const deprecatedDetail =
|
|
489
|
+
deprecated.length > 0
|
|
490
|
+
? ` Deprecated vendor(s): ${deprecated.map((vendor) => `"${vendor}"`).join(", ")}.`
|
|
491
|
+
: "";
|
|
492
|
+
throw new ProxyResolutionError(
|
|
493
|
+
"PROXY_REQUIRED",
|
|
494
|
+
`Required proxy policy has no SDK-managed adapter for provider id(s): ${providerIds}.${deprecatedDetail} Use "smartproxy" or "nodemaven".`,
|
|
495
|
+
);
|
|
496
|
+
}
|
|
497
|
+
// Deprecated decodo/custom providers have no SDK-managed adapter. Optional
|
|
498
|
+
// policies preserve the warning-only behavior and may continue directly.
|
|
516
499
|
return resolveProxyConfig({
|
|
517
500
|
...options,
|
|
518
501
|
upstream: { proxy: true },
|
|
519
502
|
});
|
|
520
503
|
}
|
|
521
504
|
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
505
|
+
// Protocol is chosen per vendor (each vendor's benchmarked-best), with an
|
|
506
|
+
// optional explicit override for the harness/tests. Both are tunnelling
|
|
507
|
+
// schemes. transportProtocols is what the calling transport can actually use.
|
|
508
|
+
const transportProtocols = options.transportProtocols ?? (["http", "socks5"] as const);
|
|
509
|
+
|
|
510
|
+
const sizes = chain.map((vendor) => vendorPoolSize(vendor, policy));
|
|
511
|
+
const total = sizes.reduce((sum, size) => sum + size, 0);
|
|
512
|
+
const normalizedAttempt = normalizeAttemptIndex(options.proxyAttempt);
|
|
513
|
+
const { vendorIndex: startVendorIndex, poolIndex: startPoolIndex } = mapFlatAttempt(
|
|
514
|
+
total > 0 ? normalizedAttempt % total : 0,
|
|
515
|
+
sizes,
|
|
516
|
+
);
|
|
517
|
+
const refreshEpoch = normalizeAttemptIndex(options.proxyRefreshEpoch);
|
|
518
|
+
|
|
519
|
+
let lastError: unknown;
|
|
520
|
+
let blockedProtocol: ProxyProtocol | undefined;
|
|
521
|
+
for (let vendorIndex = startVendorIndex; vendorIndex < chain.length; vendorIndex++) {
|
|
522
|
+
const vendor = chain[vendorIndex] as ProxyVendorName;
|
|
523
|
+
const nextVendor = chain[vendorIndex + 1];
|
|
524
|
+
const poolIndex = vendorIndex === startVendorIndex ? startPoolIndex : 0;
|
|
525
|
+
const protocol = options.protocol ?? VENDOR_DEFAULT_PROTOCOL[vendor];
|
|
526
|
+
|
|
527
|
+
if (!vendorHasCredentials(vendor)) {
|
|
528
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
529
|
+
vendor,
|
|
530
|
+
nextVendor,
|
|
531
|
+
phase: "resolution",
|
|
532
|
+
reason: "no_credentials",
|
|
533
|
+
});
|
|
534
|
+
continue;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
// The calling transport must be able to use this vendor's protocol; if not,
|
|
538
|
+
// fail over to the next vendor rather than silently downgrading.
|
|
539
|
+
if (!transportProtocols.includes(protocol)) {
|
|
540
|
+
blockedProtocol = protocol;
|
|
541
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
542
|
+
vendor,
|
|
543
|
+
nextVendor,
|
|
544
|
+
phase: "resolution",
|
|
545
|
+
reason: "protocol_unsupported",
|
|
546
|
+
});
|
|
547
|
+
continue;
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
try {
|
|
551
|
+
return await resolveWithVendor(vendor, policy, options, {
|
|
552
|
+
protocol,
|
|
553
|
+
poolIndex,
|
|
554
|
+
refreshEpoch,
|
|
555
|
+
});
|
|
556
|
+
} catch (error) {
|
|
557
|
+
// Config/programming errors (invalid filter, etc.) are not vendor
|
|
558
|
+
// outages — propagate them rather than failing over.
|
|
559
|
+
if (!(error instanceof ProxyResolutionError)) {
|
|
560
|
+
throw error;
|
|
561
|
+
}
|
|
562
|
+
if (error.telemetry) {
|
|
563
|
+
options.telemetry?.recordProxyResolution(error.telemetry);
|
|
564
|
+
}
|
|
565
|
+
lastError = error;
|
|
566
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
567
|
+
vendor,
|
|
568
|
+
nextVendor,
|
|
569
|
+
phase: "resolution",
|
|
570
|
+
reason: "allocation_failed",
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
if (policy.mode === "required") {
|
|
576
|
+
if (lastError) {
|
|
577
|
+
throw lastError instanceof ProxyResolutionError
|
|
578
|
+
? lastError
|
|
579
|
+
: new ProxyResolutionError(
|
|
580
|
+
"PROXY_ALLOCATION_FAILED",
|
|
581
|
+
`All proxy vendors [${chain.join(", ")}] failed for required proxy egress.`,
|
|
582
|
+
{ cause: lastError, vendorChain: chain },
|
|
583
|
+
);
|
|
584
|
+
}
|
|
585
|
+
if (blockedProtocol) {
|
|
525
586
|
throw new ProxyResolutionError(
|
|
526
|
-
"
|
|
527
|
-
`
|
|
587
|
+
"PROXY_PROTOCOL_UNSUPPORTED",
|
|
588
|
+
`No proxy vendor in [${chain.join(", ")}] could serve a protocol supported by this transport (supports: ${transportProtocols.join(", ")}; vendor wanted "${blockedProtocol}"). Route this provider through the stealth transport.`,
|
|
589
|
+
{ protocol: blockedProtocol, vendorChain: chain },
|
|
528
590
|
);
|
|
529
591
|
}
|
|
530
|
-
|
|
592
|
+
throw new ProxyResolutionError(
|
|
593
|
+
"PROXY_REQUIRED",
|
|
594
|
+
`Proxy egress is required but no vendor credentials are configured. Missing: ${chain
|
|
595
|
+
.map((vendor) => `${missingCredentialEnv(vendor)} (${vendor})`)
|
|
596
|
+
.join(", ")}.`,
|
|
597
|
+
{ vendorChain: chain },
|
|
598
|
+
);
|
|
531
599
|
}
|
|
532
|
-
|
|
600
|
+
return { shouldWarn: true };
|
|
601
|
+
}
|
|
533
602
|
|
|
603
|
+
/**
|
|
604
|
+
* Resolve the proxy URL for a provider-owned consumer such as a CAPTCHA solver.
|
|
605
|
+
* Vendor allocation and failover remain owned by the SDK.
|
|
606
|
+
*/
|
|
607
|
+
export async function resolveProxy(
|
|
608
|
+
options: ProxyResolutionOptions = {},
|
|
609
|
+
): Promise<ResolvedProxyConfig> {
|
|
610
|
+
const resolved = await resolveProxyConfigAsync(options);
|
|
611
|
+
const vendor = vendorFromResolvedSource(resolved.source);
|
|
612
|
+
return vendor ? { ...resolved, vendor } : resolved;
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
/**
|
|
616
|
+
* Each vendor's default egress protocol, chosen from live KR benchmarks. HTTP
|
|
617
|
+
* CONNECT wins for nodemaven (socks5 adds ~500ms through the gateway) and ties
|
|
618
|
+
* for smartproxy, and is the only protocol ctx.http (Bun native fetch) supports.
|
|
619
|
+
* Override per call via ProxyResolutionOptions.protocol (harness/tests).
|
|
620
|
+
*/
|
|
621
|
+
export const VENDOR_DEFAULT_PROTOCOL: Readonly<Record<ProxyVendorName, ProxyProtocol>> = {
|
|
622
|
+
smartproxy: "http",
|
|
623
|
+
nodemaven: NODEMAVEN_DEFAULT_PROTOCOL,
|
|
624
|
+
};
|
|
625
|
+
|
|
626
|
+
/**
|
|
627
|
+
* Guard the No-MITM invariant: a resolved proxy URL must use a tunnelling scheme
|
|
628
|
+
* (http CONNECT or socks5) so the client TLS handshake reaches the origin
|
|
629
|
+
* end-to-end. Anything else would intercept TLS and break fingerprinting.
|
|
630
|
+
*/
|
|
631
|
+
export function assertTunnelingScheme(url: string): void {
|
|
632
|
+
let scheme: string;
|
|
534
633
|
try {
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
allocated.pool.urls.length,
|
|
544
|
-
options.proxyAttempt,
|
|
634
|
+
scheme = new URL(url).protocol.replace(/:$/, "").toLowerCase();
|
|
635
|
+
} catch {
|
|
636
|
+
throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `Malformed proxy URL: ${url}`);
|
|
637
|
+
}
|
|
638
|
+
if (scheme !== "http" && scheme !== "socks5") {
|
|
639
|
+
throw new ProxyResolutionError(
|
|
640
|
+
"PROXY_ALLOCATION_FAILED",
|
|
641
|
+
`Resolved proxy scheme "${scheme}" is not a tunnelling scheme (expected http or socks5). Refusing to route TLS through a non-tunnelling proxy.`,
|
|
545
642
|
);
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
export type ProxyVendorResolutionContext = {
|
|
647
|
+
readonly protocol: ProxyProtocol;
|
|
648
|
+
readonly poolIndex: number;
|
|
649
|
+
readonly refreshEpoch: number;
|
|
650
|
+
/** Explicit vendor credentials. Omit only on the legacy ambient-env path. */
|
|
651
|
+
readonly credentials?: Readonly<Record<string, string>>;
|
|
652
|
+
/** Disable non-policy env defaults for deterministic injected adapters. */
|
|
653
|
+
readonly ambientDefaults?: boolean;
|
|
654
|
+
/** Disable env-discovered Redis sharing for deterministic injected adapters. */
|
|
655
|
+
readonly sharedCache?: boolean;
|
|
656
|
+
};
|
|
657
|
+
|
|
658
|
+
export async function resolveWithVendor(
|
|
659
|
+
vendor: ProxyVendorName,
|
|
660
|
+
policy: ProviderProxyPolicy,
|
|
661
|
+
options: ProxyResolutionOptions,
|
|
662
|
+
context: ProxyVendorResolutionContext,
|
|
663
|
+
): Promise<ResolvedProxyConfig> {
|
|
664
|
+
if (vendor === "nodemaven") {
|
|
665
|
+
const startedAt = Date.now();
|
|
666
|
+
const username = (
|
|
667
|
+
context.credentials === undefined
|
|
668
|
+
? process.env[NODEMAVEN_USERNAME_ENV]
|
|
669
|
+
: context.credentials[NODEMAVEN_USERNAME_ENV]
|
|
670
|
+
)?.trim();
|
|
671
|
+
const password = (
|
|
672
|
+
context.credentials === undefined
|
|
673
|
+
? process.env[NODEMAVEN_PASSWORD_ENV]
|
|
674
|
+
: context.credentials[NODEMAVEN_PASSWORD_ENV]
|
|
675
|
+
)?.trim();
|
|
676
|
+
const filter =
|
|
677
|
+
context.credentials === undefined
|
|
678
|
+
? process.env[NODEMAVEN_FILTER_ENV]
|
|
679
|
+
: context.credentials[NODEMAVEN_FILTER_ENV];
|
|
680
|
+
if (!username || !password) {
|
|
681
|
+
throw new ProxyResolutionError(
|
|
682
|
+
"PROXY_ALLOCATION_FAILED",
|
|
683
|
+
`NodeMaven credentials missing: set ${NODEMAVEN_USERNAME_ENV} and ${NODEMAVEN_PASSWORD_ENV}.`,
|
|
684
|
+
{ vendor: "nodemaven" },
|
|
685
|
+
);
|
|
686
|
+
}
|
|
687
|
+
const synthesized = synthesizeNodemavenProxy({
|
|
688
|
+
policy,
|
|
689
|
+
credentials: {
|
|
690
|
+
username,
|
|
691
|
+
password,
|
|
692
|
+
...(filter ? { filter } : {}),
|
|
693
|
+
},
|
|
694
|
+
affinityKey: options.affinityKey,
|
|
695
|
+
protocol: context.protocol,
|
|
696
|
+
poolIndex: context.poolIndex,
|
|
697
|
+
refreshEpoch: context.refreshEpoch,
|
|
698
|
+
country: resolveSmartproxyCountry(policy, context.ambientDefaults !== false),
|
|
699
|
+
});
|
|
700
|
+
options.telemetry?.recordProxyResolution({
|
|
701
|
+
provider: "nodemaven",
|
|
702
|
+
protocol: synthesized.protocol,
|
|
703
|
+
cacheStatus: "disabled",
|
|
704
|
+
cacheHit: false,
|
|
705
|
+
resolutionMs: Math.max(0, Date.now() - startedAt),
|
|
706
|
+
attempts: 1,
|
|
707
|
+
});
|
|
708
|
+
assertTunnelingScheme(synthesized.url);
|
|
546
709
|
return {
|
|
547
710
|
shouldWarn: false,
|
|
548
|
-
url:
|
|
549
|
-
source: "
|
|
711
|
+
url: synthesized.url,
|
|
712
|
+
source: "nodemaven-gateway",
|
|
713
|
+
protocol: synthesized.protocol,
|
|
550
714
|
diagnostics: {
|
|
551
|
-
...
|
|
552
|
-
|
|
553
|
-
poolIndex,
|
|
715
|
+
...synthesized.diagnostics,
|
|
716
|
+
poolIndex: context.poolIndex,
|
|
554
717
|
},
|
|
555
718
|
};
|
|
556
|
-
} catch (error) {
|
|
557
|
-
if (error instanceof ProxyResolutionError && error.telemetry) {
|
|
558
|
-
options.telemetry?.recordProxyResolution(error.telemetry);
|
|
559
|
-
}
|
|
560
|
-
if (policy.mode === "required") {
|
|
561
|
-
throw error instanceof ProxyResolutionError
|
|
562
|
-
? error
|
|
563
|
-
: new ProxyResolutionError(
|
|
564
|
-
"PROXY_ALLOCATION_FAILED",
|
|
565
|
-
"Smartproxy allocator failed for required proxy egress.",
|
|
566
|
-
{ cause: error },
|
|
567
|
-
);
|
|
568
|
-
}
|
|
569
|
-
return { shouldWarn: true };
|
|
570
719
|
}
|
|
720
|
+
|
|
721
|
+
// smartproxy allocation-style vendor.
|
|
722
|
+
const appKey = (
|
|
723
|
+
context.credentials === undefined
|
|
724
|
+
? process.env[SMARTPROXY_APP_KEY_ENV]
|
|
725
|
+
: context.credentials[SMARTPROXY_APP_KEY_ENV]
|
|
726
|
+
)?.trim();
|
|
727
|
+
if (!appKey) {
|
|
728
|
+
// Guarded by vendorHasCredentials; treated as a vendor-internal failure.
|
|
729
|
+
throw new ProxyResolutionError(
|
|
730
|
+
"PROXY_ALLOCATION_FAILED",
|
|
731
|
+
`${SMARTPROXY_APP_KEY_ENV} is not configured.`,
|
|
732
|
+
{ vendor: "smartproxy" },
|
|
733
|
+
);
|
|
734
|
+
}
|
|
735
|
+
const lifetimeMinutes = resolveSmartproxyLifetime(policy, context.ambientDefaults !== false);
|
|
736
|
+
const allocated = await allocateSmartproxy(
|
|
737
|
+
policy,
|
|
738
|
+
appKey,
|
|
739
|
+
lifetimeMinutes,
|
|
740
|
+
options.affinityKey,
|
|
741
|
+
context.protocol,
|
|
742
|
+
context.ambientDefaults !== false,
|
|
743
|
+
context.sharedCache !== false,
|
|
744
|
+
);
|
|
745
|
+
options.telemetry?.recordProxyResolution({ ...allocated.telemetry, protocol: context.protocol });
|
|
746
|
+
const poolIndex = selectProxyPoolIndex(allocated.pool.urls.length, context.poolIndex);
|
|
747
|
+
const url = allocated.pool.urls[poolIndex];
|
|
748
|
+
if (url) assertTunnelingScheme(url);
|
|
749
|
+
return {
|
|
750
|
+
shouldWarn: false,
|
|
751
|
+
url,
|
|
752
|
+
source: "smartproxy-allocator",
|
|
753
|
+
protocol: context.protocol,
|
|
754
|
+
diagnostics: {
|
|
755
|
+
...allocated.pool.diagnostics,
|
|
756
|
+
poolSize: allocated.pool.urls.length,
|
|
757
|
+
poolIndex,
|
|
758
|
+
},
|
|
759
|
+
};
|
|
571
760
|
}
|
|
572
761
|
|
|
573
|
-
function resolvePolicy(
|
|
574
|
-
options: ProxyResolutionOptions,
|
|
575
|
-
): ProviderProxyPolicy | undefined {
|
|
762
|
+
function resolvePolicy(options: ProxyResolutionOptions): ProviderProxyPolicy | undefined {
|
|
576
763
|
if (options.proxyPolicy) {
|
|
577
764
|
return options.proxyPolicy;
|
|
578
765
|
}
|
|
@@ -583,32 +770,219 @@ function resolvePolicy(
|
|
|
583
770
|
return undefined;
|
|
584
771
|
}
|
|
585
772
|
|
|
586
|
-
function
|
|
587
|
-
return
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
773
|
+
function isRegistryVendor(name: string | undefined): name is ProxyVendorName {
|
|
774
|
+
return name === "smartproxy" || name === "nodemaven";
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
function declaredVendorChain(policy: ProviderProxyPolicy): ProviderProxyProvider[] {
|
|
778
|
+
const declared = policy.providers?.length
|
|
779
|
+
? policy.providers
|
|
780
|
+
: [policy.provider ?? envDefaultProvider()];
|
|
781
|
+
return declared.filter((vendor): vendor is ProviderProxyProvider => vendor !== undefined);
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
/**
|
|
785
|
+
* Ordered list of SDK-native proxy vendors declared by the policy. `providers`
|
|
786
|
+
* takes precedence over the legacy singular `provider`; the platform default
|
|
787
|
+
* env is the final fallback. Non-registry names (decodo/custom) are dropped so
|
|
788
|
+
* an all-deprecated chain has no managed adapter.
|
|
789
|
+
*/
|
|
790
|
+
export function resolveVendorChain(policy: ProviderProxyPolicy): ProxyVendorName[] {
|
|
791
|
+
const chain: ProxyVendorName[] = [];
|
|
792
|
+
for (const name of declaredVendorChain(policy)) {
|
|
793
|
+
if (isRegistryVendor(name) && !chain.includes(name)) {
|
|
794
|
+
chain.push(name);
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
return chain;
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
function envDefaultProvider(): ProviderProxyProvider | undefined {
|
|
801
|
+
const raw = process.env[DEFAULT_PROXY_PROVIDER_ENV]?.trim().toLowerCase();
|
|
802
|
+
return (raw as ProviderProxyProvider | undefined) ?? undefined;
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
function vendorHasCredentials(vendor: ProxyVendorName): boolean {
|
|
806
|
+
if (vendor === "nodemaven") return hasNodemavenCredentials();
|
|
807
|
+
return Boolean(process.env[SMARTPROXY_APP_KEY_ENV]?.trim());
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
function missingCredentialEnv(vendor: ProxyVendorName): string {
|
|
811
|
+
return vendor === "nodemaven" ? "APIFUSE__PROXY__NODEMAVEN_USERNAME" : SMARTPROXY_APP_KEY_ENV;
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
function vendorPoolSize(vendor: ProxyVendorName, policy: ProviderProxyPolicy): number {
|
|
815
|
+
return vendor === "nodemaven" ? nodemavenPoolSize(policy) : resolveSmartproxyPoolSize(policy);
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
/**
|
|
819
|
+
* Total attempt span across a policy's vendor chain — the sum of each vendor's
|
|
820
|
+
* pool size. Transports use this so successive attempts rotate a vendor's pool
|
|
821
|
+
* and then fail over to the next vendor via the flat attempt index. With one
|
|
822
|
+
* vendor this equals that vendor's pool size (today's behaviour).
|
|
823
|
+
*/
|
|
824
|
+
export function resolvePolicyProxyPoolSpan(policy: ProviderProxyPolicy): number {
|
|
825
|
+
const chain = resolveVendorChain(policy);
|
|
826
|
+
if (chain.length === 0) return resolveSmartproxyPoolSize(policy);
|
|
827
|
+
return chain.reduce((sum, vendor) => sum + vendorPoolSize(vendor, policy), 0);
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
function vendorMaxPoolSize(vendor: ProxyVendorName): number {
|
|
831
|
+
return vendor === "nodemaven" ? NODEMAVEN_MAX_POOL_SIZE : SMARTPROXY_MAX_POOL_SIZE;
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
/**
|
|
835
|
+
* Absolute upper bound on a chain's attempt span — the sum of each vendor's
|
|
836
|
+
* *maximum* pool size. Unlike `resolvePolicyProxyPoolSpan` (the configured
|
|
837
|
+
* span), this backstop is independent of `session.poolSize`, so it never
|
|
838
|
+
* truncates a legitimately large pool below the point where the flat attempt
|
|
839
|
+
* index would cross into the next vendor (e.g. a 50-slot NodeMaven pool).
|
|
840
|
+
*/
|
|
841
|
+
export function maxPolicyProxyPoolSpan(policy: ProviderProxyPolicy): number {
|
|
842
|
+
const chain = resolveVendorChain(policy);
|
|
843
|
+
if (chain.length === 0) return SMARTPROXY_MAX_POOL_SIZE;
|
|
844
|
+
return chain.reduce((sum, vendor) => sum + vendorMaxPoolSize(vendor), 0);
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
const UNSAFE_TRANSPORT_RETRY_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE", "TRACE"]);
|
|
848
|
+
|
|
849
|
+
/**
|
|
850
|
+
* Transport-retry attempt cap for a policy-managed request. A transport failure
|
|
851
|
+
* rotates the flat attempt index onto the *next* endpoint (and, once the index
|
|
852
|
+
* passes the primary vendor's pool span, the *next vendor*), so the cap must be
|
|
853
|
+
* the chain's full pool span for failover to reach the fallback vendor — the
|
|
854
|
+
* per-endpoint retry budget (default 3) never gets there.
|
|
855
|
+
*
|
|
856
|
+
* The span only widens beyond the caller's retry budget when ALL hold:
|
|
857
|
+
* - the request is policy-allocator managed (not a caller-supplied proxy URL);
|
|
858
|
+
* - the caller did NOT pin an explicit retry policy — `HttpRetryOptions.attempts`
|
|
859
|
+
* is the documented total-attempt ceiling and must be honoured verbatim;
|
|
860
|
+
* - the method is safe/idempotent — an unsafe request must never be duplicated
|
|
861
|
+
* across the pool even if some framework default would allow it;
|
|
862
|
+
* - the policy resolves a non-empty *registry* vendor chain (smartproxy /
|
|
863
|
+
* nodemaven). Deprecated vendors (custom / decodo) resolve no managed pool,
|
|
864
|
+
* so there is no possible endpoint crossover — they keep the retry budget.
|
|
865
|
+
*
|
|
866
|
+
* The widened cap is bounded by the chain's true maximum span (sum of each
|
|
867
|
+
* vendor's max pool size), so a large NodeMaven pool (≤50) stays reachable and
|
|
868
|
+
* a pathological chain can never spin unbounded.
|
|
869
|
+
*/
|
|
870
|
+
/**
|
|
871
|
+
* True when a policy request is in *implicit chain-rotation* mode: successive
|
|
872
|
+
* transport attempts rotate the flat index across the concatenated vendor pool
|
|
873
|
+
* spans (and, past the primary vendor's span, into the fallback vendor). This is
|
|
874
|
+
* the ONLY mode in which the transport loop widens its attempt cap AND
|
|
875
|
+
* de-duplicates repeated endpoints — the two behaviours must share one predicate
|
|
876
|
+
* so they never diverge. It holds when ALL of the widening conditions hold:
|
|
877
|
+
* - the request is policy-allocator managed (not a caller-supplied proxy URL);
|
|
878
|
+
* - the caller did NOT pin an explicit retry policy — its `attempts` ceiling is
|
|
879
|
+
* the documented contract and must be honoured verbatim against whatever
|
|
880
|
+
* endpoint each attempt resolves (even a repeated one), so no de-duplication;
|
|
881
|
+
* - the method is safe/idempotent — an unsafe request is never duplicated;
|
|
882
|
+
* - the policy resolves a non-empty registry vendor chain (smartproxy /
|
|
883
|
+
* nodemaven). Deprecated vendors (custom / decodo) resolve no managed
|
|
884
|
+
* endpoint, so there is nothing to rotate or de-duplicate.
|
|
885
|
+
*/
|
|
886
|
+
export function policyRotatesTransportVendorChain(input: {
|
|
887
|
+
policy: ProviderProxyPolicy | undefined;
|
|
888
|
+
usesPolicyAllocator: boolean;
|
|
889
|
+
explicitRetry: boolean;
|
|
890
|
+
method: string;
|
|
891
|
+
}): boolean {
|
|
892
|
+
if (!input.usesPolicyAllocator || !input.policy || input.explicitRetry) {
|
|
893
|
+
return false;
|
|
894
|
+
}
|
|
895
|
+
if (UNSAFE_TRANSPORT_RETRY_METHODS.has(input.method.toUpperCase())) {
|
|
896
|
+
return false;
|
|
897
|
+
}
|
|
898
|
+
return resolveVendorChain(input.policy).length > 0;
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
export function resolvePolicyTransportAttemptCap(input: {
|
|
902
|
+
policy: ProviderProxyPolicy | undefined;
|
|
903
|
+
usesPolicyAllocator: boolean;
|
|
904
|
+
retryAttempts: number;
|
|
905
|
+
explicitRetry: boolean;
|
|
906
|
+
method: string;
|
|
907
|
+
}): number {
|
|
908
|
+
const budget = Math.max(1, Math.floor(input.retryAttempts));
|
|
909
|
+
if (
|
|
910
|
+
!policyRotatesTransportVendorChain({
|
|
911
|
+
policy: input.policy,
|
|
912
|
+
usesPolicyAllocator: input.usesPolicyAllocator,
|
|
913
|
+
explicitRetry: input.explicitRetry,
|
|
914
|
+
method: input.method,
|
|
915
|
+
})
|
|
916
|
+
) {
|
|
917
|
+
return budget;
|
|
918
|
+
}
|
|
919
|
+
const span = Math.min(maxPolicyProxyPoolSpan(input.policy as ProviderProxyPolicy), resolvePolicyProxyPoolSpan(input.policy as ProviderProxyPolicy));
|
|
920
|
+
return Math.max(budget, span);
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
/**
|
|
924
|
+
* A registry vendor chain (smartproxy/nodemaven) resolves a potentially
|
|
925
|
+
* *different* endpoint per flat attempt index, so a transport retry should
|
|
926
|
+
* advance across endpoints and de-duplicate once the chain stops yielding new
|
|
927
|
+
* ones. Deprecated custom/decodo policies have an empty registry chain and no
|
|
928
|
+
* managed endpoint, so the transport loop has nothing to rotate or de-duplicate.
|
|
929
|
+
*/
|
|
930
|
+
export function policyResolvesRegistryVendorChain(
|
|
931
|
+
policy: ProviderProxyPolicy | undefined,
|
|
932
|
+
): boolean {
|
|
933
|
+
return Boolean(policy) && resolveVendorChain(policy as ProviderProxyPolicy).length > 0;
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
/** Map a resolved proxy source label to the vendor that served it. */
|
|
937
|
+
export function vendorFromResolvedSource(
|
|
938
|
+
source: ResolvedProxyConfig["source"],
|
|
939
|
+
): ProxyVendorName | undefined {
|
|
940
|
+
if (source === "nodemaven-gateway") return "nodemaven";
|
|
941
|
+
if (source === "smartproxy-allocator") return "smartproxy";
|
|
942
|
+
return undefined;
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
function normalizeAttemptIndex(attempt: number | undefined): number {
|
|
946
|
+
return Number.isFinite(attempt) ? Math.max(0, Math.floor(attempt as number)) : 0;
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
/**
|
|
950
|
+
* Map a flat attempt index into (vendorIndex, poolIndex) by concatenating each
|
|
951
|
+
* vendor's pool space in chain order. With a single vendor this reduces to
|
|
952
|
+
* `attempt % poolSize`, preserving today's behaviour exactly.
|
|
953
|
+
*/
|
|
954
|
+
export function mapFlatAttempt(
|
|
955
|
+
flat: number,
|
|
956
|
+
sizes: readonly number[],
|
|
957
|
+
): { vendorIndex: number; poolIndex: number } {
|
|
958
|
+
let cursor = flat;
|
|
959
|
+
for (let vendorIndex = 0; vendorIndex < sizes.length; vendorIndex++) {
|
|
960
|
+
const size = Math.max(1, sizes[vendorIndex] ?? 1);
|
|
961
|
+
if (cursor < size) {
|
|
962
|
+
return { vendorIndex, poolIndex: cursor };
|
|
963
|
+
}
|
|
964
|
+
cursor -= size;
|
|
965
|
+
}
|
|
966
|
+
return { vendorIndex: 0, poolIndex: 0 };
|
|
592
967
|
}
|
|
593
968
|
|
|
594
969
|
function resolveSmartproxyCountry(
|
|
595
970
|
policy: ProviderProxyPolicy,
|
|
971
|
+
ambientDefaults = true,
|
|
596
972
|
): string | undefined {
|
|
597
973
|
return (
|
|
598
974
|
policy.geo?.country ??
|
|
599
|
-
|
|
600
|
-
|
|
975
|
+
(ambientDefaults
|
|
976
|
+
? process.env[DEFAULT_PROXY_COUNTRY_ENV]?.trim().toUpperCase() || undefined
|
|
977
|
+
: undefined)
|
|
601
978
|
);
|
|
602
979
|
}
|
|
603
980
|
|
|
604
|
-
function resolveSmartproxyLifetime(policy: ProviderProxyPolicy): number {
|
|
981
|
+
function resolveSmartproxyLifetime(policy: ProviderProxyPolicy, ambientDefaults = true): number {
|
|
605
982
|
const configuredLifetime =
|
|
606
983
|
policy.session?.lifetimeMinutes ??
|
|
607
|
-
readPositiveNumberEnv(DEFAULT_PROXY_LIFETIME_ENV, 30);
|
|
608
|
-
return Math.min(
|
|
609
|
-
SMARTPROXY_MAX_LIFETIME_MINUTES,
|
|
610
|
-
Math.max(1, Math.floor(configuredLifetime)),
|
|
611
|
-
);
|
|
984
|
+
(ambientDefaults ? readPositiveNumberEnv(DEFAULT_PROXY_LIFETIME_ENV, 30) : 30);
|
|
985
|
+
return Math.min(SMARTPROXY_MAX_LIFETIME_MINUTES, Math.max(1, Math.floor(configuredLifetime)));
|
|
612
986
|
}
|
|
613
987
|
|
|
614
988
|
function readPositiveNumberEnv(name: string, fallback: number): number {
|
|
@@ -624,10 +998,7 @@ function readPositiveNumberEnv(name: string, fallback: number): number {
|
|
|
624
998
|
function resolveSmartproxyPoolSize(policy: ProviderProxyPolicy): number {
|
|
625
999
|
return Math.min(
|
|
626
1000
|
SMARTPROXY_MAX_POOL_SIZE,
|
|
627
|
-
Math.max(
|
|
628
|
-
1,
|
|
629
|
-
Math.floor(policy.session?.poolSize ?? DEFAULT_SMARTPROXY_POOL_SIZE),
|
|
630
|
-
),
|
|
1001
|
+
Math.max(1, Math.floor(policy.session?.poolSize ?? DEFAULT_SMARTPROXY_POOL_SIZE)),
|
|
631
1002
|
);
|
|
632
1003
|
}
|
|
633
1004
|
|
|
@@ -635,26 +1006,29 @@ function selectProxyPoolIndex(poolSize: number, attempt = 0): number {
|
|
|
635
1006
|
if (poolSize <= 1) {
|
|
636
1007
|
return 0;
|
|
637
1008
|
}
|
|
638
|
-
const normalizedAttempt = Number.isFinite(attempt)
|
|
639
|
-
? Math.max(0, Math.floor(attempt))
|
|
640
|
-
: 0;
|
|
1009
|
+
const normalizedAttempt = Number.isFinite(attempt) ? Math.max(0, Math.floor(attempt)) : 0;
|
|
641
1010
|
return normalizedAttempt % poolSize;
|
|
642
1011
|
}
|
|
643
1012
|
|
|
644
1013
|
function buildSmartproxyCacheKey(
|
|
645
1014
|
policy: ProviderProxyPolicy,
|
|
1015
|
+
appKey: string,
|
|
646
1016
|
affinityKey: string | undefined,
|
|
647
1017
|
lifetimeMinutes: number,
|
|
1018
|
+
protocol: ProxyProtocol,
|
|
1019
|
+
ambientDefaults = true,
|
|
648
1020
|
): string {
|
|
649
1021
|
const poolSize = resolveSmartproxyPoolSize(policy);
|
|
650
1022
|
return JSON.stringify({
|
|
651
1023
|
provider: "smartproxy",
|
|
652
|
-
|
|
1024
|
+
credentialHash: createHash("sha256")
|
|
1025
|
+
.update("apifuse-smartproxy-credential:v1\0")
|
|
1026
|
+
.update(appKey)
|
|
1027
|
+
.digest("hex"),
|
|
1028
|
+
protocol,
|
|
1029
|
+
country: resolveSmartproxyCountry(policy, ambientDefaults),
|
|
653
1030
|
affinity: policy.session?.affinity ?? "request",
|
|
654
|
-
affinityKey:
|
|
655
|
-
(policy.session?.affinity ?? "request") === "request"
|
|
656
|
-
? undefined
|
|
657
|
-
: affinityKey,
|
|
1031
|
+
affinityKey: (policy.session?.affinity ?? "request") === "request" ? undefined : affinityKey,
|
|
658
1032
|
lifetimeMinutes,
|
|
659
1033
|
poolSize,
|
|
660
1034
|
});
|
|
@@ -665,11 +1039,17 @@ async function allocateSmartproxy(
|
|
|
665
1039
|
appKey: string,
|
|
666
1040
|
lifetimeMinutes: number,
|
|
667
1041
|
affinityKey: string | undefined,
|
|
1042
|
+
protocol: ProxyProtocol,
|
|
1043
|
+
ambientDefaults = true,
|
|
1044
|
+
sharedCache = true,
|
|
668
1045
|
): Promise<SmartproxyAllocationResult> {
|
|
669
1046
|
const cacheKey = buildSmartproxyCacheKey(
|
|
670
1047
|
policy,
|
|
1048
|
+
appKey,
|
|
671
1049
|
affinityKey,
|
|
672
1050
|
lifetimeMinutes,
|
|
1051
|
+
protocol,
|
|
1052
|
+
ambientDefaults,
|
|
673
1053
|
);
|
|
674
1054
|
const startedAt = Date.now();
|
|
675
1055
|
const now = startedAt;
|
|
@@ -678,7 +1058,15 @@ async function allocateSmartproxy(
|
|
|
678
1058
|
const cached = proxyCache.get(cacheKey);
|
|
679
1059
|
if (!skipCached && cached && isFresh(cached, now)) {
|
|
680
1060
|
if (shouldSoftRefresh(cached, now)) {
|
|
681
|
-
void refreshSmartproxyPool(
|
|
1061
|
+
void refreshSmartproxyPool(
|
|
1062
|
+
cacheKey,
|
|
1063
|
+
policy,
|
|
1064
|
+
appKey,
|
|
1065
|
+
lifetimeMinutes,
|
|
1066
|
+
protocol,
|
|
1067
|
+
ambientDefaults,
|
|
1068
|
+
sharedCache,
|
|
1069
|
+
);
|
|
682
1070
|
return {
|
|
683
1071
|
pool: cached,
|
|
684
1072
|
telemetry: telemetryForPool(cached, "soft_stale_refresh", startedAt, {
|
|
@@ -692,7 +1080,7 @@ async function allocateSmartproxy(
|
|
|
692
1080
|
};
|
|
693
1081
|
}
|
|
694
1082
|
|
|
695
|
-
if (!skipCached) {
|
|
1083
|
+
if (!skipCached && sharedCache) {
|
|
696
1084
|
const redisResult = await readSmartproxyRedisPool(cacheKey, startedAt);
|
|
697
1085
|
if (redisResult) return redisResult;
|
|
698
1086
|
}
|
|
@@ -714,6 +1102,9 @@ async function allocateSmartproxy(
|
|
|
714
1102
|
appKey,
|
|
715
1103
|
lifetimeMinutes,
|
|
716
1104
|
startedAt,
|
|
1105
|
+
protocol,
|
|
1106
|
+
ambientDefaults,
|
|
1107
|
+
sharedCache,
|
|
717
1108
|
).finally(() => {
|
|
718
1109
|
proxyInflight.delete(cacheKey);
|
|
719
1110
|
});
|
|
@@ -730,9 +1121,7 @@ async function readSmartproxyRedisPool(
|
|
|
730
1121
|
const redis = getProxyRedis();
|
|
731
1122
|
if (!redis || !(await ensureRedisReady(redis))) return null;
|
|
732
1123
|
const redisStartedAt = Date.now();
|
|
733
|
-
const raw = await withRedisTimeout(() =>
|
|
734
|
-
redis.get(smartproxyRedisPoolKey(cacheKey)),
|
|
735
|
-
);
|
|
1124
|
+
const raw = await withRedisTimeout(() => redis.get(smartproxyRedisPoolKey(cacheKey)));
|
|
736
1125
|
const redisReadMs = Math.max(0, Date.now() - redisStartedAt);
|
|
737
1126
|
if (typeof raw !== "string") return null;
|
|
738
1127
|
const pool = safeParseSmartproxyPool(raw);
|
|
@@ -754,6 +1143,9 @@ async function refreshSmartproxyPool(
|
|
|
754
1143
|
policy: ProviderProxyPolicy,
|
|
755
1144
|
appKey: string,
|
|
756
1145
|
lifetimeMinutes: number,
|
|
1146
|
+
protocol: ProxyProtocol,
|
|
1147
|
+
ambientDefaults: boolean,
|
|
1148
|
+
sharedCache: boolean,
|
|
757
1149
|
): Promise<void> {
|
|
758
1150
|
try {
|
|
759
1151
|
await allocateSmartproxyShared(
|
|
@@ -762,6 +1154,9 @@ async function refreshSmartproxyPool(
|
|
|
762
1154
|
appKey,
|
|
763
1155
|
lifetimeMinutes,
|
|
764
1156
|
Date.now(),
|
|
1157
|
+
protocol,
|
|
1158
|
+
ambientDefaults,
|
|
1159
|
+
sharedCache,
|
|
765
1160
|
{
|
|
766
1161
|
background: true,
|
|
767
1162
|
},
|
|
@@ -777,9 +1172,12 @@ async function allocateSmartproxyShared(
|
|
|
777
1172
|
appKey: string,
|
|
778
1173
|
lifetimeMinutes: number,
|
|
779
1174
|
startedAt: number,
|
|
1175
|
+
protocol: ProxyProtocol,
|
|
1176
|
+
ambientDefaults: boolean,
|
|
1177
|
+
sharedCache: boolean,
|
|
780
1178
|
options: { background?: boolean } = {},
|
|
781
1179
|
): Promise<SmartproxyAllocationResult> {
|
|
782
|
-
const redis = getProxyRedis();
|
|
1180
|
+
const redis = sharedCache ? getProxyRedis() : undefined;
|
|
783
1181
|
if (!redis || !(await ensureRedisReady(redis))) {
|
|
784
1182
|
return await allocateAndStoreSmartproxyPool(
|
|
785
1183
|
cacheKey,
|
|
@@ -787,7 +1185,7 @@ async function allocateSmartproxyShared(
|
|
|
787
1185
|
appKey,
|
|
788
1186
|
lifetimeMinutes,
|
|
789
1187
|
startedAt,
|
|
790
|
-
{ cacheStatus: "allocator" },
|
|
1188
|
+
{ cacheStatus: "allocator", protocol, ambientDefaults },
|
|
791
1189
|
);
|
|
792
1190
|
}
|
|
793
1191
|
|
|
@@ -809,11 +1207,11 @@ async function allocateSmartproxyShared(
|
|
|
809
1207
|
lifetimeMinutes,
|
|
810
1208
|
startedAt,
|
|
811
1209
|
{
|
|
812
|
-
cacheStatus: options.background
|
|
813
|
-
? "soft_stale_refresh"
|
|
814
|
-
: "allocator",
|
|
1210
|
+
cacheStatus: options.background ? "soft_stale_refresh" : "allocator",
|
|
815
1211
|
redis,
|
|
816
1212
|
poolKey,
|
|
1213
|
+
protocol,
|
|
1214
|
+
ambientDefaults,
|
|
817
1215
|
},
|
|
818
1216
|
);
|
|
819
1217
|
} finally {
|
|
@@ -838,15 +1236,10 @@ async function allocateSmartproxyShared(
|
|
|
838
1236
|
if (typeof pttl === "number" && pttl <= 0) {
|
|
839
1237
|
continue;
|
|
840
1238
|
}
|
|
841
|
-
if (
|
|
842
|
-
Date.now() - startedAt >
|
|
843
|
-
SMARTPROXY_LOCK_POLL_MAX_MS - SMARTPROXY_DEADLINE_MARGIN_MS
|
|
844
|
-
) {
|
|
1239
|
+
if (Date.now() - startedAt > SMARTPROXY_LOCK_POLL_MAX_MS - SMARTPROXY_DEADLINE_MARGIN_MS) {
|
|
845
1240
|
break;
|
|
846
1241
|
}
|
|
847
|
-
await sleep(
|
|
848
|
-
Math.min(500, Math.max(50, typeof pttl === "number" ? pttl : 100)),
|
|
849
|
-
);
|
|
1242
|
+
await sleep(Math.min(500, Math.max(50, typeof pttl === "number" ? pttl : 100)));
|
|
850
1243
|
}
|
|
851
1244
|
|
|
852
1245
|
throw new ProxyResolutionError(
|
|
@@ -893,9 +1286,7 @@ function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
|
893
1286
|
}
|
|
894
1287
|
|
|
895
1288
|
function smartproxyAllocatorDeadlineMs(): number {
|
|
896
|
-
return
|
|
897
|
-
smartproxyAllocatorDeadlineMsForTests ?? SMARTPROXY_ALLOCATOR_DEADLINE_MS
|
|
898
|
-
);
|
|
1289
|
+
return smartproxyAllocatorDeadlineMsForTests ?? SMARTPROXY_ALLOCATOR_DEADLINE_MS;
|
|
899
1290
|
}
|
|
900
1291
|
|
|
901
1292
|
function createDeadlineAbortController(deadlineAt: number): {
|
|
@@ -917,9 +1308,7 @@ function createDeadlineAbortController(deadlineAt: number): {
|
|
|
917
1308
|
};
|
|
918
1309
|
}
|
|
919
1310
|
|
|
920
|
-
function smartproxyAllocatorDeadlineFailure(
|
|
921
|
-
attempt: number,
|
|
922
|
-
): SmartproxyAllocatorFailure {
|
|
1311
|
+
function smartproxyAllocatorDeadlineFailure(attempt: number): SmartproxyAllocatorFailure {
|
|
923
1312
|
return {
|
|
924
1313
|
ok: false,
|
|
925
1314
|
attempt,
|
|
@@ -978,6 +1367,8 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
978
1367
|
cacheStatus: ProxyCacheStatus;
|
|
979
1368
|
redis?: ProxyRedisClient;
|
|
980
1369
|
poolKey?: string;
|
|
1370
|
+
protocol: ProxyProtocol;
|
|
1371
|
+
ambientDefaults: boolean;
|
|
981
1372
|
},
|
|
982
1373
|
): Promise<SmartproxyAllocationResult> {
|
|
983
1374
|
const poolSize = resolveSmartproxyPoolSize(policy);
|
|
@@ -986,17 +1377,14 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
986
1377
|
appKey,
|
|
987
1378
|
lifetimeMinutes,
|
|
988
1379
|
poolSize,
|
|
1380
|
+
options.protocol,
|
|
1381
|
+
options.ambientDefaults,
|
|
989
1382
|
);
|
|
990
1383
|
const allocatorStartedAt = Date.now();
|
|
991
|
-
const allocatorDeadlineAt =
|
|
992
|
-
allocatorStartedAt + smartproxyAllocatorDeadlineMs();
|
|
1384
|
+
const allocatorDeadlineAt = allocatorStartedAt + smartproxyAllocatorDeadlineMs();
|
|
993
1385
|
let allocation: SmartproxyAllocatorSuccess | undefined;
|
|
994
1386
|
let lastFailure: SmartproxyAllocatorFailure | undefined;
|
|
995
|
-
for (
|
|
996
|
-
let attempt = 1;
|
|
997
|
-
attempt <= SMARTPROXY_ALLOCATOR_MAX_ATTEMPTS;
|
|
998
|
-
attempt += 1
|
|
999
|
-
) {
|
|
1387
|
+
for (let attempt = 1; attempt <= SMARTPROXY_ALLOCATOR_MAX_ATTEMPTS; attempt += 1) {
|
|
1000
1388
|
if (Date.now() >= allocatorDeadlineAt) {
|
|
1001
1389
|
lastFailure = smartproxyAllocatorDeadlineFailure(attempt);
|
|
1002
1390
|
break;
|
|
@@ -1005,6 +1393,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
1005
1393
|
allocatorUrl,
|
|
1006
1394
|
attempt,
|
|
1007
1395
|
allocatorDeadlineAt,
|
|
1396
|
+
options.protocol,
|
|
1008
1397
|
);
|
|
1009
1398
|
if (attemptResult.ok) {
|
|
1010
1399
|
allocation = attemptResult;
|
|
@@ -1013,8 +1402,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
1013
1402
|
lastFailure = attemptResult;
|
|
1014
1403
|
if (attempt < SMARTPROXY_ALLOCATOR_MAX_ATTEMPTS) {
|
|
1015
1404
|
if (Date.now() >= allocatorDeadlineAt) break;
|
|
1016
|
-
const { controller, dispose } =
|
|
1017
|
-
createDeadlineAbortController(allocatorDeadlineAt);
|
|
1405
|
+
const { controller, dispose } = createDeadlineAbortController(allocatorDeadlineAt);
|
|
1018
1406
|
try {
|
|
1019
1407
|
await sleep(smartproxyAllocatorBackoffMs(attempt), controller.signal);
|
|
1020
1408
|
} catch {
|
|
@@ -1027,9 +1415,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
1027
1415
|
|
|
1028
1416
|
const allocatorMs = Math.max(0, Date.now() - allocatorStartedAt);
|
|
1029
1417
|
const allocatorAttempts =
|
|
1030
|
-
allocation?.attempt ??
|
|
1031
|
-
lastFailure?.attempt ??
|
|
1032
|
-
SMARTPROXY_ALLOCATOR_MAX_ATTEMPTS;
|
|
1418
|
+
allocation?.attempt ?? lastFailure?.attempt ?? SMARTPROXY_ALLOCATOR_MAX_ATTEMPTS;
|
|
1033
1419
|
const allocatorBodyClass =
|
|
1034
1420
|
lastFailure?.bodyClass ?? allocation?.bodyClass ?? "usable_proxy_endpoints";
|
|
1035
1421
|
const allocatorStatus = lastFailure ? lastFailure.status : allocation?.status;
|
|
@@ -1063,7 +1449,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
1063
1449
|
expiresAt: allocatedAt + ttlMs,
|
|
1064
1450
|
diagnostics: {
|
|
1065
1451
|
provider: "smartproxy",
|
|
1066
|
-
country: resolveSmartproxyCountry(policy) ?? "default",
|
|
1452
|
+
country: resolveSmartproxyCountry(policy, options.ambientDefaults) ?? "default",
|
|
1067
1453
|
lifetimeMinutes,
|
|
1068
1454
|
affinity: policy.session?.affinity ?? "request",
|
|
1069
1455
|
rawConnect: true,
|
|
@@ -1106,9 +1492,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
1106
1492
|
};
|
|
1107
1493
|
}
|
|
1108
1494
|
|
|
1109
|
-
type SmartproxyAllocatorAttemptResult =
|
|
1110
|
-
| SmartproxyAllocatorSuccess
|
|
1111
|
-
| SmartproxyAllocatorFailure;
|
|
1495
|
+
type SmartproxyAllocatorAttemptResult = SmartproxyAllocatorSuccess | SmartproxyAllocatorFailure;
|
|
1112
1496
|
|
|
1113
1497
|
type SmartproxyAllocatorSuccess = {
|
|
1114
1498
|
ok: true;
|
|
@@ -1130,6 +1514,7 @@ async function fetchSmartproxyAllocatorAttempt(
|
|
|
1130
1514
|
allocatorUrl: string,
|
|
1131
1515
|
attempt: number,
|
|
1132
1516
|
deadlineAt: number,
|
|
1517
|
+
protocol: ProxyProtocol,
|
|
1133
1518
|
): Promise<SmartproxyAllocatorAttemptResult> {
|
|
1134
1519
|
const { controller, dispose } = createDeadlineAbortController(deadlineAt);
|
|
1135
1520
|
let response: Response;
|
|
@@ -1150,10 +1535,7 @@ async function fetchSmartproxyAllocatorAttempt(
|
|
|
1150
1535
|
|
|
1151
1536
|
let body: string;
|
|
1152
1537
|
try {
|
|
1153
|
-
body = await readSmartproxyAllocatorBodyWithDeadline(
|
|
1154
|
-
response,
|
|
1155
|
-
controller.signal,
|
|
1156
|
-
);
|
|
1538
|
+
body = await readSmartproxyAllocatorBodyWithDeadline(response, controller.signal);
|
|
1157
1539
|
} catch (error) {
|
|
1158
1540
|
dispose();
|
|
1159
1541
|
return {
|
|
@@ -1176,7 +1558,7 @@ async function fetchSmartproxyAllocatorAttempt(
|
|
|
1176
1558
|
};
|
|
1177
1559
|
}
|
|
1178
1560
|
|
|
1179
|
-
const urls = parseSmartproxyAllocatorProxies(body);
|
|
1561
|
+
const urls = parseSmartproxyAllocatorProxies(body, protocol);
|
|
1180
1562
|
const bodyClass = classifySmartproxyAllocatorBody(body, urls);
|
|
1181
1563
|
if (urls.length === 0) {
|
|
1182
1564
|
return {
|
|
@@ -1196,8 +1578,7 @@ async function fetchSmartproxyAllocatorAttempt(
|
|
|
1196
1578
|
}
|
|
1197
1579
|
|
|
1198
1580
|
function smartproxyAllocatorBackoffMs(attempt: number): number {
|
|
1199
|
-
const base =
|
|
1200
|
-
SMARTPROXY_ALLOCATOR_RETRY_BASE_MS * 2 ** Math.max(0, attempt - 1);
|
|
1581
|
+
const base = SMARTPROXY_ALLOCATOR_RETRY_BASE_MS * 2 ** Math.max(0, attempt - 1);
|
|
1201
1582
|
const jitter = Math.floor(Math.random() * SMARTPROXY_ALLOCATOR_RETRY_BASE_MS);
|
|
1202
1583
|
return base + jitter;
|
|
1203
1584
|
}
|
|
@@ -1217,29 +1598,42 @@ function smartproxyAllocatorFailureMessage(
|
|
|
1217
1598
|
return "Smartproxy allocator response did not contain a usable proxy endpoint.";
|
|
1218
1599
|
}
|
|
1219
1600
|
|
|
1601
|
+
// Smartproxy get-ip-v3 `protocol` param: 1 = HTTP. The SOCKS5 value ("2") is a
|
|
1602
|
+
// best-effort mapping pending live vendor confirmation; http is the default and
|
|
1603
|
+
// the only value exercised in production today.
|
|
1604
|
+
const SMARTPROXY_PROTOCOL_PARAM: Record<ProxyProtocol, string> = {
|
|
1605
|
+
http: "1",
|
|
1606
|
+
socks5: "2",
|
|
1607
|
+
};
|
|
1608
|
+
|
|
1220
1609
|
function buildSmartproxyAllocatorUrl(
|
|
1221
1610
|
policy: ProviderProxyPolicy,
|
|
1222
1611
|
appKey: string,
|
|
1223
1612
|
lifetimeMinutes: number,
|
|
1224
1613
|
poolSize: number,
|
|
1614
|
+
protocol: ProxyProtocol,
|
|
1615
|
+
ambientDefaults = true,
|
|
1225
1616
|
): string {
|
|
1226
1617
|
const params = new URLSearchParams({
|
|
1227
1618
|
app_key: appKey,
|
|
1228
1619
|
pt: "9",
|
|
1229
1620
|
num: String(poolSize),
|
|
1230
1621
|
life: String(lifetimeMinutes),
|
|
1231
|
-
protocol:
|
|
1622
|
+
protocol: SMARTPROXY_PROTOCOL_PARAM[protocol],
|
|
1232
1623
|
format: "txt",
|
|
1233
1624
|
lb: "\\n",
|
|
1234
1625
|
});
|
|
1235
|
-
const country = resolveSmartproxyCountry(policy);
|
|
1626
|
+
const country = resolveSmartproxyCountry(policy, ambientDefaults);
|
|
1236
1627
|
if (country) {
|
|
1237
1628
|
params.set("cc", country);
|
|
1238
1629
|
}
|
|
1239
|
-
|
|
1630
|
+
// www.smartproxy.org stopped serving the extraction API on 2026-07-21 (the
|
|
1631
|
+
// old path 404s into the marketing site); the API lives on the api host.
|
|
1632
|
+
return `https://api.smartproxy.org/web_v1/ip/get-ip-v3?${params.toString()}`;
|
|
1240
1633
|
}
|
|
1241
1634
|
|
|
1242
|
-
function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
1635
|
+
function parseSmartproxyAllocatorProxies(body: string, protocol: ProxyProtocol): string[] {
|
|
1636
|
+
const scheme = protocol === "socks5" ? "socks5" : "http";
|
|
1243
1637
|
const trimmed = body.trim();
|
|
1244
1638
|
if (!trimmed) {
|
|
1245
1639
|
return [];
|
|
@@ -1247,14 +1641,8 @@ function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
|
1247
1641
|
|
|
1248
1642
|
try {
|
|
1249
1643
|
const parsed: unknown = JSON.parse(trimmed);
|
|
1250
|
-
const data =
|
|
1251
|
-
|
|
1252
|
-
? parsed.data
|
|
1253
|
-
: undefined;
|
|
1254
|
-
const list =
|
|
1255
|
-
data && typeof data === "object" && "list" in data
|
|
1256
|
-
? data.list
|
|
1257
|
-
: undefined;
|
|
1644
|
+
const data = parsed && typeof parsed === "object" && "data" in parsed ? parsed.data : undefined;
|
|
1645
|
+
const list = data && typeof data === "object" && "list" in data ? data.list : undefined;
|
|
1258
1646
|
if (Array.isArray(list)) {
|
|
1259
1647
|
return list
|
|
1260
1648
|
.map((item) => {
|
|
@@ -1263,11 +1651,10 @@ function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
|
1263
1651
|
}
|
|
1264
1652
|
const ip = "ip" in item && typeof item.ip === "string" ? item.ip : "";
|
|
1265
1653
|
const port =
|
|
1266
|
-
"port" in item &&
|
|
1267
|
-
(typeof item.port === "string" || typeof item.port === "number")
|
|
1654
|
+
"port" in item && (typeof item.port === "string" || typeof item.port === "number")
|
|
1268
1655
|
? item.port
|
|
1269
1656
|
: "";
|
|
1270
|
-
return ip && port ?
|
|
1657
|
+
return ip && port ? `${scheme}://${ip}:${port}` : null;
|
|
1271
1658
|
})
|
|
1272
1659
|
.filter((url): url is string => url !== null);
|
|
1273
1660
|
}
|
|
@@ -1279,7 +1666,7 @@ function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
|
1279
1666
|
.split(/\r?\n/)
|
|
1280
1667
|
.map((item) => item.trim())
|
|
1281
1668
|
.filter((item) => /^\d{1,3}(?:\.\d{1,3}){3}:\d{2,5}$/.test(item))
|
|
1282
|
-
.map((line) =>
|
|
1669
|
+
.map((line) => `${scheme}://${line}`);
|
|
1283
1670
|
}
|
|
1284
1671
|
|
|
1285
1672
|
function classifySmartproxyAllocatorBody(
|
|
@@ -1307,35 +1694,32 @@ export function clearProxyResolutionCache(): void {
|
|
|
1307
1694
|
invalidatedProxyKeys.clear();
|
|
1308
1695
|
}
|
|
1309
1696
|
|
|
1310
|
-
function markSmartproxyCacheInvalidated(
|
|
1311
|
-
options: ProxyResolutionOptions = {},
|
|
1312
|
-
): string | undefined {
|
|
1697
|
+
function markSmartproxyCacheInvalidated(options: ProxyResolutionOptions = {}): string | undefined {
|
|
1313
1698
|
const policy = resolvePolicy(options);
|
|
1314
1699
|
if (!policy || policy.mode === "disabled") {
|
|
1315
1700
|
return undefined;
|
|
1316
1701
|
}
|
|
1317
|
-
if (
|
|
1702
|
+
if (!resolveVendorChain(policy).includes("smartproxy")) {
|
|
1318
1703
|
return undefined;
|
|
1319
1704
|
}
|
|
1320
1705
|
|
|
1321
1706
|
const lifetimeMinutes = resolveSmartproxyLifetime(policy);
|
|
1707
|
+
const appKey = process.env[SMARTPROXY_APP_KEY_ENV]?.trim();
|
|
1708
|
+
if (!appKey) return undefined;
|
|
1322
1709
|
const cacheKey = buildSmartproxyCacheKey(
|
|
1323
1710
|
policy,
|
|
1711
|
+
appKey,
|
|
1324
1712
|
options.affinityKey,
|
|
1325
1713
|
lifetimeMinutes,
|
|
1714
|
+
options.protocol ?? VENDOR_DEFAULT_PROTOCOL.smartproxy,
|
|
1326
1715
|
);
|
|
1327
|
-
invalidatedProxyKeys.set(
|
|
1328
|
-
cacheKey,
|
|
1329
|
-
Date.now() + SMARTPROXY_INVALIDATION_SKIP_REDIS_MS,
|
|
1330
|
-
);
|
|
1716
|
+
invalidatedProxyKeys.set(cacheKey, Date.now() + SMARTPROXY_INVALIDATION_SKIP_REDIS_MS);
|
|
1331
1717
|
proxyCache.delete(cacheKey);
|
|
1332
1718
|
proxyInflight.delete(cacheKey);
|
|
1333
1719
|
return cacheKey;
|
|
1334
1720
|
}
|
|
1335
1721
|
|
|
1336
|
-
export function invalidateProxyResolutionCache(
|
|
1337
|
-
options: ProxyResolutionOptions = {},
|
|
1338
|
-
): boolean {
|
|
1722
|
+
export function invalidateProxyResolutionCache(options: ProxyResolutionOptions = {}): boolean {
|
|
1339
1723
|
return markSmartproxyCacheInvalidated(options) !== undefined;
|
|
1340
1724
|
}
|
|
1341
1725
|
|
|
@@ -1374,23 +1758,17 @@ async function importConfig(filePath: string): Promise<ApiFuseConfig | null> {
|
|
|
1374
1758
|
}
|
|
1375
1759
|
}
|
|
1376
1760
|
|
|
1377
|
-
export async function loadApiFuseConfig(
|
|
1378
|
-
dir: string = process.cwd(),
|
|
1379
|
-
): Promise<ApiFuseConfig> {
|
|
1761
|
+
export async function loadApiFuseConfig(dir: string = process.cwd()): Promise<ApiFuseConfig> {
|
|
1380
1762
|
const tsPath = path.resolve(dir, "apifuse.config.ts");
|
|
1381
1763
|
if (existsSync(tsPath)) {
|
|
1382
1764
|
const config = await importConfig(tsPath);
|
|
1383
|
-
|
|
1384
|
-
syncProxyEnv(resolvedConfig);
|
|
1385
|
-
return resolvedConfig;
|
|
1765
|
+
return config ?? {};
|
|
1386
1766
|
}
|
|
1387
1767
|
|
|
1388
1768
|
const jsPath = path.resolve(dir, "apifuse.config.js");
|
|
1389
1769
|
if (existsSync(jsPath)) {
|
|
1390
1770
|
const config = await importConfig(jsPath);
|
|
1391
|
-
|
|
1392
|
-
syncProxyEnv(resolvedConfig);
|
|
1393
|
-
return resolvedConfig;
|
|
1771
|
+
return config ?? {};
|
|
1394
1772
|
}
|
|
1395
1773
|
|
|
1396
1774
|
return {};
|