@apifuse/provider-sdk 2.2.0-beta.1 → 2.2.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +106 -0
- package/CHANGELOG.md +41 -0
- package/README.md +5 -1
- package/SUBMISSION.md +1 -1
- package/bin/apifuse-check.ts +44 -59
- package/bin/apifuse-create.ts +1 -1
- package/bin/apifuse-dev.ts +27 -52
- package/bin/apifuse-pack-check.ts +14 -0
- package/bin/apifuse-pack-smoke.ts +22 -81
- package/bin/apifuse-pack-types.ts +266 -0
- package/bin/apifuse-perf.ts +45 -127
- package/bin/apifuse-record.ts +53 -70
- package/bin/apifuse-submit-check.ts +510 -45
- package/bin/apifuse-sync-assets.ts +117 -0
- package/bin/apifuse.ts +1 -1
- package/bin/submit-check-delimited-text.ts +50 -0
- package/bin/submit-check-xml-semantics.ts +204 -0
- package/bin/submit-check-xml.ts +134 -0
- package/dist/auth-turn/index.d.ts +3 -3
- package/dist/auth.d.ts +2 -2
- package/dist/auth.js +9 -18
- package/dist/ceremonies/index.d.ts +1 -1
- package/dist/ceremonies/index.js +8 -18
- package/dist/cli/commands.d.ts +1 -1
- package/dist/cli/commands.js +8 -0
- package/dist/cli/create.d.ts +3 -0
- package/dist/cli/create.js +34 -35
- package/dist/cli/prompt-assets.d.ts +80 -0
- package/dist/cli/prompt-assets.js +743 -0
- package/dist/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/dist/config/loader.d.ts +149 -8
- package/dist/config/loader.js +378 -89
- package/dist/contract-serialization.d.ts +2 -2
- package/dist/contract-serialization.js +3 -6
- package/dist/contract-types.d.ts +2 -2
- package/dist/contract.d.ts +3 -3
- package/dist/contract.js +4 -6
- package/dist/define.d.ts +8 -1
- package/dist/define.js +119 -122
- package/dist/dev.d.ts +1 -1
- package/dist/dev.js +1 -1
- package/dist/errors.d.ts +4 -1
- package/dist/errors.js +48 -0
- package/dist/i18n/catalog.d.ts +2 -2
- package/dist/i18n/catalog.js +4 -10
- package/dist/i18n/index.d.ts +2 -2
- package/dist/i18n/index.js +2 -2
- package/dist/i18n/keys.d.ts +2 -2
- package/dist/index.d.ts +44 -42
- package/dist/index.js +39 -37
- package/dist/lint.js +8 -15
- package/dist/provider.d.ts +11 -11
- package/dist/provider.js +9 -9
- package/dist/public-schema-field-lint.d.ts +1 -1
- package/dist/recipes/gov-api.js +1 -1
- package/dist/runtime/auth-flow.d.ts +1 -1
- package/dist/runtime/auth-flow.js +3 -3
- package/dist/runtime/browser.d.ts +1 -1
- package/dist/runtime/browser.js +15 -29
- package/dist/runtime/cache.d.ts +1 -1
- package/dist/runtime/cache.js +4 -8
- package/dist/runtime/choice.d.ts +1 -1
- package/dist/runtime/choice.js +31 -35
- package/dist/runtime/credential.d.ts +1 -1
- package/dist/runtime/credential.js +1 -1
- package/dist/runtime/env.d.ts +1 -1
- package/dist/runtime/executor.d.ts +1 -1
- package/dist/runtime/executor.js +15 -3
- package/dist/runtime/http.d.ts +2 -2
- package/dist/runtime/http.js +144 -348
- package/dist/runtime/insights.d.ts +1 -1
- package/dist/runtime/insights.js +6 -13
- package/dist/runtime/instrumentation.d.ts +2 -2
- package/dist/runtime/instrumentation.js +56 -19
- package/dist/runtime/keyring.js +1 -1
- package/dist/runtime/namespace.js +1 -1
- package/dist/runtime/otlp.d.ts +1 -1
- package/dist/runtime/perf.d.ts +1 -1
- package/dist/runtime/provider.d.ts +1 -1
- package/dist/runtime/provider.js +1 -2
- package/dist/runtime/proxy-errors.d.ts +1 -1
- package/dist/runtime/proxy-errors.js +9 -7
- package/dist/runtime/proxy-nodemaven.d.ts +35 -0
- package/dist/runtime/proxy-nodemaven.js +128 -0
- package/dist/runtime/proxy-retry-policy.d.ts +40 -0
- package/dist/runtime/proxy-retry-policy.js +326 -0
- package/dist/runtime/proxy-telemetry.d.ts +2 -1
- package/dist/runtime/proxy-telemetry.js +55 -52
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +2 -4
- package/dist/runtime/request-options.d.ts +1 -1
- package/dist/runtime/secrets.d.ts +27 -0
- package/dist/runtime/secrets.js +51 -0
- package/dist/runtime/state.d.ts +2 -2
- package/dist/runtime/state.js +15 -4
- package/dist/runtime/stealth.d.ts +2 -2
- package/dist/runtime/stealth.js +108 -212
- package/dist/runtime/stt.d.ts +1 -1
- package/dist/runtime/stt.js +11 -15
- package/dist/runtime/trace.d.ts +2 -2
- package/dist/runtime/trace.js +2 -4
- package/dist/runtime/waterfall.d.ts +1 -1
- package/dist/schema.d.ts +1 -1
- package/dist/schema.js +7 -15
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/index.d.ts +7 -7
- package/dist/server/index.js +6 -6
- package/dist/server/self-test-redaction.d.ts +1 -1
- package/dist/server/self-test-redaction.js +1 -1
- package/dist/server/self-test.d.ts +104 -3
- package/dist/server/self-test.js +673 -115
- package/dist/server/serve.d.ts +105 -4
- package/dist/server/serve.js +588 -80
- package/dist/server/types.d.ts +10 -9
- package/dist/server/types.js +3 -7
- package/dist/stateful/errors.d.ts +14 -0
- package/dist/stateful/errors.js +14 -0
- package/dist/stateful/http-provider-event-emitter.d.ts +40 -0
- package/dist/stateful/http-provider-event-emitter.js +237 -0
- package/dist/stateful/http-session-owner-registry.d.ts +44 -0
- package/dist/stateful/http-session-owner-registry.js +210 -0
- package/dist/stateful/index.d.ts +18 -0
- package/dist/stateful/index.js +18 -0
- package/dist/stateful/provider-event-delivery-failures.d.ts +32 -0
- package/dist/stateful/provider-event-delivery-failures.js +43 -0
- package/dist/stateful/provider-event-pipeline-metrics.d.ts +46 -0
- package/dist/stateful/provider-event-pipeline-metrics.js +48 -0
- package/dist/stateful/provider-event-pipeline.d.ts +50 -0
- package/dist/stateful/provider-event-pipeline.js +1 -0
- package/dist/stateful/provider-events.d.ts +101 -0
- package/dist/stateful/provider-events.js +289 -0
- package/dist/stateful/session-key.d.ts +15 -0
- package/dist/stateful/session-key.js +86 -0
- package/dist/stateful/stateful-provider-adapter-context.d.ts +5 -0
- package/dist/stateful/stateful-provider-adapter-context.js +42 -0
- package/dist/stateful/stateful-provider-adapter-metrics.d.ts +15 -0
- package/dist/stateful/stateful-provider-adapter-metrics.js +21 -0
- package/dist/stateful/stateful-provider-adapter.d.ts +98 -0
- package/dist/stateful/stateful-provider-adapter.js +287 -0
- package/dist/stateful/stateful-provider-observability.d.ts +62 -0
- package/dist/stateful/stateful-provider-observability.js +161 -0
- package/dist/stateful/stateful-provider-owner-forwarder.d.ts +41 -0
- package/dist/stateful/stateful-provider-owner-forwarder.js +207 -0
- package/dist/stateful/stateful-provider-runtime-context.d.ts +32 -0
- package/dist/stateful/stateful-provider-runtime-context.js +60 -0
- package/dist/stateful/stateful-provider-runtime-executor.d.ts +34 -0
- package/dist/stateful/stateful-provider-runtime-executor.js +52 -0
- package/dist/stateful/stateful-provider-session-routing.d.ts +71 -0
- package/dist/stateful/stateful-provider-session-routing.js +353 -0
- package/dist/stateful/stateful-provider-session-runtime.d.ts +98 -0
- package/dist/stateful/stateful-provider-session-runtime.js +245 -0
- package/dist/stateful-signing.d.ts +18 -0
- package/dist/stateful-signing.js +27 -0
- package/dist/stealth/profiles.d.ts +1 -1
- package/dist/stealth/profiles.js +5 -14
- package/dist/stream.d.ts +1 -1
- package/dist/testing/index.d.ts +2 -2
- package/dist/testing/index.js +2 -2
- package/dist/testing/run.d.ts +1 -1
- package/dist/testing/run.js +7 -14
- package/dist/types.d.ts +87 -1
- package/dist/user-input.d.ts +30 -0
- package/dist/user-input.js +66 -0
- package/package.json +14 -5
- package/src/auth-turn/index.ts +1 -1
- package/src/auth.ts +28 -86
- package/src/ceremonies/index.ts +24 -78
- package/src/cli/commands.ts +10 -0
- package/src/cli/create.ts +42 -35
- package/src/cli/prompt-assets.ts +865 -0
- package/src/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/src/config/loader.ts +543 -208
- package/src/contract-serialization.ts +5 -11
- package/src/contract-types.ts +2 -2
- package/src/contract.ts +12 -28
- package/src/define.ts +247 -505
- package/src/dev.ts +4 -9
- package/src/errors.ts +58 -5
- package/src/i18n/catalog.ts +10 -32
- package/src/i18n/index.ts +2 -2
- package/src/i18n/keys.ts +5 -11
- package/src/index.ts +49 -42
- package/src/lint.ts +89 -159
- package/src/provider.ts +15 -11
- package/src/public-schema-field-lint.ts +7 -33
- package/src/recipes/gov-api.ts +2 -5
- package/src/runtime/auth-flow.ts +5 -7
- package/src/runtime/browser.ts +64 -187
- package/src/runtime/cache.ts +20 -67
- package/src/runtime/choice.ts +79 -132
- package/src/runtime/credential.ts +2 -2
- package/src/runtime/env.ts +1 -1
- package/src/runtime/executor.ts +23 -20
- package/src/runtime/http.ts +195 -559
- package/src/runtime/insights.ts +15 -53
- package/src/runtime/instrumentation.ts +78 -65
- package/src/runtime/keyring.ts +7 -19
- package/src/runtime/namespace.ts +2 -7
- package/src/runtime/otlp.ts +12 -23
- package/src/runtime/perf.ts +1 -1
- package/src/runtime/provider.ts +4 -9
- package/src/runtime/proxy-errors.ts +29 -42
- package/src/runtime/proxy-nodemaven.ts +178 -0
- package/src/runtime/proxy-retry-policy.ts +469 -0
- package/src/runtime/proxy-telemetry.ts +79 -77
- package/src/runtime/redis.ts +4 -12
- package/src/runtime/request-options.ts +4 -13
- package/src/runtime/secrets.ts +64 -0
- package/src/runtime/state.ts +41 -110
- package/src/runtime/stealth.ts +162 -377
- package/src/runtime/stt.ts +38 -94
- package/src/runtime/trace.ts +14 -44
- package/src/runtime/waterfall.ts +5 -18
- package/src/schema.ts +23 -84
- package/src/serve.ts +1 -1
- package/src/server/index.ts +24 -7
- package/src/server/self-test-redaction.ts +2 -2
- package/src/server/self-test.ts +857 -132
- package/src/server/serve.ts +859 -266
- package/src/server/types.ts +7 -19
- package/src/stateful/README.md +146 -0
- package/src/stateful/errors.ts +23 -0
- package/src/stateful/http-provider-event-emitter.ts +314 -0
- package/src/stateful/http-session-owner-registry.ts +306 -0
- package/src/stateful/index.ts +18 -0
- package/src/stateful/provider-event-delivery-failures.ts +80 -0
- package/src/stateful/provider-event-pipeline-metrics.ts +95 -0
- package/src/stateful/provider-event-pipeline.ts +61 -0
- package/src/stateful/provider-events.ts +462 -0
- package/src/stateful/session-key.ts +111 -0
- package/src/stateful/stateful-provider-adapter-context.ts +59 -0
- package/src/stateful/stateful-provider-adapter-metrics.ts +48 -0
- package/src/stateful/stateful-provider-adapter.ts +562 -0
- package/src/stateful/stateful-provider-observability.ts +261 -0
- package/src/stateful/stateful-provider-owner-forwarder.ts +279 -0
- package/src/stateful/stateful-provider-runtime-context.ts +92 -0
- package/src/stateful/stateful-provider-runtime-executor.ts +96 -0
- package/src/stateful/stateful-provider-session-routing.ts +555 -0
- package/src/stateful/stateful-provider-session-runtime.ts +403 -0
- package/src/stateful-signing.ts +46 -0
- package/src/stealth/profiles.ts +10 -26
- package/src/stream.ts +8 -19
- package/src/testing/index.ts +2 -2
- package/src/testing/run.ts +19 -67
- package/src/types.ts +88 -1
- package/src/user-input.ts +118 -0
- package/dist/cli/templates/provider/CLAUDE.md.tpl +0 -1
- package/src/cli/templates/provider/CLAUDE.md.tpl +0 -1
- /package/dist/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
package/src/config/loader.ts
CHANGED
|
@@ -2,22 +2,36 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
2
2
|
import { existsSync } from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
|
|
5
|
-
import Redis from "ioredis";
|
|
6
|
-
|
|
7
|
-
import type { ProviderProxyPolicy, TraceConfig } from "../types";
|
|
8
|
-
|
|
5
|
+
import { Redis } from "ioredis";
|
|
6
|
+
|
|
7
|
+
import type { ProviderProxyPolicy, ProviderProxyProvider, TraceConfig } from "../types.js";
|
|
8
|
+
import {
|
|
9
|
+
NODEMAVEN_DEFAULT_PROTOCOL,
|
|
10
|
+
NODEMAVEN_MAX_POOL_SIZE,
|
|
11
|
+
type ProxyProtocol,
|
|
12
|
+
hasNodemavenCredentials,
|
|
13
|
+
nodemavenPoolSize,
|
|
14
|
+
synthesizeNodemavenProxy,
|
|
15
|
+
} from "../runtime/proxy-nodemaven.js";
|
|
16
|
+
|
|
17
|
+
export type { ProxyProtocol } from "../runtime/proxy-nodemaven.js";
|
|
18
|
+
|
|
19
|
+
/** Proxy vendors the SDK resolves natively (as opposed to the static env path). */
|
|
20
|
+
export type ProxyVendorName = "smartproxy" | "nodemaven";
|
|
21
|
+
|
|
22
|
+
// "smartproxy" here is api.smartproxy.org — a residential proxy with an IP
|
|
23
|
+
// extraction API (app_key → raw ip:port pool). It is NOT the company formerly
|
|
24
|
+
// named Smartproxy (smartproxy.com), which rebranded to Decodo in 2025 and is
|
|
25
|
+
// modelled separately as the `decodo` gateway vendor. Do not conflate them.
|
|
9
26
|
export const SMARTPROXY_APP_KEY_ENV = "APIFUSE__PROXY__SMARTPROXY_APP_KEY";
|
|
10
27
|
export const SMARTPROXY_MAX_LIFETIME_MINUTES = 2000;
|
|
11
28
|
export const DEFAULT_SMARTPROXY_POOL_SIZE = 20;
|
|
12
29
|
export const SMARTPROXY_MAX_POOL_SIZE = 20;
|
|
13
30
|
export const DEFAULT_PROXY_PROVIDER_ENV = "APIFUSE__PROXY__PROVIDER";
|
|
14
31
|
export const DEFAULT_PROXY_COUNTRY_ENV = "APIFUSE__PROXY__DEFAULT_COUNTRY";
|
|
15
|
-
export const DEFAULT_PROXY_LIFETIME_ENV =
|
|
16
|
-
|
|
17
|
-
export const
|
|
18
|
-
"APIFUSE__PROVIDER__CACHE_REDIS_URL";
|
|
19
|
-
export const PROVIDER_STATE_REDIS_URL_ENV =
|
|
20
|
-
"APIFUSE__PROVIDER__STATE_REDIS_URL";
|
|
32
|
+
export const DEFAULT_PROXY_LIFETIME_ENV = "APIFUSE__PROXY__DEFAULT_LIFETIME_MINUTES";
|
|
33
|
+
export const PROVIDER_CACHE_REDIS_URL_ENV = "APIFUSE__PROVIDER__CACHE_REDIS_URL";
|
|
34
|
+
export const PROVIDER_STATE_REDIS_URL_ENV = "APIFUSE__PROVIDER__STATE_REDIS_URL";
|
|
21
35
|
export const REDIS_URL_ENV = "APIFUSE__REDIS__URL";
|
|
22
36
|
|
|
23
37
|
export type ProxyOptions = {
|
|
@@ -55,6 +69,26 @@ export type ProxyResolutionOptions = {
|
|
|
55
69
|
affinityKey?: string;
|
|
56
70
|
/** Zero-based proxy-pool attempt index used by SDK transports for failover. */
|
|
57
71
|
proxyAttempt?: number;
|
|
72
|
+
/**
|
|
73
|
+
* Tunnelling protocols the calling transport can use. When a resolved
|
|
74
|
+
* protocol is not in this set the resolver fails with
|
|
75
|
+
* `PROXY_PROTOCOL_UNSUPPORTED` instead of silently downgrading. Unset means
|
|
76
|
+
* permissive (both protocols allowed).
|
|
77
|
+
*/
|
|
78
|
+
transportProtocols?: readonly ProxyProtocol[];
|
|
79
|
+
/**
|
|
80
|
+
* Explicit protocol override. Internal — for the verification harness and
|
|
81
|
+
* tests, or an advanced caller. Normal callers omit it and each vendor uses
|
|
82
|
+
* its own benchmarked default protocol (see VENDOR_DEFAULT_PROTOCOL). Not an
|
|
83
|
+
* env var and not a provider-policy field.
|
|
84
|
+
*/
|
|
85
|
+
protocol?: ProxyProtocol;
|
|
86
|
+
/**
|
|
87
|
+
* Gateway pool "refresh" generation. Bumped by transports on pool refresh to
|
|
88
|
+
* derive a fresh gateway session set (ignored by allocation-style vendors,
|
|
89
|
+
* whose refresh is driven by cache invalidation).
|
|
90
|
+
*/
|
|
91
|
+
proxyRefreshEpoch?: number;
|
|
58
92
|
telemetry?: ProxyTelemetrySink;
|
|
59
93
|
};
|
|
60
94
|
|
|
@@ -77,7 +111,8 @@ export type SmartproxyAllocatorBodyClass =
|
|
|
77
111
|
| "usable_proxy_endpoints";
|
|
78
112
|
|
|
79
113
|
export type ProxyResolutionTelemetryEvent = {
|
|
80
|
-
provider:
|
|
114
|
+
provider: ProxyVendorName;
|
|
115
|
+
protocol?: ProxyProtocol;
|
|
81
116
|
cacheStatus: ProxyCacheStatus;
|
|
82
117
|
cacheHit: boolean;
|
|
83
118
|
resolutionMs: number;
|
|
@@ -95,7 +130,7 @@ export type ProxyResolutionTelemetryEvent = {
|
|
|
95
130
|
};
|
|
96
131
|
|
|
97
132
|
export type ProxyAttemptTelemetryEvent = {
|
|
98
|
-
provider:
|
|
133
|
+
provider: ProxyVendorName;
|
|
99
134
|
attempt: number;
|
|
100
135
|
poolIndex?: number;
|
|
101
136
|
proxyHash?: string;
|
|
@@ -105,31 +140,60 @@ export type ProxyAttemptTelemetryEvent = {
|
|
|
105
140
|
durationMs?: number;
|
|
106
141
|
};
|
|
107
142
|
|
|
143
|
+
export type ProxyVendorFailoverTelemetryEvent = {
|
|
144
|
+
/** Vendor that failed or was skipped. */
|
|
145
|
+
vendor: ProxyVendorName;
|
|
146
|
+
/** Vendor tried next, or undefined when the chain is exhausted. */
|
|
147
|
+
nextVendor?: ProxyVendorName;
|
|
148
|
+
phase: "resolution" | "transport";
|
|
149
|
+
reason: "no_credentials" | "allocation_failed" | "pool_exhausted" | "protocol_unsupported";
|
|
150
|
+
attempt?: number;
|
|
151
|
+
};
|
|
152
|
+
|
|
108
153
|
export type ProxyTelemetrySink = {
|
|
109
154
|
recordProxyResolution(event: ProxyResolutionTelemetryEvent): void;
|
|
110
155
|
recordProxyAttempt?(event: ProxyAttemptTelemetryEvent): void;
|
|
156
|
+
recordProxyVendorFailover?(event: ProxyVendorFailoverTelemetryEvent): void;
|
|
111
157
|
};
|
|
112
158
|
|
|
113
159
|
export type ResolvedProxyConfig = {
|
|
114
160
|
shouldWarn: boolean;
|
|
115
161
|
url?: string;
|
|
116
|
-
source?: "explicit" | "env" | "config" | "smartproxy-allocator";
|
|
162
|
+
source?: "explicit" | "env" | "config" | "smartproxy-allocator" | "nodemaven-gateway";
|
|
163
|
+
protocol?: ProxyProtocol;
|
|
117
164
|
diagnostics?: Record<string, string | number | boolean>;
|
|
118
165
|
};
|
|
119
166
|
|
|
167
|
+
export type ProxyResolutionErrorCode =
|
|
168
|
+
| "PROXY_REQUIRED"
|
|
169
|
+
| "PROXY_ALLOCATION_FAILED"
|
|
170
|
+
| "PROXY_PROTOCOL_UNSUPPORTED";
|
|
171
|
+
|
|
120
172
|
export class ProxyResolutionError extends Error {
|
|
121
|
-
readonly code:
|
|
173
|
+
readonly code: ProxyResolutionErrorCode;
|
|
122
174
|
readonly telemetry?: ProxyResolutionTelemetryEvent;
|
|
175
|
+
readonly vendor?: ProxyVendorName;
|
|
176
|
+
readonly vendorChain?: ProxyVendorName[];
|
|
177
|
+
readonly protocol?: ProxyProtocol;
|
|
123
178
|
|
|
124
179
|
constructor(
|
|
125
|
-
code:
|
|
180
|
+
code: ProxyResolutionErrorCode,
|
|
126
181
|
message: string,
|
|
127
|
-
options?: {
|
|
182
|
+
options?: {
|
|
183
|
+
cause?: unknown;
|
|
184
|
+
telemetry?: ProxyResolutionTelemetryEvent;
|
|
185
|
+
vendor?: ProxyVendorName;
|
|
186
|
+
vendorChain?: ProxyVendorName[];
|
|
187
|
+
protocol?: ProxyProtocol;
|
|
188
|
+
},
|
|
128
189
|
) {
|
|
129
190
|
super(message, options);
|
|
130
191
|
this.name = "ProxyResolutionError";
|
|
131
192
|
this.code = code;
|
|
132
193
|
this.telemetry = options?.telemetry;
|
|
194
|
+
this.vendor = options?.vendor;
|
|
195
|
+
this.vendorChain = options?.vendorChain;
|
|
196
|
+
this.protocol = options?.protocol;
|
|
133
197
|
}
|
|
134
198
|
}
|
|
135
199
|
|
|
@@ -165,8 +229,7 @@ const SMARTPROXY_LOCK_POLL_MAX_MS = 9_000;
|
|
|
165
229
|
const SMARTPROXY_DEADLINE_MARGIN_MS = 1_000;
|
|
166
230
|
const SMARTPROXY_INVALIDATION_SKIP_REDIS_MS = 30_000;
|
|
167
231
|
const SMARTPROXY_ALLOCATOR_MAX_ATTEMPTS = 3;
|
|
168
|
-
const SMARTPROXY_ALLOCATOR_DEADLINE_MS =
|
|
169
|
-
SMARTPROXY_LOCK_TTL_MS - SMARTPROXY_DEADLINE_MARGIN_MS;
|
|
232
|
+
const SMARTPROXY_ALLOCATOR_DEADLINE_MS = SMARTPROXY_LOCK_TTL_MS - SMARTPROXY_DEADLINE_MARGIN_MS;
|
|
170
233
|
const SMARTPROXY_ALLOCATOR_RETRY_BASE_MS = 25;
|
|
171
234
|
// Smartproxy API extraction returns fresh IP:port candidates; the `life`
|
|
172
235
|
// parameter controls session duration intent, not a hard endpoint lease. Keep
|
|
@@ -197,15 +260,11 @@ export function providerStateRedisUrlFromEnv(): string | undefined {
|
|
|
197
260
|
}
|
|
198
261
|
|
|
199
262
|
/** @internal Test-only hook for exercising shared proxy-cache behavior. */
|
|
200
|
-
export function __setProxyRedisForTests(
|
|
201
|
-
redis: ProxyRedisClient | undefined,
|
|
202
|
-
): void {
|
|
263
|
+
export function __setProxyRedisForTests(redis: ProxyRedisClient | undefined): void {
|
|
203
264
|
proxyRedisForTests = redis;
|
|
204
265
|
}
|
|
205
266
|
|
|
206
|
-
export function __setSmartproxyAllocatorDeadlineMsForTests(
|
|
207
|
-
deadlineMs: number | undefined,
|
|
208
|
-
): void {
|
|
267
|
+
export function __setSmartproxyAllocatorDeadlineMsForTests(deadlineMs: number | undefined): void {
|
|
209
268
|
smartproxyAllocatorDeadlineMsForTests = deadlineMs;
|
|
210
269
|
}
|
|
211
270
|
|
|
@@ -230,9 +289,7 @@ function getProxyRedis(): ProxyRedisClient | undefined {
|
|
|
230
289
|
return redis;
|
|
231
290
|
}
|
|
232
291
|
|
|
233
|
-
async function withRedisTimeout<T>(
|
|
234
|
-
operation: () => Promise<T>,
|
|
235
|
-
): Promise<T | undefined> {
|
|
292
|
+
async function withRedisTimeout<T>(operation: () => Promise<T>): Promise<T | undefined> {
|
|
236
293
|
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
|
237
294
|
try {
|
|
238
295
|
const timeout = new Promise<undefined>((resolve) => {
|
|
@@ -318,17 +375,11 @@ function isRecord(value: unknown): value is Record<string, unknown> {
|
|
|
318
375
|
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
319
376
|
}
|
|
320
377
|
|
|
321
|
-
function toProxyDiagnostics(
|
|
322
|
-
value: unknown,
|
|
323
|
-
): Record<string, string | number | boolean> | undefined {
|
|
378
|
+
function toProxyDiagnostics(value: unknown): Record<string, string | number | boolean> | undefined {
|
|
324
379
|
if (!isRecord(value)) return undefined;
|
|
325
380
|
const diagnostics: Record<string, string | number | boolean> = {};
|
|
326
381
|
for (const [key, item] of Object.entries(value)) {
|
|
327
|
-
if (
|
|
328
|
-
typeof item === "string" ||
|
|
329
|
-
typeof item === "number" ||
|
|
330
|
-
typeof item === "boolean"
|
|
331
|
-
) {
|
|
382
|
+
if (typeof item === "string" || typeof item === "number" || typeof item === "boolean") {
|
|
332
383
|
diagnostics[key] = item;
|
|
333
384
|
}
|
|
334
385
|
}
|
|
@@ -405,17 +456,19 @@ function applyStickyProxySession(proxyUrl: string): string {
|
|
|
405
456
|
return proxyUrl;
|
|
406
457
|
}
|
|
407
458
|
|
|
459
|
+
// This rewrites sticky-session usernames for a bring-your-own *gateway* URL
|
|
460
|
+
// (APIFUSE__PROXY__URL). The `smartproxy` host here means a smartproxy.com /
|
|
461
|
+
// Decodo-family gateway that authenticates by username — NOT the
|
|
462
|
+
// api.smartproxy.org allocation vendor, whose endpoints are raw ip:port with
|
|
463
|
+
// no credentials and therefore return early above.
|
|
408
464
|
const host = parsed.hostname.toLowerCase();
|
|
409
465
|
if (!host.includes("smartproxy") && !host.includes("decodo")) {
|
|
410
466
|
return proxyUrl;
|
|
411
467
|
}
|
|
412
468
|
|
|
413
469
|
const username = decodeURIComponent(parsed.username);
|
|
414
|
-
const sessionId =
|
|
415
|
-
|
|
416
|
-
const sessionDuration = readPositiveIntegerEnv(
|
|
417
|
-
"APIFUSE__PROXY__SESSION_DURATION",
|
|
418
|
-
);
|
|
470
|
+
const sessionId = process.env.APIFUSE__PROXY__SESSION_ID?.trim() || "apifuse-shared";
|
|
471
|
+
const sessionDuration = readPositiveIntegerEnv("APIFUSE__PROXY__SESSION_DURATION");
|
|
419
472
|
const stickyUsername = host.includes("smartproxy")
|
|
420
473
|
? buildSmartproxyUsername(username, sessionId, sessionDuration)
|
|
421
474
|
: buildDecodoUsername(username, sessionId, sessionDuration ?? "60");
|
|
@@ -430,27 +483,16 @@ function buildSmartproxyUsername(
|
|
|
430
483
|
sessionDuration?: string,
|
|
431
484
|
): string {
|
|
432
485
|
const parts = username.split("_");
|
|
433
|
-
const configuredLife = parts
|
|
434
|
-
.find((part) => part.startsWith("life-"))
|
|
435
|
-
?.slice("life-".length);
|
|
486
|
+
const configuredLife = parts.find((part) => part.startsWith("life-"))?.slice("life-".length);
|
|
436
487
|
const baseUsername = parts
|
|
437
488
|
.filter((part) => !part.startsWith("session-") && !part.startsWith("life-"))
|
|
438
489
|
.join("_");
|
|
439
490
|
return `${baseUsername}_session-${sessionId}_life-${sessionDuration ?? configuredLife ?? "60"}`;
|
|
440
491
|
}
|
|
441
492
|
|
|
442
|
-
function buildDecodoUsername(
|
|
443
|
-
username
|
|
444
|
-
|
|
445
|
-
sessionDuration: string,
|
|
446
|
-
): string {
|
|
447
|
-
const withoutSticky = username.replace(
|
|
448
|
-
/-session-.+-sessionduration-\d+$/,
|
|
449
|
-
"",
|
|
450
|
-
);
|
|
451
|
-
const baseUsername = withoutSticky.startsWith("user-")
|
|
452
|
-
? withoutSticky
|
|
453
|
-
: `user-${withoutSticky}`;
|
|
493
|
+
function buildDecodoUsername(username: string, sessionId: string, sessionDuration: string): string {
|
|
494
|
+
const withoutSticky = username.replace(/-session-.+-sessionduration-\d+$/, "");
|
|
495
|
+
const baseUsername = withoutSticky.startsWith("user-") ? withoutSticky : `user-${withoutSticky}`;
|
|
454
496
|
return `${baseUsername}-session-${sessionId}-sessionduration-${sessionDuration}`;
|
|
455
497
|
}
|
|
456
498
|
|
|
@@ -461,9 +503,7 @@ function syncProxyEnv(config: ApiFuseConfig): void {
|
|
|
461
503
|
}
|
|
462
504
|
}
|
|
463
505
|
|
|
464
|
-
export function resolveProxyConfig(
|
|
465
|
-
options: ProxyResolutionOptions = {},
|
|
466
|
-
): ResolvedProxyConfig {
|
|
506
|
+
export function resolveProxyConfig(options: ProxyResolutionOptions = {}): ResolvedProxyConfig {
|
|
467
507
|
const explicitProxyUrl = normalizeProxyUrl(options.proxy);
|
|
468
508
|
if (explicitProxyUrl) {
|
|
469
509
|
return { shouldWarn: false, url: explicitProxyUrl };
|
|
@@ -485,9 +525,7 @@ export function resolveProxyConfig(
|
|
|
485
525
|
return { shouldWarn: false, url: envProxyUrl };
|
|
486
526
|
}
|
|
487
527
|
|
|
488
|
-
const configuredProxyUrl = normalizeProxyUrl(
|
|
489
|
-
options.apifuseConfig?.proxy?.url,
|
|
490
|
-
);
|
|
528
|
+
const configuredProxyUrl = normalizeProxyUrl(options.apifuseConfig?.proxy?.url);
|
|
491
529
|
if (configuredProxyUrl) {
|
|
492
530
|
return { shouldWarn: false, url: configuredProxyUrl };
|
|
493
531
|
}
|
|
@@ -511,68 +549,217 @@ export async function resolveProxyConfigAsync(
|
|
|
511
549
|
return { shouldWarn: false };
|
|
512
550
|
}
|
|
513
551
|
|
|
514
|
-
const
|
|
515
|
-
if (
|
|
552
|
+
const chain = resolveVendorChain(policy);
|
|
553
|
+
if (chain.length === 0) {
|
|
554
|
+
// decodo/custom/env-static providers keep the legacy static-URL path.
|
|
516
555
|
return resolveProxyConfig({
|
|
517
556
|
...options,
|
|
518
557
|
upstream: { proxy: true },
|
|
519
558
|
});
|
|
520
559
|
}
|
|
521
560
|
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
561
|
+
// Protocol is chosen per vendor (each vendor's benchmarked-best), with an
|
|
562
|
+
// optional explicit override for the harness/tests. Both are tunnelling
|
|
563
|
+
// schemes. transportProtocols is what the calling transport can actually use.
|
|
564
|
+
const transportProtocols = options.transportProtocols ?? (["http", "socks5"] as const);
|
|
565
|
+
|
|
566
|
+
const sizes = chain.map((vendor) => vendorPoolSize(vendor, policy));
|
|
567
|
+
const total = sizes.reduce((sum, size) => sum + size, 0);
|
|
568
|
+
const normalizedAttempt = normalizeAttemptIndex(options.proxyAttempt);
|
|
569
|
+
const { vendorIndex: startVendorIndex, poolIndex: startPoolIndex } = mapFlatAttempt(
|
|
570
|
+
total > 0 ? normalizedAttempt % total : 0,
|
|
571
|
+
sizes,
|
|
572
|
+
);
|
|
573
|
+
const refreshEpoch = normalizeAttemptIndex(options.proxyRefreshEpoch);
|
|
574
|
+
|
|
575
|
+
let lastError: unknown;
|
|
576
|
+
let blockedProtocol: ProxyProtocol | undefined;
|
|
577
|
+
for (let vendorIndex = startVendorIndex; vendorIndex < chain.length; vendorIndex++) {
|
|
578
|
+
const vendor = chain[vendorIndex] as ProxyVendorName;
|
|
579
|
+
const nextVendor = chain[vendorIndex + 1];
|
|
580
|
+
const poolIndex = vendorIndex === startVendorIndex ? startPoolIndex : 0;
|
|
581
|
+
const protocol = options.protocol ?? VENDOR_DEFAULT_PROTOCOL[vendor];
|
|
582
|
+
|
|
583
|
+
if (!vendorHasCredentials(vendor)) {
|
|
584
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
585
|
+
vendor,
|
|
586
|
+
nextVendor,
|
|
587
|
+
phase: "resolution",
|
|
588
|
+
reason: "no_credentials",
|
|
589
|
+
});
|
|
590
|
+
continue;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
// The calling transport must be able to use this vendor's protocol; if not,
|
|
594
|
+
// fail over to the next vendor rather than silently downgrading.
|
|
595
|
+
if (!transportProtocols.includes(protocol)) {
|
|
596
|
+
blockedProtocol = protocol;
|
|
597
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
598
|
+
vendor,
|
|
599
|
+
nextVendor,
|
|
600
|
+
phase: "resolution",
|
|
601
|
+
reason: "protocol_unsupported",
|
|
602
|
+
});
|
|
603
|
+
continue;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
try {
|
|
607
|
+
return await resolveWithVendor(vendor, policy, options, {
|
|
608
|
+
protocol,
|
|
609
|
+
poolIndex,
|
|
610
|
+
refreshEpoch,
|
|
611
|
+
});
|
|
612
|
+
} catch (error) {
|
|
613
|
+
// Config/programming errors (invalid filter, etc.) are not vendor
|
|
614
|
+
// outages — propagate them rather than failing over.
|
|
615
|
+
if (!(error instanceof ProxyResolutionError)) {
|
|
616
|
+
throw error;
|
|
617
|
+
}
|
|
618
|
+
if (error.telemetry) {
|
|
619
|
+
options.telemetry?.recordProxyResolution(error.telemetry);
|
|
620
|
+
}
|
|
621
|
+
lastError = error;
|
|
622
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
623
|
+
vendor,
|
|
624
|
+
nextVendor,
|
|
625
|
+
phase: "resolution",
|
|
626
|
+
reason: "allocation_failed",
|
|
627
|
+
});
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
if (policy.mode === "required") {
|
|
632
|
+
if (lastError) {
|
|
633
|
+
throw lastError instanceof ProxyResolutionError
|
|
634
|
+
? lastError
|
|
635
|
+
: new ProxyResolutionError(
|
|
636
|
+
"PROXY_ALLOCATION_FAILED",
|
|
637
|
+
`All proxy vendors [${chain.join(", ")}] failed for required proxy egress.`,
|
|
638
|
+
{ cause: lastError, vendorChain: chain },
|
|
639
|
+
);
|
|
640
|
+
}
|
|
641
|
+
if (blockedProtocol) {
|
|
525
642
|
throw new ProxyResolutionError(
|
|
526
|
-
"
|
|
527
|
-
`
|
|
643
|
+
"PROXY_PROTOCOL_UNSUPPORTED",
|
|
644
|
+
`No proxy vendor in [${chain.join(", ")}] could serve a protocol supported by this transport (supports: ${transportProtocols.join(", ")}; vendor wanted "${blockedProtocol}"). Route this provider through the stealth transport.`,
|
|
645
|
+
{ protocol: blockedProtocol, vendorChain: chain },
|
|
528
646
|
);
|
|
529
647
|
}
|
|
530
|
-
|
|
648
|
+
throw new ProxyResolutionError(
|
|
649
|
+
"PROXY_REQUIRED",
|
|
650
|
+
`Proxy egress is required but no vendor credentials are configured. Missing: ${chain
|
|
651
|
+
.map((vendor) => `${missingCredentialEnv(vendor)} (${vendor})`)
|
|
652
|
+
.join(", ")}.`,
|
|
653
|
+
{ vendorChain: chain },
|
|
654
|
+
);
|
|
531
655
|
}
|
|
532
|
-
|
|
656
|
+
return { shouldWarn: true };
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
/**
|
|
660
|
+
* Each vendor's default egress protocol, chosen from live KR benchmarks. HTTP
|
|
661
|
+
* CONNECT wins for nodemaven (socks5 adds ~500ms through the gateway) and ties
|
|
662
|
+
* for smartproxy, and is the only protocol ctx.http (Bun native fetch) supports.
|
|
663
|
+
* Override per call via ProxyResolutionOptions.protocol (harness/tests).
|
|
664
|
+
*/
|
|
665
|
+
const VENDOR_DEFAULT_PROTOCOL: Record<ProxyVendorName, ProxyProtocol> = {
|
|
666
|
+
smartproxy: "http",
|
|
667
|
+
nodemaven: NODEMAVEN_DEFAULT_PROTOCOL,
|
|
668
|
+
};
|
|
533
669
|
|
|
670
|
+
/**
|
|
671
|
+
* Guard the No-MITM invariant: a resolved proxy URL must use a tunnelling scheme
|
|
672
|
+
* (http CONNECT or socks5) so the client TLS handshake reaches the origin
|
|
673
|
+
* end-to-end. Anything else would intercept TLS and break fingerprinting.
|
|
674
|
+
*/
|
|
675
|
+
export function assertTunnelingScheme(url: string): void {
|
|
676
|
+
let scheme: string;
|
|
534
677
|
try {
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
allocated.pool.urls.length,
|
|
544
|
-
options.proxyAttempt,
|
|
678
|
+
scheme = new URL(url).protocol.replace(/:$/, "").toLowerCase();
|
|
679
|
+
} catch {
|
|
680
|
+
throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `Malformed proxy URL: ${url}`);
|
|
681
|
+
}
|
|
682
|
+
if (scheme !== "http" && scheme !== "socks5") {
|
|
683
|
+
throw new ProxyResolutionError(
|
|
684
|
+
"PROXY_ALLOCATION_FAILED",
|
|
685
|
+
`Resolved proxy scheme "${scheme}" is not a tunnelling scheme (expected http or socks5). Refusing to route TLS through a non-tunnelling proxy.`,
|
|
545
686
|
);
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
async function resolveWithVendor(
|
|
691
|
+
vendor: ProxyVendorName,
|
|
692
|
+
policy: ProviderProxyPolicy,
|
|
693
|
+
options: ProxyResolutionOptions,
|
|
694
|
+
context: { protocol: ProxyProtocol; poolIndex: number; refreshEpoch: number },
|
|
695
|
+
): Promise<ResolvedProxyConfig> {
|
|
696
|
+
if (vendor === "nodemaven") {
|
|
697
|
+
const startedAt = Date.now();
|
|
698
|
+
const synthesized = synthesizeNodemavenProxy({
|
|
699
|
+
policy,
|
|
700
|
+
affinityKey: options.affinityKey,
|
|
701
|
+
protocol: context.protocol,
|
|
702
|
+
poolIndex: context.poolIndex,
|
|
703
|
+
refreshEpoch: context.refreshEpoch,
|
|
704
|
+
country: resolveSmartproxyCountry(policy),
|
|
705
|
+
});
|
|
706
|
+
options.telemetry?.recordProxyResolution({
|
|
707
|
+
provider: "nodemaven",
|
|
708
|
+
protocol: synthesized.protocol,
|
|
709
|
+
cacheStatus: "disabled",
|
|
710
|
+
cacheHit: false,
|
|
711
|
+
resolutionMs: Math.max(0, Date.now() - startedAt),
|
|
712
|
+
attempts: 1,
|
|
713
|
+
});
|
|
714
|
+
assertTunnelingScheme(synthesized.url);
|
|
546
715
|
return {
|
|
547
716
|
shouldWarn: false,
|
|
548
|
-
url:
|
|
549
|
-
source: "
|
|
717
|
+
url: synthesized.url,
|
|
718
|
+
source: "nodemaven-gateway",
|
|
719
|
+
protocol: synthesized.protocol,
|
|
550
720
|
diagnostics: {
|
|
551
|
-
...
|
|
552
|
-
|
|
553
|
-
poolIndex,
|
|
721
|
+
...synthesized.diagnostics,
|
|
722
|
+
poolIndex: context.poolIndex,
|
|
554
723
|
},
|
|
555
724
|
};
|
|
556
|
-
} catch (error) {
|
|
557
|
-
if (error instanceof ProxyResolutionError && error.telemetry) {
|
|
558
|
-
options.telemetry?.recordProxyResolution(error.telemetry);
|
|
559
|
-
}
|
|
560
|
-
if (policy.mode === "required") {
|
|
561
|
-
throw error instanceof ProxyResolutionError
|
|
562
|
-
? error
|
|
563
|
-
: new ProxyResolutionError(
|
|
564
|
-
"PROXY_ALLOCATION_FAILED",
|
|
565
|
-
"Smartproxy allocator failed for required proxy egress.",
|
|
566
|
-
{ cause: error },
|
|
567
|
-
);
|
|
568
|
-
}
|
|
569
|
-
return { shouldWarn: true };
|
|
570
725
|
}
|
|
726
|
+
|
|
727
|
+
// smartproxy allocation-style vendor.
|
|
728
|
+
const appKey = process.env[SMARTPROXY_APP_KEY_ENV]?.trim();
|
|
729
|
+
if (!appKey) {
|
|
730
|
+
// Guarded by vendorHasCredentials; treated as a vendor-internal failure.
|
|
731
|
+
throw new ProxyResolutionError(
|
|
732
|
+
"PROXY_ALLOCATION_FAILED",
|
|
733
|
+
`${SMARTPROXY_APP_KEY_ENV} is not configured.`,
|
|
734
|
+
{ vendor: "smartproxy" },
|
|
735
|
+
);
|
|
736
|
+
}
|
|
737
|
+
const lifetimeMinutes = resolveSmartproxyLifetime(policy);
|
|
738
|
+
const allocated = await allocateSmartproxy(
|
|
739
|
+
policy,
|
|
740
|
+
appKey,
|
|
741
|
+
lifetimeMinutes,
|
|
742
|
+
options.affinityKey,
|
|
743
|
+
context.protocol,
|
|
744
|
+
);
|
|
745
|
+
options.telemetry?.recordProxyResolution({ ...allocated.telemetry, protocol: context.protocol });
|
|
746
|
+
const poolIndex = selectProxyPoolIndex(allocated.pool.urls.length, context.poolIndex);
|
|
747
|
+
const url = allocated.pool.urls[poolIndex];
|
|
748
|
+
if (url) assertTunnelingScheme(url);
|
|
749
|
+
return {
|
|
750
|
+
shouldWarn: false,
|
|
751
|
+
url,
|
|
752
|
+
source: "smartproxy-allocator",
|
|
753
|
+
protocol: context.protocol,
|
|
754
|
+
diagnostics: {
|
|
755
|
+
...allocated.pool.diagnostics,
|
|
756
|
+
poolSize: allocated.pool.urls.length,
|
|
757
|
+
poolIndex,
|
|
758
|
+
},
|
|
759
|
+
};
|
|
571
760
|
}
|
|
572
761
|
|
|
573
|
-
function resolvePolicy(
|
|
574
|
-
options: ProxyResolutionOptions,
|
|
575
|
-
): ProviderProxyPolicy | undefined {
|
|
762
|
+
function resolvePolicy(options: ProxyResolutionOptions): ProviderProxyPolicy | undefined {
|
|
576
763
|
if (options.proxyPolicy) {
|
|
577
764
|
return options.proxyPolicy;
|
|
578
765
|
}
|
|
@@ -583,32 +770,210 @@ function resolvePolicy(
|
|
|
583
770
|
return undefined;
|
|
584
771
|
}
|
|
585
772
|
|
|
586
|
-
function
|
|
587
|
-
return
|
|
588
|
-
policy.provider ??
|
|
589
|
-
process.env[DEFAULT_PROXY_PROVIDER_ENV]?.trim().toLowerCase() ??
|
|
590
|
-
"custom"
|
|
591
|
-
);
|
|
773
|
+
function isRegistryVendor(name: string | undefined): name is ProxyVendorName {
|
|
774
|
+
return name === "smartproxy" || name === "nodemaven";
|
|
592
775
|
}
|
|
593
776
|
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
777
|
+
/**
|
|
778
|
+
* Ordered list of SDK-native proxy vendors declared by the policy. `providers`
|
|
779
|
+
* takes precedence over the legacy singular `provider`; the platform default
|
|
780
|
+
* env is the final fallback. Non-registry names (decodo/custom) are dropped so
|
|
781
|
+
* an all-static chain falls through to the legacy env-URL path unchanged.
|
|
782
|
+
*/
|
|
783
|
+
export function resolveVendorChain(policy: ProviderProxyPolicy): ProxyVendorName[] {
|
|
784
|
+
const declared: (ProviderProxyProvider | undefined)[] = policy.providers?.length
|
|
785
|
+
? policy.providers
|
|
786
|
+
: [policy.provider ?? envDefaultProvider()];
|
|
787
|
+
const chain: ProxyVendorName[] = [];
|
|
788
|
+
for (const name of declared) {
|
|
789
|
+
if (isRegistryVendor(name) && !chain.includes(name)) {
|
|
790
|
+
chain.push(name);
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
return chain;
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
function envDefaultProvider(): ProviderProxyProvider | undefined {
|
|
797
|
+
const raw = process.env[DEFAULT_PROXY_PROVIDER_ENV]?.trim().toLowerCase();
|
|
798
|
+
return (raw as ProviderProxyProvider | undefined) ?? undefined;
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
function vendorHasCredentials(vendor: ProxyVendorName): boolean {
|
|
802
|
+
if (vendor === "nodemaven") return hasNodemavenCredentials();
|
|
803
|
+
return Boolean(process.env[SMARTPROXY_APP_KEY_ENV]?.trim());
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
function missingCredentialEnv(vendor: ProxyVendorName): string {
|
|
807
|
+
return vendor === "nodemaven" ? "APIFUSE__PROXY__NODEMAVEN_USERNAME" : SMARTPROXY_APP_KEY_ENV;
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
function vendorPoolSize(vendor: ProxyVendorName, policy: ProviderProxyPolicy): number {
|
|
811
|
+
return vendor === "nodemaven" ? nodemavenPoolSize(policy) : resolveSmartproxyPoolSize(policy);
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
/**
|
|
815
|
+
* Total attempt span across a policy's vendor chain — the sum of each vendor's
|
|
816
|
+
* pool size. Transports use this so successive attempts rotate a vendor's pool
|
|
817
|
+
* and then fail over to the next vendor via the flat attempt index. With one
|
|
818
|
+
* vendor this equals that vendor's pool size (today's behaviour).
|
|
819
|
+
*/
|
|
820
|
+
export function resolvePolicyProxyPoolSpan(policy: ProviderProxyPolicy): number {
|
|
821
|
+
const chain = resolveVendorChain(policy);
|
|
822
|
+
if (chain.length === 0) return resolveSmartproxyPoolSize(policy);
|
|
823
|
+
return chain.reduce((sum, vendor) => sum + vendorPoolSize(vendor, policy), 0);
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
function vendorMaxPoolSize(vendor: ProxyVendorName): number {
|
|
827
|
+
return vendor === "nodemaven" ? NODEMAVEN_MAX_POOL_SIZE : SMARTPROXY_MAX_POOL_SIZE;
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
/**
|
|
831
|
+
* Absolute upper bound on a chain's attempt span — the sum of each vendor's
|
|
832
|
+
* *maximum* pool size. Unlike `resolvePolicyProxyPoolSpan` (the configured
|
|
833
|
+
* span), this backstop is independent of `session.poolSize`, so it never
|
|
834
|
+
* truncates a legitimately large pool below the point where the flat attempt
|
|
835
|
+
* index would cross into the next vendor (e.g. a 50-slot NodeMaven pool).
|
|
836
|
+
*/
|
|
837
|
+
export function maxPolicyProxyPoolSpan(policy: ProviderProxyPolicy): number {
|
|
838
|
+
const chain = resolveVendorChain(policy);
|
|
839
|
+
if (chain.length === 0) return SMARTPROXY_MAX_POOL_SIZE;
|
|
840
|
+
return chain.reduce((sum, vendor) => sum + vendorMaxPoolSize(vendor), 0);
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
const UNSAFE_TRANSPORT_RETRY_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE", "TRACE"]);
|
|
844
|
+
|
|
845
|
+
/**
|
|
846
|
+
* Transport-retry attempt cap for a policy-managed request. A transport failure
|
|
847
|
+
* rotates the flat attempt index onto the *next* endpoint (and, once the index
|
|
848
|
+
* passes the primary vendor's pool span, the *next vendor*), so the cap must be
|
|
849
|
+
* the chain's full pool span for failover to reach the fallback vendor — the
|
|
850
|
+
* per-endpoint retry budget (default 3) never gets there.
|
|
851
|
+
*
|
|
852
|
+
* The span only widens beyond the caller's retry budget when ALL hold:
|
|
853
|
+
* - the request is policy-allocator managed (not a caller-supplied proxy URL);
|
|
854
|
+
* - the caller did NOT pin an explicit retry policy — `HttpRetryOptions.attempts`
|
|
855
|
+
* is the documented total-attempt ceiling and must be honoured verbatim;
|
|
856
|
+
* - the method is safe/idempotent — an unsafe request must never be duplicated
|
|
857
|
+
* across the pool even if some framework default would allow it;
|
|
858
|
+
* - the policy resolves a non-empty *registry* vendor chain (smartproxy /
|
|
859
|
+
* nodemaven). Static vendors (custom / decodo) and credential-less policies
|
|
860
|
+
* resolve no allocator pool, so every attempt would hit the same endpoint
|
|
861
|
+
* with no possible crossover — they keep the retry budget.
|
|
862
|
+
*
|
|
863
|
+
* The widened cap is bounded by the chain's true maximum span (sum of each
|
|
864
|
+
* vendor's max pool size), so a large NodeMaven pool (≤50) stays reachable and
|
|
865
|
+
* a pathological chain can never spin unbounded.
|
|
866
|
+
*/
|
|
867
|
+
/**
|
|
868
|
+
* True when a policy request is in *implicit chain-rotation* mode: successive
|
|
869
|
+
* transport attempts rotate the flat index across the concatenated vendor pool
|
|
870
|
+
* spans (and, past the primary vendor's span, into the fallback vendor). This is
|
|
871
|
+
* the ONLY mode in which the transport loop widens its attempt cap AND
|
|
872
|
+
* de-duplicates repeated endpoints — the two behaviours must share one predicate
|
|
873
|
+
* so they never diverge. It holds when ALL of the widening conditions hold:
|
|
874
|
+
* - the request is policy-allocator managed (not a caller-supplied proxy URL);
|
|
875
|
+
* - the caller did NOT pin an explicit retry policy — its `attempts` ceiling is
|
|
876
|
+
* the documented contract and must be honoured verbatim against whatever
|
|
877
|
+
* endpoint each attempt resolves (even a repeated one), so no de-duplication;
|
|
878
|
+
* - the method is safe/idempotent — an unsafe request is never duplicated;
|
|
879
|
+
* - the policy resolves a non-empty registry vendor chain (smartproxy /
|
|
880
|
+
* nodemaven). Static vendors (custom / decodo) resolve the same URL every
|
|
881
|
+
* attempt, so there is nothing to rotate or de-duplicate.
|
|
882
|
+
*/
|
|
883
|
+
export function policyRotatesTransportVendorChain(input: {
|
|
884
|
+
policy: ProviderProxyPolicy | undefined;
|
|
885
|
+
usesPolicyAllocator: boolean;
|
|
886
|
+
explicitRetry: boolean;
|
|
887
|
+
method: string;
|
|
888
|
+
}): boolean {
|
|
889
|
+
if (!input.usesPolicyAllocator || !input.policy || input.explicitRetry) {
|
|
890
|
+
return false;
|
|
891
|
+
}
|
|
892
|
+
if (UNSAFE_TRANSPORT_RETRY_METHODS.has(input.method.toUpperCase())) {
|
|
893
|
+
return false;
|
|
894
|
+
}
|
|
895
|
+
return resolveVendorChain(input.policy).length > 0;
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
export function resolvePolicyTransportAttemptCap(input: {
|
|
899
|
+
policy: ProviderProxyPolicy | undefined;
|
|
900
|
+
usesPolicyAllocator: boolean;
|
|
901
|
+
retryAttempts: number;
|
|
902
|
+
explicitRetry: boolean;
|
|
903
|
+
method: string;
|
|
904
|
+
}): number {
|
|
905
|
+
const budget = Math.max(1, Math.floor(input.retryAttempts));
|
|
906
|
+
if (
|
|
907
|
+
!policyRotatesTransportVendorChain({
|
|
908
|
+
policy: input.policy,
|
|
909
|
+
usesPolicyAllocator: input.usesPolicyAllocator,
|
|
910
|
+
explicitRetry: input.explicitRetry,
|
|
911
|
+
method: input.method,
|
|
912
|
+
})
|
|
913
|
+
) {
|
|
914
|
+
return budget;
|
|
915
|
+
}
|
|
916
|
+
const span = Math.min(maxPolicyProxyPoolSpan(input.policy as ProviderProxyPolicy), resolvePolicyProxyPoolSpan(input.policy as ProviderProxyPolicy));
|
|
917
|
+
return Math.max(budget, span);
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
/**
|
|
921
|
+
* A registry vendor chain (smartproxy/nodemaven) resolves a potentially
|
|
922
|
+
* *different* endpoint per flat attempt index, so a transport retry should
|
|
923
|
+
* advance across endpoints and de-duplicate once the chain stops yielding new
|
|
924
|
+
* ones. Static/custom/decodo policies (empty registry chain) resolve the *same*
|
|
925
|
+
* URL every attempt by design — retrying that same endpoint is intended, so the
|
|
926
|
+
* transport loop must not de-duplicate them.
|
|
927
|
+
*/
|
|
928
|
+
export function policyResolvesRegistryVendorChain(
|
|
929
|
+
policy: ProviderProxyPolicy | undefined,
|
|
930
|
+
): boolean {
|
|
931
|
+
return Boolean(policy) && resolveVendorChain(policy as ProviderProxyPolicy).length > 0;
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
/** Map a resolved proxy source label to the vendor that served it. */
|
|
935
|
+
export function vendorFromResolvedSource(
|
|
936
|
+
source: ResolvedProxyConfig["source"],
|
|
937
|
+
): ProxyVendorName | undefined {
|
|
938
|
+
if (source === "nodemaven-gateway") return "nodemaven";
|
|
939
|
+
if (source === "smartproxy-allocator") return "smartproxy";
|
|
940
|
+
return undefined;
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
function normalizeAttemptIndex(attempt: number | undefined): number {
|
|
944
|
+
return Number.isFinite(attempt) ? Math.max(0, Math.floor(attempt as number)) : 0;
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
/**
|
|
948
|
+
* Map a flat attempt index into (vendorIndex, poolIndex) by concatenating each
|
|
949
|
+
* vendor's pool space in chain order. With a single vendor this reduces to
|
|
950
|
+
* `attempt % poolSize`, preserving today's behaviour exactly.
|
|
951
|
+
*/
|
|
952
|
+
export function mapFlatAttempt(
|
|
953
|
+
flat: number,
|
|
954
|
+
sizes: readonly number[],
|
|
955
|
+
): { vendorIndex: number; poolIndex: number } {
|
|
956
|
+
let cursor = flat;
|
|
957
|
+
for (let vendorIndex = 0; vendorIndex < sizes.length; vendorIndex++) {
|
|
958
|
+
const size = Math.max(1, sizes[vendorIndex] ?? 1);
|
|
959
|
+
if (cursor < size) {
|
|
960
|
+
return { vendorIndex, poolIndex: cursor };
|
|
961
|
+
}
|
|
962
|
+
cursor -= size;
|
|
963
|
+
}
|
|
964
|
+
return { vendorIndex: 0, poolIndex: 0 };
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
function resolveSmartproxyCountry(policy: ProviderProxyPolicy): string | undefined {
|
|
597
968
|
return (
|
|
598
|
-
policy.geo?.country ??
|
|
599
|
-
process.env[DEFAULT_PROXY_COUNTRY_ENV]?.trim().toUpperCase() ??
|
|
600
|
-
undefined
|
|
969
|
+
policy.geo?.country ?? process.env[DEFAULT_PROXY_COUNTRY_ENV]?.trim().toUpperCase() ?? undefined
|
|
601
970
|
);
|
|
602
971
|
}
|
|
603
972
|
|
|
604
973
|
function resolveSmartproxyLifetime(policy: ProviderProxyPolicy): number {
|
|
605
974
|
const configuredLifetime =
|
|
606
|
-
policy.session?.lifetimeMinutes ??
|
|
607
|
-
|
|
608
|
-
return Math.min(
|
|
609
|
-
SMARTPROXY_MAX_LIFETIME_MINUTES,
|
|
610
|
-
Math.max(1, Math.floor(configuredLifetime)),
|
|
611
|
-
);
|
|
975
|
+
policy.session?.lifetimeMinutes ?? readPositiveNumberEnv(DEFAULT_PROXY_LIFETIME_ENV, 30);
|
|
976
|
+
return Math.min(SMARTPROXY_MAX_LIFETIME_MINUTES, Math.max(1, Math.floor(configuredLifetime)));
|
|
612
977
|
}
|
|
613
978
|
|
|
614
979
|
function readPositiveNumberEnv(name: string, fallback: number): number {
|
|
@@ -624,10 +989,7 @@ function readPositiveNumberEnv(name: string, fallback: number): number {
|
|
|
624
989
|
function resolveSmartproxyPoolSize(policy: ProviderProxyPolicy): number {
|
|
625
990
|
return Math.min(
|
|
626
991
|
SMARTPROXY_MAX_POOL_SIZE,
|
|
627
|
-
Math.max(
|
|
628
|
-
1,
|
|
629
|
-
Math.floor(policy.session?.poolSize ?? DEFAULT_SMARTPROXY_POOL_SIZE),
|
|
630
|
-
),
|
|
992
|
+
Math.max(1, Math.floor(policy.session?.poolSize ?? DEFAULT_SMARTPROXY_POOL_SIZE)),
|
|
631
993
|
);
|
|
632
994
|
}
|
|
633
995
|
|
|
@@ -635,9 +997,7 @@ function selectProxyPoolIndex(poolSize: number, attempt = 0): number {
|
|
|
635
997
|
if (poolSize <= 1) {
|
|
636
998
|
return 0;
|
|
637
999
|
}
|
|
638
|
-
const normalizedAttempt = Number.isFinite(attempt)
|
|
639
|
-
? Math.max(0, Math.floor(attempt))
|
|
640
|
-
: 0;
|
|
1000
|
+
const normalizedAttempt = Number.isFinite(attempt) ? Math.max(0, Math.floor(attempt)) : 0;
|
|
641
1001
|
return normalizedAttempt % poolSize;
|
|
642
1002
|
}
|
|
643
1003
|
|
|
@@ -645,16 +1005,15 @@ function buildSmartproxyCacheKey(
|
|
|
645
1005
|
policy: ProviderProxyPolicy,
|
|
646
1006
|
affinityKey: string | undefined,
|
|
647
1007
|
lifetimeMinutes: number,
|
|
1008
|
+
protocol: ProxyProtocol,
|
|
648
1009
|
): string {
|
|
649
1010
|
const poolSize = resolveSmartproxyPoolSize(policy);
|
|
650
1011
|
return JSON.stringify({
|
|
651
1012
|
provider: "smartproxy",
|
|
1013
|
+
protocol,
|
|
652
1014
|
country: resolveSmartproxyCountry(policy),
|
|
653
1015
|
affinity: policy.session?.affinity ?? "request",
|
|
654
|
-
affinityKey:
|
|
655
|
-
(policy.session?.affinity ?? "request") === "request"
|
|
656
|
-
? undefined
|
|
657
|
-
: affinityKey,
|
|
1016
|
+
affinityKey: (policy.session?.affinity ?? "request") === "request" ? undefined : affinityKey,
|
|
658
1017
|
lifetimeMinutes,
|
|
659
1018
|
poolSize,
|
|
660
1019
|
});
|
|
@@ -665,12 +1024,9 @@ async function allocateSmartproxy(
|
|
|
665
1024
|
appKey: string,
|
|
666
1025
|
lifetimeMinutes: number,
|
|
667
1026
|
affinityKey: string | undefined,
|
|
1027
|
+
protocol: ProxyProtocol,
|
|
668
1028
|
): Promise<SmartproxyAllocationResult> {
|
|
669
|
-
const cacheKey = buildSmartproxyCacheKey(
|
|
670
|
-
policy,
|
|
671
|
-
affinityKey,
|
|
672
|
-
lifetimeMinutes,
|
|
673
|
-
);
|
|
1029
|
+
const cacheKey = buildSmartproxyCacheKey(policy, affinityKey, lifetimeMinutes, protocol);
|
|
674
1030
|
const startedAt = Date.now();
|
|
675
1031
|
const now = startedAt;
|
|
676
1032
|
const invalidatedUntil = invalidatedProxyKeys.get(cacheKey) ?? 0;
|
|
@@ -678,7 +1034,7 @@ async function allocateSmartproxy(
|
|
|
678
1034
|
const cached = proxyCache.get(cacheKey);
|
|
679
1035
|
if (!skipCached && cached && isFresh(cached, now)) {
|
|
680
1036
|
if (shouldSoftRefresh(cached, now)) {
|
|
681
|
-
void refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes);
|
|
1037
|
+
void refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, protocol);
|
|
682
1038
|
return {
|
|
683
1039
|
pool: cached,
|
|
684
1040
|
telemetry: telemetryForPool(cached, "soft_stale_refresh", startedAt, {
|
|
@@ -714,6 +1070,7 @@ async function allocateSmartproxy(
|
|
|
714
1070
|
appKey,
|
|
715
1071
|
lifetimeMinutes,
|
|
716
1072
|
startedAt,
|
|
1073
|
+
protocol,
|
|
717
1074
|
).finally(() => {
|
|
718
1075
|
proxyInflight.delete(cacheKey);
|
|
719
1076
|
});
|
|
@@ -730,9 +1087,7 @@ async function readSmartproxyRedisPool(
|
|
|
730
1087
|
const redis = getProxyRedis();
|
|
731
1088
|
if (!redis || !(await ensureRedisReady(redis))) return null;
|
|
732
1089
|
const redisStartedAt = Date.now();
|
|
733
|
-
const raw = await withRedisTimeout(() =>
|
|
734
|
-
redis.get(smartproxyRedisPoolKey(cacheKey)),
|
|
735
|
-
);
|
|
1090
|
+
const raw = await withRedisTimeout(() => redis.get(smartproxyRedisPoolKey(cacheKey)));
|
|
736
1091
|
const redisReadMs = Math.max(0, Date.now() - redisStartedAt);
|
|
737
1092
|
if (typeof raw !== "string") return null;
|
|
738
1093
|
const pool = safeParseSmartproxyPool(raw);
|
|
@@ -754,6 +1109,7 @@ async function refreshSmartproxyPool(
|
|
|
754
1109
|
policy: ProviderProxyPolicy,
|
|
755
1110
|
appKey: string,
|
|
756
1111
|
lifetimeMinutes: number,
|
|
1112
|
+
protocol: ProxyProtocol,
|
|
757
1113
|
): Promise<void> {
|
|
758
1114
|
try {
|
|
759
1115
|
await allocateSmartproxyShared(
|
|
@@ -762,6 +1118,7 @@ async function refreshSmartproxyPool(
|
|
|
762
1118
|
appKey,
|
|
763
1119
|
lifetimeMinutes,
|
|
764
1120
|
Date.now(),
|
|
1121
|
+
protocol,
|
|
765
1122
|
{
|
|
766
1123
|
background: true,
|
|
767
1124
|
},
|
|
@@ -777,6 +1134,7 @@ async function allocateSmartproxyShared(
|
|
|
777
1134
|
appKey: string,
|
|
778
1135
|
lifetimeMinutes: number,
|
|
779
1136
|
startedAt: number,
|
|
1137
|
+
protocol: ProxyProtocol,
|
|
780
1138
|
options: { background?: boolean } = {},
|
|
781
1139
|
): Promise<SmartproxyAllocationResult> {
|
|
782
1140
|
const redis = getProxyRedis();
|
|
@@ -787,7 +1145,7 @@ async function allocateSmartproxyShared(
|
|
|
787
1145
|
appKey,
|
|
788
1146
|
lifetimeMinutes,
|
|
789
1147
|
startedAt,
|
|
790
|
-
{ cacheStatus: "allocator" },
|
|
1148
|
+
{ cacheStatus: "allocator", protocol },
|
|
791
1149
|
);
|
|
792
1150
|
}
|
|
793
1151
|
|
|
@@ -809,11 +1167,10 @@ async function allocateSmartproxyShared(
|
|
|
809
1167
|
lifetimeMinutes,
|
|
810
1168
|
startedAt,
|
|
811
1169
|
{
|
|
812
|
-
cacheStatus: options.background
|
|
813
|
-
? "soft_stale_refresh"
|
|
814
|
-
: "allocator",
|
|
1170
|
+
cacheStatus: options.background ? "soft_stale_refresh" : "allocator",
|
|
815
1171
|
redis,
|
|
816
1172
|
poolKey,
|
|
1173
|
+
protocol,
|
|
817
1174
|
},
|
|
818
1175
|
);
|
|
819
1176
|
} finally {
|
|
@@ -838,15 +1195,10 @@ async function allocateSmartproxyShared(
|
|
|
838
1195
|
if (typeof pttl === "number" && pttl <= 0) {
|
|
839
1196
|
continue;
|
|
840
1197
|
}
|
|
841
|
-
if (
|
|
842
|
-
Date.now() - startedAt >
|
|
843
|
-
SMARTPROXY_LOCK_POLL_MAX_MS - SMARTPROXY_DEADLINE_MARGIN_MS
|
|
844
|
-
) {
|
|
1198
|
+
if (Date.now() - startedAt > SMARTPROXY_LOCK_POLL_MAX_MS - SMARTPROXY_DEADLINE_MARGIN_MS) {
|
|
845
1199
|
break;
|
|
846
1200
|
}
|
|
847
|
-
await sleep(
|
|
848
|
-
Math.min(500, Math.max(50, typeof pttl === "number" ? pttl : 100)),
|
|
849
|
-
);
|
|
1201
|
+
await sleep(Math.min(500, Math.max(50, typeof pttl === "number" ? pttl : 100)));
|
|
850
1202
|
}
|
|
851
1203
|
|
|
852
1204
|
throw new ProxyResolutionError(
|
|
@@ -893,9 +1245,7 @@ function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
|
893
1245
|
}
|
|
894
1246
|
|
|
895
1247
|
function smartproxyAllocatorDeadlineMs(): number {
|
|
896
|
-
return
|
|
897
|
-
smartproxyAllocatorDeadlineMsForTests ?? SMARTPROXY_ALLOCATOR_DEADLINE_MS
|
|
898
|
-
);
|
|
1248
|
+
return smartproxyAllocatorDeadlineMsForTests ?? SMARTPROXY_ALLOCATOR_DEADLINE_MS;
|
|
899
1249
|
}
|
|
900
1250
|
|
|
901
1251
|
function createDeadlineAbortController(deadlineAt: number): {
|
|
@@ -917,9 +1267,7 @@ function createDeadlineAbortController(deadlineAt: number): {
|
|
|
917
1267
|
};
|
|
918
1268
|
}
|
|
919
1269
|
|
|
920
|
-
function smartproxyAllocatorDeadlineFailure(
|
|
921
|
-
attempt: number,
|
|
922
|
-
): SmartproxyAllocatorFailure {
|
|
1270
|
+
function smartproxyAllocatorDeadlineFailure(attempt: number): SmartproxyAllocatorFailure {
|
|
923
1271
|
return {
|
|
924
1272
|
ok: false,
|
|
925
1273
|
attempt,
|
|
@@ -978,6 +1326,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
978
1326
|
cacheStatus: ProxyCacheStatus;
|
|
979
1327
|
redis?: ProxyRedisClient;
|
|
980
1328
|
poolKey?: string;
|
|
1329
|
+
protocol: ProxyProtocol;
|
|
981
1330
|
},
|
|
982
1331
|
): Promise<SmartproxyAllocationResult> {
|
|
983
1332
|
const poolSize = resolveSmartproxyPoolSize(policy);
|
|
@@ -986,17 +1335,13 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
986
1335
|
appKey,
|
|
987
1336
|
lifetimeMinutes,
|
|
988
1337
|
poolSize,
|
|
1338
|
+
options.protocol,
|
|
989
1339
|
);
|
|
990
1340
|
const allocatorStartedAt = Date.now();
|
|
991
|
-
const allocatorDeadlineAt =
|
|
992
|
-
allocatorStartedAt + smartproxyAllocatorDeadlineMs();
|
|
1341
|
+
const allocatorDeadlineAt = allocatorStartedAt + smartproxyAllocatorDeadlineMs();
|
|
993
1342
|
let allocation: SmartproxyAllocatorSuccess | undefined;
|
|
994
1343
|
let lastFailure: SmartproxyAllocatorFailure | undefined;
|
|
995
|
-
for (
|
|
996
|
-
let attempt = 1;
|
|
997
|
-
attempt <= SMARTPROXY_ALLOCATOR_MAX_ATTEMPTS;
|
|
998
|
-
attempt += 1
|
|
999
|
-
) {
|
|
1344
|
+
for (let attempt = 1; attempt <= SMARTPROXY_ALLOCATOR_MAX_ATTEMPTS; attempt += 1) {
|
|
1000
1345
|
if (Date.now() >= allocatorDeadlineAt) {
|
|
1001
1346
|
lastFailure = smartproxyAllocatorDeadlineFailure(attempt);
|
|
1002
1347
|
break;
|
|
@@ -1005,6 +1350,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
1005
1350
|
allocatorUrl,
|
|
1006
1351
|
attempt,
|
|
1007
1352
|
allocatorDeadlineAt,
|
|
1353
|
+
options.protocol,
|
|
1008
1354
|
);
|
|
1009
1355
|
if (attemptResult.ok) {
|
|
1010
1356
|
allocation = attemptResult;
|
|
@@ -1013,8 +1359,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
1013
1359
|
lastFailure = attemptResult;
|
|
1014
1360
|
if (attempt < SMARTPROXY_ALLOCATOR_MAX_ATTEMPTS) {
|
|
1015
1361
|
if (Date.now() >= allocatorDeadlineAt) break;
|
|
1016
|
-
const { controller, dispose } =
|
|
1017
|
-
createDeadlineAbortController(allocatorDeadlineAt);
|
|
1362
|
+
const { controller, dispose } = createDeadlineAbortController(allocatorDeadlineAt);
|
|
1018
1363
|
try {
|
|
1019
1364
|
await sleep(smartproxyAllocatorBackoffMs(attempt), controller.signal);
|
|
1020
1365
|
} catch {
|
|
@@ -1027,9 +1372,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
1027
1372
|
|
|
1028
1373
|
const allocatorMs = Math.max(0, Date.now() - allocatorStartedAt);
|
|
1029
1374
|
const allocatorAttempts =
|
|
1030
|
-
allocation?.attempt ??
|
|
1031
|
-
lastFailure?.attempt ??
|
|
1032
|
-
SMARTPROXY_ALLOCATOR_MAX_ATTEMPTS;
|
|
1375
|
+
allocation?.attempt ?? lastFailure?.attempt ?? SMARTPROXY_ALLOCATOR_MAX_ATTEMPTS;
|
|
1033
1376
|
const allocatorBodyClass =
|
|
1034
1377
|
lastFailure?.bodyClass ?? allocation?.bodyClass ?? "usable_proxy_endpoints";
|
|
1035
1378
|
const allocatorStatus = lastFailure ? lastFailure.status : allocation?.status;
|
|
@@ -1106,9 +1449,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
1106
1449
|
};
|
|
1107
1450
|
}
|
|
1108
1451
|
|
|
1109
|
-
type SmartproxyAllocatorAttemptResult =
|
|
1110
|
-
| SmartproxyAllocatorSuccess
|
|
1111
|
-
| SmartproxyAllocatorFailure;
|
|
1452
|
+
type SmartproxyAllocatorAttemptResult = SmartproxyAllocatorSuccess | SmartproxyAllocatorFailure;
|
|
1112
1453
|
|
|
1113
1454
|
type SmartproxyAllocatorSuccess = {
|
|
1114
1455
|
ok: true;
|
|
@@ -1130,6 +1471,7 @@ async function fetchSmartproxyAllocatorAttempt(
|
|
|
1130
1471
|
allocatorUrl: string,
|
|
1131
1472
|
attempt: number,
|
|
1132
1473
|
deadlineAt: number,
|
|
1474
|
+
protocol: ProxyProtocol,
|
|
1133
1475
|
): Promise<SmartproxyAllocatorAttemptResult> {
|
|
1134
1476
|
const { controller, dispose } = createDeadlineAbortController(deadlineAt);
|
|
1135
1477
|
let response: Response;
|
|
@@ -1150,10 +1492,7 @@ async function fetchSmartproxyAllocatorAttempt(
|
|
|
1150
1492
|
|
|
1151
1493
|
let body: string;
|
|
1152
1494
|
try {
|
|
1153
|
-
body = await readSmartproxyAllocatorBodyWithDeadline(
|
|
1154
|
-
response,
|
|
1155
|
-
controller.signal,
|
|
1156
|
-
);
|
|
1495
|
+
body = await readSmartproxyAllocatorBodyWithDeadline(response, controller.signal);
|
|
1157
1496
|
} catch (error) {
|
|
1158
1497
|
dispose();
|
|
1159
1498
|
return {
|
|
@@ -1176,7 +1515,7 @@ async function fetchSmartproxyAllocatorAttempt(
|
|
|
1176
1515
|
};
|
|
1177
1516
|
}
|
|
1178
1517
|
|
|
1179
|
-
const urls = parseSmartproxyAllocatorProxies(body);
|
|
1518
|
+
const urls = parseSmartproxyAllocatorProxies(body, protocol);
|
|
1180
1519
|
const bodyClass = classifySmartproxyAllocatorBody(body, urls);
|
|
1181
1520
|
if (urls.length === 0) {
|
|
1182
1521
|
return {
|
|
@@ -1196,8 +1535,7 @@ async function fetchSmartproxyAllocatorAttempt(
|
|
|
1196
1535
|
}
|
|
1197
1536
|
|
|
1198
1537
|
function smartproxyAllocatorBackoffMs(attempt: number): number {
|
|
1199
|
-
const base =
|
|
1200
|
-
SMARTPROXY_ALLOCATOR_RETRY_BASE_MS * 2 ** Math.max(0, attempt - 1);
|
|
1538
|
+
const base = SMARTPROXY_ALLOCATOR_RETRY_BASE_MS * 2 ** Math.max(0, attempt - 1);
|
|
1201
1539
|
const jitter = Math.floor(Math.random() * SMARTPROXY_ALLOCATOR_RETRY_BASE_MS);
|
|
1202
1540
|
return base + jitter;
|
|
1203
1541
|
}
|
|
@@ -1217,18 +1555,27 @@ function smartproxyAllocatorFailureMessage(
|
|
|
1217
1555
|
return "Smartproxy allocator response did not contain a usable proxy endpoint.";
|
|
1218
1556
|
}
|
|
1219
1557
|
|
|
1558
|
+
// Smartproxy get-ip-v3 `protocol` param: 1 = HTTP. The SOCKS5 value ("2") is a
|
|
1559
|
+
// best-effort mapping pending live vendor confirmation; http is the default and
|
|
1560
|
+
// the only value exercised in production today.
|
|
1561
|
+
const SMARTPROXY_PROTOCOL_PARAM: Record<ProxyProtocol, string> = {
|
|
1562
|
+
http: "1",
|
|
1563
|
+
socks5: "2",
|
|
1564
|
+
};
|
|
1565
|
+
|
|
1220
1566
|
function buildSmartproxyAllocatorUrl(
|
|
1221
1567
|
policy: ProviderProxyPolicy,
|
|
1222
1568
|
appKey: string,
|
|
1223
1569
|
lifetimeMinutes: number,
|
|
1224
1570
|
poolSize: number,
|
|
1571
|
+
protocol: ProxyProtocol,
|
|
1225
1572
|
): string {
|
|
1226
1573
|
const params = new URLSearchParams({
|
|
1227
1574
|
app_key: appKey,
|
|
1228
1575
|
pt: "9",
|
|
1229
1576
|
num: String(poolSize),
|
|
1230
1577
|
life: String(lifetimeMinutes),
|
|
1231
|
-
protocol:
|
|
1578
|
+
protocol: SMARTPROXY_PROTOCOL_PARAM[protocol],
|
|
1232
1579
|
format: "txt",
|
|
1233
1580
|
lb: "\\n",
|
|
1234
1581
|
});
|
|
@@ -1236,10 +1583,13 @@ function buildSmartproxyAllocatorUrl(
|
|
|
1236
1583
|
if (country) {
|
|
1237
1584
|
params.set("cc", country);
|
|
1238
1585
|
}
|
|
1239
|
-
|
|
1586
|
+
// www.smartproxy.org stopped serving the extraction API on 2026-07-21 (the
|
|
1587
|
+
// old path 404s into the marketing site); the API lives on the api host.
|
|
1588
|
+
return `https://api.smartproxy.org/web_v1/ip/get-ip-v3?${params.toString()}`;
|
|
1240
1589
|
}
|
|
1241
1590
|
|
|
1242
|
-
function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
1591
|
+
function parseSmartproxyAllocatorProxies(body: string, protocol: ProxyProtocol): string[] {
|
|
1592
|
+
const scheme = protocol === "socks5" ? "socks5" : "http";
|
|
1243
1593
|
const trimmed = body.trim();
|
|
1244
1594
|
if (!trimmed) {
|
|
1245
1595
|
return [];
|
|
@@ -1247,14 +1597,8 @@ function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
|
1247
1597
|
|
|
1248
1598
|
try {
|
|
1249
1599
|
const parsed: unknown = JSON.parse(trimmed);
|
|
1250
|
-
const data =
|
|
1251
|
-
|
|
1252
|
-
? parsed.data
|
|
1253
|
-
: undefined;
|
|
1254
|
-
const list =
|
|
1255
|
-
data && typeof data === "object" && "list" in data
|
|
1256
|
-
? data.list
|
|
1257
|
-
: undefined;
|
|
1600
|
+
const data = parsed && typeof parsed === "object" && "data" in parsed ? parsed.data : undefined;
|
|
1601
|
+
const list = data && typeof data === "object" && "list" in data ? data.list : undefined;
|
|
1258
1602
|
if (Array.isArray(list)) {
|
|
1259
1603
|
return list
|
|
1260
1604
|
.map((item) => {
|
|
@@ -1263,11 +1607,10 @@ function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
|
1263
1607
|
}
|
|
1264
1608
|
const ip = "ip" in item && typeof item.ip === "string" ? item.ip : "";
|
|
1265
1609
|
const port =
|
|
1266
|
-
"port" in item &&
|
|
1267
|
-
(typeof item.port === "string" || typeof item.port === "number")
|
|
1610
|
+
"port" in item && (typeof item.port === "string" || typeof item.port === "number")
|
|
1268
1611
|
? item.port
|
|
1269
1612
|
: "";
|
|
1270
|
-
return ip && port ?
|
|
1613
|
+
return ip && port ? `${scheme}://${ip}:${port}` : null;
|
|
1271
1614
|
})
|
|
1272
1615
|
.filter((url): url is string => url !== null);
|
|
1273
1616
|
}
|
|
@@ -1279,7 +1622,7 @@ function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
|
1279
1622
|
.split(/\r?\n/)
|
|
1280
1623
|
.map((item) => item.trim())
|
|
1281
1624
|
.filter((item) => /^\d{1,3}(?:\.\d{1,3}){3}:\d{2,5}$/.test(item))
|
|
1282
|
-
.map((line) =>
|
|
1625
|
+
.map((line) => `${scheme}://${line}`);
|
|
1283
1626
|
}
|
|
1284
1627
|
|
|
1285
1628
|
function classifySmartproxyAllocatorBody(
|
|
@@ -1307,14 +1650,12 @@ export function clearProxyResolutionCache(): void {
|
|
|
1307
1650
|
invalidatedProxyKeys.clear();
|
|
1308
1651
|
}
|
|
1309
1652
|
|
|
1310
|
-
function markSmartproxyCacheInvalidated(
|
|
1311
|
-
options: ProxyResolutionOptions = {},
|
|
1312
|
-
): string | undefined {
|
|
1653
|
+
function markSmartproxyCacheInvalidated(options: ProxyResolutionOptions = {}): string | undefined {
|
|
1313
1654
|
const policy = resolvePolicy(options);
|
|
1314
1655
|
if (!policy || policy.mode === "disabled") {
|
|
1315
1656
|
return undefined;
|
|
1316
1657
|
}
|
|
1317
|
-
if (
|
|
1658
|
+
if (!resolveVendorChain(policy).includes("smartproxy")) {
|
|
1318
1659
|
return undefined;
|
|
1319
1660
|
}
|
|
1320
1661
|
|
|
@@ -1323,19 +1664,15 @@ function markSmartproxyCacheInvalidated(
|
|
|
1323
1664
|
policy,
|
|
1324
1665
|
options.affinityKey,
|
|
1325
1666
|
lifetimeMinutes,
|
|
1667
|
+
options.protocol ?? VENDOR_DEFAULT_PROTOCOL.smartproxy,
|
|
1326
1668
|
);
|
|
1327
|
-
invalidatedProxyKeys.set(
|
|
1328
|
-
cacheKey,
|
|
1329
|
-
Date.now() + SMARTPROXY_INVALIDATION_SKIP_REDIS_MS,
|
|
1330
|
-
);
|
|
1669
|
+
invalidatedProxyKeys.set(cacheKey, Date.now() + SMARTPROXY_INVALIDATION_SKIP_REDIS_MS);
|
|
1331
1670
|
proxyCache.delete(cacheKey);
|
|
1332
1671
|
proxyInflight.delete(cacheKey);
|
|
1333
1672
|
return cacheKey;
|
|
1334
1673
|
}
|
|
1335
1674
|
|
|
1336
|
-
export function invalidateProxyResolutionCache(
|
|
1337
|
-
options: ProxyResolutionOptions = {},
|
|
1338
|
-
): boolean {
|
|
1675
|
+
export function invalidateProxyResolutionCache(options: ProxyResolutionOptions = {}): boolean {
|
|
1339
1676
|
return markSmartproxyCacheInvalidated(options) !== undefined;
|
|
1340
1677
|
}
|
|
1341
1678
|
|
|
@@ -1374,9 +1711,7 @@ async function importConfig(filePath: string): Promise<ApiFuseConfig | null> {
|
|
|
1374
1711
|
}
|
|
1375
1712
|
}
|
|
1376
1713
|
|
|
1377
|
-
export async function loadApiFuseConfig(
|
|
1378
|
-
dir: string = process.cwd(),
|
|
1379
|
-
): Promise<ApiFuseConfig> {
|
|
1714
|
+
export async function loadApiFuseConfig(dir: string = process.cwd()): Promise<ApiFuseConfig> {
|
|
1380
1715
|
const tsPath = path.resolve(dir, "apifuse.config.ts");
|
|
1381
1716
|
if (existsSync(tsPath)) {
|
|
1382
1717
|
const config = await importConfig(tsPath);
|