@apifuse/provider-sdk 2.2.0-beta.7 → 2.2.0-beta.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/dist/auth-turn/index.d.ts +2 -2
- package/dist/config/loader.d.ts +79 -6
- package/dist/config/loader.js +272 -48
- package/dist/define.js +27 -3
- package/dist/runtime/http.js +3 -0
- package/dist/runtime/proxy-errors.js +6 -2
- package/dist/runtime/proxy-nodemaven.d.ts +34 -0
- package/dist/runtime/proxy-nodemaven.js +128 -0
- package/dist/runtime/proxy-telemetry.d.ts +2 -1
- package/dist/runtime/proxy-telemetry.js +39 -4
- package/dist/runtime/stealth.js +20 -9
- package/dist/server/types.d.ts +9 -9
- package/dist/types.d.ts +30 -1
- package/package.json +4 -3
- package/src/config/loader.ts +405 -61
- package/src/define.ts +35 -3
- package/src/runtime/http.ts +3 -0
- package/src/runtime/proxy-errors.ts +12 -4
- package/src/runtime/proxy-nodemaven.ts +178 -0
- package/src/runtime/proxy-telemetry.ts +56 -5
- package/src/runtime/stealth.ts +26 -10
- package/src/types.ts +30 -1
package/src/define.ts
CHANGED
|
@@ -78,7 +78,7 @@ const VALID_RUNTIMES = ["standard", "shared", "browser"] as const;
|
|
|
78
78
|
const VALID_AUTH_MODES = ["none", "platform-managed", "credentials", "oauth2"] as const;
|
|
79
79
|
const VALID_PROVIDER_ACCESS_VISIBILITIES = ["public", "early_access"] as const;
|
|
80
80
|
const VALID_PROVIDER_PROXY_MODES = ["disabled", "optional", "required"] as const;
|
|
81
|
-
const VALID_PROVIDER_PROXY_PROVIDERS = ["smartproxy", "decodo", "custom"] as const;
|
|
81
|
+
const VALID_PROVIDER_PROXY_PROVIDERS = ["smartproxy", "nodemaven", "decodo", "custom"] as const;
|
|
82
82
|
const VALID_PROVIDER_PROXY_AFFINITIES = [
|
|
83
83
|
"request",
|
|
84
84
|
"operation",
|
|
@@ -371,11 +371,24 @@ function validateProviderProxy(config: {
|
|
|
371
371
|
},
|
|
372
372
|
);
|
|
373
373
|
}
|
|
374
|
-
rejectUnknownFields(proxy, new Set(["mode", "provider", "geo", "session"]), "proxy");
|
|
374
|
+
rejectUnknownFields(proxy, new Set(["mode", "provider", "providers", "geo", "session"]), "proxy");
|
|
375
375
|
assertLiteralField(proxy.mode, "proxy.mode", VALID_PROVIDER_PROXY_MODES, config.id);
|
|
376
376
|
if (proxy.provider !== undefined) {
|
|
377
377
|
assertLiteralField(proxy.provider, "proxy.provider", VALID_PROVIDER_PROXY_PROVIDERS, config.id);
|
|
378
378
|
}
|
|
379
|
+
if (proxy.providers !== undefined) {
|
|
380
|
+
if (!Array.isArray(proxy.providers) || proxy.providers.length === 0) {
|
|
381
|
+
throw new ValidationError(
|
|
382
|
+
`Provider "${config.id}" has invalid proxy.providers: must be a non-empty array of proxy vendors.`,
|
|
383
|
+
{
|
|
384
|
+
fix: `Use proxy.providers: ["smartproxy", "nodemaven"] to declare an ordered fallback chain.`,
|
|
385
|
+
},
|
|
386
|
+
);
|
|
387
|
+
}
|
|
388
|
+
for (const vendor of proxy.providers) {
|
|
389
|
+
assertLiteralField(vendor, "proxy.providers[]", VALID_PROVIDER_PROXY_PROVIDERS, config.id);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
379
392
|
if (proxy.geo !== undefined) {
|
|
380
393
|
if (!proxy.geo || typeof proxy.geo !== "object" || Array.isArray(proxy.geo)) {
|
|
381
394
|
throw new ValidationError(
|
|
@@ -433,7 +446,16 @@ function validateProviderProxy(config: {
|
|
|
433
446
|
);
|
|
434
447
|
}
|
|
435
448
|
}
|
|
436
|
-
|
|
449
|
+
// Smartproxy uses a provider-declared secret; when it is a required-mode
|
|
450
|
+
// vendor (singular or in the chain) the app key must be declared so a missing
|
|
451
|
+
// credential fails at build/validation time, not during a live outage.
|
|
452
|
+
const vendorChain =
|
|
453
|
+
proxy.providers && proxy.providers.length > 0
|
|
454
|
+
? proxy.providers
|
|
455
|
+
: proxy.provider
|
|
456
|
+
? [proxy.provider]
|
|
457
|
+
: [];
|
|
458
|
+
if (proxy.mode === "required" && vendorChain.includes("smartproxy")) {
|
|
437
459
|
const hasSmartproxySecret = config.secrets?.some(
|
|
438
460
|
(secret) => secret.name === SMARTPROXY_APP_KEY_SECRET && secret.required !== false,
|
|
439
461
|
);
|
|
@@ -446,6 +468,16 @@ function validateProviderProxy(config: {
|
|
|
446
468
|
);
|
|
447
469
|
}
|
|
448
470
|
}
|
|
471
|
+
// `decodo`/`custom` are deprecated vendor values (string-union members, so the
|
|
472
|
+
// @deprecated symbol gate can't catch them — warn at validation time instead).
|
|
473
|
+
const deprecatedVendors = vendorChain.filter(
|
|
474
|
+
(vendor) => vendor === "decodo" || vendor === "custom",
|
|
475
|
+
);
|
|
476
|
+
if (deprecatedVendors.length > 0) {
|
|
477
|
+
console.warn(
|
|
478
|
+
`[provider-sdk] Provider "${config.id}" uses deprecated proxy vendor(s): ${deprecatedVendors.join(", ")}. Use "smartproxy"/"nodemaven", or the APIFUSE__PROXY__URL bring-your-own escape hatch.`,
|
|
479
|
+
);
|
|
480
|
+
}
|
|
449
481
|
}
|
|
450
482
|
|
|
451
483
|
function validateProviderStt(config: { id: string; stt?: ProviderSttConfig }): void {
|
package/src/runtime/http.ts
CHANGED
|
@@ -256,6 +256,9 @@ async function resolveNativeProxy(
|
|
|
256
256
|
baseProxyAttempt: clientOptions.proxyAttempt,
|
|
257
257
|
retryAttemptOffset: proxyAttemptOffset,
|
|
258
258
|
}),
|
|
259
|
+
// Bun's native fetch proxy option tunnels HTTP CONNECT only; SOCKS5 is not
|
|
260
|
+
// supported here, so a socks5 policy fails loudly rather than downgrading.
|
|
261
|
+
transportProtocols: ["http"],
|
|
259
262
|
telemetry: clientOptions.telemetry,
|
|
260
263
|
});
|
|
261
264
|
if (resolvedProxy.shouldWarn) {
|
|
@@ -19,10 +19,18 @@ const PROXY_AUTH_IP_DENIED_PATTERN =
|
|
|
19
19
|
/\b(?:source|egress|client)\s+ip\b.{0,120}\b(?:deny|denied|unauthori[sz]ed|not\s+authori[sz]ed|white\s*list|allow\s*list)\b|\b(?:white\s*list|allow\s*list)\b.{0,120}\b(?:source|egress|client)\s+ip\b/i;
|
|
20
20
|
const PROXY_EDGE_AUTH_REJECTED_PATTERN =
|
|
21
21
|
/\bauth\s+ip\s+err\b|\bproxy\b.{0,120}\bauth(?:entication)?\b.{0,120}\b(?:reject(?:ed)?|fail(?:ed)?|invalid|den(?:y|ied)|unauthori[sz]ed)\b|\bauth(?:entication)?\b.{0,120}\b(?:reject(?:ed)?|fail(?:ed)?|invalid|den(?:y|ied)|unauthori[sz]ed)\b.{0,120}\bproxy\b/i;
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
22
|
+
// Vendor host tokens that can appear in upstream error strings. Adding a proxy
|
|
23
|
+
// vendor updates every classifier below in one place. `proxy` is the generic
|
|
24
|
+
// fallback so vendor-agnostic messages still classify.
|
|
25
|
+
const PROXY_VENDOR_ALTERNATION = "smartproxy|nodemaven|proxy";
|
|
26
|
+
const PROXY_POOL_STALE_MESSAGE_PATTERN = new RegExp(
|
|
27
|
+
`\\bproxy\\b.{0,120}\\b(?:pool|lease|expired|unavailable|exhausted|non[\\s-]?200\\s+code:\\s*(?:509|512))\\b|\\bnon[\\s-]?200\\s+code:\\s*(?:509|512)\\b.{0,120}\\bproxy\\b|\\b(?:${PROXY_VENDOR_ALTERNATION})\\b.{0,120}\\b(?:509|512)\\b`,
|
|
28
|
+
"i",
|
|
29
|
+
);
|
|
30
|
+
const PROXY_EDGE_TLS_REJECTED_MESSAGE_PATTERN = new RegExp(
|
|
31
|
+
`\\b(?:${PROXY_VENDOR_ALTERNATION})\\b.{0,160}\\b(?:495|ssl|tls|cert(?:ificate)?|handshake|edge|connect|non[\\s-]?200)\\b|\\b(?:495|ssl|tls|cert(?:ificate)?|handshake|edge|connect|non[\\s-]?200)\\b.{0,160}\\b(?:${PROXY_VENDOR_ALTERNATION})\\b`,
|
|
32
|
+
"i",
|
|
33
|
+
);
|
|
26
34
|
|
|
27
35
|
export function isProxyAuthIpDeniedMessage(message: string): boolean {
|
|
28
36
|
return PROXY_AUTH_IP_DENIED_PATTERN.test(message);
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
import type { ProviderProxyPolicy } from "../types.js";
|
|
4
|
+
|
|
5
|
+
export const NODEMAVEN_USERNAME_ENV = "APIFUSE__PROXY__NODEMAVEN_USERNAME";
|
|
6
|
+
export const NODEMAVEN_PASSWORD_ENV = "APIFUSE__PROXY__NODEMAVEN_PASSWORD";
|
|
7
|
+
export const NODEMAVEN_FILTER_ENV = "APIFUSE__PROXY__NODEMAVEN_FILTER";
|
|
8
|
+
|
|
9
|
+
export const NODEMAVEN_GATEWAY_HOST = "gate.nodemaven.com";
|
|
10
|
+
|
|
11
|
+
/** Both schemes tunnel bytes end-to-end, preserving the client TLS handshake. */
|
|
12
|
+
export type ProxyProtocol = "http" | "socks5";
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* NodeMaven's fastest protocol: HTTP CONNECT. Benchmarks (KR, cold + warm)
|
|
16
|
+
* showed socks5 through the gateway adds ~500ms per request over http, so
|
|
17
|
+
* NodeMaven never defaults to socks5.
|
|
18
|
+
*/
|
|
19
|
+
export const NODEMAVEN_DEFAULT_PROTOCOL: ProxyProtocol = "http";
|
|
20
|
+
|
|
21
|
+
/** NodeMaven gateway port ranges per protocol (docs: HTTP 8080-9080, SOCKS5 1080-2080). */
|
|
22
|
+
const NODEMAVEN_PORTS: Record<ProxyProtocol, { min: number; max: number }> = {
|
|
23
|
+
http: { min: 8080, max: 9080 },
|
|
24
|
+
socks5: { min: 1080, max: 2080 },
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const NODEMAVEN_FILTERS = new Set(["medium", "high"]);
|
|
28
|
+
const DEFAULT_NODEMAVEN_FILTER = "medium";
|
|
29
|
+
const DEFAULT_NODEMAVEN_POOL_SIZE = 20;
|
|
30
|
+
const NODEMAVEN_MAX_POOL_SIZE = 50;
|
|
31
|
+
/** NodeMaven sticky sessions persist up to 24h server-side, keyed by the sid. */
|
|
32
|
+
const NODEMAVEN_MAX_LIFETIME_MINUTES = 1440;
|
|
33
|
+
const SID_LENGTH = 10;
|
|
34
|
+
|
|
35
|
+
export function hasNodemavenCredentials(): boolean {
|
|
36
|
+
return Boolean(readNodemavenUsername() && readNodemavenPassword());
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function readNodemavenUsername(): string | undefined {
|
|
40
|
+
return process.env[NODEMAVEN_USERNAME_ENV]?.trim() || undefined;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function readNodemavenPassword(): string | undefined {
|
|
44
|
+
return process.env[NODEMAVEN_PASSWORD_ENV]?.trim() || undefined;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function resolveNodemavenFilter(): string {
|
|
48
|
+
const raw = process.env[NODEMAVEN_FILTER_ENV]?.trim().toLowerCase();
|
|
49
|
+
if (!raw) return DEFAULT_NODEMAVEN_FILTER;
|
|
50
|
+
if (!NODEMAVEN_FILTERS.has(raw)) {
|
|
51
|
+
throw new Error(`${NODEMAVEN_FILTER_ENV} must be "medium" or "high"`);
|
|
52
|
+
}
|
|
53
|
+
return raw;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function nodemavenPoolSize(policy: ProviderProxyPolicy): number {
|
|
57
|
+
return Math.min(
|
|
58
|
+
NODEMAVEN_MAX_POOL_SIZE,
|
|
59
|
+
Math.max(1, Math.floor(policy.session?.poolSize ?? DEFAULT_NODEMAVEN_POOL_SIZE)),
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function nodemavenLifetimeMinutes(policy: ProviderProxyPolicy): number {
|
|
64
|
+
const configured = policy.session?.lifetimeMinutes;
|
|
65
|
+
if (typeof configured !== "number" || !Number.isFinite(configured) || configured <= 0) {
|
|
66
|
+
return NODEMAVEN_MAX_LIFETIME_MINUTES;
|
|
67
|
+
}
|
|
68
|
+
return Math.min(NODEMAVEN_MAX_LIFETIME_MINUTES, Math.max(1, Math.floor(configured)));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** NodeMaven username tokens accept `[a-z0-9]`; slugify geo values to that set. */
|
|
72
|
+
function slugifyGeo(value: string | undefined): string | undefined {
|
|
73
|
+
if (!value) return undefined;
|
|
74
|
+
const slug = value
|
|
75
|
+
.trim()
|
|
76
|
+
.toLowerCase()
|
|
77
|
+
.replace(/[^a-z0-9]+/g, "");
|
|
78
|
+
return slug || undefined;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function isStickyAffinity(policy: ProviderProxyPolicy): boolean {
|
|
82
|
+
return (policy.session?.affinity ?? "request") !== "request";
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* A sticky sid is deterministic from the affinity key so every process serving
|
|
87
|
+
* the same connection derives the same egress IP without shared storage. A
|
|
88
|
+
* rotating sid is random per call (a fresh egress IP per request).
|
|
89
|
+
*/
|
|
90
|
+
function deriveSid(
|
|
91
|
+
policy: ProviderProxyPolicy,
|
|
92
|
+
affinityKey: string | undefined,
|
|
93
|
+
poolIndex: number,
|
|
94
|
+
refreshEpoch: number,
|
|
95
|
+
): string {
|
|
96
|
+
if (!isStickyAffinity(policy) || !affinityKey) {
|
|
97
|
+
return randomBytes(SID_LENGTH).toString("hex").slice(0, SID_LENGTH);
|
|
98
|
+
}
|
|
99
|
+
const digest = createHash("sha256")
|
|
100
|
+
.update(`${affinityKey}:${refreshEpoch}:${poolIndex}`)
|
|
101
|
+
.digest("hex");
|
|
102
|
+
// hex digits are a subset of the allowed [a-z0-9] sid charset.
|
|
103
|
+
return digest.slice(0, SID_LENGTH);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function selectPort(protocol: ProxyProtocol, sid: string, poolIndex: number): number {
|
|
107
|
+
const { min, max } = NODEMAVEN_PORTS[protocol];
|
|
108
|
+
const span = max - min + 1;
|
|
109
|
+
const hashInt = Number.parseInt(
|
|
110
|
+
createHash("sha256").update(`${sid}:${poolIndex}`).digest("hex").slice(0, 8),
|
|
111
|
+
16,
|
|
112
|
+
);
|
|
113
|
+
return min + (hashInt % span);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export type NodemavenSynthesisInput = {
|
|
117
|
+
policy: ProviderProxyPolicy;
|
|
118
|
+
affinityKey: string | undefined;
|
|
119
|
+
protocol: ProxyProtocol;
|
|
120
|
+
poolIndex: number;
|
|
121
|
+
refreshEpoch: number;
|
|
122
|
+
/** ISO 3166-1 alpha-2, already resolved by the caller (falls back to env). */
|
|
123
|
+
country?: string;
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
export type NodemavenSynthesis = {
|
|
127
|
+
url: string;
|
|
128
|
+
protocol: ProxyProtocol;
|
|
129
|
+
diagnostics: Record<string, string | number | boolean>;
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Synthesize a NodeMaven gateway proxy URL locally from static credentials.
|
|
134
|
+
* There is no allocation API — geo/session are encoded in the username.
|
|
135
|
+
*/
|
|
136
|
+
export function synthesizeNodemavenProxy(input: NodemavenSynthesisInput): NodemavenSynthesis {
|
|
137
|
+
const username = readNodemavenUsername();
|
|
138
|
+
const password = readNodemavenPassword();
|
|
139
|
+
if (!username || !password) {
|
|
140
|
+
throw new Error(
|
|
141
|
+
`NodeMaven credentials missing: set ${NODEMAVEN_USERNAME_ENV} and ${NODEMAVEN_PASSWORD_ENV}.`,
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const filter = resolveNodemavenFilter();
|
|
146
|
+
const sid = deriveSid(input.policy, input.affinityKey, input.poolIndex, input.refreshEpoch);
|
|
147
|
+
const port = selectPort(input.protocol, sid, input.poolIndex);
|
|
148
|
+
const lifetimeMinutes = nodemavenLifetimeMinutes(input.policy);
|
|
149
|
+
|
|
150
|
+
const country = slugifyGeo(input.country ?? input.policy.geo?.country);
|
|
151
|
+
const region = slugifyGeo(input.policy.geo?.subdivision);
|
|
152
|
+
const city = slugifyGeo(input.policy.geo?.city);
|
|
153
|
+
|
|
154
|
+
const tokens = [username];
|
|
155
|
+
if (country) tokens.push("country", country);
|
|
156
|
+
if (region) tokens.push("region", region);
|
|
157
|
+
if (city) tokens.push("city", city);
|
|
158
|
+
tokens.push("sid", sid);
|
|
159
|
+
tokens.push("filter", filter);
|
|
160
|
+
tokens.push("ipv4", "true");
|
|
161
|
+
const proxyUsername = tokens.join("-");
|
|
162
|
+
|
|
163
|
+
// Username tokens are [a-z0-9-] only, which survive URL encoding unchanged.
|
|
164
|
+
const url = `${input.protocol}://${proxyUsername}:${encodeURIComponent(password)}@${NODEMAVEN_GATEWAY_HOST}:${port}`;
|
|
165
|
+
|
|
166
|
+
return {
|
|
167
|
+
url,
|
|
168
|
+
protocol: input.protocol,
|
|
169
|
+
diagnostics: {
|
|
170
|
+
vendor: "nodemaven",
|
|
171
|
+
protocol: input.protocol,
|
|
172
|
+
sticky: isStickyAffinity(input.policy),
|
|
173
|
+
filter,
|
|
174
|
+
lifetimeMinutes,
|
|
175
|
+
...(country ? { country } : {}),
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
}
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import type {
|
|
2
2
|
ProxyAttemptTelemetryEvent,
|
|
3
3
|
ProxyCacheStatus,
|
|
4
|
+
ProxyProtocol,
|
|
4
5
|
ProxyResolutionTelemetryEvent,
|
|
5
6
|
ProxyTelemetrySink,
|
|
7
|
+
ProxyVendorFailoverTelemetryEvent,
|
|
8
|
+
ProxyVendorName,
|
|
6
9
|
SmartproxyAllocatorBodyClass,
|
|
7
10
|
} from "../config/loader.js";
|
|
8
11
|
|
|
@@ -11,7 +14,8 @@ export const PROVIDER_TELEMETRY_HEADER = "X-ApiFuse-Provider-Telemetry";
|
|
|
11
14
|
type ProviderTelemetryHeader = {
|
|
12
15
|
v: 1;
|
|
13
16
|
proxy?: {
|
|
14
|
-
provider:
|
|
17
|
+
provider: ProxyVendorName;
|
|
18
|
+
protocol?: ProxyProtocol;
|
|
15
19
|
cacheStatus: ProxyCacheStatus;
|
|
16
20
|
cacheHit: boolean;
|
|
17
21
|
resolutionMs: number;
|
|
@@ -27,6 +31,10 @@ type ProviderTelemetryHeader = {
|
|
|
27
31
|
attempts: number;
|
|
28
32
|
refreshes?: number;
|
|
29
33
|
attemptSamples?: CompactProxyAttemptSample[];
|
|
34
|
+
/** Distinct vendors attempted across the resolution chain, in order seen. */
|
|
35
|
+
vendors?: ProxyVendorName[];
|
|
36
|
+
/** Cross-vendor failover events (bounded). */
|
|
37
|
+
failovers?: CompactVendorFailoverSample[];
|
|
30
38
|
};
|
|
31
39
|
};
|
|
32
40
|
|
|
@@ -41,8 +49,17 @@ type CompactProxyAttemptSample = {
|
|
|
41
49
|
d?: number;
|
|
42
50
|
};
|
|
43
51
|
|
|
52
|
+
type CompactVendorFailoverSample = {
|
|
53
|
+
v: ProxyVendorName;
|
|
54
|
+
nx?: ProxyVendorName;
|
|
55
|
+
p: "resolution" | "transport";
|
|
56
|
+
r: ProxyVendorFailoverTelemetryEvent["reason"];
|
|
57
|
+
a?: number;
|
|
58
|
+
};
|
|
59
|
+
|
|
44
60
|
const MAX_HEADER_BYTES = 4_096;
|
|
45
61
|
const MAX_PROXY_ATTEMPT_SAMPLES = 24;
|
|
62
|
+
const MAX_PROXY_FAILOVER_SAMPLES = 12;
|
|
46
63
|
|
|
47
64
|
const CACHE_STATUS_SEVERITY: Record<ProxyCacheStatus, number> = {
|
|
48
65
|
disabled: 0,
|
|
@@ -76,10 +93,12 @@ function encodeBase64Url(value: string): string {
|
|
|
76
93
|
export class ProxyTelemetryCollector implements ProxyTelemetrySink {
|
|
77
94
|
#events: ProxyResolutionTelemetryEvent[] = [];
|
|
78
95
|
#attempts: ProxyAttemptTelemetryEvent[] = [];
|
|
96
|
+
#failovers: ProxyVendorFailoverTelemetryEvent[] = [];
|
|
79
97
|
|
|
80
98
|
recordProxyResolution(event: ProxyResolutionTelemetryEvent): void {
|
|
81
99
|
this.#events.push({
|
|
82
|
-
provider:
|
|
100
|
+
provider: event.provider,
|
|
101
|
+
...(event.protocol ? { protocol: event.protocol } : {}),
|
|
83
102
|
cacheStatus: event.cacheStatus,
|
|
84
103
|
cacheHit: event.cacheHit,
|
|
85
104
|
resolutionMs: Math.max(0, Math.floor(event.resolutionMs)),
|
|
@@ -112,10 +131,21 @@ export class ProxyTelemetryCollector implements ProxyTelemetrySink {
|
|
|
112
131
|
});
|
|
113
132
|
}
|
|
114
133
|
|
|
134
|
+
recordProxyVendorFailover(event: ProxyVendorFailoverTelemetryEvent): void {
|
|
135
|
+
if (this.#failovers.length >= MAX_PROXY_FAILOVER_SAMPLES) return;
|
|
136
|
+
this.#failovers.push({
|
|
137
|
+
vendor: event.vendor,
|
|
138
|
+
...(event.nextVendor ? { nextVendor: event.nextVendor } : {}),
|
|
139
|
+
phase: event.phase,
|
|
140
|
+
reason: event.reason,
|
|
141
|
+
...(event.attempt === undefined ? {} : { attempt: Math.max(0, Math.floor(event.attempt)) }),
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
|
|
115
145
|
recordProxyAttempt(event: ProxyAttemptTelemetryEvent): void {
|
|
116
146
|
if (this.#attempts.length >= MAX_PROXY_ATTEMPT_SAMPLES) return;
|
|
117
147
|
this.#attempts.push({
|
|
118
|
-
provider:
|
|
148
|
+
provider: event.provider,
|
|
119
149
|
attempt: Math.max(1, Math.floor(event.attempt || 1)),
|
|
120
150
|
...(event.poolIndex === undefined
|
|
121
151
|
? {}
|
|
@@ -134,9 +164,17 @@ export class ProxyTelemetryCollector implements ProxyTelemetrySink {
|
|
|
134
164
|
const [first, ...rest] = this.#events;
|
|
135
165
|
if (!first) return undefined;
|
|
136
166
|
|
|
167
|
+
// The serving vendor/protocol is the last recorded resolution (a failed
|
|
168
|
+
// vendor records first, the vendor that served records last).
|
|
169
|
+
const serving = this.#events[this.#events.length - 1] ?? first;
|
|
170
|
+
const vendors: ProxyVendorName[] = [];
|
|
171
|
+
for (const event of this.#events) {
|
|
172
|
+
if (!vendors.includes(event.provider)) vendors.push(event.provider);
|
|
173
|
+
}
|
|
174
|
+
|
|
137
175
|
const aggregate = rest.reduce<ProxyResolutionTelemetryEvent>(
|
|
138
176
|
(acc, event) => ({
|
|
139
|
-
provider:
|
|
177
|
+
provider: event.provider,
|
|
140
178
|
cacheStatus: worseStatus(acc.cacheStatus, event.cacheStatus),
|
|
141
179
|
cacheHit: acc.cacheHit && event.cacheHit,
|
|
142
180
|
resolutionMs: acc.resolutionMs + event.resolutionMs,
|
|
@@ -157,7 +195,8 @@ export class ProxyTelemetryCollector implements ProxyTelemetrySink {
|
|
|
157
195
|
const payload: ProviderTelemetryHeader = {
|
|
158
196
|
v: 1,
|
|
159
197
|
proxy: {
|
|
160
|
-
provider:
|
|
198
|
+
provider: serving.provider,
|
|
199
|
+
...(serving.protocol ? { protocol: serving.protocol } : {}),
|
|
161
200
|
cacheStatus: aggregate.cacheStatus,
|
|
162
201
|
cacheHit: aggregate.cacheHit,
|
|
163
202
|
resolutionMs: aggregate.resolutionMs,
|
|
@@ -194,6 +233,18 @@ export class ProxyTelemetryCollector implements ProxyTelemetrySink {
|
|
|
194
233
|
})),
|
|
195
234
|
}
|
|
196
235
|
: {}),
|
|
236
|
+
...(vendors.length > 1 ? { vendors } : {}),
|
|
237
|
+
...(this.#failovers.length > 0
|
|
238
|
+
? {
|
|
239
|
+
failovers: this.#failovers.map((failover) => ({
|
|
240
|
+
v: failover.vendor,
|
|
241
|
+
...(failover.nextVendor ? { nx: failover.nextVendor } : {}),
|
|
242
|
+
p: failover.phase,
|
|
243
|
+
r: failover.reason,
|
|
244
|
+
...(failover.attempt === undefined ? {} : { a: failover.attempt }),
|
|
245
|
+
})),
|
|
246
|
+
}
|
|
247
|
+
: {}),
|
|
197
248
|
},
|
|
198
249
|
};
|
|
199
250
|
|
package/src/runtime/stealth.ts
CHANGED
|
@@ -2,13 +2,15 @@ import { createHash } from "node:crypto";
|
|
|
2
2
|
import type { Browser, ImpitOptions, ImpitResponse, RequestInit } from "impit";
|
|
3
3
|
import { Impit } from "impit";
|
|
4
4
|
|
|
5
|
-
import type { ProxyResolutionOptions } from "../config/loader.js";
|
|
5
|
+
import type { ProxyResolutionOptions, ProxyVendorName } from "../config/loader.js";
|
|
6
6
|
import {
|
|
7
7
|
DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
8
8
|
invalidateProxyResolutionCacheAsync,
|
|
9
9
|
ProxyResolutionError,
|
|
10
|
+
resolvePolicyProxyPoolSpan,
|
|
10
11
|
resolveProxyConfigAsync,
|
|
11
12
|
SMARTPROXY_MAX_POOL_SIZE,
|
|
13
|
+
vendorFromResolvedSource,
|
|
12
14
|
} from "../config/loader.js";
|
|
13
15
|
import { SDKError, TransportError } from "../errors.js";
|
|
14
16
|
import { getStealthProfile } from "../stealth/profiles.js";
|
|
@@ -51,7 +53,11 @@ const DEFAULT_PROFILE = "chrome-146";
|
|
|
51
53
|
const MISSING_PROXY_WARNING =
|
|
52
54
|
"[provider-sdk] Provider requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
|
|
53
55
|
|
|
54
|
-
|
|
56
|
+
/**
|
|
57
|
+
* Upper bound on attempts across a multi-vendor chain, so a two-vendor chain can
|
|
58
|
+
* exhaust each vendor's pool before failing over and finally throwing.
|
|
59
|
+
*/
|
|
60
|
+
const MAX_POLICY_PROXY_TOTAL_ATTEMPTS = SMARTPROXY_MAX_POOL_SIZE * 2;
|
|
55
61
|
const MAX_POLICY_PROXY_POOL_REFRESHES = 1;
|
|
56
62
|
const PROXY_CONNECT_FAILURE_CODE = "proxy_connect_failed";
|
|
57
63
|
const PROXY_CONNECT_FAILURE_BODY_PATTERN =
|
|
@@ -426,6 +432,7 @@ type ResolvedAttemptProxy = {
|
|
|
426
432
|
url?: string;
|
|
427
433
|
poolIndex?: number;
|
|
428
434
|
proxyHash?: string;
|
|
435
|
+
vendor?: ProxyVendorName;
|
|
429
436
|
};
|
|
430
437
|
|
|
431
438
|
function proxyPoolIndexFromDiagnostics(
|
|
@@ -615,6 +622,7 @@ function createSessionFetcher(
|
|
|
615
622
|
async function resolveRequestProxy(
|
|
616
623
|
options?: StealthFetchOptions,
|
|
617
624
|
proxyAttempt?: number,
|
|
625
|
+
refreshEpoch?: number,
|
|
618
626
|
): Promise<ResolvedAttemptProxy> {
|
|
619
627
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
620
628
|
proxy: options?.proxy ?? clientOptions.proxy,
|
|
@@ -626,6 +634,10 @@ function createSessionFetcher(
|
|
|
626
634
|
proxyAttemptOffset: options?.proxyAttemptOffset,
|
|
627
635
|
retryAttemptOffset: proxyAttempt,
|
|
628
636
|
}),
|
|
637
|
+
// The impit stealth transport tunnels both HTTP CONNECT and SOCKS5,
|
|
638
|
+
// preserving the client TLS fingerprint end-to-end.
|
|
639
|
+
transportProtocols: ["http", "socks5"],
|
|
640
|
+
...(refreshEpoch === undefined ? {} : { proxyRefreshEpoch: refreshEpoch }),
|
|
629
641
|
telemetry: clientOptions.telemetry,
|
|
630
642
|
});
|
|
631
643
|
|
|
@@ -638,6 +650,7 @@ function createSessionFetcher(
|
|
|
638
650
|
url: resolvedProxy.url,
|
|
639
651
|
poolIndex: proxyPoolIndexFromDiagnostics(resolvedProxy.diagnostics),
|
|
640
652
|
proxyHash: proxyEndpointHash(resolvedProxy.url),
|
|
653
|
+
vendor: vendorFromResolvedSource(resolvedProxy.source),
|
|
641
654
|
};
|
|
642
655
|
}
|
|
643
656
|
|
|
@@ -662,15 +675,18 @@ function createSessionFetcher(
|
|
|
662
675
|
const hasPolicyProxy = isPolicyManagedProxy(clientOptions);
|
|
663
676
|
const usesPolicyAllocator = hasPolicyProxy && !options.proxy && !clientOptions.proxy;
|
|
664
677
|
const retryAttemptCap = Math.max(1, stealthRetryOptions?.attempts ?? 1);
|
|
678
|
+
// Span the whole vendor chain: successive attempts rotate one vendor's
|
|
679
|
+
// pool, then fail over to the next vendor via the flat attempt index.
|
|
680
|
+
const policyProxy =
|
|
681
|
+
clientOptions.proxyPolicy ??
|
|
682
|
+
(typeof clientOptions.upstream?.proxy === "object"
|
|
683
|
+
? clientOptions.upstream.proxy
|
|
684
|
+
: undefined);
|
|
665
685
|
const policyProxyAttemptCap = Math.max(
|
|
666
686
|
1,
|
|
667
687
|
Math.min(
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
(typeof clientOptions.upstream?.proxy === "object"
|
|
671
|
-
? clientOptions.upstream.proxy.session?.poolSize
|
|
672
|
-
: undefined) ??
|
|
673
|
-
DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
688
|
+
MAX_POLICY_PROXY_TOTAL_ATTEMPTS,
|
|
689
|
+
policyProxy ? resolvePolicyProxyPoolSpan(policyProxy) : DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
674
690
|
),
|
|
675
691
|
);
|
|
676
692
|
const maxAttempts = usesPolicyAllocator ? policyProxyAttemptCap : retryAttemptCap;
|
|
@@ -698,7 +714,7 @@ function createSessionFetcher(
|
|
|
698
714
|
if (attemptRecorded || !proxy) return;
|
|
699
715
|
attemptRecorded = true;
|
|
700
716
|
clientOptions.telemetry?.recordProxyAttempt?.({
|
|
701
|
-
provider: "smartproxy",
|
|
717
|
+
provider: attemptProxy?.vendor ?? "smartproxy",
|
|
702
718
|
attempt: attempt + 1,
|
|
703
719
|
...(attemptProxy?.poolIndex === undefined
|
|
704
720
|
? {}
|
|
@@ -712,7 +728,7 @@ function createSessionFetcher(
|
|
|
712
728
|
};
|
|
713
729
|
try {
|
|
714
730
|
assertNoUnsupportedFingerprintOverrides(options);
|
|
715
|
-
attemptProxy = await resolveRequestProxy(options, attempt);
|
|
731
|
+
attemptProxy = await resolveRequestProxy(options, attempt, refreshAttempt);
|
|
716
732
|
proxy = attemptProxy.url;
|
|
717
733
|
if (proxy && usesPolicyAllocator) {
|
|
718
734
|
if (attemptedProxies.has(proxy)) {
|
package/src/types.ts
CHANGED
|
@@ -769,7 +769,25 @@ export type ProviderAccessVisibility = "public" | "early_access";
|
|
|
769
769
|
|
|
770
770
|
export type ProviderProxyMode = "disabled" | "optional" | "required";
|
|
771
771
|
|
|
772
|
-
|
|
772
|
+
/**
|
|
773
|
+
* Proxy egress vendors. These are FOUR DISTINCT services — do not conflate them
|
|
774
|
+
* (a common mistake because the names collide with a well-known rebrand):
|
|
775
|
+
*
|
|
776
|
+
* - `smartproxy` — **api.smartproxy.org**, a residential proxy with an IP
|
|
777
|
+
* *extraction/allocation* API (app_key → a pool of raw `ip:port` CONNECT
|
|
778
|
+
* endpoints). This is our own vendor. It is NOT the company formerly named
|
|
779
|
+
* "Smartproxy". Credentials: `APIFUSE__PROXY__SMARTPROXY_APP_KEY`.
|
|
780
|
+
* - `nodemaven` — **gate.nodemaven.com**, a *gateway* proxy with static
|
|
781
|
+
* credentials; geo/session encoded in the username, no allocation API.
|
|
782
|
+
* - `decodo` — **decodo.com**, the *gateway* proxy that was named "Smartproxy"
|
|
783
|
+
* (smartproxy.com) before its 2025 rebrand to Decodo. Sticky sessions via
|
|
784
|
+
* username params. A different company from `smartproxy` above.
|
|
785
|
+
* **@deprecated** — unused; no managed adapter. Use `smartproxy`/`nodemaven`,
|
|
786
|
+
* or the `APIFUSE__PROXY__URL` bring-your-own escape hatch.
|
|
787
|
+
* - `custom` — **@deprecated** bring-your-own static proxy URL marker. The
|
|
788
|
+
* `APIFUSE__PROXY__URL` env still works without declaring this value.
|
|
789
|
+
*/
|
|
790
|
+
export type ProviderProxyProvider = "smartproxy" | "nodemaven" | "decodo" | "custom";
|
|
773
791
|
|
|
774
792
|
export type ProviderProxySessionAffinity =
|
|
775
793
|
| "request"
|
|
@@ -783,7 +801,18 @@ export interface ProviderProxyPolicy {
|
|
|
783
801
|
* certificate verification, and vendor allocator endpoints are SDK-owned.
|
|
784
802
|
*/
|
|
785
803
|
mode: ProviderProxyMode;
|
|
804
|
+
/**
|
|
805
|
+
* @deprecated Use `providers: [...]` to declare an ordered vendor fallback
|
|
806
|
+
* chain. A single-element `providers` list is equivalent to this field.
|
|
807
|
+
*/
|
|
786
808
|
provider?: ProviderProxyProvider;
|
|
809
|
+
/**
|
|
810
|
+
* Ordered proxy-vendor fallback chain. The SDK tries each vendor in order and
|
|
811
|
+
* fails over to the next when a vendor lacks credentials or its allocation /
|
|
812
|
+
* transport is exhausted. When omitted, `provider` (or the platform default)
|
|
813
|
+
* is used as a single-vendor chain.
|
|
814
|
+
*/
|
|
815
|
+
providers?: ProviderProxyProvider[];
|
|
787
816
|
geo?: {
|
|
788
817
|
/** ISO 3166-1 alpha-2 country code, for example KR or US. */
|
|
789
818
|
country?: Iso3166Alpha2CountryCode;
|