@apifuse/provider-sdk 2.2.0-beta.7 → 2.2.0-beta.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/dist/auth-turn/index.d.ts +2 -2
- package/dist/config/loader.d.ts +79 -6
- package/dist/config/loader.js +272 -48
- package/dist/define.js +27 -3
- package/dist/runtime/http.js +3 -0
- package/dist/runtime/proxy-errors.js +6 -2
- package/dist/runtime/proxy-nodemaven.d.ts +34 -0
- package/dist/runtime/proxy-nodemaven.js +128 -0
- package/dist/runtime/proxy-telemetry.d.ts +2 -1
- package/dist/runtime/proxy-telemetry.js +39 -4
- package/dist/runtime/stealth.js +20 -9
- package/dist/server/types.d.ts +9 -9
- package/dist/types.d.ts +30 -1
- package/package.json +4 -3
- package/src/config/loader.ts +405 -61
- package/src/define.ts +35 -3
- package/src/runtime/http.ts +3 -0
- package/src/runtime/proxy-errors.ts +12 -4
- package/src/runtime/proxy-nodemaven.ts +178 -0
- package/src/runtime/proxy-telemetry.ts +56 -5
- package/src/runtime/stealth.ts +26 -10
- package/src/types.ts +30 -1
package/dist/define.js
CHANGED
|
@@ -8,7 +8,7 @@ const VALID_RUNTIMES = ["standard", "shared", "browser"];
|
|
|
8
8
|
const VALID_AUTH_MODES = ["none", "platform-managed", "credentials", "oauth2"];
|
|
9
9
|
const VALID_PROVIDER_ACCESS_VISIBILITIES = ["public", "early_access"];
|
|
10
10
|
const VALID_PROVIDER_PROXY_MODES = ["disabled", "optional", "required"];
|
|
11
|
-
const VALID_PROVIDER_PROXY_PROVIDERS = ["smartproxy", "decodo", "custom"];
|
|
11
|
+
const VALID_PROVIDER_PROXY_PROVIDERS = ["smartproxy", "nodemaven", "decodo", "custom"];
|
|
12
12
|
const VALID_PROVIDER_PROXY_AFFINITIES = [
|
|
13
13
|
"request",
|
|
14
14
|
"operation",
|
|
@@ -137,11 +137,21 @@ function validateProviderProxy(config) {
|
|
|
137
137
|
fix: `Use proxy: { mode: "required", provider: "smartproxy", geo: { country: "KR" }, session: { affinity: "connection", lifetimeMinutes: 30 } }`,
|
|
138
138
|
});
|
|
139
139
|
}
|
|
140
|
-
rejectUnknownFields(proxy, new Set(["mode", "provider", "geo", "session"]), "proxy");
|
|
140
|
+
rejectUnknownFields(proxy, new Set(["mode", "provider", "providers", "geo", "session"]), "proxy");
|
|
141
141
|
assertLiteralField(proxy.mode, "proxy.mode", VALID_PROVIDER_PROXY_MODES, config.id);
|
|
142
142
|
if (proxy.provider !== undefined) {
|
|
143
143
|
assertLiteralField(proxy.provider, "proxy.provider", VALID_PROVIDER_PROXY_PROVIDERS, config.id);
|
|
144
144
|
}
|
|
145
|
+
if (proxy.providers !== undefined) {
|
|
146
|
+
if (!Array.isArray(proxy.providers) || proxy.providers.length === 0) {
|
|
147
|
+
throw new ValidationError(`Provider "${config.id}" has invalid proxy.providers: must be a non-empty array of proxy vendors.`, {
|
|
148
|
+
fix: `Use proxy.providers: ["smartproxy", "nodemaven"] to declare an ordered fallback chain.`,
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
for (const vendor of proxy.providers) {
|
|
152
|
+
assertLiteralField(vendor, "proxy.providers[]", VALID_PROVIDER_PROXY_PROVIDERS, config.id);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
145
155
|
if (proxy.geo !== undefined) {
|
|
146
156
|
if (!proxy.geo || typeof proxy.geo !== "object" || Array.isArray(proxy.geo)) {
|
|
147
157
|
throw new ValidationError(`Provider "${config.id}" has invalid proxy.geo: must be an object.`, {
|
|
@@ -178,7 +188,15 @@ function validateProviderProxy(config) {
|
|
|
178
188
|
throw new ValidationError(`Provider "${config.id}" has invalid proxy.session.poolSize: must be a positive integer.`);
|
|
179
189
|
}
|
|
180
190
|
}
|
|
181
|
-
|
|
191
|
+
// Smartproxy uses a provider-declared secret; when it is a required-mode
|
|
192
|
+
// vendor (singular or in the chain) the app key must be declared so a missing
|
|
193
|
+
// credential fails at build/validation time, not during a live outage.
|
|
194
|
+
const vendorChain = proxy.providers && proxy.providers.length > 0
|
|
195
|
+
? proxy.providers
|
|
196
|
+
: proxy.provider
|
|
197
|
+
? [proxy.provider]
|
|
198
|
+
: [];
|
|
199
|
+
if (proxy.mode === "required" && vendorChain.includes("smartproxy")) {
|
|
182
200
|
const hasSmartproxySecret = config.secrets?.some((secret) => secret.name === SMARTPROXY_APP_KEY_SECRET && secret.required !== false);
|
|
183
201
|
if (!hasSmartproxySecret) {
|
|
184
202
|
throw new ValidationError(`Provider "${config.id}" requires Smartproxy egress but does not declare ${SMARTPROXY_APP_KEY_SECRET}.`, {
|
|
@@ -186,6 +204,12 @@ function validateProviderProxy(config) {
|
|
|
186
204
|
});
|
|
187
205
|
}
|
|
188
206
|
}
|
|
207
|
+
// `decodo`/`custom` are deprecated vendor values (string-union members, so the
|
|
208
|
+
// @deprecated symbol gate can't catch them — warn at validation time instead).
|
|
209
|
+
const deprecatedVendors = vendorChain.filter((vendor) => vendor === "decodo" || vendor === "custom");
|
|
210
|
+
if (deprecatedVendors.length > 0) {
|
|
211
|
+
console.warn(`[provider-sdk] Provider "${config.id}" uses deprecated proxy vendor(s): ${deprecatedVendors.join(", ")}. Use "smartproxy"/"nodemaven", or the APIFUSE__PROXY__URL bring-your-own escape hatch.`);
|
|
212
|
+
}
|
|
189
213
|
}
|
|
190
214
|
function validateProviderStt(config) {
|
|
191
215
|
const stt = config.stt;
|
package/dist/runtime/http.js
CHANGED
|
@@ -175,6 +175,9 @@ async function resolveNativeProxy(options, clientOptions, warn, proxyAttemptOffs
|
|
|
175
175
|
baseProxyAttempt: clientOptions.proxyAttempt,
|
|
176
176
|
retryAttemptOffset: proxyAttemptOffset,
|
|
177
177
|
}),
|
|
178
|
+
// Bun's native fetch proxy option tunnels HTTP CONNECT only; SOCKS5 is not
|
|
179
|
+
// supported here, so a socks5 policy fails loudly rather than downgrading.
|
|
180
|
+
transportProtocols: ["http"],
|
|
178
181
|
telemetry: clientOptions.telemetry,
|
|
179
182
|
});
|
|
180
183
|
if (resolvedProxy.shouldWarn) {
|
|
@@ -11,8 +11,12 @@ const PROXY_POOL_STALE_STATUS_CODES = new Set([509, 512]);
|
|
|
11
11
|
const PROXY_EDGE_TLS_REJECTED_STATUS_CODES = new Set([495]);
|
|
12
12
|
const PROXY_AUTH_IP_DENIED_PATTERN = /\b(?:source|egress|client)\s+ip\b.{0,120}\b(?:deny|denied|unauthori[sz]ed|not\s+authori[sz]ed|white\s*list|allow\s*list)\b|\b(?:white\s*list|allow\s*list)\b.{0,120}\b(?:source|egress|client)\s+ip\b/i;
|
|
13
13
|
const PROXY_EDGE_AUTH_REJECTED_PATTERN = /\bauth\s+ip\s+err\b|\bproxy\b.{0,120}\bauth(?:entication)?\b.{0,120}\b(?:reject(?:ed)?|fail(?:ed)?|invalid|den(?:y|ied)|unauthori[sz]ed)\b|\bauth(?:entication)?\b.{0,120}\b(?:reject(?:ed)?|fail(?:ed)?|invalid|den(?:y|ied)|unauthori[sz]ed)\b.{0,120}\bproxy\b/i;
|
|
14
|
-
|
|
15
|
-
|
|
14
|
+
// Vendor host tokens that can appear in upstream error strings. Adding a proxy
|
|
15
|
+
// vendor updates every classifier below in one place. `proxy` is the generic
|
|
16
|
+
// fallback so vendor-agnostic messages still classify.
|
|
17
|
+
const PROXY_VENDOR_ALTERNATION = "smartproxy|nodemaven|proxy";
|
|
18
|
+
const PROXY_POOL_STALE_MESSAGE_PATTERN = new RegExp(`\\bproxy\\b.{0,120}\\b(?:pool|lease|expired|unavailable|exhausted|non[\\s-]?200\\s+code:\\s*(?:509|512))\\b|\\bnon[\\s-]?200\\s+code:\\s*(?:509|512)\\b.{0,120}\\bproxy\\b|\\b(?:${PROXY_VENDOR_ALTERNATION})\\b.{0,120}\\b(?:509|512)\\b`, "i");
|
|
19
|
+
const PROXY_EDGE_TLS_REJECTED_MESSAGE_PATTERN = new RegExp(`\\b(?:${PROXY_VENDOR_ALTERNATION})\\b.{0,160}\\b(?:495|ssl|tls|cert(?:ificate)?|handshake|edge|connect|non[\\s-]?200)\\b|\\b(?:495|ssl|tls|cert(?:ificate)?|handshake|edge|connect|non[\\s-]?200)\\b.{0,160}\\b(?:${PROXY_VENDOR_ALTERNATION})\\b`, "i");
|
|
16
20
|
export function isProxyAuthIpDeniedMessage(message) {
|
|
17
21
|
return PROXY_AUTH_IP_DENIED_PATTERN.test(message);
|
|
18
22
|
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { ProviderProxyPolicy } from "../types.js";
|
|
2
|
+
export declare const NODEMAVEN_USERNAME_ENV = "APIFUSE__PROXY__NODEMAVEN_USERNAME";
|
|
3
|
+
export declare const NODEMAVEN_PASSWORD_ENV = "APIFUSE__PROXY__NODEMAVEN_PASSWORD";
|
|
4
|
+
export declare const NODEMAVEN_FILTER_ENV = "APIFUSE__PROXY__NODEMAVEN_FILTER";
|
|
5
|
+
export declare const NODEMAVEN_GATEWAY_HOST = "gate.nodemaven.com";
|
|
6
|
+
/** Both schemes tunnel bytes end-to-end, preserving the client TLS handshake. */
|
|
7
|
+
export type ProxyProtocol = "http" | "socks5";
|
|
8
|
+
/**
|
|
9
|
+
* NodeMaven's fastest protocol: HTTP CONNECT. Benchmarks (KR, cold + warm)
|
|
10
|
+
* showed socks5 through the gateway adds ~500ms per request over http, so
|
|
11
|
+
* NodeMaven never defaults to socks5.
|
|
12
|
+
*/
|
|
13
|
+
export declare const NODEMAVEN_DEFAULT_PROTOCOL: ProxyProtocol;
|
|
14
|
+
export declare function hasNodemavenCredentials(): boolean;
|
|
15
|
+
export declare function nodemavenPoolSize(policy: ProviderProxyPolicy): number;
|
|
16
|
+
export type NodemavenSynthesisInput = {
|
|
17
|
+
policy: ProviderProxyPolicy;
|
|
18
|
+
affinityKey: string | undefined;
|
|
19
|
+
protocol: ProxyProtocol;
|
|
20
|
+
poolIndex: number;
|
|
21
|
+
refreshEpoch: number;
|
|
22
|
+
/** ISO 3166-1 alpha-2, already resolved by the caller (falls back to env). */
|
|
23
|
+
country?: string;
|
|
24
|
+
};
|
|
25
|
+
export type NodemavenSynthesis = {
|
|
26
|
+
url: string;
|
|
27
|
+
protocol: ProxyProtocol;
|
|
28
|
+
diagnostics: Record<string, string | number | boolean>;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Synthesize a NodeMaven gateway proxy URL locally from static credentials.
|
|
32
|
+
* There is no allocation API — geo/session are encoded in the username.
|
|
33
|
+
*/
|
|
34
|
+
export declare function synthesizeNodemavenProxy(input: NodemavenSynthesisInput): NodemavenSynthesis;
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
2
|
+
export const NODEMAVEN_USERNAME_ENV = "APIFUSE__PROXY__NODEMAVEN_USERNAME";
|
|
3
|
+
export const NODEMAVEN_PASSWORD_ENV = "APIFUSE__PROXY__NODEMAVEN_PASSWORD";
|
|
4
|
+
export const NODEMAVEN_FILTER_ENV = "APIFUSE__PROXY__NODEMAVEN_FILTER";
|
|
5
|
+
export const NODEMAVEN_GATEWAY_HOST = "gate.nodemaven.com";
|
|
6
|
+
/**
|
|
7
|
+
* NodeMaven's fastest protocol: HTTP CONNECT. Benchmarks (KR, cold + warm)
|
|
8
|
+
* showed socks5 through the gateway adds ~500ms per request over http, so
|
|
9
|
+
* NodeMaven never defaults to socks5.
|
|
10
|
+
*/
|
|
11
|
+
export const NODEMAVEN_DEFAULT_PROTOCOL = "http";
|
|
12
|
+
/** NodeMaven gateway port ranges per protocol (docs: HTTP 8080-9080, SOCKS5 1080-2080). */
|
|
13
|
+
const NODEMAVEN_PORTS = {
|
|
14
|
+
http: { min: 8080, max: 9080 },
|
|
15
|
+
socks5: { min: 1080, max: 2080 },
|
|
16
|
+
};
|
|
17
|
+
const NODEMAVEN_FILTERS = new Set(["medium", "high"]);
|
|
18
|
+
const DEFAULT_NODEMAVEN_FILTER = "medium";
|
|
19
|
+
const DEFAULT_NODEMAVEN_POOL_SIZE = 20;
|
|
20
|
+
const NODEMAVEN_MAX_POOL_SIZE = 50;
|
|
21
|
+
/** NodeMaven sticky sessions persist up to 24h server-side, keyed by the sid. */
|
|
22
|
+
const NODEMAVEN_MAX_LIFETIME_MINUTES = 1440;
|
|
23
|
+
const SID_LENGTH = 10;
|
|
24
|
+
export function hasNodemavenCredentials() {
|
|
25
|
+
return Boolean(readNodemavenUsername() && readNodemavenPassword());
|
|
26
|
+
}
|
|
27
|
+
function readNodemavenUsername() {
|
|
28
|
+
return process.env[NODEMAVEN_USERNAME_ENV]?.trim() || undefined;
|
|
29
|
+
}
|
|
30
|
+
function readNodemavenPassword() {
|
|
31
|
+
return process.env[NODEMAVEN_PASSWORD_ENV]?.trim() || undefined;
|
|
32
|
+
}
|
|
33
|
+
function resolveNodemavenFilter() {
|
|
34
|
+
const raw = process.env[NODEMAVEN_FILTER_ENV]?.trim().toLowerCase();
|
|
35
|
+
if (!raw)
|
|
36
|
+
return DEFAULT_NODEMAVEN_FILTER;
|
|
37
|
+
if (!NODEMAVEN_FILTERS.has(raw)) {
|
|
38
|
+
throw new Error(`${NODEMAVEN_FILTER_ENV} must be "medium" or "high"`);
|
|
39
|
+
}
|
|
40
|
+
return raw;
|
|
41
|
+
}
|
|
42
|
+
export function nodemavenPoolSize(policy) {
|
|
43
|
+
return Math.min(NODEMAVEN_MAX_POOL_SIZE, Math.max(1, Math.floor(policy.session?.poolSize ?? DEFAULT_NODEMAVEN_POOL_SIZE)));
|
|
44
|
+
}
|
|
45
|
+
function nodemavenLifetimeMinutes(policy) {
|
|
46
|
+
const configured = policy.session?.lifetimeMinutes;
|
|
47
|
+
if (typeof configured !== "number" || !Number.isFinite(configured) || configured <= 0) {
|
|
48
|
+
return NODEMAVEN_MAX_LIFETIME_MINUTES;
|
|
49
|
+
}
|
|
50
|
+
return Math.min(NODEMAVEN_MAX_LIFETIME_MINUTES, Math.max(1, Math.floor(configured)));
|
|
51
|
+
}
|
|
52
|
+
/** NodeMaven username tokens accept `[a-z0-9]`; slugify geo values to that set. */
|
|
53
|
+
function slugifyGeo(value) {
|
|
54
|
+
if (!value)
|
|
55
|
+
return undefined;
|
|
56
|
+
const slug = value
|
|
57
|
+
.trim()
|
|
58
|
+
.toLowerCase()
|
|
59
|
+
.replace(/[^a-z0-9]+/g, "");
|
|
60
|
+
return slug || undefined;
|
|
61
|
+
}
|
|
62
|
+
function isStickyAffinity(policy) {
|
|
63
|
+
return (policy.session?.affinity ?? "request") !== "request";
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* A sticky sid is deterministic from the affinity key so every process serving
|
|
67
|
+
* the same connection derives the same egress IP without shared storage. A
|
|
68
|
+
* rotating sid is random per call (a fresh egress IP per request).
|
|
69
|
+
*/
|
|
70
|
+
function deriveSid(policy, affinityKey, poolIndex, refreshEpoch) {
|
|
71
|
+
if (!isStickyAffinity(policy) || !affinityKey) {
|
|
72
|
+
return randomBytes(SID_LENGTH).toString("hex").slice(0, SID_LENGTH);
|
|
73
|
+
}
|
|
74
|
+
const digest = createHash("sha256")
|
|
75
|
+
.update(`${affinityKey}:${refreshEpoch}:${poolIndex}`)
|
|
76
|
+
.digest("hex");
|
|
77
|
+
// hex digits are a subset of the allowed [a-z0-9] sid charset.
|
|
78
|
+
return digest.slice(0, SID_LENGTH);
|
|
79
|
+
}
|
|
80
|
+
function selectPort(protocol, sid, poolIndex) {
|
|
81
|
+
const { min, max } = NODEMAVEN_PORTS[protocol];
|
|
82
|
+
const span = max - min + 1;
|
|
83
|
+
const hashInt = Number.parseInt(createHash("sha256").update(`${sid}:${poolIndex}`).digest("hex").slice(0, 8), 16);
|
|
84
|
+
return min + (hashInt % span);
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Synthesize a NodeMaven gateway proxy URL locally from static credentials.
|
|
88
|
+
* There is no allocation API — geo/session are encoded in the username.
|
|
89
|
+
*/
|
|
90
|
+
export function synthesizeNodemavenProxy(input) {
|
|
91
|
+
const username = readNodemavenUsername();
|
|
92
|
+
const password = readNodemavenPassword();
|
|
93
|
+
if (!username || !password) {
|
|
94
|
+
throw new Error(`NodeMaven credentials missing: set ${NODEMAVEN_USERNAME_ENV} and ${NODEMAVEN_PASSWORD_ENV}.`);
|
|
95
|
+
}
|
|
96
|
+
const filter = resolveNodemavenFilter();
|
|
97
|
+
const sid = deriveSid(input.policy, input.affinityKey, input.poolIndex, input.refreshEpoch);
|
|
98
|
+
const port = selectPort(input.protocol, sid, input.poolIndex);
|
|
99
|
+
const lifetimeMinutes = nodemavenLifetimeMinutes(input.policy);
|
|
100
|
+
const country = slugifyGeo(input.country ?? input.policy.geo?.country);
|
|
101
|
+
const region = slugifyGeo(input.policy.geo?.subdivision);
|
|
102
|
+
const city = slugifyGeo(input.policy.geo?.city);
|
|
103
|
+
const tokens = [username];
|
|
104
|
+
if (country)
|
|
105
|
+
tokens.push("country", country);
|
|
106
|
+
if (region)
|
|
107
|
+
tokens.push("region", region);
|
|
108
|
+
if (city)
|
|
109
|
+
tokens.push("city", city);
|
|
110
|
+
tokens.push("sid", sid);
|
|
111
|
+
tokens.push("filter", filter);
|
|
112
|
+
tokens.push("ipv4", "true");
|
|
113
|
+
const proxyUsername = tokens.join("-");
|
|
114
|
+
// Username tokens are [a-z0-9-] only, which survive URL encoding unchanged.
|
|
115
|
+
const url = `${input.protocol}://${proxyUsername}:${encodeURIComponent(password)}@${NODEMAVEN_GATEWAY_HOST}:${port}`;
|
|
116
|
+
return {
|
|
117
|
+
url,
|
|
118
|
+
protocol: input.protocol,
|
|
119
|
+
diagnostics: {
|
|
120
|
+
vendor: "nodemaven",
|
|
121
|
+
protocol: input.protocol,
|
|
122
|
+
sticky: isStickyAffinity(input.policy),
|
|
123
|
+
filter,
|
|
124
|
+
lifetimeMinutes,
|
|
125
|
+
...(country ? { country } : {}),
|
|
126
|
+
},
|
|
127
|
+
};
|
|
128
|
+
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import type { ProxyAttemptTelemetryEvent, ProxyResolutionTelemetryEvent, ProxyTelemetrySink } from "../config/loader.js";
|
|
1
|
+
import type { ProxyAttemptTelemetryEvent, ProxyResolutionTelemetryEvent, ProxyTelemetrySink, ProxyVendorFailoverTelemetryEvent } from "../config/loader.js";
|
|
2
2
|
export declare const PROVIDER_TELEMETRY_HEADER = "X-ApiFuse-Provider-Telemetry";
|
|
3
3
|
export declare class ProxyTelemetryCollector implements ProxyTelemetrySink {
|
|
4
4
|
#private;
|
|
5
5
|
recordProxyResolution(event: ProxyResolutionTelemetryEvent): void;
|
|
6
|
+
recordProxyVendorFailover(event: ProxyVendorFailoverTelemetryEvent): void;
|
|
6
7
|
recordProxyAttempt(event: ProxyAttemptTelemetryEvent): void;
|
|
7
8
|
toHeaderValue(): string | undefined;
|
|
8
9
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export const PROVIDER_TELEMETRY_HEADER = "X-ApiFuse-Provider-Telemetry";
|
|
2
2
|
const MAX_HEADER_BYTES = 4_096;
|
|
3
3
|
const MAX_PROXY_ATTEMPT_SAMPLES = 24;
|
|
4
|
+
const MAX_PROXY_FAILOVER_SAMPLES = 12;
|
|
4
5
|
const CACHE_STATUS_SEVERITY = {
|
|
5
6
|
disabled: 0,
|
|
6
7
|
memory_hit: 1,
|
|
@@ -28,9 +29,11 @@ function encodeBase64Url(value) {
|
|
|
28
29
|
export class ProxyTelemetryCollector {
|
|
29
30
|
#events = [];
|
|
30
31
|
#attempts = [];
|
|
32
|
+
#failovers = [];
|
|
31
33
|
recordProxyResolution(event) {
|
|
32
34
|
this.#events.push({
|
|
33
|
-
provider:
|
|
35
|
+
provider: event.provider,
|
|
36
|
+
...(event.protocol ? { protocol: event.protocol } : {}),
|
|
34
37
|
cacheStatus: event.cacheStatus,
|
|
35
38
|
cacheHit: event.cacheHit,
|
|
36
39
|
resolutionMs: Math.max(0, Math.floor(event.resolutionMs)),
|
|
@@ -53,11 +56,22 @@ export class ProxyTelemetryCollector {
|
|
|
53
56
|
refreshes: event.refreshes === undefined ? undefined : Math.max(0, Math.floor(event.refreshes)),
|
|
54
57
|
});
|
|
55
58
|
}
|
|
59
|
+
recordProxyVendorFailover(event) {
|
|
60
|
+
if (this.#failovers.length >= MAX_PROXY_FAILOVER_SAMPLES)
|
|
61
|
+
return;
|
|
62
|
+
this.#failovers.push({
|
|
63
|
+
vendor: event.vendor,
|
|
64
|
+
...(event.nextVendor ? { nextVendor: event.nextVendor } : {}),
|
|
65
|
+
phase: event.phase,
|
|
66
|
+
reason: event.reason,
|
|
67
|
+
...(event.attempt === undefined ? {} : { attempt: Math.max(0, Math.floor(event.attempt)) }),
|
|
68
|
+
});
|
|
69
|
+
}
|
|
56
70
|
recordProxyAttempt(event) {
|
|
57
71
|
if (this.#attempts.length >= MAX_PROXY_ATTEMPT_SAMPLES)
|
|
58
72
|
return;
|
|
59
73
|
this.#attempts.push({
|
|
60
|
-
provider:
|
|
74
|
+
provider: event.provider,
|
|
61
75
|
attempt: Math.max(1, Math.floor(event.attempt || 1)),
|
|
62
76
|
...(event.poolIndex === undefined
|
|
63
77
|
? {}
|
|
@@ -75,8 +89,16 @@ export class ProxyTelemetryCollector {
|
|
|
75
89
|
const [first, ...rest] = this.#events;
|
|
76
90
|
if (!first)
|
|
77
91
|
return undefined;
|
|
92
|
+
// The serving vendor/protocol is the last recorded resolution (a failed
|
|
93
|
+
// vendor records first, the vendor that served records last).
|
|
94
|
+
const serving = this.#events[this.#events.length - 1] ?? first;
|
|
95
|
+
const vendors = [];
|
|
96
|
+
for (const event of this.#events) {
|
|
97
|
+
if (!vendors.includes(event.provider))
|
|
98
|
+
vendors.push(event.provider);
|
|
99
|
+
}
|
|
78
100
|
const aggregate = rest.reduce((acc, event) => ({
|
|
79
|
-
provider:
|
|
101
|
+
provider: event.provider,
|
|
80
102
|
cacheStatus: worseStatus(acc.cacheStatus, event.cacheStatus),
|
|
81
103
|
cacheHit: acc.cacheHit && event.cacheHit,
|
|
82
104
|
resolutionMs: acc.resolutionMs + event.resolutionMs,
|
|
@@ -95,7 +117,8 @@ export class ProxyTelemetryCollector {
|
|
|
95
117
|
const payload = {
|
|
96
118
|
v: 1,
|
|
97
119
|
proxy: {
|
|
98
|
-
provider:
|
|
120
|
+
provider: serving.provider,
|
|
121
|
+
...(serving.protocol ? { protocol: serving.protocol } : {}),
|
|
99
122
|
cacheStatus: aggregate.cacheStatus,
|
|
100
123
|
cacheHit: aggregate.cacheHit,
|
|
101
124
|
resolutionMs: aggregate.resolutionMs,
|
|
@@ -132,6 +155,18 @@ export class ProxyTelemetryCollector {
|
|
|
132
155
|
})),
|
|
133
156
|
}
|
|
134
157
|
: {}),
|
|
158
|
+
...(vendors.length > 1 ? { vendors } : {}),
|
|
159
|
+
...(this.#failovers.length > 0
|
|
160
|
+
? {
|
|
161
|
+
failovers: this.#failovers.map((failover) => ({
|
|
162
|
+
v: failover.vendor,
|
|
163
|
+
...(failover.nextVendor ? { nx: failover.nextVendor } : {}),
|
|
164
|
+
p: failover.phase,
|
|
165
|
+
r: failover.reason,
|
|
166
|
+
...(failover.attempt === undefined ? {} : { a: failover.attempt }),
|
|
167
|
+
})),
|
|
168
|
+
}
|
|
169
|
+
: {}),
|
|
135
170
|
},
|
|
136
171
|
};
|
|
137
172
|
const encoded = encodeBase64Url(JSON.stringify(payload));
|
package/dist/runtime/stealth.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { Impit } from "impit";
|
|
3
|
-
import { DEFAULT_SMARTPROXY_POOL_SIZE, invalidateProxyResolutionCacheAsync, ProxyResolutionError, resolveProxyConfigAsync, SMARTPROXY_MAX_POOL_SIZE, } from "../config/loader.js";
|
|
3
|
+
import { DEFAULT_SMARTPROXY_POOL_SIZE, invalidateProxyResolutionCacheAsync, ProxyResolutionError, resolvePolicyProxyPoolSpan, resolveProxyConfigAsync, SMARTPROXY_MAX_POOL_SIZE, vendorFromResolvedSource, } from "../config/loader.js";
|
|
4
4
|
import { SDKError, TransportError } from "../errors.js";
|
|
5
5
|
import { getStealthProfile } from "../stealth/profiles.js";
|
|
6
6
|
import { createProxyAuthIpDeniedError, createProxyEdgeAuthRejectedError, createProxyEdgeTlsRejectedError, createProxyPoolExhaustedError, createProxyPoolStaleError, isProxyAuthIpDeniedMessage, isProxyEdgeAuthRejectedMessage, isProxyEdgeTlsRejectedResponse, isProxyPoolRefreshableError, isProxyPoolStaleMessage, isProxyPoolStaleStatus, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_STALE_CODE, } from "./proxy-errors.js";
|
|
@@ -8,7 +8,11 @@ import { computeProxyAttemptIndex, computeProxyTransportRetryDelayMs, createDefa
|
|
|
8
8
|
import { appendQueryParams } from "./request-options.js";
|
|
9
9
|
const DEFAULT_PROFILE = "chrome-146";
|
|
10
10
|
const MISSING_PROXY_WARNING = "[provider-sdk] Provider requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
|
|
11
|
-
|
|
11
|
+
/**
|
|
12
|
+
* Upper bound on attempts across a multi-vendor chain, so a two-vendor chain can
|
|
13
|
+
* exhaust each vendor's pool before failing over and finally throwing.
|
|
14
|
+
*/
|
|
15
|
+
const MAX_POLICY_PROXY_TOTAL_ATTEMPTS = SMARTPROXY_MAX_POOL_SIZE * 2;
|
|
12
16
|
const MAX_POLICY_PROXY_POOL_REFRESHES = 1;
|
|
13
17
|
const PROXY_CONNECT_FAILURE_CODE = "proxy_connect_failed";
|
|
14
18
|
const PROXY_CONNECT_FAILURE_BODY_PATTERN = /\bproxy\b.*\b(non[\s-]?200|connect|tunnel)|\bconnect\b.*\bproxy\b|\btunnel\b/i;
|
|
@@ -463,7 +467,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
463
467
|
}
|
|
464
468
|
return client;
|
|
465
469
|
}
|
|
466
|
-
async function resolveRequestProxy(options, proxyAttempt) {
|
|
470
|
+
async function resolveRequestProxy(options, proxyAttempt, refreshEpoch) {
|
|
467
471
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
468
472
|
proxy: options?.proxy ?? clientOptions.proxy,
|
|
469
473
|
upstream: clientOptions.upstream,
|
|
@@ -474,6 +478,10 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
474
478
|
proxyAttemptOffset: options?.proxyAttemptOffset,
|
|
475
479
|
retryAttemptOffset: proxyAttempt,
|
|
476
480
|
}),
|
|
481
|
+
// The impit stealth transport tunnels both HTTP CONNECT and SOCKS5,
|
|
482
|
+
// preserving the client TLS fingerprint end-to-end.
|
|
483
|
+
transportProtocols: ["http", "socks5"],
|
|
484
|
+
...(refreshEpoch === undefined ? {} : { proxyRefreshEpoch: refreshEpoch }),
|
|
477
485
|
telemetry: clientOptions.telemetry,
|
|
478
486
|
});
|
|
479
487
|
if (resolvedProxy.shouldWarn && !hasWarnedMissingProxy) {
|
|
@@ -484,6 +492,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
484
492
|
url: resolvedProxy.url,
|
|
485
493
|
poolIndex: proxyPoolIndexFromDiagnostics(resolvedProxy.diagnostics),
|
|
486
494
|
proxyHash: proxyEndpointHash(resolvedProxy.url),
|
|
495
|
+
vendor: vendorFromResolvedSource(resolvedProxy.source),
|
|
487
496
|
};
|
|
488
497
|
}
|
|
489
498
|
const session = {
|
|
@@ -506,11 +515,13 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
506
515
|
const hasPolicyProxy = isPolicyManagedProxy(clientOptions);
|
|
507
516
|
const usesPolicyAllocator = hasPolicyProxy && !options.proxy && !clientOptions.proxy;
|
|
508
517
|
const retryAttemptCap = Math.max(1, stealthRetryOptions?.attempts ?? 1);
|
|
509
|
-
|
|
518
|
+
// Span the whole vendor chain: successive attempts rotate one vendor's
|
|
519
|
+
// pool, then fail over to the next vendor via the flat attempt index.
|
|
520
|
+
const policyProxy = clientOptions.proxyPolicy ??
|
|
510
521
|
(typeof clientOptions.upstream?.proxy === "object"
|
|
511
|
-
? clientOptions.upstream.proxy
|
|
512
|
-
: undefined)
|
|
513
|
-
|
|
522
|
+
? clientOptions.upstream.proxy
|
|
523
|
+
: undefined);
|
|
524
|
+
const policyProxyAttemptCap = Math.max(1, Math.min(MAX_POLICY_PROXY_TOTAL_ATTEMPTS, policyProxy ? resolvePolicyProxyPoolSpan(policyProxy) : DEFAULT_SMARTPROXY_POOL_SIZE));
|
|
514
525
|
const maxAttempts = usesPolicyAllocator ? policyProxyAttemptCap : retryAttemptCap;
|
|
515
526
|
let lastError;
|
|
516
527
|
for (let refreshAttempt = 0; refreshAttempt <= MAX_POLICY_PROXY_POOL_REFRESHES; refreshAttempt += 1) {
|
|
@@ -527,7 +538,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
527
538
|
return;
|
|
528
539
|
attemptRecorded = true;
|
|
529
540
|
clientOptions.telemetry?.recordProxyAttempt?.({
|
|
530
|
-
provider: "smartproxy",
|
|
541
|
+
provider: attemptProxy?.vendor ?? "smartproxy",
|
|
531
542
|
attempt: attempt + 1,
|
|
532
543
|
...(attemptProxy?.poolIndex === undefined
|
|
533
544
|
? {}
|
|
@@ -541,7 +552,7 @@ function createSessionFetcher(baseUrl, defaultProfile, clientOptions) {
|
|
|
541
552
|
};
|
|
542
553
|
try {
|
|
543
554
|
assertNoUnsupportedFingerprintOverrides(options);
|
|
544
|
-
attemptProxy = await resolveRequestProxy(options, attempt);
|
|
555
|
+
attemptProxy = await resolveRequestProxy(options, attempt, refreshAttempt);
|
|
545
556
|
proxy = attemptProxy.url;
|
|
546
557
|
if (proxy && usesPolicyAllocator) {
|
|
547
558
|
if (attemptedProxies.has(proxy)) {
|
package/dist/server/types.d.ts
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
export declare const ConnectionModeSchema: z.ZodEnum<{
|
|
3
|
-
credentials: "credentials";
|
|
4
3
|
none: "none";
|
|
4
|
+
credentials: "credentials";
|
|
5
5
|
oauth2: "oauth2";
|
|
6
6
|
"platform-managed": "platform-managed";
|
|
7
7
|
}>;
|
|
8
8
|
export declare const OperationConnectionSchema: z.ZodObject<{
|
|
9
9
|
id: z.ZodString;
|
|
10
10
|
mode: z.ZodEnum<{
|
|
11
|
-
credentials: "credentials";
|
|
12
11
|
none: "none";
|
|
12
|
+
credentials: "credentials";
|
|
13
13
|
oauth2: "oauth2";
|
|
14
14
|
"platform-managed": "platform-managed";
|
|
15
15
|
}>;
|
|
@@ -25,8 +25,8 @@ export declare const OperationRequestSchema: z.ZodObject<{
|
|
|
25
25
|
connection: z.ZodOptional<z.ZodObject<{
|
|
26
26
|
id: z.ZodString;
|
|
27
27
|
mode: z.ZodEnum<{
|
|
28
|
-
credentials: "credentials";
|
|
29
28
|
none: "none";
|
|
29
|
+
credentials: "credentials";
|
|
30
30
|
oauth2: "oauth2";
|
|
31
31
|
"platform-managed": "platform-managed";
|
|
32
32
|
}>;
|
|
@@ -55,21 +55,21 @@ export declare const OperationSuccessResponseSchema: z.ZodObject<{
|
|
|
55
55
|
stale: z.ZodBoolean;
|
|
56
56
|
keys: z.ZodArray<z.ZodString>;
|
|
57
57
|
source: z.ZodOptional<z.ZodEnum<{
|
|
58
|
-
loader: "loader";
|
|
59
|
-
memory: "memory";
|
|
60
58
|
mixed: "mixed";
|
|
61
59
|
redis: "redis";
|
|
60
|
+
memory: "memory";
|
|
61
|
+
loader: "loader";
|
|
62
62
|
}>>;
|
|
63
63
|
}, z.core.$strip>>;
|
|
64
64
|
retry: z.ZodOptional<z.ZodObject<{
|
|
65
65
|
attempts: z.ZodNumber;
|
|
66
66
|
retries: z.ZodNumber;
|
|
67
67
|
preset: z.ZodOptional<z.ZodEnum<{
|
|
68
|
-
aggressive_read: "aggressive_read";
|
|
69
68
|
off: "off";
|
|
70
|
-
rate_limit_aware: "rate_limit_aware";
|
|
71
|
-
safe_read: "safe_read";
|
|
72
69
|
transport_transient: "transport_transient";
|
|
70
|
+
safe_read: "safe_read";
|
|
71
|
+
aggressive_read: "aggressive_read";
|
|
72
|
+
rate_limit_aware: "rate_limit_aware";
|
|
73
73
|
}>>;
|
|
74
74
|
transport: z.ZodEnum<{
|
|
75
75
|
native: "native";
|
|
@@ -101,8 +101,8 @@ export declare const AuthFlowRequestSchema: z.ZodObject<{
|
|
|
101
101
|
connection: z.ZodOptional<z.ZodObject<{
|
|
102
102
|
id: z.ZodString;
|
|
103
103
|
mode: z.ZodEnum<{
|
|
104
|
-
credentials: "credentials";
|
|
105
104
|
none: "none";
|
|
105
|
+
credentials: "credentials";
|
|
106
106
|
oauth2: "oauth2";
|
|
107
107
|
"platform-managed": "platform-managed";
|
|
108
108
|
}>;
|
package/dist/types.d.ts
CHANGED
|
@@ -650,7 +650,25 @@ export type ConnectionMode = AuthMode;
|
|
|
650
650
|
export type ProviderReviewed = "first-party" | "community" | "staging";
|
|
651
651
|
export type ProviderAccessVisibility = "public" | "early_access";
|
|
652
652
|
export type ProviderProxyMode = "disabled" | "optional" | "required";
|
|
653
|
-
|
|
653
|
+
/**
|
|
654
|
+
* Proxy egress vendors. These are FOUR DISTINCT services — do not conflate them
|
|
655
|
+
* (a common mistake because the names collide with a well-known rebrand):
|
|
656
|
+
*
|
|
657
|
+
* - `smartproxy` — **api.smartproxy.org**, a residential proxy with an IP
|
|
658
|
+
* *extraction/allocation* API (app_key → a pool of raw `ip:port` CONNECT
|
|
659
|
+
* endpoints). This is our own vendor. It is NOT the company formerly named
|
|
660
|
+
* "Smartproxy". Credentials: `APIFUSE__PROXY__SMARTPROXY_APP_KEY`.
|
|
661
|
+
* - `nodemaven` — **gate.nodemaven.com**, a *gateway* proxy with static
|
|
662
|
+
* credentials; geo/session encoded in the username, no allocation API.
|
|
663
|
+
* - `decodo` — **decodo.com**, the *gateway* proxy that was named "Smartproxy"
|
|
664
|
+
* (smartproxy.com) before its 2025 rebrand to Decodo. Sticky sessions via
|
|
665
|
+
* username params. A different company from `smartproxy` above.
|
|
666
|
+
* **@deprecated** — unused; no managed adapter. Use `smartproxy`/`nodemaven`,
|
|
667
|
+
* or the `APIFUSE__PROXY__URL` bring-your-own escape hatch.
|
|
668
|
+
* - `custom` — **@deprecated** bring-your-own static proxy URL marker. The
|
|
669
|
+
* `APIFUSE__PROXY__URL` env still works without declaring this value.
|
|
670
|
+
*/
|
|
671
|
+
export type ProviderProxyProvider = "smartproxy" | "nodemaven" | "decodo" | "custom";
|
|
654
672
|
export type ProviderProxySessionAffinity = "request" | "operation" | "auth-flow" | "connection";
|
|
655
673
|
export interface ProviderProxyPolicy {
|
|
656
674
|
/**
|
|
@@ -658,7 +676,18 @@ export interface ProviderProxyPolicy {
|
|
|
658
676
|
* certificate verification, and vendor allocator endpoints are SDK-owned.
|
|
659
677
|
*/
|
|
660
678
|
mode: ProviderProxyMode;
|
|
679
|
+
/**
|
|
680
|
+
* @deprecated Use `providers: [...]` to declare an ordered vendor fallback
|
|
681
|
+
* chain. A single-element `providers` list is equivalent to this field.
|
|
682
|
+
*/
|
|
661
683
|
provider?: ProviderProxyProvider;
|
|
684
|
+
/**
|
|
685
|
+
* Ordered proxy-vendor fallback chain. The SDK tries each vendor in order and
|
|
686
|
+
* fails over to the next when a vendor lacks credentials or its allocation /
|
|
687
|
+
* transport is exhausted. When omitted, `provider` (or the platform default)
|
|
688
|
+
* is used as a single-vendor chain.
|
|
689
|
+
*/
|
|
690
|
+
providers?: ProviderProxyProvider[];
|
|
662
691
|
geo?: {
|
|
663
692
|
/** ISO 3166-1 alpha-2 country code, for example KR or US. */
|
|
664
693
|
country?: Iso3166Alpha2CountryCode;
|
package/package.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version": "2.2.0-beta.
|
|
2
|
+
"version": "2.2.0-beta.8",
|
|
3
3
|
"name": "@apifuse/provider-sdk",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
@@ -75,10 +75,11 @@
|
|
|
75
75
|
"scripts": {
|
|
76
76
|
"lint": "biome lint .",
|
|
77
77
|
"lint:fix": "biome lint --write",
|
|
78
|
+
"lint:deprecated": "bun run scripts/lint-deprecated-usage.ts",
|
|
78
79
|
"format": "biome format --write",
|
|
79
80
|
"type-check": "tsc --noEmit",
|
|
80
81
|
"test": "bun test",
|
|
81
|
-
"check": "bun run lint && bun run type-check && bun run build",
|
|
82
|
+
"check": "bun run lint && bun run type-check && bun run lint:deprecated && bun run build",
|
|
82
83
|
"pack:check": "bun run build && bun bin/apifuse-pack-check.ts",
|
|
83
84
|
"pack:smoke": "bun run build && bun bin/apifuse-pack-smoke.ts",
|
|
84
85
|
"pack:types": "bun run build && bun bin/apifuse-pack-types.ts",
|
|
@@ -91,7 +92,7 @@
|
|
|
91
92
|
"@biomejs/biome": "^2.5.0",
|
|
92
93
|
"@types/bun": "latest",
|
|
93
94
|
"@types/node": "^25.9.3",
|
|
94
|
-
"typescript": "
|
|
95
|
+
"typescript": "6.0.3"
|
|
95
96
|
},
|
|
96
97
|
"dependencies": {
|
|
97
98
|
"@clack/prompts": "^1.5.1",
|