@apifuse/provider-sdk 2.2.0-beta.5 → 2.2.0-beta.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +53 -0
- package/CHANGELOG.md +12 -0
- package/README.md +5 -1
- package/SUBMISSION.md +1 -1
- package/bin/apifuse-check.ts +26 -1
- package/bin/apifuse-pack-check.ts +14 -0
- package/bin/apifuse-submit-check.ts +193 -2
- package/bin/apifuse-sync-assets.ts +117 -0
- package/dist/auth-turn/index.d.ts +2 -2
- package/dist/cli/commands.d.ts +1 -1
- package/dist/cli/commands.js +8 -0
- package/dist/cli/create.d.ts +3 -0
- package/dist/cli/create.js +34 -35
- package/dist/cli/prompt-assets.d.ts +80 -0
- package/dist/cli/prompt-assets.js +743 -0
- package/dist/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/dist/config/loader.d.ts +79 -6
- package/dist/config/loader.js +272 -48
- package/dist/define.js +27 -3
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/runtime/executor.js +7 -0
- package/dist/runtime/http.js +3 -0
- package/dist/runtime/proxy-errors.js +6 -2
- package/dist/runtime/proxy-nodemaven.d.ts +34 -0
- package/dist/runtime/proxy-nodemaven.js +128 -0
- package/dist/runtime/proxy-telemetry.d.ts +2 -1
- package/dist/runtime/proxy-telemetry.js +39 -4
- package/dist/runtime/secrets.d.ts +27 -0
- package/dist/runtime/secrets.js +51 -0
- package/dist/runtime/stealth.js +20 -9
- package/dist/server/serve.d.ts +5 -0
- package/dist/server/serve.js +39 -0
- package/dist/server/types.d.ts +9 -9
- package/dist/types.d.ts +30 -1
- package/package.json +4 -3
- package/src/cli/commands.ts +10 -0
- package/src/cli/create.ts +42 -35
- package/src/cli/prompt-assets.ts +865 -0
- package/src/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/src/config/loader.ts +405 -61
- package/src/define.ts +35 -3
- package/src/index.ts +5 -0
- package/src/runtime/executor.ts +8 -0
- package/src/runtime/http.ts +3 -0
- package/src/runtime/proxy-errors.ts +12 -4
- package/src/runtime/proxy-nodemaven.ts +178 -0
- package/src/runtime/proxy-telemetry.ts +56 -5
- package/src/runtime/secrets.ts +64 -0
- package/src/runtime/stealth.ts +26 -10
- package/src/server/serve.ts +53 -0
- package/src/types.ts +30 -1
- package/dist/cli/templates/provider/CLAUDE.md.tpl +0 -1
- package/src/cli/templates/provider/CLAUDE.md.tpl +0 -1
- /package/dist/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
package/src/config/loader.ts
CHANGED
|
@@ -4,8 +4,24 @@ import path from "node:path";
|
|
|
4
4
|
|
|
5
5
|
import { Redis } from "ioredis";
|
|
6
6
|
|
|
7
|
-
import type { ProviderProxyPolicy, TraceConfig } from "../types.js";
|
|
8
|
-
|
|
7
|
+
import type { ProviderProxyPolicy, ProviderProxyProvider, TraceConfig } from "../types.js";
|
|
8
|
+
import {
|
|
9
|
+
NODEMAVEN_DEFAULT_PROTOCOL,
|
|
10
|
+
type ProxyProtocol,
|
|
11
|
+
hasNodemavenCredentials,
|
|
12
|
+
nodemavenPoolSize,
|
|
13
|
+
synthesizeNodemavenProxy,
|
|
14
|
+
} from "../runtime/proxy-nodemaven.js";
|
|
15
|
+
|
|
16
|
+
export type { ProxyProtocol } from "../runtime/proxy-nodemaven.js";
|
|
17
|
+
|
|
18
|
+
/** Proxy vendors the SDK resolves natively (as opposed to the static env path). */
|
|
19
|
+
export type ProxyVendorName = "smartproxy" | "nodemaven";
|
|
20
|
+
|
|
21
|
+
// "smartproxy" here is api.smartproxy.org — a residential proxy with an IP
|
|
22
|
+
// extraction API (app_key → raw ip:port pool). It is NOT the company formerly
|
|
23
|
+
// named Smartproxy (smartproxy.com), which rebranded to Decodo in 2025 and is
|
|
24
|
+
// modelled separately as the `decodo` gateway vendor. Do not conflate them.
|
|
9
25
|
export const SMARTPROXY_APP_KEY_ENV = "APIFUSE__PROXY__SMARTPROXY_APP_KEY";
|
|
10
26
|
export const SMARTPROXY_MAX_LIFETIME_MINUTES = 2000;
|
|
11
27
|
export const DEFAULT_SMARTPROXY_POOL_SIZE = 20;
|
|
@@ -52,6 +68,26 @@ export type ProxyResolutionOptions = {
|
|
|
52
68
|
affinityKey?: string;
|
|
53
69
|
/** Zero-based proxy-pool attempt index used by SDK transports for failover. */
|
|
54
70
|
proxyAttempt?: number;
|
|
71
|
+
/**
|
|
72
|
+
* Tunnelling protocols the calling transport can use. When a resolved
|
|
73
|
+
* protocol is not in this set the resolver fails with
|
|
74
|
+
* `PROXY_PROTOCOL_UNSUPPORTED` instead of silently downgrading. Unset means
|
|
75
|
+
* permissive (both protocols allowed).
|
|
76
|
+
*/
|
|
77
|
+
transportProtocols?: readonly ProxyProtocol[];
|
|
78
|
+
/**
|
|
79
|
+
* Explicit protocol override. Internal — for the verification harness and
|
|
80
|
+
* tests, or an advanced caller. Normal callers omit it and each vendor uses
|
|
81
|
+
* its own benchmarked default protocol (see VENDOR_DEFAULT_PROTOCOL). Not an
|
|
82
|
+
* env var and not a provider-policy field.
|
|
83
|
+
*/
|
|
84
|
+
protocol?: ProxyProtocol;
|
|
85
|
+
/**
|
|
86
|
+
* Gateway pool "refresh" generation. Bumped by transports on pool refresh to
|
|
87
|
+
* derive a fresh gateway session set (ignored by allocation-style vendors,
|
|
88
|
+
* whose refresh is driven by cache invalidation).
|
|
89
|
+
*/
|
|
90
|
+
proxyRefreshEpoch?: number;
|
|
55
91
|
telemetry?: ProxyTelemetrySink;
|
|
56
92
|
};
|
|
57
93
|
|
|
@@ -74,7 +110,8 @@ export type SmartproxyAllocatorBodyClass =
|
|
|
74
110
|
| "usable_proxy_endpoints";
|
|
75
111
|
|
|
76
112
|
export type ProxyResolutionTelemetryEvent = {
|
|
77
|
-
provider:
|
|
113
|
+
provider: ProxyVendorName;
|
|
114
|
+
protocol?: ProxyProtocol;
|
|
78
115
|
cacheStatus: ProxyCacheStatus;
|
|
79
116
|
cacheHit: boolean;
|
|
80
117
|
resolutionMs: number;
|
|
@@ -92,7 +129,7 @@ export type ProxyResolutionTelemetryEvent = {
|
|
|
92
129
|
};
|
|
93
130
|
|
|
94
131
|
export type ProxyAttemptTelemetryEvent = {
|
|
95
|
-
provider:
|
|
132
|
+
provider: ProxyVendorName;
|
|
96
133
|
attempt: number;
|
|
97
134
|
poolIndex?: number;
|
|
98
135
|
proxyHash?: string;
|
|
@@ -102,31 +139,60 @@ export type ProxyAttemptTelemetryEvent = {
|
|
|
102
139
|
durationMs?: number;
|
|
103
140
|
};
|
|
104
141
|
|
|
142
|
+
export type ProxyVendorFailoverTelemetryEvent = {
|
|
143
|
+
/** Vendor that failed or was skipped. */
|
|
144
|
+
vendor: ProxyVendorName;
|
|
145
|
+
/** Vendor tried next, or undefined when the chain is exhausted. */
|
|
146
|
+
nextVendor?: ProxyVendorName;
|
|
147
|
+
phase: "resolution" | "transport";
|
|
148
|
+
reason: "no_credentials" | "allocation_failed" | "pool_exhausted" | "protocol_unsupported";
|
|
149
|
+
attempt?: number;
|
|
150
|
+
};
|
|
151
|
+
|
|
105
152
|
export type ProxyTelemetrySink = {
|
|
106
153
|
recordProxyResolution(event: ProxyResolutionTelemetryEvent): void;
|
|
107
154
|
recordProxyAttempt?(event: ProxyAttemptTelemetryEvent): void;
|
|
155
|
+
recordProxyVendorFailover?(event: ProxyVendorFailoverTelemetryEvent): void;
|
|
108
156
|
};
|
|
109
157
|
|
|
110
158
|
export type ResolvedProxyConfig = {
|
|
111
159
|
shouldWarn: boolean;
|
|
112
160
|
url?: string;
|
|
113
|
-
source?: "explicit" | "env" | "config" | "smartproxy-allocator";
|
|
161
|
+
source?: "explicit" | "env" | "config" | "smartproxy-allocator" | "nodemaven-gateway";
|
|
162
|
+
protocol?: ProxyProtocol;
|
|
114
163
|
diagnostics?: Record<string, string | number | boolean>;
|
|
115
164
|
};
|
|
116
165
|
|
|
166
|
+
export type ProxyResolutionErrorCode =
|
|
167
|
+
| "PROXY_REQUIRED"
|
|
168
|
+
| "PROXY_ALLOCATION_FAILED"
|
|
169
|
+
| "PROXY_PROTOCOL_UNSUPPORTED";
|
|
170
|
+
|
|
117
171
|
export class ProxyResolutionError extends Error {
|
|
118
|
-
readonly code:
|
|
172
|
+
readonly code: ProxyResolutionErrorCode;
|
|
119
173
|
readonly telemetry?: ProxyResolutionTelemetryEvent;
|
|
174
|
+
readonly vendor?: ProxyVendorName;
|
|
175
|
+
readonly vendorChain?: ProxyVendorName[];
|
|
176
|
+
readonly protocol?: ProxyProtocol;
|
|
120
177
|
|
|
121
178
|
constructor(
|
|
122
|
-
code:
|
|
179
|
+
code: ProxyResolutionErrorCode,
|
|
123
180
|
message: string,
|
|
124
|
-
options?: {
|
|
181
|
+
options?: {
|
|
182
|
+
cause?: unknown;
|
|
183
|
+
telemetry?: ProxyResolutionTelemetryEvent;
|
|
184
|
+
vendor?: ProxyVendorName;
|
|
185
|
+
vendorChain?: ProxyVendorName[];
|
|
186
|
+
protocol?: ProxyProtocol;
|
|
187
|
+
},
|
|
125
188
|
) {
|
|
126
189
|
super(message, options);
|
|
127
190
|
this.name = "ProxyResolutionError";
|
|
128
191
|
this.code = code;
|
|
129
192
|
this.telemetry = options?.telemetry;
|
|
193
|
+
this.vendor = options?.vendor;
|
|
194
|
+
this.vendorChain = options?.vendorChain;
|
|
195
|
+
this.protocol = options?.protocol;
|
|
130
196
|
}
|
|
131
197
|
}
|
|
132
198
|
|
|
@@ -389,6 +455,11 @@ function applyStickyProxySession(proxyUrl: string): string {
|
|
|
389
455
|
return proxyUrl;
|
|
390
456
|
}
|
|
391
457
|
|
|
458
|
+
// This rewrites sticky-session usernames for a bring-your-own *gateway* URL
|
|
459
|
+
// (APIFUSE__PROXY__URL). The `smartproxy` host here means a smartproxy.com /
|
|
460
|
+
// Decodo-family gateway that authenticates by username — NOT the
|
|
461
|
+
// api.smartproxy.org allocation vendor, whose endpoints are raw ip:port with
|
|
462
|
+
// no credentials and therefore return early above.
|
|
392
463
|
const host = parsed.hostname.toLowerCase();
|
|
393
464
|
if (!host.includes("smartproxy") && !host.includes("decodo")) {
|
|
394
465
|
return proxyUrl;
|
|
@@ -477,60 +548,214 @@ export async function resolveProxyConfigAsync(
|
|
|
477
548
|
return { shouldWarn: false };
|
|
478
549
|
}
|
|
479
550
|
|
|
480
|
-
const
|
|
481
|
-
if (
|
|
551
|
+
const chain = resolveVendorChain(policy);
|
|
552
|
+
if (chain.length === 0) {
|
|
553
|
+
// decodo/custom/env-static providers keep the legacy static-URL path.
|
|
482
554
|
return resolveProxyConfig({
|
|
483
555
|
...options,
|
|
484
556
|
upstream: { proxy: true },
|
|
485
557
|
});
|
|
486
558
|
}
|
|
487
559
|
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
560
|
+
// Protocol is chosen per vendor (each vendor's benchmarked-best), with an
|
|
561
|
+
// optional explicit override for the harness/tests. Both are tunnelling
|
|
562
|
+
// schemes. transportProtocols is what the calling transport can actually use.
|
|
563
|
+
const transportProtocols = options.transportProtocols ?? (["http", "socks5"] as const);
|
|
564
|
+
|
|
565
|
+
const sizes = chain.map((vendor) => vendorPoolSize(vendor, policy));
|
|
566
|
+
const total = sizes.reduce((sum, size) => sum + size, 0);
|
|
567
|
+
const normalizedAttempt = normalizeAttemptIndex(options.proxyAttempt);
|
|
568
|
+
const { vendorIndex: startVendorIndex, poolIndex: startPoolIndex } = mapFlatAttempt(
|
|
569
|
+
total > 0 ? normalizedAttempt % total : 0,
|
|
570
|
+
sizes,
|
|
571
|
+
);
|
|
572
|
+
const refreshEpoch = normalizeAttemptIndex(options.proxyRefreshEpoch);
|
|
573
|
+
|
|
574
|
+
let lastError: unknown;
|
|
575
|
+
let blockedProtocol: ProxyProtocol | undefined;
|
|
576
|
+
for (let vendorIndex = startVendorIndex; vendorIndex < chain.length; vendorIndex++) {
|
|
577
|
+
const vendor = chain[vendorIndex] as ProxyVendorName;
|
|
578
|
+
const nextVendor = chain[vendorIndex + 1];
|
|
579
|
+
const poolIndex = vendorIndex === startVendorIndex ? startPoolIndex : 0;
|
|
580
|
+
const protocol = options.protocol ?? VENDOR_DEFAULT_PROTOCOL[vendor];
|
|
581
|
+
|
|
582
|
+
if (!vendorHasCredentials(vendor)) {
|
|
583
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
584
|
+
vendor,
|
|
585
|
+
nextVendor,
|
|
586
|
+
phase: "resolution",
|
|
587
|
+
reason: "no_credentials",
|
|
588
|
+
});
|
|
589
|
+
continue;
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
// The calling transport must be able to use this vendor's protocol; if not,
|
|
593
|
+
// fail over to the next vendor rather than silently downgrading.
|
|
594
|
+
if (!transportProtocols.includes(protocol)) {
|
|
595
|
+
blockedProtocol = protocol;
|
|
596
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
597
|
+
vendor,
|
|
598
|
+
nextVendor,
|
|
599
|
+
phase: "resolution",
|
|
600
|
+
reason: "protocol_unsupported",
|
|
601
|
+
});
|
|
602
|
+
continue;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
try {
|
|
606
|
+
return await resolveWithVendor(vendor, policy, options, {
|
|
607
|
+
protocol,
|
|
608
|
+
poolIndex,
|
|
609
|
+
refreshEpoch,
|
|
610
|
+
});
|
|
611
|
+
} catch (error) {
|
|
612
|
+
// Config/programming errors (invalid filter, etc.) are not vendor
|
|
613
|
+
// outages — propagate them rather than failing over.
|
|
614
|
+
if (!(error instanceof ProxyResolutionError)) {
|
|
615
|
+
throw error;
|
|
616
|
+
}
|
|
617
|
+
if (error.telemetry) {
|
|
618
|
+
options.telemetry?.recordProxyResolution(error.telemetry);
|
|
619
|
+
}
|
|
620
|
+
lastError = error;
|
|
621
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
622
|
+
vendor,
|
|
623
|
+
nextVendor,
|
|
624
|
+
phase: "resolution",
|
|
625
|
+
reason: "allocation_failed",
|
|
626
|
+
});
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
if (policy.mode === "required") {
|
|
631
|
+
if (lastError) {
|
|
632
|
+
throw lastError instanceof ProxyResolutionError
|
|
633
|
+
? lastError
|
|
634
|
+
: new ProxyResolutionError(
|
|
635
|
+
"PROXY_ALLOCATION_FAILED",
|
|
636
|
+
`All proxy vendors [${chain.join(", ")}] failed for required proxy egress.`,
|
|
637
|
+
{ cause: lastError, vendorChain: chain },
|
|
638
|
+
);
|
|
639
|
+
}
|
|
640
|
+
if (blockedProtocol) {
|
|
491
641
|
throw new ProxyResolutionError(
|
|
492
|
-
"
|
|
493
|
-
`
|
|
642
|
+
"PROXY_PROTOCOL_UNSUPPORTED",
|
|
643
|
+
`No proxy vendor in [${chain.join(", ")}] could serve a protocol supported by this transport (supports: ${transportProtocols.join(", ")}; vendor wanted "${blockedProtocol}"). Route this provider through the stealth transport.`,
|
|
644
|
+
{ protocol: blockedProtocol, vendorChain: chain },
|
|
494
645
|
);
|
|
495
646
|
}
|
|
496
|
-
|
|
647
|
+
throw new ProxyResolutionError(
|
|
648
|
+
"PROXY_REQUIRED",
|
|
649
|
+
`Proxy egress is required but no vendor credentials are configured. Missing: ${chain
|
|
650
|
+
.map((vendor) => `${missingCredentialEnv(vendor)} (${vendor})`)
|
|
651
|
+
.join(", ")}.`,
|
|
652
|
+
{ vendorChain: chain },
|
|
653
|
+
);
|
|
497
654
|
}
|
|
498
|
-
|
|
655
|
+
return { shouldWarn: true };
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
/**
|
|
659
|
+
* Each vendor's default egress protocol, chosen from live KR benchmarks. HTTP
|
|
660
|
+
* CONNECT wins for nodemaven (socks5 adds ~500ms through the gateway) and ties
|
|
661
|
+
* for smartproxy, and is the only protocol ctx.http (Bun native fetch) supports.
|
|
662
|
+
* Override per call via ProxyResolutionOptions.protocol (harness/tests).
|
|
663
|
+
*/
|
|
664
|
+
const VENDOR_DEFAULT_PROTOCOL: Record<ProxyVendorName, ProxyProtocol> = {
|
|
665
|
+
smartproxy: "http",
|
|
666
|
+
nodemaven: NODEMAVEN_DEFAULT_PROTOCOL,
|
|
667
|
+
};
|
|
499
668
|
|
|
669
|
+
/**
|
|
670
|
+
* Guard the No-MITM invariant: a resolved proxy URL must use a tunnelling scheme
|
|
671
|
+
* (http CONNECT or socks5) so the client TLS handshake reaches the origin
|
|
672
|
+
* end-to-end. Anything else would intercept TLS and break fingerprinting.
|
|
673
|
+
*/
|
|
674
|
+
export function assertTunnelingScheme(url: string): void {
|
|
675
|
+
let scheme: string;
|
|
500
676
|
try {
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
677
|
+
scheme = new URL(url).protocol.replace(/:$/, "").toLowerCase();
|
|
678
|
+
} catch {
|
|
679
|
+
throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `Malformed proxy URL: ${url}`);
|
|
680
|
+
}
|
|
681
|
+
if (scheme !== "http" && scheme !== "socks5") {
|
|
682
|
+
throw new ProxyResolutionError(
|
|
683
|
+
"PROXY_ALLOCATION_FAILED",
|
|
684
|
+
`Resolved proxy scheme "${scheme}" is not a tunnelling scheme (expected http or socks5). Refusing to route TLS through a non-tunnelling proxy.`,
|
|
506
685
|
);
|
|
507
|
-
|
|
508
|
-
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
async function resolveWithVendor(
|
|
690
|
+
vendor: ProxyVendorName,
|
|
691
|
+
policy: ProviderProxyPolicy,
|
|
692
|
+
options: ProxyResolutionOptions,
|
|
693
|
+
context: { protocol: ProxyProtocol; poolIndex: number; refreshEpoch: number },
|
|
694
|
+
): Promise<ResolvedProxyConfig> {
|
|
695
|
+
if (vendor === "nodemaven") {
|
|
696
|
+
const startedAt = Date.now();
|
|
697
|
+
const synthesized = synthesizeNodemavenProxy({
|
|
698
|
+
policy,
|
|
699
|
+
affinityKey: options.affinityKey,
|
|
700
|
+
protocol: context.protocol,
|
|
701
|
+
poolIndex: context.poolIndex,
|
|
702
|
+
refreshEpoch: context.refreshEpoch,
|
|
703
|
+
country: resolveSmartproxyCountry(policy),
|
|
704
|
+
});
|
|
705
|
+
options.telemetry?.recordProxyResolution({
|
|
706
|
+
provider: "nodemaven",
|
|
707
|
+
protocol: synthesized.protocol,
|
|
708
|
+
cacheStatus: "disabled",
|
|
709
|
+
cacheHit: false,
|
|
710
|
+
resolutionMs: Math.max(0, Date.now() - startedAt),
|
|
711
|
+
attempts: 1,
|
|
712
|
+
});
|
|
713
|
+
assertTunnelingScheme(synthesized.url);
|
|
509
714
|
return {
|
|
510
715
|
shouldWarn: false,
|
|
511
|
-
url:
|
|
512
|
-
source: "
|
|
716
|
+
url: synthesized.url,
|
|
717
|
+
source: "nodemaven-gateway",
|
|
718
|
+
protocol: synthesized.protocol,
|
|
513
719
|
diagnostics: {
|
|
514
|
-
...
|
|
515
|
-
|
|
516
|
-
poolIndex,
|
|
720
|
+
...synthesized.diagnostics,
|
|
721
|
+
poolIndex: context.poolIndex,
|
|
517
722
|
},
|
|
518
723
|
};
|
|
519
|
-
} catch (error) {
|
|
520
|
-
if (error instanceof ProxyResolutionError && error.telemetry) {
|
|
521
|
-
options.telemetry?.recordProxyResolution(error.telemetry);
|
|
522
|
-
}
|
|
523
|
-
if (policy.mode === "required") {
|
|
524
|
-
throw error instanceof ProxyResolutionError
|
|
525
|
-
? error
|
|
526
|
-
: new ProxyResolutionError(
|
|
527
|
-
"PROXY_ALLOCATION_FAILED",
|
|
528
|
-
"Smartproxy allocator failed for required proxy egress.",
|
|
529
|
-
{ cause: error },
|
|
530
|
-
);
|
|
531
|
-
}
|
|
532
|
-
return { shouldWarn: true };
|
|
533
724
|
}
|
|
725
|
+
|
|
726
|
+
// smartproxy allocation-style vendor.
|
|
727
|
+
const appKey = process.env[SMARTPROXY_APP_KEY_ENV]?.trim();
|
|
728
|
+
if (!appKey) {
|
|
729
|
+
// Guarded by vendorHasCredentials; treated as a vendor-internal failure.
|
|
730
|
+
throw new ProxyResolutionError(
|
|
731
|
+
"PROXY_ALLOCATION_FAILED",
|
|
732
|
+
`${SMARTPROXY_APP_KEY_ENV} is not configured.`,
|
|
733
|
+
{ vendor: "smartproxy" },
|
|
734
|
+
);
|
|
735
|
+
}
|
|
736
|
+
const lifetimeMinutes = resolveSmartproxyLifetime(policy);
|
|
737
|
+
const allocated = await allocateSmartproxy(
|
|
738
|
+
policy,
|
|
739
|
+
appKey,
|
|
740
|
+
lifetimeMinutes,
|
|
741
|
+
options.affinityKey,
|
|
742
|
+
context.protocol,
|
|
743
|
+
);
|
|
744
|
+
options.telemetry?.recordProxyResolution({ ...allocated.telemetry, protocol: context.protocol });
|
|
745
|
+
const poolIndex = selectProxyPoolIndex(allocated.pool.urls.length, context.poolIndex);
|
|
746
|
+
const url = allocated.pool.urls[poolIndex];
|
|
747
|
+
if (url) assertTunnelingScheme(url);
|
|
748
|
+
return {
|
|
749
|
+
shouldWarn: false,
|
|
750
|
+
url,
|
|
751
|
+
source: "smartproxy-allocator",
|
|
752
|
+
protocol: context.protocol,
|
|
753
|
+
diagnostics: {
|
|
754
|
+
...allocated.pool.diagnostics,
|
|
755
|
+
poolSize: allocated.pool.urls.length,
|
|
756
|
+
poolIndex,
|
|
757
|
+
},
|
|
758
|
+
};
|
|
534
759
|
}
|
|
535
760
|
|
|
536
761
|
function resolvePolicy(options: ProxyResolutionOptions): ProviderProxyPolicy | undefined {
|
|
@@ -544,10 +769,90 @@ function resolvePolicy(options: ProxyResolutionOptions): ProviderProxyPolicy | u
|
|
|
544
769
|
return undefined;
|
|
545
770
|
}
|
|
546
771
|
|
|
547
|
-
function
|
|
548
|
-
return
|
|
549
|
-
|
|
550
|
-
|
|
772
|
+
function isRegistryVendor(name: string | undefined): name is ProxyVendorName {
|
|
773
|
+
return name === "smartproxy" || name === "nodemaven";
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
/**
|
|
777
|
+
* Ordered list of SDK-native proxy vendors declared by the policy. `providers`
|
|
778
|
+
* takes precedence over the legacy singular `provider`; the platform default
|
|
779
|
+
* env is the final fallback. Non-registry names (decodo/custom) are dropped so
|
|
780
|
+
* an all-static chain falls through to the legacy env-URL path unchanged.
|
|
781
|
+
*/
|
|
782
|
+
export function resolveVendorChain(policy: ProviderProxyPolicy): ProxyVendorName[] {
|
|
783
|
+
const declared: (ProviderProxyProvider | undefined)[] = policy.providers?.length
|
|
784
|
+
? policy.providers
|
|
785
|
+
: [policy.provider ?? envDefaultProvider()];
|
|
786
|
+
const chain: ProxyVendorName[] = [];
|
|
787
|
+
for (const name of declared) {
|
|
788
|
+
if (isRegistryVendor(name) && !chain.includes(name)) {
|
|
789
|
+
chain.push(name);
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
return chain;
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
function envDefaultProvider(): ProviderProxyProvider | undefined {
|
|
796
|
+
const raw = process.env[DEFAULT_PROXY_PROVIDER_ENV]?.trim().toLowerCase();
|
|
797
|
+
return (raw as ProviderProxyProvider | undefined) ?? undefined;
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
function vendorHasCredentials(vendor: ProxyVendorName): boolean {
|
|
801
|
+
if (vendor === "nodemaven") return hasNodemavenCredentials();
|
|
802
|
+
return Boolean(process.env[SMARTPROXY_APP_KEY_ENV]?.trim());
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
function missingCredentialEnv(vendor: ProxyVendorName): string {
|
|
806
|
+
return vendor === "nodemaven" ? "APIFUSE__PROXY__NODEMAVEN_USERNAME" : SMARTPROXY_APP_KEY_ENV;
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
function vendorPoolSize(vendor: ProxyVendorName, policy: ProviderProxyPolicy): number {
|
|
810
|
+
return vendor === "nodemaven" ? nodemavenPoolSize(policy) : resolveSmartproxyPoolSize(policy);
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
/**
|
|
814
|
+
* Total attempt span across a policy's vendor chain — the sum of each vendor's
|
|
815
|
+
* pool size. Transports use this so successive attempts rotate a vendor's pool
|
|
816
|
+
* and then fail over to the next vendor via the flat attempt index. With one
|
|
817
|
+
* vendor this equals that vendor's pool size (today's behaviour).
|
|
818
|
+
*/
|
|
819
|
+
export function resolvePolicyProxyPoolSpan(policy: ProviderProxyPolicy): number {
|
|
820
|
+
const chain = resolveVendorChain(policy);
|
|
821
|
+
if (chain.length === 0) return resolveSmartproxyPoolSize(policy);
|
|
822
|
+
return chain.reduce((sum, vendor) => sum + vendorPoolSize(vendor, policy), 0);
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
/** Map a resolved proxy source label to the vendor that served it. */
|
|
826
|
+
export function vendorFromResolvedSource(
|
|
827
|
+
source: ResolvedProxyConfig["source"],
|
|
828
|
+
): ProxyVendorName | undefined {
|
|
829
|
+
if (source === "nodemaven-gateway") return "nodemaven";
|
|
830
|
+
if (source === "smartproxy-allocator") return "smartproxy";
|
|
831
|
+
return undefined;
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
function normalizeAttemptIndex(attempt: number | undefined): number {
|
|
835
|
+
return Number.isFinite(attempt) ? Math.max(0, Math.floor(attempt as number)) : 0;
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
/**
|
|
839
|
+
* Map a flat attempt index into (vendorIndex, poolIndex) by concatenating each
|
|
840
|
+
* vendor's pool space in chain order. With a single vendor this reduces to
|
|
841
|
+
* `attempt % poolSize`, preserving today's behaviour exactly.
|
|
842
|
+
*/
|
|
843
|
+
export function mapFlatAttempt(
|
|
844
|
+
flat: number,
|
|
845
|
+
sizes: readonly number[],
|
|
846
|
+
): { vendorIndex: number; poolIndex: number } {
|
|
847
|
+
let cursor = flat;
|
|
848
|
+
for (let vendorIndex = 0; vendorIndex < sizes.length; vendorIndex++) {
|
|
849
|
+
const size = Math.max(1, sizes[vendorIndex] ?? 1);
|
|
850
|
+
if (cursor < size) {
|
|
851
|
+
return { vendorIndex, poolIndex: cursor };
|
|
852
|
+
}
|
|
853
|
+
cursor -= size;
|
|
854
|
+
}
|
|
855
|
+
return { vendorIndex: 0, poolIndex: 0 };
|
|
551
856
|
}
|
|
552
857
|
|
|
553
858
|
function resolveSmartproxyCountry(policy: ProviderProxyPolicy): string | undefined {
|
|
@@ -591,10 +896,12 @@ function buildSmartproxyCacheKey(
|
|
|
591
896
|
policy: ProviderProxyPolicy,
|
|
592
897
|
affinityKey: string | undefined,
|
|
593
898
|
lifetimeMinutes: number,
|
|
899
|
+
protocol: ProxyProtocol,
|
|
594
900
|
): string {
|
|
595
901
|
const poolSize = resolveSmartproxyPoolSize(policy);
|
|
596
902
|
return JSON.stringify({
|
|
597
903
|
provider: "smartproxy",
|
|
904
|
+
protocol,
|
|
598
905
|
country: resolveSmartproxyCountry(policy),
|
|
599
906
|
affinity: policy.session?.affinity ?? "request",
|
|
600
907
|
affinityKey: (policy.session?.affinity ?? "request") === "request" ? undefined : affinityKey,
|
|
@@ -608,8 +915,9 @@ async function allocateSmartproxy(
|
|
|
608
915
|
appKey: string,
|
|
609
916
|
lifetimeMinutes: number,
|
|
610
917
|
affinityKey: string | undefined,
|
|
918
|
+
protocol: ProxyProtocol,
|
|
611
919
|
): Promise<SmartproxyAllocationResult> {
|
|
612
|
-
const cacheKey = buildSmartproxyCacheKey(policy, affinityKey, lifetimeMinutes);
|
|
920
|
+
const cacheKey = buildSmartproxyCacheKey(policy, affinityKey, lifetimeMinutes, protocol);
|
|
613
921
|
const startedAt = Date.now();
|
|
614
922
|
const now = startedAt;
|
|
615
923
|
const invalidatedUntil = invalidatedProxyKeys.get(cacheKey) ?? 0;
|
|
@@ -617,7 +925,7 @@ async function allocateSmartproxy(
|
|
|
617
925
|
const cached = proxyCache.get(cacheKey);
|
|
618
926
|
if (!skipCached && cached && isFresh(cached, now)) {
|
|
619
927
|
if (shouldSoftRefresh(cached, now)) {
|
|
620
|
-
void refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes);
|
|
928
|
+
void refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, protocol);
|
|
621
929
|
return {
|
|
622
930
|
pool: cached,
|
|
623
931
|
telemetry: telemetryForPool(cached, "soft_stale_refresh", startedAt, {
|
|
@@ -653,6 +961,7 @@ async function allocateSmartproxy(
|
|
|
653
961
|
appKey,
|
|
654
962
|
lifetimeMinutes,
|
|
655
963
|
startedAt,
|
|
964
|
+
protocol,
|
|
656
965
|
).finally(() => {
|
|
657
966
|
proxyInflight.delete(cacheKey);
|
|
658
967
|
});
|
|
@@ -691,11 +1000,20 @@ async function refreshSmartproxyPool(
|
|
|
691
1000
|
policy: ProviderProxyPolicy,
|
|
692
1001
|
appKey: string,
|
|
693
1002
|
lifetimeMinutes: number,
|
|
1003
|
+
protocol: ProxyProtocol,
|
|
694
1004
|
): Promise<void> {
|
|
695
1005
|
try {
|
|
696
|
-
await allocateSmartproxyShared(
|
|
697
|
-
|
|
698
|
-
|
|
1006
|
+
await allocateSmartproxyShared(
|
|
1007
|
+
cacheKey,
|
|
1008
|
+
policy,
|
|
1009
|
+
appKey,
|
|
1010
|
+
lifetimeMinutes,
|
|
1011
|
+
Date.now(),
|
|
1012
|
+
protocol,
|
|
1013
|
+
{
|
|
1014
|
+
background: true,
|
|
1015
|
+
},
|
|
1016
|
+
);
|
|
699
1017
|
} catch {
|
|
700
1018
|
// Soft refresh is opportunistic; current fresh pool remains usable.
|
|
701
1019
|
}
|
|
@@ -707,6 +1025,7 @@ async function allocateSmartproxyShared(
|
|
|
707
1025
|
appKey: string,
|
|
708
1026
|
lifetimeMinutes: number,
|
|
709
1027
|
startedAt: number,
|
|
1028
|
+
protocol: ProxyProtocol,
|
|
710
1029
|
options: { background?: boolean } = {},
|
|
711
1030
|
): Promise<SmartproxyAllocationResult> {
|
|
712
1031
|
const redis = getProxyRedis();
|
|
@@ -717,7 +1036,7 @@ async function allocateSmartproxyShared(
|
|
|
717
1036
|
appKey,
|
|
718
1037
|
lifetimeMinutes,
|
|
719
1038
|
startedAt,
|
|
720
|
-
{ cacheStatus: "allocator" },
|
|
1039
|
+
{ cacheStatus: "allocator", protocol },
|
|
721
1040
|
);
|
|
722
1041
|
}
|
|
723
1042
|
|
|
@@ -742,6 +1061,7 @@ async function allocateSmartproxyShared(
|
|
|
742
1061
|
cacheStatus: options.background ? "soft_stale_refresh" : "allocator",
|
|
743
1062
|
redis,
|
|
744
1063
|
poolKey,
|
|
1064
|
+
protocol,
|
|
745
1065
|
},
|
|
746
1066
|
);
|
|
747
1067
|
} finally {
|
|
@@ -897,10 +1217,17 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
897
1217
|
cacheStatus: ProxyCacheStatus;
|
|
898
1218
|
redis?: ProxyRedisClient;
|
|
899
1219
|
poolKey?: string;
|
|
1220
|
+
protocol: ProxyProtocol;
|
|
900
1221
|
},
|
|
901
1222
|
): Promise<SmartproxyAllocationResult> {
|
|
902
1223
|
const poolSize = resolveSmartproxyPoolSize(policy);
|
|
903
|
-
const allocatorUrl = buildSmartproxyAllocatorUrl(
|
|
1224
|
+
const allocatorUrl = buildSmartproxyAllocatorUrl(
|
|
1225
|
+
policy,
|
|
1226
|
+
appKey,
|
|
1227
|
+
lifetimeMinutes,
|
|
1228
|
+
poolSize,
|
|
1229
|
+
options.protocol,
|
|
1230
|
+
);
|
|
904
1231
|
const allocatorStartedAt = Date.now();
|
|
905
1232
|
const allocatorDeadlineAt = allocatorStartedAt + smartproxyAllocatorDeadlineMs();
|
|
906
1233
|
let allocation: SmartproxyAllocatorSuccess | undefined;
|
|
@@ -914,6 +1241,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
914
1241
|
allocatorUrl,
|
|
915
1242
|
attempt,
|
|
916
1243
|
allocatorDeadlineAt,
|
|
1244
|
+
options.protocol,
|
|
917
1245
|
);
|
|
918
1246
|
if (attemptResult.ok) {
|
|
919
1247
|
allocation = attemptResult;
|
|
@@ -1034,6 +1362,7 @@ async function fetchSmartproxyAllocatorAttempt(
|
|
|
1034
1362
|
allocatorUrl: string,
|
|
1035
1363
|
attempt: number,
|
|
1036
1364
|
deadlineAt: number,
|
|
1365
|
+
protocol: ProxyProtocol,
|
|
1037
1366
|
): Promise<SmartproxyAllocatorAttemptResult> {
|
|
1038
1367
|
const { controller, dispose } = createDeadlineAbortController(deadlineAt);
|
|
1039
1368
|
let response: Response;
|
|
@@ -1077,7 +1406,7 @@ async function fetchSmartproxyAllocatorAttempt(
|
|
|
1077
1406
|
};
|
|
1078
1407
|
}
|
|
1079
1408
|
|
|
1080
|
-
const urls = parseSmartproxyAllocatorProxies(body);
|
|
1409
|
+
const urls = parseSmartproxyAllocatorProxies(body, protocol);
|
|
1081
1410
|
const bodyClass = classifySmartproxyAllocatorBody(body, urls);
|
|
1082
1411
|
if (urls.length === 0) {
|
|
1083
1412
|
return {
|
|
@@ -1117,18 +1446,27 @@ function smartproxyAllocatorFailureMessage(
|
|
|
1117
1446
|
return "Smartproxy allocator response did not contain a usable proxy endpoint.";
|
|
1118
1447
|
}
|
|
1119
1448
|
|
|
1449
|
+
// Smartproxy get-ip-v3 `protocol` param: 1 = HTTP. The SOCKS5 value ("2") is a
|
|
1450
|
+
// best-effort mapping pending live vendor confirmation; http is the default and
|
|
1451
|
+
// the only value exercised in production today.
|
|
1452
|
+
const SMARTPROXY_PROTOCOL_PARAM: Record<ProxyProtocol, string> = {
|
|
1453
|
+
http: "1",
|
|
1454
|
+
socks5: "2",
|
|
1455
|
+
};
|
|
1456
|
+
|
|
1120
1457
|
function buildSmartproxyAllocatorUrl(
|
|
1121
1458
|
policy: ProviderProxyPolicy,
|
|
1122
1459
|
appKey: string,
|
|
1123
1460
|
lifetimeMinutes: number,
|
|
1124
1461
|
poolSize: number,
|
|
1462
|
+
protocol: ProxyProtocol,
|
|
1125
1463
|
): string {
|
|
1126
1464
|
const params = new URLSearchParams({
|
|
1127
1465
|
app_key: appKey,
|
|
1128
1466
|
pt: "9",
|
|
1129
1467
|
num: String(poolSize),
|
|
1130
1468
|
life: String(lifetimeMinutes),
|
|
1131
|
-
protocol:
|
|
1469
|
+
protocol: SMARTPROXY_PROTOCOL_PARAM[protocol],
|
|
1132
1470
|
format: "txt",
|
|
1133
1471
|
lb: "\\n",
|
|
1134
1472
|
});
|
|
@@ -1141,7 +1479,8 @@ function buildSmartproxyAllocatorUrl(
|
|
|
1141
1479
|
return `https://api.smartproxy.org/web_v1/ip/get-ip-v3?${params.toString()}`;
|
|
1142
1480
|
}
|
|
1143
1481
|
|
|
1144
|
-
function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
1482
|
+
function parseSmartproxyAllocatorProxies(body: string, protocol: ProxyProtocol): string[] {
|
|
1483
|
+
const scheme = protocol === "socks5" ? "socks5" : "http";
|
|
1145
1484
|
const trimmed = body.trim();
|
|
1146
1485
|
if (!trimmed) {
|
|
1147
1486
|
return [];
|
|
@@ -1162,7 +1501,7 @@ function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
|
1162
1501
|
"port" in item && (typeof item.port === "string" || typeof item.port === "number")
|
|
1163
1502
|
? item.port
|
|
1164
1503
|
: "";
|
|
1165
|
-
return ip && port ?
|
|
1504
|
+
return ip && port ? `${scheme}://${ip}:${port}` : null;
|
|
1166
1505
|
})
|
|
1167
1506
|
.filter((url): url is string => url !== null);
|
|
1168
1507
|
}
|
|
@@ -1174,7 +1513,7 @@ function parseSmartproxyAllocatorProxies(body: string): string[] {
|
|
|
1174
1513
|
.split(/\r?\n/)
|
|
1175
1514
|
.map((item) => item.trim())
|
|
1176
1515
|
.filter((item) => /^\d{1,3}(?:\.\d{1,3}){3}:\d{2,5}$/.test(item))
|
|
1177
|
-
.map((line) =>
|
|
1516
|
+
.map((line) => `${scheme}://${line}`);
|
|
1178
1517
|
}
|
|
1179
1518
|
|
|
1180
1519
|
function classifySmartproxyAllocatorBody(
|
|
@@ -1207,12 +1546,17 @@ function markSmartproxyCacheInvalidated(options: ProxyResolutionOptions = {}): s
|
|
|
1207
1546
|
if (!policy || policy.mode === "disabled") {
|
|
1208
1547
|
return undefined;
|
|
1209
1548
|
}
|
|
1210
|
-
if (
|
|
1549
|
+
if (!resolveVendorChain(policy).includes("smartproxy")) {
|
|
1211
1550
|
return undefined;
|
|
1212
1551
|
}
|
|
1213
1552
|
|
|
1214
1553
|
const lifetimeMinutes = resolveSmartproxyLifetime(policy);
|
|
1215
|
-
const cacheKey = buildSmartproxyCacheKey(
|
|
1554
|
+
const cacheKey = buildSmartproxyCacheKey(
|
|
1555
|
+
policy,
|
|
1556
|
+
options.affinityKey,
|
|
1557
|
+
lifetimeMinutes,
|
|
1558
|
+
options.protocol ?? VENDOR_DEFAULT_PROTOCOL.smartproxy,
|
|
1559
|
+
);
|
|
1216
1560
|
invalidatedProxyKeys.set(cacheKey, Date.now() + SMARTPROXY_INVALIDATION_SKIP_REDIS_MS);
|
|
1217
1561
|
proxyCache.delete(cacheKey);
|
|
1218
1562
|
proxyInflight.delete(cacheKey);
|