@apifuse/provider-sdk 2.2.0-beta.7 → 2.2.0-beta.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/auth-turn/index.d.ts +2 -2
- package/dist/config/loader.d.ts +147 -6
- package/dist/config/loader.js +360 -48
- package/dist/define.js +63 -8
- package/dist/runtime/choice.js +20 -2
- package/dist/runtime/http.js +115 -9
- package/dist/runtime/proxy-errors.js +6 -2
- package/dist/runtime/proxy-nodemaven.d.ts +35 -0
- package/dist/runtime/proxy-nodemaven.js +128 -0
- package/dist/runtime/proxy-telemetry.d.ts +2 -1
- package/dist/runtime/proxy-telemetry.js +39 -4
- package/dist/runtime/state.js +12 -1
- package/dist/runtime/stealth.js +60 -18
- package/dist/server/serve.js +11 -0
- package/dist/server/types.d.ts +9 -9
- package/dist/types.d.ts +30 -1
- package/package.json +4 -3
- package/src/config/loader.ts +514 -61
- package/src/define.ts +76 -13
- package/src/runtime/choice.ts +25 -4
- package/src/runtime/http.ts +134 -11
- package/src/runtime/proxy-errors.ts +12 -4
- package/src/runtime/proxy-nodemaven.ts +178 -0
- package/src/runtime/proxy-telemetry.ts +56 -5
- package/src/runtime/state.ts +11 -1
- package/src/runtime/stealth.ts +68 -22
- package/src/server/serve.ts +11 -0
- package/src/types.ts +30 -1
package/dist/config/loader.js
CHANGED
|
@@ -2,6 +2,11 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
2
2
|
import { existsSync } from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { Redis } from "ioredis";
|
|
5
|
+
import { NODEMAVEN_DEFAULT_PROTOCOL, NODEMAVEN_MAX_POOL_SIZE, hasNodemavenCredentials, nodemavenPoolSize, synthesizeNodemavenProxy, } from "../runtime/proxy-nodemaven.js";
|
|
6
|
+
// "smartproxy" here is api.smartproxy.org — a residential proxy with an IP
|
|
7
|
+
// extraction API (app_key → raw ip:port pool). It is NOT the company formerly
|
|
8
|
+
// named Smartproxy (smartproxy.com), which rebranded to Decodo in 2025 and is
|
|
9
|
+
// modelled separately as the `decodo` gateway vendor. Do not conflate them.
|
|
5
10
|
export const SMARTPROXY_APP_KEY_ENV = "APIFUSE__PROXY__SMARTPROXY_APP_KEY";
|
|
6
11
|
export const SMARTPROXY_MAX_LIFETIME_MINUTES = 2000;
|
|
7
12
|
export const DEFAULT_SMARTPROXY_POOL_SIZE = 20;
|
|
@@ -15,11 +20,17 @@ export const REDIS_URL_ENV = "APIFUSE__REDIS__URL";
|
|
|
15
20
|
export class ProxyResolutionError extends Error {
|
|
16
21
|
code;
|
|
17
22
|
telemetry;
|
|
23
|
+
vendor;
|
|
24
|
+
vendorChain;
|
|
25
|
+
protocol;
|
|
18
26
|
constructor(code, message, options) {
|
|
19
27
|
super(message, options);
|
|
20
28
|
this.name = "ProxyResolutionError";
|
|
21
29
|
this.code = code;
|
|
22
30
|
this.telemetry = options?.telemetry;
|
|
31
|
+
this.vendor = options?.vendor;
|
|
32
|
+
this.vendorChain = options?.vendorChain;
|
|
33
|
+
this.protocol = options?.protocol;
|
|
23
34
|
}
|
|
24
35
|
}
|
|
25
36
|
const proxyCache = new Map();
|
|
@@ -232,6 +243,11 @@ function applyStickyProxySession(proxyUrl) {
|
|
|
232
243
|
if (!parsed.hostname || !parsed.username || !parsed.password) {
|
|
233
244
|
return proxyUrl;
|
|
234
245
|
}
|
|
246
|
+
// This rewrites sticky-session usernames for a bring-your-own *gateway* URL
|
|
247
|
+
// (APIFUSE__PROXY__URL). The `smartproxy` host here means a smartproxy.com /
|
|
248
|
+
// Decodo-family gateway that authenticates by username — NOT the
|
|
249
|
+
// api.smartproxy.org allocation vendor, whose endpoints are raw ip:port with
|
|
250
|
+
// no credentials and therefore return early above.
|
|
235
251
|
const host = parsed.hostname.toLowerCase();
|
|
236
252
|
if (!host.includes("smartproxy") && !host.includes("decodo")) {
|
|
237
253
|
return proxyUrl;
|
|
@@ -299,47 +315,173 @@ export async function resolveProxyConfigAsync(options = {}) {
|
|
|
299
315
|
if (policy.mode === "disabled") {
|
|
300
316
|
return { shouldWarn: false };
|
|
301
317
|
}
|
|
302
|
-
const
|
|
303
|
-
if (
|
|
318
|
+
const chain = resolveVendorChain(policy);
|
|
319
|
+
if (chain.length === 0) {
|
|
320
|
+
// decodo/custom/env-static providers keep the legacy static-URL path.
|
|
304
321
|
return resolveProxyConfig({
|
|
305
322
|
...options,
|
|
306
323
|
upstream: { proxy: true },
|
|
307
324
|
});
|
|
308
325
|
}
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
326
|
+
// Protocol is chosen per vendor (each vendor's benchmarked-best), with an
|
|
327
|
+
// optional explicit override for the harness/tests. Both are tunnelling
|
|
328
|
+
// schemes. transportProtocols is what the calling transport can actually use.
|
|
329
|
+
const transportProtocols = options.transportProtocols ?? ["http", "socks5"];
|
|
330
|
+
const sizes = chain.map((vendor) => vendorPoolSize(vendor, policy));
|
|
331
|
+
const total = sizes.reduce((sum, size) => sum + size, 0);
|
|
332
|
+
const normalizedAttempt = normalizeAttemptIndex(options.proxyAttempt);
|
|
333
|
+
const { vendorIndex: startVendorIndex, poolIndex: startPoolIndex } = mapFlatAttempt(total > 0 ? normalizedAttempt % total : 0, sizes);
|
|
334
|
+
const refreshEpoch = normalizeAttemptIndex(options.proxyRefreshEpoch);
|
|
335
|
+
let lastError;
|
|
336
|
+
let blockedProtocol;
|
|
337
|
+
for (let vendorIndex = startVendorIndex; vendorIndex < chain.length; vendorIndex++) {
|
|
338
|
+
const vendor = chain[vendorIndex];
|
|
339
|
+
const nextVendor = chain[vendorIndex + 1];
|
|
340
|
+
const poolIndex = vendorIndex === startVendorIndex ? startPoolIndex : 0;
|
|
341
|
+
const protocol = options.protocol ?? VENDOR_DEFAULT_PROTOCOL[vendor];
|
|
342
|
+
if (!vendorHasCredentials(vendor)) {
|
|
343
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
344
|
+
vendor,
|
|
345
|
+
nextVendor,
|
|
346
|
+
phase: "resolution",
|
|
347
|
+
reason: "no_credentials",
|
|
348
|
+
});
|
|
349
|
+
continue;
|
|
350
|
+
}
|
|
351
|
+
// The calling transport must be able to use this vendor's protocol; if not,
|
|
352
|
+
// fail over to the next vendor rather than silently downgrading.
|
|
353
|
+
if (!transportProtocols.includes(protocol)) {
|
|
354
|
+
blockedProtocol = protocol;
|
|
355
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
356
|
+
vendor,
|
|
357
|
+
nextVendor,
|
|
358
|
+
phase: "resolution",
|
|
359
|
+
reason: "protocol_unsupported",
|
|
360
|
+
});
|
|
361
|
+
continue;
|
|
362
|
+
}
|
|
363
|
+
try {
|
|
364
|
+
return await resolveWithVendor(vendor, policy, options, {
|
|
365
|
+
protocol,
|
|
366
|
+
poolIndex,
|
|
367
|
+
refreshEpoch,
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
catch (error) {
|
|
371
|
+
// Config/programming errors (invalid filter, etc.) are not vendor
|
|
372
|
+
// outages — propagate them rather than failing over.
|
|
373
|
+
if (!(error instanceof ProxyResolutionError)) {
|
|
374
|
+
throw error;
|
|
375
|
+
}
|
|
376
|
+
if (error.telemetry) {
|
|
377
|
+
options.telemetry?.recordProxyResolution(error.telemetry);
|
|
378
|
+
}
|
|
379
|
+
lastError = error;
|
|
380
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
381
|
+
vendor,
|
|
382
|
+
nextVendor,
|
|
383
|
+
phase: "resolution",
|
|
384
|
+
reason: "allocation_failed",
|
|
385
|
+
});
|
|
313
386
|
}
|
|
314
|
-
return { shouldWarn: true };
|
|
315
387
|
}
|
|
316
|
-
|
|
388
|
+
if (policy.mode === "required") {
|
|
389
|
+
if (lastError) {
|
|
390
|
+
throw lastError instanceof ProxyResolutionError
|
|
391
|
+
? lastError
|
|
392
|
+
: new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `All proxy vendors [${chain.join(", ")}] failed for required proxy egress.`, { cause: lastError, vendorChain: chain });
|
|
393
|
+
}
|
|
394
|
+
if (blockedProtocol) {
|
|
395
|
+
throw new ProxyResolutionError("PROXY_PROTOCOL_UNSUPPORTED", `No proxy vendor in [${chain.join(", ")}] could serve a protocol supported by this transport (supports: ${transportProtocols.join(", ")}; vendor wanted "${blockedProtocol}"). Route this provider through the stealth transport.`, { protocol: blockedProtocol, vendorChain: chain });
|
|
396
|
+
}
|
|
397
|
+
throw new ProxyResolutionError("PROXY_REQUIRED", `Proxy egress is required but no vendor credentials are configured. Missing: ${chain
|
|
398
|
+
.map((vendor) => `${missingCredentialEnv(vendor)} (${vendor})`)
|
|
399
|
+
.join(", ")}.`, { vendorChain: chain });
|
|
400
|
+
}
|
|
401
|
+
return { shouldWarn: true };
|
|
402
|
+
}
|
|
403
|
+
/**
|
|
404
|
+
* Each vendor's default egress protocol, chosen from live KR benchmarks. HTTP
|
|
405
|
+
* CONNECT wins for nodemaven (socks5 adds ~500ms through the gateway) and ties
|
|
406
|
+
* for smartproxy, and is the only protocol ctx.http (Bun native fetch) supports.
|
|
407
|
+
* Override per call via ProxyResolutionOptions.protocol (harness/tests).
|
|
408
|
+
*/
|
|
409
|
+
const VENDOR_DEFAULT_PROTOCOL = {
|
|
410
|
+
smartproxy: "http",
|
|
411
|
+
nodemaven: NODEMAVEN_DEFAULT_PROTOCOL,
|
|
412
|
+
};
|
|
413
|
+
/**
|
|
414
|
+
* Guard the No-MITM invariant: a resolved proxy URL must use a tunnelling scheme
|
|
415
|
+
* (http CONNECT or socks5) so the client TLS handshake reaches the origin
|
|
416
|
+
* end-to-end. Anything else would intercept TLS and break fingerprinting.
|
|
417
|
+
*/
|
|
418
|
+
export function assertTunnelingScheme(url) {
|
|
419
|
+
let scheme;
|
|
317
420
|
try {
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
421
|
+
scheme = new URL(url).protocol.replace(/:$/, "").toLowerCase();
|
|
422
|
+
}
|
|
423
|
+
catch {
|
|
424
|
+
throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `Malformed proxy URL: ${url}`);
|
|
425
|
+
}
|
|
426
|
+
if (scheme !== "http" && scheme !== "socks5") {
|
|
427
|
+
throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `Resolved proxy scheme "${scheme}" is not a tunnelling scheme (expected http or socks5). Refusing to route TLS through a non-tunnelling proxy.`);
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
async function resolveWithVendor(vendor, policy, options, context) {
|
|
431
|
+
if (vendor === "nodemaven") {
|
|
432
|
+
const startedAt = Date.now();
|
|
433
|
+
const synthesized = synthesizeNodemavenProxy({
|
|
434
|
+
policy,
|
|
435
|
+
affinityKey: options.affinityKey,
|
|
436
|
+
protocol: context.protocol,
|
|
437
|
+
poolIndex: context.poolIndex,
|
|
438
|
+
refreshEpoch: context.refreshEpoch,
|
|
439
|
+
country: resolveSmartproxyCountry(policy),
|
|
440
|
+
});
|
|
441
|
+
options.telemetry?.recordProxyResolution({
|
|
442
|
+
provider: "nodemaven",
|
|
443
|
+
protocol: synthesized.protocol,
|
|
444
|
+
cacheStatus: "disabled",
|
|
445
|
+
cacheHit: false,
|
|
446
|
+
resolutionMs: Math.max(0, Date.now() - startedAt),
|
|
447
|
+
attempts: 1,
|
|
448
|
+
});
|
|
449
|
+
assertTunnelingScheme(synthesized.url);
|
|
321
450
|
return {
|
|
322
451
|
shouldWarn: false,
|
|
323
|
-
url:
|
|
324
|
-
source: "
|
|
452
|
+
url: synthesized.url,
|
|
453
|
+
source: "nodemaven-gateway",
|
|
454
|
+
protocol: synthesized.protocol,
|
|
325
455
|
diagnostics: {
|
|
326
|
-
...
|
|
327
|
-
|
|
328
|
-
poolIndex,
|
|
456
|
+
...synthesized.diagnostics,
|
|
457
|
+
poolIndex: context.poolIndex,
|
|
329
458
|
},
|
|
330
459
|
};
|
|
331
460
|
}
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
throw error instanceof ProxyResolutionError
|
|
338
|
-
? error
|
|
339
|
-
: new ProxyResolutionError("PROXY_ALLOCATION_FAILED", "Smartproxy allocator failed for required proxy egress.", { cause: error });
|
|
340
|
-
}
|
|
341
|
-
return { shouldWarn: true };
|
|
461
|
+
// smartproxy allocation-style vendor.
|
|
462
|
+
const appKey = process.env[SMARTPROXY_APP_KEY_ENV]?.trim();
|
|
463
|
+
if (!appKey) {
|
|
464
|
+
// Guarded by vendorHasCredentials; treated as a vendor-internal failure.
|
|
465
|
+
throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `${SMARTPROXY_APP_KEY_ENV} is not configured.`, { vendor: "smartproxy" });
|
|
342
466
|
}
|
|
467
|
+
const lifetimeMinutes = resolveSmartproxyLifetime(policy);
|
|
468
|
+
const allocated = await allocateSmartproxy(policy, appKey, lifetimeMinutes, options.affinityKey, context.protocol);
|
|
469
|
+
options.telemetry?.recordProxyResolution({ ...allocated.telemetry, protocol: context.protocol });
|
|
470
|
+
const poolIndex = selectProxyPoolIndex(allocated.pool.urls.length, context.poolIndex);
|
|
471
|
+
const url = allocated.pool.urls[poolIndex];
|
|
472
|
+
if (url)
|
|
473
|
+
assertTunnelingScheme(url);
|
|
474
|
+
return {
|
|
475
|
+
shouldWarn: false,
|
|
476
|
+
url,
|
|
477
|
+
source: "smartproxy-allocator",
|
|
478
|
+
protocol: context.protocol,
|
|
479
|
+
diagnostics: {
|
|
480
|
+
...allocated.pool.diagnostics,
|
|
481
|
+
poolSize: allocated.pool.urls.length,
|
|
482
|
+
poolIndex,
|
|
483
|
+
},
|
|
484
|
+
};
|
|
343
485
|
}
|
|
344
486
|
function resolvePolicy(options) {
|
|
345
487
|
if (options.proxyPolicy) {
|
|
@@ -351,8 +493,168 @@ function resolvePolicy(options) {
|
|
|
351
493
|
}
|
|
352
494
|
return undefined;
|
|
353
495
|
}
|
|
354
|
-
function
|
|
355
|
-
return
|
|
496
|
+
function isRegistryVendor(name) {
|
|
497
|
+
return name === "smartproxy" || name === "nodemaven";
|
|
498
|
+
}
|
|
499
|
+
/**
|
|
500
|
+
* Ordered list of SDK-native proxy vendors declared by the policy. `providers`
|
|
501
|
+
* takes precedence over the legacy singular `provider`; the platform default
|
|
502
|
+
* env is the final fallback. Non-registry names (decodo/custom) are dropped so
|
|
503
|
+
* an all-static chain falls through to the legacy env-URL path unchanged.
|
|
504
|
+
*/
|
|
505
|
+
export function resolveVendorChain(policy) {
|
|
506
|
+
const declared = policy.providers?.length
|
|
507
|
+
? policy.providers
|
|
508
|
+
: [policy.provider ?? envDefaultProvider()];
|
|
509
|
+
const chain = [];
|
|
510
|
+
for (const name of declared) {
|
|
511
|
+
if (isRegistryVendor(name) && !chain.includes(name)) {
|
|
512
|
+
chain.push(name);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
return chain;
|
|
516
|
+
}
|
|
517
|
+
function envDefaultProvider() {
|
|
518
|
+
const raw = process.env[DEFAULT_PROXY_PROVIDER_ENV]?.trim().toLowerCase();
|
|
519
|
+
return raw ?? undefined;
|
|
520
|
+
}
|
|
521
|
+
function vendorHasCredentials(vendor) {
|
|
522
|
+
if (vendor === "nodemaven")
|
|
523
|
+
return hasNodemavenCredentials();
|
|
524
|
+
return Boolean(process.env[SMARTPROXY_APP_KEY_ENV]?.trim());
|
|
525
|
+
}
|
|
526
|
+
function missingCredentialEnv(vendor) {
|
|
527
|
+
return vendor === "nodemaven" ? "APIFUSE__PROXY__NODEMAVEN_USERNAME" : SMARTPROXY_APP_KEY_ENV;
|
|
528
|
+
}
|
|
529
|
+
function vendorPoolSize(vendor, policy) {
|
|
530
|
+
return vendor === "nodemaven" ? nodemavenPoolSize(policy) : resolveSmartproxyPoolSize(policy);
|
|
531
|
+
}
|
|
532
|
+
/**
|
|
533
|
+
* Total attempt span across a policy's vendor chain — the sum of each vendor's
|
|
534
|
+
* pool size. Transports use this so successive attempts rotate a vendor's pool
|
|
535
|
+
* and then fail over to the next vendor via the flat attempt index. With one
|
|
536
|
+
* vendor this equals that vendor's pool size (today's behaviour).
|
|
537
|
+
*/
|
|
538
|
+
export function resolvePolicyProxyPoolSpan(policy) {
|
|
539
|
+
const chain = resolveVendorChain(policy);
|
|
540
|
+
if (chain.length === 0)
|
|
541
|
+
return resolveSmartproxyPoolSize(policy);
|
|
542
|
+
return chain.reduce((sum, vendor) => sum + vendorPoolSize(vendor, policy), 0);
|
|
543
|
+
}
|
|
544
|
+
function vendorMaxPoolSize(vendor) {
|
|
545
|
+
return vendor === "nodemaven" ? NODEMAVEN_MAX_POOL_SIZE : SMARTPROXY_MAX_POOL_SIZE;
|
|
546
|
+
}
|
|
547
|
+
/**
|
|
548
|
+
* Absolute upper bound on a chain's attempt span — the sum of each vendor's
|
|
549
|
+
* *maximum* pool size. Unlike `resolvePolicyProxyPoolSpan` (the configured
|
|
550
|
+
* span), this backstop is independent of `session.poolSize`, so it never
|
|
551
|
+
* truncates a legitimately large pool below the point where the flat attempt
|
|
552
|
+
* index would cross into the next vendor (e.g. a 50-slot NodeMaven pool).
|
|
553
|
+
*/
|
|
554
|
+
export function maxPolicyProxyPoolSpan(policy) {
|
|
555
|
+
const chain = resolveVendorChain(policy);
|
|
556
|
+
if (chain.length === 0)
|
|
557
|
+
return SMARTPROXY_MAX_POOL_SIZE;
|
|
558
|
+
return chain.reduce((sum, vendor) => sum + vendorMaxPoolSize(vendor), 0);
|
|
559
|
+
}
|
|
560
|
+
const UNSAFE_TRANSPORT_RETRY_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE", "TRACE"]);
|
|
561
|
+
/**
|
|
562
|
+
* Transport-retry attempt cap for a policy-managed request. A transport failure
|
|
563
|
+
* rotates the flat attempt index onto the *next* endpoint (and, once the index
|
|
564
|
+
* passes the primary vendor's pool span, the *next vendor*), so the cap must be
|
|
565
|
+
* the chain's full pool span for failover to reach the fallback vendor — the
|
|
566
|
+
* per-endpoint retry budget (default 3) never gets there.
|
|
567
|
+
*
|
|
568
|
+
* The span only widens beyond the caller's retry budget when ALL hold:
|
|
569
|
+
* - the request is policy-allocator managed (not a caller-supplied proxy URL);
|
|
570
|
+
* - the caller did NOT pin an explicit retry policy — `HttpRetryOptions.attempts`
|
|
571
|
+
* is the documented total-attempt ceiling and must be honoured verbatim;
|
|
572
|
+
* - the method is safe/idempotent — an unsafe request must never be duplicated
|
|
573
|
+
* across the pool even if some framework default would allow it;
|
|
574
|
+
* - the policy resolves a non-empty *registry* vendor chain (smartproxy /
|
|
575
|
+
* nodemaven). Static vendors (custom / decodo) and credential-less policies
|
|
576
|
+
* resolve no allocator pool, so every attempt would hit the same endpoint
|
|
577
|
+
* with no possible crossover — they keep the retry budget.
|
|
578
|
+
*
|
|
579
|
+
* The widened cap is bounded by the chain's true maximum span (sum of each
|
|
580
|
+
* vendor's max pool size), so a large NodeMaven pool (≤50) stays reachable and
|
|
581
|
+
* a pathological chain can never spin unbounded.
|
|
582
|
+
*/
|
|
583
|
+
/**
|
|
584
|
+
* True when a policy request is in *implicit chain-rotation* mode: successive
|
|
585
|
+
* transport attempts rotate the flat index across the concatenated vendor pool
|
|
586
|
+
* spans (and, past the primary vendor's span, into the fallback vendor). This is
|
|
587
|
+
* the ONLY mode in which the transport loop widens its attempt cap AND
|
|
588
|
+
* de-duplicates repeated endpoints — the two behaviours must share one predicate
|
|
589
|
+
* so they never diverge. It holds when ALL of the widening conditions hold:
|
|
590
|
+
* - the request is policy-allocator managed (not a caller-supplied proxy URL);
|
|
591
|
+
* - the caller did NOT pin an explicit retry policy — its `attempts` ceiling is
|
|
592
|
+
* the documented contract and must be honoured verbatim against whatever
|
|
593
|
+
* endpoint each attempt resolves (even a repeated one), so no de-duplication;
|
|
594
|
+
* - the method is safe/idempotent — an unsafe request is never duplicated;
|
|
595
|
+
* - the policy resolves a non-empty registry vendor chain (smartproxy /
|
|
596
|
+
* nodemaven). Static vendors (custom / decodo) resolve the same URL every
|
|
597
|
+
* attempt, so there is nothing to rotate or de-duplicate.
|
|
598
|
+
*/
|
|
599
|
+
export function policyRotatesTransportVendorChain(input) {
|
|
600
|
+
if (!input.usesPolicyAllocator || !input.policy || input.explicitRetry) {
|
|
601
|
+
return false;
|
|
602
|
+
}
|
|
603
|
+
if (UNSAFE_TRANSPORT_RETRY_METHODS.has(input.method.toUpperCase())) {
|
|
604
|
+
return false;
|
|
605
|
+
}
|
|
606
|
+
return resolveVendorChain(input.policy).length > 0;
|
|
607
|
+
}
|
|
608
|
+
export function resolvePolicyTransportAttemptCap(input) {
|
|
609
|
+
const budget = Math.max(1, Math.floor(input.retryAttempts));
|
|
610
|
+
if (!policyRotatesTransportVendorChain({
|
|
611
|
+
policy: input.policy,
|
|
612
|
+
usesPolicyAllocator: input.usesPolicyAllocator,
|
|
613
|
+
explicitRetry: input.explicitRetry,
|
|
614
|
+
method: input.method,
|
|
615
|
+
})) {
|
|
616
|
+
return budget;
|
|
617
|
+
}
|
|
618
|
+
const span = Math.min(maxPolicyProxyPoolSpan(input.policy), resolvePolicyProxyPoolSpan(input.policy));
|
|
619
|
+
return Math.max(budget, span);
|
|
620
|
+
}
|
|
621
|
+
/**
|
|
622
|
+
* A registry vendor chain (smartproxy/nodemaven) resolves a potentially
|
|
623
|
+
* *different* endpoint per flat attempt index, so a transport retry should
|
|
624
|
+
* advance across endpoints and de-duplicate once the chain stops yielding new
|
|
625
|
+
* ones. Static/custom/decodo policies (empty registry chain) resolve the *same*
|
|
626
|
+
* URL every attempt by design — retrying that same endpoint is intended, so the
|
|
627
|
+
* transport loop must not de-duplicate them.
|
|
628
|
+
*/
|
|
629
|
+
export function policyResolvesRegistryVendorChain(policy) {
|
|
630
|
+
return Boolean(policy) && resolveVendorChain(policy).length > 0;
|
|
631
|
+
}
|
|
632
|
+
/** Map a resolved proxy source label to the vendor that served it. */
|
|
633
|
+
export function vendorFromResolvedSource(source) {
|
|
634
|
+
if (source === "nodemaven-gateway")
|
|
635
|
+
return "nodemaven";
|
|
636
|
+
if (source === "smartproxy-allocator")
|
|
637
|
+
return "smartproxy";
|
|
638
|
+
return undefined;
|
|
639
|
+
}
|
|
640
|
+
function normalizeAttemptIndex(attempt) {
|
|
641
|
+
return Number.isFinite(attempt) ? Math.max(0, Math.floor(attempt)) : 0;
|
|
642
|
+
}
|
|
643
|
+
/**
|
|
644
|
+
* Map a flat attempt index into (vendorIndex, poolIndex) by concatenating each
|
|
645
|
+
* vendor's pool space in chain order. With a single vendor this reduces to
|
|
646
|
+
* `attempt % poolSize`, preserving today's behaviour exactly.
|
|
647
|
+
*/
|
|
648
|
+
export function mapFlatAttempt(flat, sizes) {
|
|
649
|
+
let cursor = flat;
|
|
650
|
+
for (let vendorIndex = 0; vendorIndex < sizes.length; vendorIndex++) {
|
|
651
|
+
const size = Math.max(1, sizes[vendorIndex] ?? 1);
|
|
652
|
+
if (cursor < size) {
|
|
653
|
+
return { vendorIndex, poolIndex: cursor };
|
|
654
|
+
}
|
|
655
|
+
cursor -= size;
|
|
656
|
+
}
|
|
657
|
+
return { vendorIndex: 0, poolIndex: 0 };
|
|
356
658
|
}
|
|
357
659
|
function resolveSmartproxyCountry(policy) {
|
|
358
660
|
return (policy.geo?.country ?? process.env[DEFAULT_PROXY_COUNTRY_ENV]?.trim().toUpperCase() ?? undefined);
|
|
@@ -381,10 +683,11 @@ function selectProxyPoolIndex(poolSize, attempt = 0) {
|
|
|
381
683
|
const normalizedAttempt = Number.isFinite(attempt) ? Math.max(0, Math.floor(attempt)) : 0;
|
|
382
684
|
return normalizedAttempt % poolSize;
|
|
383
685
|
}
|
|
384
|
-
function buildSmartproxyCacheKey(policy, affinityKey, lifetimeMinutes) {
|
|
686
|
+
function buildSmartproxyCacheKey(policy, affinityKey, lifetimeMinutes, protocol) {
|
|
385
687
|
const poolSize = resolveSmartproxyPoolSize(policy);
|
|
386
688
|
return JSON.stringify({
|
|
387
689
|
provider: "smartproxy",
|
|
690
|
+
protocol,
|
|
388
691
|
country: resolveSmartproxyCountry(policy),
|
|
389
692
|
affinity: policy.session?.affinity ?? "request",
|
|
390
693
|
affinityKey: (policy.session?.affinity ?? "request") === "request" ? undefined : affinityKey,
|
|
@@ -392,8 +695,8 @@ function buildSmartproxyCacheKey(policy, affinityKey, lifetimeMinutes) {
|
|
|
392
695
|
poolSize,
|
|
393
696
|
});
|
|
394
697
|
}
|
|
395
|
-
async function allocateSmartproxy(policy, appKey, lifetimeMinutes, affinityKey) {
|
|
396
|
-
const cacheKey = buildSmartproxyCacheKey(policy, affinityKey, lifetimeMinutes);
|
|
698
|
+
async function allocateSmartproxy(policy, appKey, lifetimeMinutes, affinityKey, protocol) {
|
|
699
|
+
const cacheKey = buildSmartproxyCacheKey(policy, affinityKey, lifetimeMinutes, protocol);
|
|
397
700
|
const startedAt = Date.now();
|
|
398
701
|
const now = startedAt;
|
|
399
702
|
const invalidatedUntil = invalidatedProxyKeys.get(cacheKey) ?? 0;
|
|
@@ -401,7 +704,7 @@ async function allocateSmartproxy(policy, appKey, lifetimeMinutes, affinityKey)
|
|
|
401
704
|
const cached = proxyCache.get(cacheKey);
|
|
402
705
|
if (!skipCached && cached && isFresh(cached, now)) {
|
|
403
706
|
if (shouldSoftRefresh(cached, now)) {
|
|
404
|
-
void refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes);
|
|
707
|
+
void refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, protocol);
|
|
405
708
|
return {
|
|
406
709
|
pool: cached,
|
|
407
710
|
telemetry: telemetryForPool(cached, "soft_stale_refresh", startedAt, {
|
|
@@ -429,7 +732,7 @@ async function allocateSmartproxy(policy, appKey, lifetimeMinutes, affinityKey)
|
|
|
429
732
|
}),
|
|
430
733
|
};
|
|
431
734
|
}
|
|
432
|
-
const promise = allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, startedAt).finally(() => {
|
|
735
|
+
const promise = allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, startedAt, protocol).finally(() => {
|
|
433
736
|
proxyInflight.delete(cacheKey);
|
|
434
737
|
});
|
|
435
738
|
proxyInflight.set(cacheKey, promise);
|
|
@@ -460,9 +763,9 @@ async function readSmartproxyRedisPool(cacheKey, startedAt) {
|
|
|
460
763
|
telemetry: telemetryForPool(pool, "redis_hit", startedAt, { redisReadMs }),
|
|
461
764
|
};
|
|
462
765
|
}
|
|
463
|
-
async function refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes) {
|
|
766
|
+
async function refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, protocol) {
|
|
464
767
|
try {
|
|
465
|
-
await allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, Date.now(), {
|
|
768
|
+
await allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, Date.now(), protocol, {
|
|
466
769
|
background: true,
|
|
467
770
|
});
|
|
468
771
|
}
|
|
@@ -470,10 +773,10 @@ async function refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes)
|
|
|
470
773
|
// Soft refresh is opportunistic; current fresh pool remains usable.
|
|
471
774
|
}
|
|
472
775
|
}
|
|
473
|
-
async function allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, startedAt, options = {}) {
|
|
776
|
+
async function allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, startedAt, protocol, options = {}) {
|
|
474
777
|
const redis = getProxyRedis();
|
|
475
778
|
if (!redis || !(await ensureRedisReady(redis))) {
|
|
476
|
-
return await allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, startedAt, { cacheStatus: "allocator" });
|
|
779
|
+
return await allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, startedAt, { cacheStatus: "allocator", protocol });
|
|
477
780
|
}
|
|
478
781
|
const poolKey = smartproxyRedisPoolKey(cacheKey);
|
|
479
782
|
const lockKey = smartproxyRedisLockKey(cacheKey);
|
|
@@ -487,6 +790,7 @@ async function allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinute
|
|
|
487
790
|
cacheStatus: options.background ? "soft_stale_refresh" : "allocator",
|
|
488
791
|
redis,
|
|
489
792
|
poolKey,
|
|
793
|
+
protocol,
|
|
490
794
|
});
|
|
491
795
|
}
|
|
492
796
|
finally {
|
|
@@ -599,7 +903,7 @@ async function readSmartproxyAllocatorBodyWithDeadline(response, signal) {
|
|
|
599
903
|
}
|
|
600
904
|
async function allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, startedAt, options) {
|
|
601
905
|
const poolSize = resolveSmartproxyPoolSize(policy);
|
|
602
|
-
const allocatorUrl = buildSmartproxyAllocatorUrl(policy, appKey, lifetimeMinutes, poolSize);
|
|
906
|
+
const allocatorUrl = buildSmartproxyAllocatorUrl(policy, appKey, lifetimeMinutes, poolSize, options.protocol);
|
|
603
907
|
const allocatorStartedAt = Date.now();
|
|
604
908
|
const allocatorDeadlineAt = allocatorStartedAt + smartproxyAllocatorDeadlineMs();
|
|
605
909
|
let allocation;
|
|
@@ -609,7 +913,7 @@ async function allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetime
|
|
|
609
913
|
lastFailure = smartproxyAllocatorDeadlineFailure(attempt);
|
|
610
914
|
break;
|
|
611
915
|
}
|
|
612
|
-
const attemptResult = await fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, allocatorDeadlineAt);
|
|
916
|
+
const attemptResult = await fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, allocatorDeadlineAt, options.protocol);
|
|
613
917
|
if (attemptResult.ok) {
|
|
614
918
|
allocation = attemptResult;
|
|
615
919
|
break;
|
|
@@ -692,7 +996,7 @@ async function allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetime
|
|
|
692
996
|
}),
|
|
693
997
|
};
|
|
694
998
|
}
|
|
695
|
-
async function fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, deadlineAt) {
|
|
999
|
+
async function fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, deadlineAt, protocol) {
|
|
696
1000
|
const { controller, dispose } = createDeadlineAbortController(deadlineAt);
|
|
697
1001
|
let response;
|
|
698
1002
|
try {
|
|
@@ -735,7 +1039,7 @@ async function fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, deadlineAt
|
|
|
735
1039
|
bodyClass: "http_error",
|
|
736
1040
|
};
|
|
737
1041
|
}
|
|
738
|
-
const urls = parseSmartproxyAllocatorProxies(body);
|
|
1042
|
+
const urls = parseSmartproxyAllocatorProxies(body, protocol);
|
|
739
1043
|
const bodyClass = classifySmartproxyAllocatorBody(body, urls);
|
|
740
1044
|
if (urls.length === 0) {
|
|
741
1045
|
return {
|
|
@@ -770,13 +1074,20 @@ function smartproxyAllocatorFailureMessage(failure) {
|
|
|
770
1074
|
}
|
|
771
1075
|
return "Smartproxy allocator response did not contain a usable proxy endpoint.";
|
|
772
1076
|
}
|
|
773
|
-
|
|
1077
|
+
// Smartproxy get-ip-v3 `protocol` param: 1 = HTTP. The SOCKS5 value ("2") is a
|
|
1078
|
+
// best-effort mapping pending live vendor confirmation; http is the default and
|
|
1079
|
+
// the only value exercised in production today.
|
|
1080
|
+
const SMARTPROXY_PROTOCOL_PARAM = {
|
|
1081
|
+
http: "1",
|
|
1082
|
+
socks5: "2",
|
|
1083
|
+
};
|
|
1084
|
+
function buildSmartproxyAllocatorUrl(policy, appKey, lifetimeMinutes, poolSize, protocol) {
|
|
774
1085
|
const params = new URLSearchParams({
|
|
775
1086
|
app_key: appKey,
|
|
776
1087
|
pt: "9",
|
|
777
1088
|
num: String(poolSize),
|
|
778
1089
|
life: String(lifetimeMinutes),
|
|
779
|
-
protocol:
|
|
1090
|
+
protocol: SMARTPROXY_PROTOCOL_PARAM[protocol],
|
|
780
1091
|
format: "txt",
|
|
781
1092
|
lb: "\\n",
|
|
782
1093
|
});
|
|
@@ -788,7 +1099,8 @@ function buildSmartproxyAllocatorUrl(policy, appKey, lifetimeMinutes, poolSize)
|
|
|
788
1099
|
// old path 404s into the marketing site); the API lives on the api host.
|
|
789
1100
|
return `https://api.smartproxy.org/web_v1/ip/get-ip-v3?${params.toString()}`;
|
|
790
1101
|
}
|
|
791
|
-
function parseSmartproxyAllocatorProxies(body) {
|
|
1102
|
+
function parseSmartproxyAllocatorProxies(body, protocol) {
|
|
1103
|
+
const scheme = protocol === "socks5" ? "socks5" : "http";
|
|
792
1104
|
const trimmed = body.trim();
|
|
793
1105
|
if (!trimmed) {
|
|
794
1106
|
return [];
|
|
@@ -807,7 +1119,7 @@ function parseSmartproxyAllocatorProxies(body) {
|
|
|
807
1119
|
const port = "port" in item && (typeof item.port === "string" || typeof item.port === "number")
|
|
808
1120
|
? item.port
|
|
809
1121
|
: "";
|
|
810
|
-
return ip && port ?
|
|
1122
|
+
return ip && port ? `${scheme}://${ip}:${port}` : null;
|
|
811
1123
|
})
|
|
812
1124
|
.filter((url) => url !== null);
|
|
813
1125
|
}
|
|
@@ -819,7 +1131,7 @@ function parseSmartproxyAllocatorProxies(body) {
|
|
|
819
1131
|
.split(/\r?\n/)
|
|
820
1132
|
.map((item) => item.trim())
|
|
821
1133
|
.filter((item) => /^\d{1,3}(?:\.\d{1,3}){3}:\d{2,5}$/.test(item))
|
|
822
|
-
.map((line) =>
|
|
1134
|
+
.map((line) => `${scheme}://${line}`);
|
|
823
1135
|
}
|
|
824
1136
|
function classifySmartproxyAllocatorBody(body, urls) {
|
|
825
1137
|
if (urls.length > 0) {
|
|
@@ -847,11 +1159,11 @@ function markSmartproxyCacheInvalidated(options = {}) {
|
|
|
847
1159
|
if (!policy || policy.mode === "disabled") {
|
|
848
1160
|
return undefined;
|
|
849
1161
|
}
|
|
850
|
-
if (
|
|
1162
|
+
if (!resolveVendorChain(policy).includes("smartproxy")) {
|
|
851
1163
|
return undefined;
|
|
852
1164
|
}
|
|
853
1165
|
const lifetimeMinutes = resolveSmartproxyLifetime(policy);
|
|
854
|
-
const cacheKey = buildSmartproxyCacheKey(policy, options.affinityKey, lifetimeMinutes);
|
|
1166
|
+
const cacheKey = buildSmartproxyCacheKey(policy, options.affinityKey, lifetimeMinutes, options.protocol ?? VENDOR_DEFAULT_PROTOCOL.smartproxy);
|
|
855
1167
|
invalidatedProxyKeys.set(cacheKey, Date.now() + SMARTPROXY_INVALIDATION_SKIP_REDIS_MS);
|
|
856
1168
|
proxyCache.delete(cacheKey);
|
|
857
1169
|
proxyInflight.delete(cacheKey);
|