@apifuse/provider-sdk 2.2.0-beta.7 → 2.2.0-beta.70

package/dist/config/loader.js

@@ -1,7 +1,14 @@
+import { readDiagnosticEnv } from "../runtime/diagnostic-env.js";
 import { createHash, randomUUID } from "node:crypto";
 import { existsSync } from "node:fs";
+import { createRequire } from "node:module";
 import path from "node:path";
-import { Redis } from "ioredis";
+import { BoundedExpiringMap } from "../runtime/bounded-expiring-map.js";
+import { NODEMAVEN_DEFAULT_PROTOCOL, NODEMAVEN_FILTER_ENV, NODEMAVEN_MAX_POOL_SIZE, NODEMAVEN_PASSWORD_ENV, NODEMAVEN_USERNAME_ENV, nodemavenPoolSize, synthesizeNodemavenProxy, } from "../runtime/proxy-nodemaven.js";
+// "smartproxy" here is api.smartproxy.org — a residential proxy with an IP
+// extraction API (app_key → raw ip:port pool). It is NOT the company formerly
+// named Smartproxy (smartproxy.com), which rebranded to Decodo in 2025 and is
+// modelled separately as the `decodo` gateway vendor. Do not conflate them.
 export const SMARTPROXY_APP_KEY_ENV = "APIFUSE__PROXY__SMARTPROXY_APP_KEY";
 export const SMARTPROXY_MAX_LIFETIME_MINUTES = 2000;
 export const DEFAULT_SMARTPROXY_POOL_SIZE = 20;
@@ -15,14 +22,27 @@ export const REDIS_URL_ENV = "APIFUSE__REDIS__URL";
 export class ProxyResolutionError extends Error {
     code;
     telemetry;
+    vendor;
+    vendorChain;
+    protocol;
     constructor(code, message, options) {
         super(message, options);
         this.name = "ProxyResolutionError";
         this.code = code;
         this.telemetry = options?.telemetry;
+        this.vendor = options?.vendor;
+        this.vendorChain = options?.vendorChain;
+        this.protocol = options?.protocol;
     }
 }
-const proxyCache = new Map();
+// Keyed per (policy, affinity). Connection affinities are minted by callers, so
+// without a cap a long-lived process accumulates one pool per affinity it ever
+// saw. Evicting a still-fresh pool forces a fresh allocation (a different
+// egress endpoint) for that affinity, so the cap is generous and every
+// capacity eviction is reported as `diagnostics.poolCacheEvictions`.
+const SMARTPROXY_POOL_CACHE_MAX_ENTRIES = 10_000;
+let proxyCache = createSmartproxyPoolCache(SMARTPROXY_POOL_CACHE_MAX_ENTRIES);
+const require = createRequire(import.meta.url);
 const proxyInflight = new Map();
 const invalidatedProxyKeys = new Map();
 const redisClients = new Map();
@@ -43,19 +63,36 @@ const SMARTPROXY_ALLOCATOR_RETRY_BASE_MS = 25;
 // reusing stale raw CONNECT endpoints as if they were valid for `life` minutes.
 const SMARTPROXY_EXTRACTION_CACHE_TTL_MS = 15_000;
 const SMARTPROXY_EXTRACTION_SOFT_REFRESH_MS = 10_000;
+// Precedence is declared once here so the URL accessors, the boot report and
+// any future consumer can never disagree about which env wins.
+const CACHE_REDIS_URL_PRECEDENCE = [PROVIDER_CACHE_REDIS_URL_ENV, REDIS_URL_ENV];
+const STATE_REDIS_URL_PRECEDENCE = [
+    PROVIDER_STATE_REDIS_URL_ENV,
+    PROVIDER_CACHE_REDIS_URL_ENV,
+    REDIS_URL_ENV,
+];
+function resolveRedisUrlFromEnv(precedence) {
+    for (const [index, envName] of precedence.entries()) {
+        const url = readDiagnosticEnv(envName)?.trim();
+        if (url)
+            return { url, envName, fallback: index > 0 };
+    }
+    return undefined;
+}
 function redisUrlFromEnv() {
-    return (process.env.APIFUSE__PROVIDER__CACHE_REDIS_URL?.trim() ||
-        process.env[REDIS_URL_ENV]?.trim() ||
-        undefined);
+    return providerCacheRedisResolutionFromEnv()?.url;
+}
+export function providerCacheRedisResolutionFromEnv() {
+    return resolveRedisUrlFromEnv(CACHE_REDIS_URL_PRECEDENCE);
+}
+export function providerStateRedisResolutionFromEnv() {
+    return resolveRedisUrlFromEnv(STATE_REDIS_URL_PRECEDENCE);
 }
 export function providerCacheRedisUrlFromEnv() {
-    return redisUrlFromEnv();
+    return providerCacheRedisResolutionFromEnv()?.url;
 }
 export function providerStateRedisUrlFromEnv() {
-    return (process.env[PROVIDER_STATE_REDIS_URL_ENV]?.trim() ||
-        process.env[PROVIDER_CACHE_REDIS_URL_ENV]?.trim() ||
-        process.env[REDIS_URL_ENV]?.trim() ||
-        undefined);
+    return providerStateRedisResolutionFromEnv()?.url;
 }
 /** @internal Test-only hook for exercising shared proxy-cache behavior. */
 export function __setProxyRedisForTests(redis) {
@@ -64,6 +101,17 @@ export function __setProxyRedisForTests(redis) {
 export function __setSmartproxyAllocatorDeadlineMsForTests(deadlineMs) {
     smartproxyAllocatorDeadlineMsForTests = deadlineMs;
 }
+/** @internal Test-only hook for exercising the pool-cache capacity bound at a small size. */
+export function __setSmartproxyPoolCacheMaxEntriesForTests(maxEntries) {
+    proxyCache = createSmartproxyPoolCache(maxEntries ?? SMARTPROXY_POOL_CACHE_MAX_ENTRIES);
+}
+/** @internal Test-only view of the in-memory Smartproxy cache sizes. */
+export function __getSmartproxyCacheSizesForTests() {
+    return { pools: proxyCache.size, tombstones: invalidatedProxyKeys.size };
+}
+function createSmartproxyPoolCache(maxEntries) {
+    return new BoundedExpiringMap(maxEntries, (pool) => pool.expiresAt);
+}
 function getProxyRedis() {
     if (proxyRedisForTests)
         return proxyRedisForTests;
@@ -73,7 +121,8 @@ function getProxyRedis() {
     const existing = redisClients.get(redisUrl);
     if (existing)
         return existing;
-    const redis = new Redis(redisUrl, {
+    const { Redis: RedisClient } = require("ioredis");
+    const redis = new RedisClient(redisUrl, {
         connectTimeout: REDIS_TIMEOUT_MS,
         enableOfflineQueue: false,
         lazyConnect: true,
@@ -145,6 +194,7 @@ function telemetryForPool(pool, cacheStatus, startedAt, extra = {}) {
 function telemetryForFailure(cacheStatus, startedAt, extra = {}) {
     return {
         provider: "smartproxy",
+        outcome: "error",
         cacheStatus,
         cacheHit: false,
         resolutionMs: Math.max(0, Date.now() - startedAt),
@@ -210,59 +260,7 @@ function serializeSmartproxyPool(pool) {
 }
 function normalizeProxyUrl(url) {
     const normalized = url?.trim();
-    return normalized ? applyStickyProxySession(normalized) : undefined;
-}
-function readPositiveIntegerEnv(name) {
-    const raw = process.env[name]?.trim();
-    if (!raw)
-        return undefined;
-    if (!/^[1-9]\d*$/.test(raw)) {
-        throw new Error(`${name} must be a positive integer`);
-    }
-    return raw;
-}
-function applyStickyProxySession(proxyUrl) {
-    let parsed;
-    try {
-        parsed = new URL(proxyUrl);
-    }
-    catch {
-        return proxyUrl;
-    }
-    if (!parsed.hostname || !parsed.username || !parsed.password) {
-        return proxyUrl;
-    }
-    const host = parsed.hostname.toLowerCase();
-    if (!host.includes("smartproxy") && !host.includes("decodo")) {
-        return proxyUrl;
-    }
-    const username = decodeURIComponent(parsed.username);
-    const sessionId = process.env.APIFUSE__PROXY__SESSION_ID?.trim() || "apifuse-shared";
-    const sessionDuration = readPositiveIntegerEnv("APIFUSE__PROXY__SESSION_DURATION");
-    const stickyUsername = host.includes("smartproxy")
-        ? buildSmartproxyUsername(username, sessionId, sessionDuration)
-        : buildDecodoUsername(username, sessionId, sessionDuration ?? "60");
-    parsed.username = stickyUsername;
-    return parsed.toString();
-}
-function buildSmartproxyUsername(username, sessionId, sessionDuration) {
-    const parts = username.split("_");
-    const configuredLife = parts.find((part) => part.startsWith("life-"))?.slice("life-".length);
-    const baseUsername = parts
-        .filter((part) => !part.startsWith("session-") && !part.startsWith("life-"))
-        .join("_");
-    return `${baseUsername}_session-${sessionId}_life-${sessionDuration ?? configuredLife ?? "60"}`;
-}
-function buildDecodoUsername(username, sessionId, sessionDuration) {
-    const withoutSticky = username.replace(/-session-.+-sessionduration-\d+$/, "");
-    const baseUsername = withoutSticky.startsWith("user-") ? withoutSticky : `user-${withoutSticky}`;
-    return `${baseUsername}-session-${sessionId}-sessionduration-${sessionDuration}`;
-}
-function syncProxyEnv(config) {
-    const configProxyUrl = normalizeProxyUrl(config.proxy?.url);
-    if (!process.env.APIFUSE__PROXY__URL && configProxyUrl) {
-        process.env.APIFUSE__PROXY__URL = configProxyUrl;
-    }
+    return normalized || undefined;
 }
 export function resolveProxyConfig(options = {}) {
     const explicitProxyUrl = normalizeProxyUrl(options.proxy);
@@ -277,14 +275,6 @@ export function resolveProxyConfig(options = {}) {
     if (!legacyProxyRequested) {
         return { shouldWarn: false };
     }
-    const envProxyUrl = normalizeProxyUrl(process.env.APIFUSE__PROXY__URL);
-    if (envProxyUrl) {
-        return { shouldWarn: false, url: envProxyUrl };
-    }
-    const configuredProxyUrl = normalizeProxyUrl(options.apifuseConfig?.proxy?.url);
-    if (configuredProxyUrl) {
-        return { shouldWarn: false, url: configuredProxyUrl };
-    }
     return { shouldWarn: true };
 }
 export async function resolveProxyConfigAsync(options = {}) {
@@ -299,47 +289,213 @@ export async function resolveProxyConfigAsync(options = {}) {
     if (policy.mode === "disabled") {
         return { shouldWarn: false };
     }
-    const provider = resolveProxyProvider(policy);
-    if (provider !== "smartproxy") {
+    const chain = resolveVendorChain(policy);
+    if (chain.length === 0) {
+        const declared = declaredVendorChain(policy);
+        const deprecated = declared.filter((vendor) => vendor === "decodo" || vendor === "custom");
+        if (policy.mode === "required") {
+            const providerIds = declared.length > 0 ? declared.map((vendor) => `"${vendor}"`).join(", ") : "none";
+            const deprecatedDetail = deprecated.length > 0
+                ? ` Deprecated vendor(s): ${deprecated.map((vendor) => `"${vendor}"`).join(", ")}.`
+                : "";
+            throw new ProxyResolutionError("PROXY_REQUIRED", `Required proxy policy has no SDK-managed adapter for provider id(s): ${providerIds}.${deprecatedDetail} Use "smartproxy" or "nodemaven".`);
+        }
+        // Deprecated decodo/custom providers have no SDK-managed adapter. Optional
+        // policies preserve the warning-only behavior and may continue directly.
         return resolveProxyConfig({
             ...options,
             upstream: { proxy: true },
         });
     }
-    const appKey = process.env[SMARTPROXY_APP_KEY_ENV]?.trim();
-    if (!appKey) {
-        if (policy.mode === "required") {
-            throw new ProxyResolutionError("PROXY_REQUIRED", `Smartproxy egress is required but ${SMARTPROXY_APP_KEY_ENV} is not configured.`);
+    // Protocol is chosen per vendor (each vendor's benchmarked-best), with an
+    // optional explicit override for the harness/tests. Both are tunnelling
+    // schemes. transportProtocols is what the calling transport can actually use.
+    const transportProtocols = options.transportProtocols ?? ["http", "socks5"];
+    const sizes = chain.map((vendor) => vendorPoolSize(vendor, policy));
+    const total = sizes.reduce((sum, size) => sum + size, 0);
+    const normalizedAttempt = normalizeAttemptIndex(options.proxyAttempt);
+    const { vendorIndex: startVendorIndex, poolIndex: startPoolIndex } = mapFlatAttempt(total > 0 ? normalizedAttempt % total : 0, sizes);
+    const refreshEpoch = normalizeAttemptIndex(options.proxyRefreshEpoch);
+    let lastError;
+    let blockedProtocol;
+    for (let vendorIndex = startVendorIndex; vendorIndex < chain.length; vendorIndex++) {
+        const vendor = chain[vendorIndex];
+        const nextVendor = chain[vendorIndex + 1];
+        const poolIndex = vendorIndex === startVendorIndex ? startPoolIndex : 0;
+        const protocol = options.protocol ?? VENDOR_DEFAULT_PROTOCOL[vendor];
+        if (!vendorHasCredentials(vendor, options.engineCredentials)) {
+            options.telemetry?.recordProxyVendorFailover?.({
+                vendor,
+                nextVendor,
+                phase: "resolution",
+                reason: "no_credentials",
+            });
+            continue;
+        }
+        // The calling transport must be able to use this vendor's protocol; if not,
+        // fail over to the next vendor rather than silently downgrading.
+        if (!transportProtocols.includes(protocol)) {
+            blockedProtocol = protocol;
+            options.telemetry?.recordProxyVendorFailover?.({
+                vendor,
+                nextVendor,
+                phase: "resolution",
+                reason: "protocol_unsupported",
+            });
+            continue;
+        }
+        try {
+            return await resolveWithVendor(vendor, policy, options, {
+                protocol,
+                poolIndex,
+                refreshEpoch,
+                credentials: options.engineCredentials,
+            });
+        }
+        catch (error) {
+            // Config/programming errors (invalid filter, etc.) are not vendor
+            // outages — propagate them rather than failing over.
+            if (!(error instanceof ProxyResolutionError)) {
+                throw error;
+            }
+            if (error.telemetry) {
+                options.telemetry?.recordProxyResolution(error.telemetry);
+            }
+            lastError = error;
+            options.telemetry?.recordProxyVendorFailover?.({
+                vendor,
+                nextVendor,
+                phase: "resolution",
+                reason: "allocation_failed",
+            });
         }
-        return { shouldWarn: true };
     }
-    const lifetimeMinutes = resolveSmartproxyLifetime(policy);
+    if (policy.mode === "required") {
+        if (lastError) {
+            throw lastError instanceof ProxyResolutionError
+                ? lastError
+                : new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `All proxy vendors [${chain.join(", ")}] failed for required proxy egress.`, { cause: lastError, vendorChain: chain });
+        }
+        if (blockedProtocol) {
+            throw new ProxyResolutionError("PROXY_PROTOCOL_UNSUPPORTED", `No proxy vendor in [${chain.join(", ")}] could serve a protocol supported by this transport (supports: ${transportProtocols.join(", ")}; vendor wanted "${blockedProtocol}"). Route this provider through the stealth transport.`, { protocol: blockedProtocol, vendorChain: chain });
+        }
+        throw new ProxyResolutionError("PROXY_REQUIRED", `Proxy egress is required but no vendor credentials are configured. Missing: ${chain
+            .map((vendor) => `${missingCredentialEnv(vendor)} (${vendor})`)
+            .join(", ")}.`, { vendorChain: chain });
+    }
+    return { shouldWarn: true };
+}
+/**
+ * Resolve the proxy URL for a provider-owned consumer such as a CAPTCHA solver.
+ * Vendor allocation and failover remain owned by the SDK.
+ */
+export async function resolveProxy(options = {}) {
+    const resolved = await resolveProxyConfigAsync(options);
+    const vendor = vendorFromResolvedSource(resolved.source);
+    return vendor ? { ...resolved, vendor } : resolved;
+}
+/**
+ * Each vendor's default egress protocol, chosen from live KR benchmarks. HTTP
+ * CONNECT wins for nodemaven (socks5 adds ~500ms through the gateway) and ties
+ * for smartproxy, and is the only protocol ctx.http (Bun native fetch) supports.
+ * Override per call via ProxyResolutionOptions.protocol (harness/tests).
+ */
+export const VENDOR_DEFAULT_PROTOCOL = {
+    smartproxy: "http",
+    nodemaven: NODEMAVEN_DEFAULT_PROTOCOL,
+};
+/**
+ * Guard the No-MITM invariant: a resolved proxy URL must use a tunnelling scheme
+ * (http CONNECT or socks5) so the client TLS handshake reaches the origin
+ * end-to-end. Anything else would intercept TLS and break fingerprinting.
+ */
+export function assertTunnelingScheme(url) {
+    let scheme;
     try {
-        const allocated = await allocateSmartproxy(policy, appKey, lifetimeMinutes, options.affinityKey);
-        options.telemetry?.recordProxyResolution(allocated.telemetry);
-        const poolIndex = selectProxyPoolIndex(allocated.pool.urls.length, options.proxyAttempt);
+        scheme = new URL(url).protocol.replace(/:$/, "").toLowerCase();
+    }
+    catch {
+        throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `Malformed proxy URL: ${url}`);
+    }
+    if (scheme !== "http" && scheme !== "socks5") {
+        throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `Resolved proxy scheme "${scheme}" is not a tunnelling scheme (expected http or socks5). Refusing to route TLS through a non-tunnelling proxy.`);
+    }
+}
+export async function resolveWithVendor(vendor, policy, options, context) {
+    if (vendor === "nodemaven") {
+        const startedAt = Date.now();
+        const username = (context.credentials === undefined
+            ? readDiagnosticEnv(NODEMAVEN_USERNAME_ENV)
+            : context.credentials[NODEMAVEN_USERNAME_ENV])?.trim();
+        const password = (context.credentials === undefined
+            ? readDiagnosticEnv(NODEMAVEN_PASSWORD_ENV)
+            : context.credentials[NODEMAVEN_PASSWORD_ENV])?.trim();
+        const filter = context.credentials === undefined
+            ? process.env[NODEMAVEN_FILTER_ENV]
+            : context.credentials[NODEMAVEN_FILTER_ENV];
+        if (!username || !password) {
+            throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `NodeMaven credentials missing: set ${NODEMAVEN_USERNAME_ENV} and ${NODEMAVEN_PASSWORD_ENV}.`, { vendor: "nodemaven" });
+        }
+        const synthesized = synthesizeNodemavenProxy({
+            policy,
+            credentials: {
+                username,
+                password,
+                ...(filter ? { filter } : {}),
+            },
+            affinityKey: options.affinityKey,
+            protocol: context.protocol,
+            poolIndex: context.poolIndex,
+            refreshEpoch: context.refreshEpoch,
+            country: resolveSmartproxyCountry(policy, context.ambientDefaults !== false),
+        });
+        options.telemetry?.recordProxyResolution({
+            provider: "nodemaven",
+            protocol: synthesized.protocol,
+            cacheStatus: "disabled",
+            cacheHit: false,
+            resolutionMs: Math.max(0, Date.now() - startedAt),
+            attempts: 1,
+        });
+        assertTunnelingScheme(synthesized.url);
         return {
             shouldWarn: false,
-            url: allocated.pool.urls[poolIndex],
-            source: "smartproxy-allocator",
+            url: synthesized.url,
+            source: "nodemaven-gateway",
+            protocol: synthesized.protocol,
             diagnostics: {
-                ...allocated.pool.diagnostics,
-                poolSize: allocated.pool.urls.length,
-                poolIndex,
+                ...synthesized.diagnostics,
+                poolIndex: context.poolIndex,
             },
         };
     }
-    catch (error) {
-        if (error instanceof ProxyResolutionError && error.telemetry) {
-            options.telemetry?.recordProxyResolution(error.telemetry);
-        }
-        if (policy.mode === "required") {
-            throw error instanceof ProxyResolutionError
-                ? error
-                : new ProxyResolutionError("PROXY_ALLOCATION_FAILED", "Smartproxy allocator failed for required proxy egress.", { cause: error });
-        }
-        return { shouldWarn: true };
-    }
+    // smartproxy allocation-style vendor.
+    const appKey = (context.credentials === undefined
+        ? readDiagnosticEnv(SMARTPROXY_APP_KEY_ENV)
+        : context.credentials[SMARTPROXY_APP_KEY_ENV])?.trim();
+    if (!appKey) {
+        // Guarded by vendorHasCredentials; treated as a vendor-internal failure.
+        throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `${SMARTPROXY_APP_KEY_ENV} is not configured.`, { vendor: "smartproxy" });
+    }
+    const lifetimeMinutes = resolveSmartproxyLifetime(policy, context.ambientDefaults !== false);
+    const allocated = await allocateSmartproxy(policy, appKey, lifetimeMinutes, options.affinityKey, context.protocol, context.ambientDefaults !== false, context.sharedCache !== false);
+    options.telemetry?.recordProxyResolution({ ...allocated.telemetry, protocol: context.protocol });
+    const poolIndex = selectProxyPoolIndex(allocated.pool.urls.length, context.poolIndex);
+    const url = allocated.pool.urls[poolIndex];
+    if (url)
+        assertTunnelingScheme(url);
+    return {
+        shouldWarn: false,
+        url,
+        source: "smartproxy-allocator",
+        protocol: context.protocol,
+        diagnostics: {
+            ...allocated.pool.diagnostics,
+            poolSize: allocated.pool.urls.length,
+            poolIndex,
+            poolCacheEvictions: proxyCache.capacityEvictions,
+        },
+    };
 }
 function resolvePolicy(options) {
     if (options.proxyPolicy) {
@@ -351,14 +507,187 @@ function resolvePolicy(options) {
     }
     return undefined;
 }
-function resolveProxyProvider(policy) {
-    return (policy.provider ?? process.env[DEFAULT_PROXY_PROVIDER_ENV]?.trim().toLowerCase() ?? "custom");
+function isRegistryVendor(name) {
+    return name === "smartproxy" || name === "nodemaven";
+}
+function declaredVendorChain(policy) {
+    const declared = policy.providers?.length
+        ? policy.providers
+        : [policy.provider ?? envDefaultProvider()];
+    return declared.filter((vendor) => vendor !== undefined);
+}
+/**
+ * Ordered list of SDK-native proxy vendors declared by the policy. `providers`
+ * takes precedence over the legacy singular `provider`; the platform default
+ * env is the final fallback. Non-registry names (decodo/custom) are dropped so
+ * an all-deprecated chain has no managed adapter.
+ */
+export function resolveVendorChain(policy) {
+    const chain = [];
+    for (const name of declaredVendorChain(policy)) {
+        if (isRegistryVendor(name) && !chain.includes(name)) {
+            chain.push(name);
+        }
+    }
+    return chain;
+}
+function envDefaultProvider() {
+    const raw = process.env[DEFAULT_PROXY_PROVIDER_ENV]?.trim().toLowerCase();
+    return raw ?? undefined;
+}
+function vendorHasCredentials(vendor, credentials) {
+    if (vendor === "nodemaven") {
+        return Boolean((credentials === undefined
+            ? readDiagnosticEnv(NODEMAVEN_USERNAME_ENV)
+            : credentials[NODEMAVEN_USERNAME_ENV])?.trim() &&
+            (credentials === undefined
+                ? readDiagnosticEnv(NODEMAVEN_PASSWORD_ENV)
+                : credentials[NODEMAVEN_PASSWORD_ENV])?.trim());
+    }
+    return Boolean((credentials === undefined
+        ? readDiagnosticEnv(SMARTPROXY_APP_KEY_ENV)
+        : credentials[SMARTPROXY_APP_KEY_ENV])?.trim());
+}
+function missingCredentialEnv(vendor) {
+    return vendor === "nodemaven" ? "APIFUSE__PROXY__NODEMAVEN_USERNAME" : SMARTPROXY_APP_KEY_ENV;
+}
+function vendorPoolSize(vendor, policy) {
+    return vendor === "nodemaven" ? nodemavenPoolSize(policy) : resolveSmartproxyPoolSize(policy);
+}
+/**
+ * Total attempt span across a policy's vendor chain — the sum of each vendor's
+ * pool size. Transports use this so successive attempts rotate a vendor's pool
+ * and then fail over to the next vendor via the flat attempt index. With one
+ * vendor this equals that vendor's pool size (today's behaviour).
+ */
+export function resolvePolicyProxyPoolSpan(policy) {
+    const chain = resolveVendorChain(policy);
+    if (chain.length === 0)
+        return resolveSmartproxyPoolSize(policy);
+    return chain.reduce((sum, vendor) => sum + vendorPoolSize(vendor, policy), 0);
+}
+function vendorMaxPoolSize(vendor) {
+    return vendor === "nodemaven" ? NODEMAVEN_MAX_POOL_SIZE : SMARTPROXY_MAX_POOL_SIZE;
+}
+/**
+ * Absolute upper bound on a chain's attempt span — the sum of each vendor's
+ * *maximum* pool size. Unlike `resolvePolicyProxyPoolSpan` (the configured
+ * span), this backstop is independent of `session.poolSize`, so it never
+ * truncates a legitimately large pool below the point where the flat attempt
+ * index would cross into the next vendor (e.g. a 50-slot NodeMaven pool).
+ */
+export function maxPolicyProxyPoolSpan(policy) {
+    const chain = resolveVendorChain(policy);
+    if (chain.length === 0)
+        return SMARTPROXY_MAX_POOL_SIZE;
+    return chain.reduce((sum, vendor) => sum + vendorMaxPoolSize(vendor), 0);
+}
+const UNSAFE_TRANSPORT_RETRY_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE", "TRACE"]);
+/**
+ * Transport-retry attempt cap for a policy-managed request. A transport failure
+ * rotates the flat attempt index onto the *next* endpoint (and, once the index
+ * passes the primary vendor's pool span, the *next vendor*), so the cap must be
+ * the chain's full pool span for failover to reach the fallback vendor — the
+ * per-endpoint retry budget (default 3) never gets there.
+ *
+ * The span only widens beyond the caller's retry budget when ALL hold:
+ *  - the request is policy-allocator managed (not a caller-supplied proxy URL);
+ *  - the caller did NOT pin an explicit retry policy — `HttpRetryOptions.attempts`
+ *    is the documented total-attempt ceiling and must be honoured verbatim;
+ *  - the method is safe/idempotent — an unsafe request must never be duplicated
+ *    across the pool even if some framework default would allow it;
+ *  - the policy resolves a non-empty *registry* vendor chain (smartproxy /
+ *    nodemaven). Deprecated vendors (custom / decodo) resolve no managed pool,
+ *    so there is no possible endpoint crossover — they keep the retry budget.
+ *
+ * The widened cap is bounded by the chain's true maximum span (sum of each
+ * vendor's max pool size), so a large NodeMaven pool (≤50) stays reachable and
+ * a pathological chain can never spin unbounded.
+ */
+/**
+ * True when a policy request is in *implicit chain-rotation* mode: successive
+ * transport attempts rotate the flat index across the concatenated vendor pool
+ * spans (and, past the primary vendor's span, into the fallback vendor). This is
+ * the ONLY mode in which the transport loop widens its attempt cap AND
+ * de-duplicates repeated endpoints — the two behaviours must share one predicate
+ * so they never diverge. It holds when ALL of the widening conditions hold:
+ *  - the request is policy-allocator managed (not a caller-supplied proxy URL);
+ *  - the caller did NOT pin an explicit retry policy — its `attempts` ceiling is
+ *    the documented contract and must be honoured verbatim against whatever
+ *    endpoint each attempt resolves (even a repeated one), so no de-duplication;
+ *  - the method is safe/idempotent — an unsafe request is never duplicated;
+ *  - the policy resolves a non-empty registry vendor chain (smartproxy /
+ *    nodemaven). Deprecated vendors (custom / decodo) resolve no managed
+ *    endpoint, so there is nothing to rotate or de-duplicate.
+ */
+export function policyRotatesTransportVendorChain(input) {
+    if (!input.usesPolicyAllocator || !input.policy || input.explicitRetry) {
+        return false;
+    }
+    if (UNSAFE_TRANSPORT_RETRY_METHODS.has(input.method.toUpperCase())) {
+        return false;
+    }
+    return resolveVendorChain(input.policy).length > 0;
+}
+export function resolvePolicyTransportAttemptCap(input) {
+    const budget = Math.max(1, Math.floor(input.retryAttempts));
+    if (!policyRotatesTransportVendorChain({
+        policy: input.policy,
+        usesPolicyAllocator: input.usesPolicyAllocator,
+        explicitRetry: input.explicitRetry,
+        method: input.method,
+    })) {
+        return budget;
+    }
+    const span = Math.min(maxPolicyProxyPoolSpan(input.policy), resolvePolicyProxyPoolSpan(input.policy));
+    return Math.max(budget, span);
+}
+/**
+ * A registry vendor chain (smartproxy/nodemaven) resolves a potentially
+ * *different* endpoint per flat attempt index, so a transport retry should
+ * advance across endpoints and de-duplicate once the chain stops yielding new
+ * ones. Deprecated custom/decodo policies have an empty registry chain and no
+ * managed endpoint, so the transport loop has nothing to rotate or de-duplicate.
+ */
+export function policyResolvesRegistryVendorChain(policy) {
+    return Boolean(policy) && resolveVendorChain(policy).length > 0;
+}
+/** Map a resolved proxy source label to the vendor that served it. */
+export function vendorFromResolvedSource(source) {
+    if (source === "nodemaven-gateway")
+        return "nodemaven";
+    if (source === "smartproxy-allocator")
+        return "smartproxy";
+    return undefined;
+}
+function normalizeAttemptIndex(attempt) {
+    return Number.isFinite(attempt) ? Math.max(0, Math.floor(attempt)) : 0;
 }
-function resolveSmartproxyCountry(policy) {
-    return (policy.geo?.country ?? process.env[DEFAULT_PROXY_COUNTRY_ENV]?.trim().toUpperCase() ?? undefined);
+/**
+ * Map a flat attempt index into (vendorIndex, poolIndex) by concatenating each
+ * vendor's pool space in chain order. With a single vendor this reduces to
+ * `attempt % poolSize`, preserving today's behaviour exactly.
+ */
+export function mapFlatAttempt(flat, sizes) {
+    let cursor = flat;
+    for (let vendorIndex = 0; vendorIndex < sizes.length; vendorIndex++) {
+        const size = Math.max(1, sizes[vendorIndex] ?? 1);
+        if (cursor < size) {
+            return { vendorIndex, poolIndex: cursor };
+        }
+        cursor -= size;
+    }
+    return { vendorIndex: 0, poolIndex: 0 };
 }
-function resolveSmartproxyLifetime(policy) {
-    const configuredLifetime = policy.session?.lifetimeMinutes ?? readPositiveNumberEnv(DEFAULT_PROXY_LIFETIME_ENV, 30);
+function resolveSmartproxyCountry(policy, ambientDefaults = true) {
+    return (policy.geo?.country ??
+        (ambientDefaults
+            ? process.env[DEFAULT_PROXY_COUNTRY_ENV]?.trim().toUpperCase() || undefined
+            : undefined));
+}
+function resolveSmartproxyLifetime(policy, ambientDefaults = true) {
+    const configuredLifetime = policy.session?.lifetimeMinutes ??
+        (ambientDefaults ? readPositiveNumberEnv(DEFAULT_PROXY_LIFETIME_ENV, 30) : 30);
     return Math.min(SMARTPROXY_MAX_LIFETIME_MINUTES, Math.max(1, Math.floor(configuredLifetime)));
 }
 function readPositiveNumberEnv(name, fallback) {
@@ -381,27 +710,32 @@ function selectProxyPoolIndex(poolSize, attempt = 0) {
     const normalizedAttempt = Number.isFinite(attempt) ? Math.max(0, Math.floor(attempt)) : 0;
     return normalizedAttempt % poolSize;
 }
-function buildSmartproxyCacheKey(policy, affinityKey, lifetimeMinutes) {
+function buildSmartproxyCacheKey(policy, appKey, affinityKey, lifetimeMinutes, protocol, ambientDefaults = true) {
     const poolSize = resolveSmartproxyPoolSize(policy);
     return JSON.stringify({
         provider: "smartproxy",
-        country: resolveSmartproxyCountry(policy),
+        credentialHash: createHash("sha256")
+            .update("apifuse-smartproxy-credential:v1\0")
+            .update(appKey)
+            .digest("hex"),
+        protocol,
+        country: resolveSmartproxyCountry(policy, ambientDefaults),
         affinity: policy.session?.affinity ?? "request",
         affinityKey: (policy.session?.affinity ?? "request") === "request" ? undefined : affinityKey,
         lifetimeMinutes,
         poolSize,
     });
 }
-async function allocateSmartproxy(policy, appKey, lifetimeMinutes, affinityKey) {
-    const cacheKey = buildSmartproxyCacheKey(policy, affinityKey, lifetimeMinutes);
+async function allocateSmartproxy(policy, appKey, lifetimeMinutes, affinityKey, protocol, ambientDefaults = true, sharedCache = true) {
+    const cacheKey = buildSmartproxyCacheKey(policy, appKey, affinityKey, lifetimeMinutes, protocol, ambientDefaults);
     const startedAt = Date.now();
     const now = startedAt;
     const invalidatedUntil = invalidatedProxyKeys.get(cacheKey) ?? 0;
     const skipCached = invalidatedUntil > now;
-    const cached = proxyCache.get(cacheKey);
-    if (!skipCached && cached && isFresh(cached, now)) {
+    const cached = skipCached ? undefined : proxyCache.get(cacheKey, now);
+    if (cached) {
         if (shouldSoftRefresh(cached, now)) {
-            void refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes);
+            void refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, protocol, ambientDefaults, sharedCache);
             return {
                 pool: cached,
                 telemetry: telemetryForPool(cached, "soft_stale_refresh", startedAt, {
@@ -414,7 +748,7 @@ async function allocateSmartproxy(policy, appKey, lifetimeMinutes, affinityKey)
             telemetry: telemetryForPool(cached, "memory_hit", startedAt),
         };
     }
-    if (!skipCached) {
+    if (!skipCached && sharedCache) {
         const redisResult = await readSmartproxyRedisPool(cacheKey, startedAt);
         if (redisResult)
             return redisResult;
@@ -429,7 +763,7 @@ async function allocateSmartproxy(policy, appKey, lifetimeMinutes, affinityKey)
             }),
         };
     }
-    const promise = allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, startedAt).finally(() => {
+    const promise = allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, startedAt, protocol, ambientDefaults, sharedCache).finally(() => {
         proxyInflight.delete(cacheKey);
     });
     proxyInflight.set(cacheKey, promise);
@@ -454,15 +788,15 @@ async function readSmartproxyRedisPool(cacheKey, startedAt) {
     const now = Date.now();
     if (!isFresh(pool, now))
         return null;
-    proxyCache.set(cacheKey, pool);
+    proxyCache.set(cacheKey, pool, now);
     return {
         pool,
         telemetry: telemetryForPool(pool, "redis_hit", startedAt, { redisReadMs }),
     };
 }
-async function refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes) {
+async function refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, protocol, ambientDefaults, sharedCache) {
     try {
-        await allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, Date.now(), {
+        await allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, Date.now(), protocol, ambientDefaults, sharedCache, {
             background: true,
         });
     }
@@ -470,10 +804,10 @@ async function refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes)
         // Soft refresh is opportunistic; current fresh pool remains usable.
     }
 }
-async function allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, startedAt, options = {}) {
-    const redis = getProxyRedis();
+async function allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, startedAt, protocol, ambientDefaults, sharedCache, options = {}) {
+    const redis = sharedCache ? getProxyRedis() : undefined;
     if (!redis || !(await ensureRedisReady(redis))) {
-        return await allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, startedAt, { cacheStatus: "allocator" });
+        return await allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, startedAt, { cacheStatus: "allocator", protocol, ambientDefaults });
     }
     const poolKey = smartproxyRedisPoolKey(cacheKey);
     const lockKey = smartproxyRedisLockKey(cacheKey);
@@ -487,6 +821,8 @@ async function allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinute
                     cacheStatus: options.background ? "soft_stale_refresh" : "allocator",
                     redis,
                     poolKey,
+                    protocol,
+                    ambientDefaults,
                 });
             }
             finally {
@@ -599,7 +935,7 @@ async function readSmartproxyAllocatorBodyWithDeadline(response, signal) {
 }
 async function allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, startedAt, options) {
     const poolSize = resolveSmartproxyPoolSize(policy);
-    const allocatorUrl = buildSmartproxyAllocatorUrl(policy, appKey, lifetimeMinutes, poolSize);
+    const allocatorUrl = buildSmartproxyAllocatorUrl(policy, appKey, lifetimeMinutes, poolSize, options.protocol, options.ambientDefaults);
     const allocatorStartedAt = Date.now();
     const allocatorDeadlineAt = allocatorStartedAt + smartproxyAllocatorDeadlineMs();
     let allocation;
@@ -609,7 +945,7 @@ async function allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetime
             lastFailure = smartproxyAllocatorDeadlineFailure(attempt);
             break;
         }
-        const attemptResult = await fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, allocatorDeadlineAt);
+        const attemptResult = await fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, allocatorDeadlineAt, options.protocol);
         if (attemptResult.ok) {
             allocation = attemptResult;
             break;
@@ -657,13 +993,13 @@ async function allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetime
         expiresAt: allocatedAt + ttlMs,
         diagnostics: {
             provider: "smartproxy",
-            country: resolveSmartproxyCountry(policy) ?? "default",
+            country: resolveSmartproxyCountry(policy, options.ambientDefaults) ?? "default",
             lifetimeMinutes,
             affinity: policy.session?.affinity ?? "request",
             rawConnect: true,
         },
     };
-    proxyCache.set(cacheKey, result);
+    proxyCache.set(cacheKey, result, allocatedAt);
     if (options.redis && options.poolKey) {
         const redis = options.redis;
         const poolKey = options.poolKey;
@@ -692,7 +1028,7 @@ async function allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetime
         }),
     };
 }
-async function fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, deadlineAt) {
+async function fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, deadlineAt, protocol) {
     const { controller, dispose } = createDeadlineAbortController(deadlineAt);
     let response;
     try {
@@ -735,7 +1071,7 @@ async function fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, deadlineAt
             bodyClass: "http_error",
         };
     }
-    const urls = parseSmartproxyAllocatorProxies(body);
+    const urls = parseSmartproxyAllocatorProxies(body, protocol);
     const bodyClass = classifySmartproxyAllocatorBody(body, urls);
     if (urls.length === 0) {
         return {
@@ -770,17 +1106,24 @@ function smartproxyAllocatorFailureMessage(failure) {
     }
     return "Smartproxy allocator response did not contain a usable proxy endpoint.";
 }
-function buildSmartproxyAllocatorUrl(policy, appKey, lifetimeMinutes, poolSize) {
+// Smartproxy get-ip-v3 `protocol` param: 1 = HTTP. The SOCKS5 value ("2") is a
+// best-effort mapping pending live vendor confirmation; http is the default and
+// the only value exercised in production today.
+const SMARTPROXY_PROTOCOL_PARAM = {
+    http: "1",
+    socks5: "2",
+};
+function buildSmartproxyAllocatorUrl(policy, appKey, lifetimeMinutes, poolSize, protocol, ambientDefaults = true) {
     const params = new URLSearchParams({
         app_key: appKey,
         pt: "9",
         num: String(poolSize),
         life: String(lifetimeMinutes),
-        protocol: "1",
+        protocol: SMARTPROXY_PROTOCOL_PARAM[protocol],
         format: "txt",
         lb: "\\n",
     });
-    const country = resolveSmartproxyCountry(policy);
+    const country = resolveSmartproxyCountry(policy, ambientDefaults);
     if (country) {
         params.set("cc", country);
     }
@@ -788,7 +1131,8 @@ function buildSmartproxyAllocatorUrl(policy, appKey, lifetimeMinutes, poolSize)
     // old path 404s into the marketing site); the API lives on the api host.
     return `https://api.smartproxy.org/web_v1/ip/get-ip-v3?${params.toString()}`;
 }
-function parseSmartproxyAllocatorProxies(body) {
+function parseSmartproxyAllocatorProxies(body, protocol) {
+    const scheme = protocol === "socks5" ? "socks5" : "http";
     const trimmed = body.trim();
     if (!trimmed) {
         return [];
@@ -807,7 +1151,7 @@ function parseSmartproxyAllocatorProxies(body) {
                 const port = "port" in item && (typeof item.port === "string" || typeof item.port === "number")
                     ? item.port
                     : "";
-                return ip && port ? `http://${ip}:${port}` : null;
+                return ip && port ? `${scheme}://${ip}:${port}` : null;
             })
                 .filter((url) => url !== null);
         }
@@ -819,7 +1163,7 @@ function parseSmartproxyAllocatorProxies(body) {
         .split(/\r?\n/)
         .map((item) => item.trim())
         .filter((item) => /^\d{1,3}(?:\.\d{1,3}){3}:\d{2,5}$/.test(item))
-        .map((line) => `http://${line}`);
+        .map((line) => `${scheme}://${line}`);
 }
 function classifySmartproxyAllocatorBody(body, urls) {
     if (urls.length > 0) {
@@ -847,12 +1191,22 @@ function markSmartproxyCacheInvalidated(options = {}) {
     if (!policy || policy.mode === "disabled") {
         return undefined;
     }
-    if (resolveProxyProvider(policy) !== "smartproxy") {
+    if (!resolveVendorChain(policy).includes("smartproxy")) {
         return undefined;
     }
     const lifetimeMinutes = resolveSmartproxyLifetime(policy);
-    const cacheKey = buildSmartproxyCacheKey(policy, options.affinityKey, lifetimeMinutes);
-    invalidatedProxyKeys.set(cacheKey, Date.now() + SMARTPROXY_INVALIDATION_SKIP_REDIS_MS);
+    const appKey = (options.engineCredentials === undefined
+        ? readDiagnosticEnv(SMARTPROXY_APP_KEY_ENV)
+        : options.engineCredentials[SMARTPROXY_APP_KEY_ENV])?.trim();
+    if (!appKey)
+        return undefined;
+    const cacheKey = buildSmartproxyCacheKey(policy, appKey, options.affinityKey, lifetimeMinutes, options.protocol ?? VENDOR_DEFAULT_PROTOCOL.smartproxy);
+    const now = Date.now();
+    for (const [key, invalidatedUntil] of invalidatedProxyKeys) {
+        if (invalidatedUntil <= now)
+            invalidatedProxyKeys.delete(key);
+    }
+    invalidatedProxyKeys.set(cacheKey, now + SMARTPROXY_INVALIDATION_SKIP_REDIS_MS);
     proxyCache.delete(cacheKey);
     proxyInflight.delete(cacheKey);
     return cacheKey;
@@ -897,16 +1251,12 @@ export async function loadApiFuseConfig(dir = process.cwd()) {
     const tsPath = path.resolve(dir, "apifuse.config.ts");
     if (existsSync(tsPath)) {
         const config = await importConfig(tsPath);
-        const resolvedConfig = config ?? {};
-        syncProxyEnv(resolvedConfig);
-        return resolvedConfig;
+        return config ?? {};
     }
     const jsPath = path.resolve(dir, "apifuse.config.js");
     if (existsSync(jsPath)) {
         const config = await importConfig(jsPath);
-        const resolvedConfig = config ?? {};
-        syncProxyEnv(resolvedConfig);
-        return resolvedConfig;
+        return config ?? {};
     }
     return {};
 }