@apifuse/provider-sdk 2.2.0-beta.1 → 2.2.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +106 -0
- package/CHANGELOG.md +37 -0
- package/README.md +5 -1
- package/SUBMISSION.md +1 -1
- package/bin/apifuse-check.ts +44 -59
- package/bin/apifuse-create.ts +1 -1
- package/bin/apifuse-dev.ts +27 -52
- package/bin/apifuse-pack-check.ts +14 -0
- package/bin/apifuse-pack-smoke.ts +22 -81
- package/bin/apifuse-pack-types.ts +266 -0
- package/bin/apifuse-perf.ts +45 -127
- package/bin/apifuse-record.ts +53 -70
- package/bin/apifuse-submit-check.ts +510 -45
- package/bin/apifuse-sync-assets.ts +117 -0
- package/bin/apifuse.ts +1 -1
- package/bin/submit-check-delimited-text.ts +50 -0
- package/bin/submit-check-xml-semantics.ts +204 -0
- package/bin/submit-check-xml.ts +134 -0
- package/dist/auth-turn/index.d.ts +3 -3
- package/dist/auth.d.ts +2 -2
- package/dist/auth.js +9 -18
- package/dist/ceremonies/index.d.ts +1 -1
- package/dist/ceremonies/index.js +8 -18
- package/dist/cli/commands.d.ts +1 -1
- package/dist/cli/commands.js +8 -0
- package/dist/cli/create.d.ts +3 -0
- package/dist/cli/create.js +34 -35
- package/dist/cli/prompt-assets.d.ts +80 -0
- package/dist/cli/prompt-assets.js +743 -0
- package/dist/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/dist/config/loader.d.ts +149 -8
- package/dist/config/loader.js +378 -89
- package/dist/contract-serialization.d.ts +2 -2
- package/dist/contract-serialization.js +3 -6
- package/dist/contract-types.d.ts +2 -2
- package/dist/contract.d.ts +3 -3
- package/dist/contract.js +4 -6
- package/dist/define.d.ts +8 -1
- package/dist/define.js +119 -122
- package/dist/dev.d.ts +1 -1
- package/dist/dev.js +1 -1
- package/dist/errors.d.ts +4 -1
- package/dist/errors.js +48 -0
- package/dist/i18n/catalog.d.ts +2 -2
- package/dist/i18n/catalog.js +4 -10
- package/dist/i18n/index.d.ts +2 -2
- package/dist/i18n/index.js +2 -2
- package/dist/i18n/keys.d.ts +2 -2
- package/dist/index.d.ts +44 -42
- package/dist/index.js +39 -37
- package/dist/lint.js +8 -15
- package/dist/provider.d.ts +11 -11
- package/dist/provider.js +9 -9
- package/dist/public-schema-field-lint.d.ts +1 -1
- package/dist/recipes/gov-api.js +1 -1
- package/dist/runtime/auth-flow.d.ts +1 -1
- package/dist/runtime/auth-flow.js +3 -3
- package/dist/runtime/browser.d.ts +1 -1
- package/dist/runtime/browser.js +15 -29
- package/dist/runtime/cache.d.ts +1 -1
- package/dist/runtime/cache.js +4 -8
- package/dist/runtime/choice.d.ts +1 -1
- package/dist/runtime/choice.js +31 -35
- package/dist/runtime/credential.d.ts +1 -1
- package/dist/runtime/credential.js +1 -1
- package/dist/runtime/env.d.ts +1 -1
- package/dist/runtime/executor.d.ts +1 -1
- package/dist/runtime/executor.js +15 -3
- package/dist/runtime/http.d.ts +2 -2
- package/dist/runtime/http.js +144 -348
- package/dist/runtime/insights.d.ts +1 -1
- package/dist/runtime/insights.js +6 -13
- package/dist/runtime/instrumentation.d.ts +2 -2
- package/dist/runtime/instrumentation.js +56 -19
- package/dist/runtime/keyring.js +1 -1
- package/dist/runtime/namespace.js +1 -1
- package/dist/runtime/otlp.d.ts +1 -1
- package/dist/runtime/perf.d.ts +1 -1
- package/dist/runtime/provider.d.ts +1 -1
- package/dist/runtime/provider.js +1 -2
- package/dist/runtime/proxy-errors.d.ts +1 -1
- package/dist/runtime/proxy-errors.js +9 -7
- package/dist/runtime/proxy-nodemaven.d.ts +35 -0
- package/dist/runtime/proxy-nodemaven.js +128 -0
- package/dist/runtime/proxy-retry-policy.d.ts +40 -0
- package/dist/runtime/proxy-retry-policy.js +326 -0
- package/dist/runtime/proxy-telemetry.d.ts +2 -1
- package/dist/runtime/proxy-telemetry.js +55 -52
- package/dist/runtime/redis.d.ts +1 -1
- package/dist/runtime/redis.js +2 -4
- package/dist/runtime/request-options.d.ts +1 -1
- package/dist/runtime/secrets.d.ts +27 -0
- package/dist/runtime/secrets.js +51 -0
- package/dist/runtime/state.d.ts +2 -2
- package/dist/runtime/state.js +15 -4
- package/dist/runtime/stealth.d.ts +2 -2
- package/dist/runtime/stealth.js +108 -212
- package/dist/runtime/stt.d.ts +1 -1
- package/dist/runtime/stt.js +11 -15
- package/dist/runtime/trace.d.ts +2 -2
- package/dist/runtime/trace.js +2 -4
- package/dist/runtime/waterfall.d.ts +1 -1
- package/dist/schema.d.ts +1 -1
- package/dist/schema.js +7 -15
- package/dist/serve.d.ts +1 -1
- package/dist/serve.js +1 -1
- package/dist/server/index.d.ts +7 -7
- package/dist/server/index.js +6 -6
- package/dist/server/self-test-redaction.d.ts +1 -1
- package/dist/server/self-test-redaction.js +1 -1
- package/dist/server/self-test.d.ts +104 -3
- package/dist/server/self-test.js +673 -115
- package/dist/server/serve.d.ts +8 -3
- package/dist/server/serve.js +105 -59
- package/dist/server/types.d.ts +10 -9
- package/dist/server/types.js +3 -7
- package/dist/stealth/profiles.d.ts +1 -1
- package/dist/stealth/profiles.js +5 -14
- package/dist/stream.d.ts +1 -1
- package/dist/testing/index.d.ts +2 -2
- package/dist/testing/index.js +2 -2
- package/dist/testing/run.d.ts +1 -1
- package/dist/testing/run.js +7 -14
- package/dist/types.d.ts +87 -1
- package/dist/user-input.d.ts +30 -0
- package/dist/user-input.js +66 -0
- package/package.json +9 -5
- package/src/auth-turn/index.ts +1 -1
- package/src/auth.ts +28 -86
- package/src/ceremonies/index.ts +24 -78
- package/src/cli/commands.ts +10 -0
- package/src/cli/create.ts +42 -35
- package/src/cli/prompt-assets.ts +865 -0
- package/src/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/src/config/loader.ts +543 -208
- package/src/contract-serialization.ts +5 -11
- package/src/contract-types.ts +2 -2
- package/src/contract.ts +12 -28
- package/src/define.ts +247 -505
- package/src/dev.ts +4 -9
- package/src/errors.ts +58 -5
- package/src/i18n/catalog.ts +10 -32
- package/src/i18n/index.ts +2 -2
- package/src/i18n/keys.ts +5 -11
- package/src/index.ts +49 -42
- package/src/lint.ts +89 -159
- package/src/provider.ts +15 -11
- package/src/public-schema-field-lint.ts +7 -33
- package/src/recipes/gov-api.ts +2 -5
- package/src/runtime/auth-flow.ts +5 -7
- package/src/runtime/browser.ts +64 -187
- package/src/runtime/cache.ts +20 -67
- package/src/runtime/choice.ts +79 -132
- package/src/runtime/credential.ts +2 -2
- package/src/runtime/env.ts +1 -1
- package/src/runtime/executor.ts +23 -20
- package/src/runtime/http.ts +195 -559
- package/src/runtime/insights.ts +15 -53
- package/src/runtime/instrumentation.ts +78 -65
- package/src/runtime/keyring.ts +7 -19
- package/src/runtime/namespace.ts +2 -7
- package/src/runtime/otlp.ts +12 -23
- package/src/runtime/perf.ts +1 -1
- package/src/runtime/provider.ts +4 -9
- package/src/runtime/proxy-errors.ts +29 -42
- package/src/runtime/proxy-nodemaven.ts +178 -0
- package/src/runtime/proxy-retry-policy.ts +469 -0
- package/src/runtime/proxy-telemetry.ts +79 -77
- package/src/runtime/redis.ts +4 -12
- package/src/runtime/request-options.ts +4 -13
- package/src/runtime/secrets.ts +64 -0
- package/src/runtime/state.ts +41 -110
- package/src/runtime/stealth.ts +162 -377
- package/src/runtime/stt.ts +38 -94
- package/src/runtime/trace.ts +14 -44
- package/src/runtime/waterfall.ts +5 -18
- package/src/schema.ts +23 -84
- package/src/serve.ts +1 -1
- package/src/server/index.ts +12 -7
- package/src/server/self-test-redaction.ts +2 -2
- package/src/server/self-test.ts +857 -132
- package/src/server/serve.ts +172 -245
- package/src/server/types.ts +7 -19
- package/src/stealth/profiles.ts +10 -26
- package/src/stream.ts +8 -19
- package/src/testing/index.ts +2 -2
- package/src/testing/run.ts +19 -67
- package/src/types.ts +88 -1
- package/src/user-input.ts +118 -0
- package/dist/cli/templates/provider/CLAUDE.md.tpl +0 -1
- package/src/cli/templates/provider/CLAUDE.md.tpl +0 -1
- /package/dist/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
package/dist/config/loader.js
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { existsSync } from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
-
import Redis from "ioredis";
|
|
4
|
+
import { Redis } from "ioredis";
|
|
5
|
+
import { NODEMAVEN_DEFAULT_PROTOCOL, NODEMAVEN_MAX_POOL_SIZE, hasNodemavenCredentials, nodemavenPoolSize, synthesizeNodemavenProxy, } from "../runtime/proxy-nodemaven.js";
|
|
6
|
+
// "smartproxy" here is api.smartproxy.org — a residential proxy with an IP
|
|
7
|
+
// extraction API (app_key → raw ip:port pool). It is NOT the company formerly
|
|
8
|
+
// named Smartproxy (smartproxy.com), which rebranded to Decodo in 2025 and is
|
|
9
|
+
// modelled separately as the `decodo` gateway vendor. Do not conflate them.
|
|
5
10
|
export const SMARTPROXY_APP_KEY_ENV = "APIFUSE__PROXY__SMARTPROXY_APP_KEY";
|
|
6
11
|
export const SMARTPROXY_MAX_LIFETIME_MINUTES = 2000;
|
|
7
12
|
export const DEFAULT_SMARTPROXY_POOL_SIZE = 20;
|
|
@@ -15,11 +20,17 @@ export const REDIS_URL_ENV = "APIFUSE__REDIS__URL";
|
|
|
15
20
|
export class ProxyResolutionError extends Error {
|
|
16
21
|
code;
|
|
17
22
|
telemetry;
|
|
23
|
+
vendor;
|
|
24
|
+
vendorChain;
|
|
25
|
+
protocol;
|
|
18
26
|
constructor(code, message, options) {
|
|
19
27
|
super(message, options);
|
|
20
28
|
this.name = "ProxyResolutionError";
|
|
21
29
|
this.code = code;
|
|
22
30
|
this.telemetry = options?.telemetry;
|
|
31
|
+
this.vendor = options?.vendor;
|
|
32
|
+
this.vendorChain = options?.vendorChain;
|
|
33
|
+
this.protocol = options?.protocol;
|
|
23
34
|
}
|
|
24
35
|
}
|
|
25
36
|
const proxyCache = new Map();
|
|
@@ -160,9 +171,7 @@ function toProxyDiagnostics(value) {
|
|
|
160
171
|
return undefined;
|
|
161
172
|
const diagnostics = {};
|
|
162
173
|
for (const [key, item] of Object.entries(value)) {
|
|
163
|
-
if (typeof item === "string" ||
|
|
164
|
-
typeof item === "number" ||
|
|
165
|
-
typeof item === "boolean") {
|
|
174
|
+
if (typeof item === "string" || typeof item === "number" || typeof item === "boolean") {
|
|
166
175
|
diagnostics[key] = item;
|
|
167
176
|
}
|
|
168
177
|
}
|
|
@@ -234,6 +243,11 @@ function applyStickyProxySession(proxyUrl) {
|
|
|
234
243
|
if (!parsed.hostname || !parsed.username || !parsed.password) {
|
|
235
244
|
return proxyUrl;
|
|
236
245
|
}
|
|
246
|
+
// This rewrites sticky-session usernames for a bring-your-own *gateway* URL
|
|
247
|
+
// (APIFUSE__PROXY__URL). The `smartproxy` host here means a smartproxy.com /
|
|
248
|
+
// Decodo-family gateway that authenticates by username — NOT the
|
|
249
|
+
// api.smartproxy.org allocation vendor, whose endpoints are raw ip:port with
|
|
250
|
+
// no credentials and therefore return early above.
|
|
237
251
|
const host = parsed.hostname.toLowerCase();
|
|
238
252
|
if (!host.includes("smartproxy") && !host.includes("decodo")) {
|
|
239
253
|
return proxyUrl;
|
|
@@ -249,9 +263,7 @@ function applyStickyProxySession(proxyUrl) {
|
|
|
249
263
|
}
|
|
250
264
|
function buildSmartproxyUsername(username, sessionId, sessionDuration) {
|
|
251
265
|
const parts = username.split("_");
|
|
252
|
-
const configuredLife = parts
|
|
253
|
-
.find((part) => part.startsWith("life-"))
|
|
254
|
-
?.slice("life-".length);
|
|
266
|
+
const configuredLife = parts.find((part) => part.startsWith("life-"))?.slice("life-".length);
|
|
255
267
|
const baseUsername = parts
|
|
256
268
|
.filter((part) => !part.startsWith("session-") && !part.startsWith("life-"))
|
|
257
269
|
.join("_");
|
|
@@ -259,9 +271,7 @@ function buildSmartproxyUsername(username, sessionId, sessionDuration) {
|
|
|
259
271
|
}
|
|
260
272
|
function buildDecodoUsername(username, sessionId, sessionDuration) {
|
|
261
273
|
const withoutSticky = username.replace(/-session-.+-sessionduration-\d+$/, "");
|
|
262
|
-
const baseUsername = withoutSticky.startsWith("user-")
|
|
263
|
-
? withoutSticky
|
|
264
|
-
: `user-${withoutSticky}`;
|
|
274
|
+
const baseUsername = withoutSticky.startsWith("user-") ? withoutSticky : `user-${withoutSticky}`;
|
|
265
275
|
return `${baseUsername}-session-${sessionId}-sessionduration-${sessionDuration}`;
|
|
266
276
|
}
|
|
267
277
|
function syncProxyEnv(config) {
|
|
@@ -305,47 +315,173 @@ export async function resolveProxyConfigAsync(options = {}) {
|
|
|
305
315
|
if (policy.mode === "disabled") {
|
|
306
316
|
return { shouldWarn: false };
|
|
307
317
|
}
|
|
308
|
-
const
|
|
309
|
-
if (
|
|
318
|
+
const chain = resolveVendorChain(policy);
|
|
319
|
+
if (chain.length === 0) {
|
|
320
|
+
// decodo/custom/env-static providers keep the legacy static-URL path.
|
|
310
321
|
return resolveProxyConfig({
|
|
311
322
|
...options,
|
|
312
323
|
upstream: { proxy: true },
|
|
313
324
|
});
|
|
314
325
|
}
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
326
|
+
// Protocol is chosen per vendor (each vendor's benchmarked-best), with an
|
|
327
|
+
// optional explicit override for the harness/tests. Both are tunnelling
|
|
328
|
+
// schemes. transportProtocols is what the calling transport can actually use.
|
|
329
|
+
const transportProtocols = options.transportProtocols ?? ["http", "socks5"];
|
|
330
|
+
const sizes = chain.map((vendor) => vendorPoolSize(vendor, policy));
|
|
331
|
+
const total = sizes.reduce((sum, size) => sum + size, 0);
|
|
332
|
+
const normalizedAttempt = normalizeAttemptIndex(options.proxyAttempt);
|
|
333
|
+
const { vendorIndex: startVendorIndex, poolIndex: startPoolIndex } = mapFlatAttempt(total > 0 ? normalizedAttempt % total : 0, sizes);
|
|
334
|
+
const refreshEpoch = normalizeAttemptIndex(options.proxyRefreshEpoch);
|
|
335
|
+
let lastError;
|
|
336
|
+
let blockedProtocol;
|
|
337
|
+
for (let vendorIndex = startVendorIndex; vendorIndex < chain.length; vendorIndex++) {
|
|
338
|
+
const vendor = chain[vendorIndex];
|
|
339
|
+
const nextVendor = chain[vendorIndex + 1];
|
|
340
|
+
const poolIndex = vendorIndex === startVendorIndex ? startPoolIndex : 0;
|
|
341
|
+
const protocol = options.protocol ?? VENDOR_DEFAULT_PROTOCOL[vendor];
|
|
342
|
+
if (!vendorHasCredentials(vendor)) {
|
|
343
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
344
|
+
vendor,
|
|
345
|
+
nextVendor,
|
|
346
|
+
phase: "resolution",
|
|
347
|
+
reason: "no_credentials",
|
|
348
|
+
});
|
|
349
|
+
continue;
|
|
350
|
+
}
|
|
351
|
+
// The calling transport must be able to use this vendor's protocol; if not,
|
|
352
|
+
// fail over to the next vendor rather than silently downgrading.
|
|
353
|
+
if (!transportProtocols.includes(protocol)) {
|
|
354
|
+
blockedProtocol = protocol;
|
|
355
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
356
|
+
vendor,
|
|
357
|
+
nextVendor,
|
|
358
|
+
phase: "resolution",
|
|
359
|
+
reason: "protocol_unsupported",
|
|
360
|
+
});
|
|
361
|
+
continue;
|
|
362
|
+
}
|
|
363
|
+
try {
|
|
364
|
+
return await resolveWithVendor(vendor, policy, options, {
|
|
365
|
+
protocol,
|
|
366
|
+
poolIndex,
|
|
367
|
+
refreshEpoch,
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
catch (error) {
|
|
371
|
+
// Config/programming errors (invalid filter, etc.) are not vendor
|
|
372
|
+
// outages — propagate them rather than failing over.
|
|
373
|
+
if (!(error instanceof ProxyResolutionError)) {
|
|
374
|
+
throw error;
|
|
375
|
+
}
|
|
376
|
+
if (error.telemetry) {
|
|
377
|
+
options.telemetry?.recordProxyResolution(error.telemetry);
|
|
378
|
+
}
|
|
379
|
+
lastError = error;
|
|
380
|
+
options.telemetry?.recordProxyVendorFailover?.({
|
|
381
|
+
vendor,
|
|
382
|
+
nextVendor,
|
|
383
|
+
phase: "resolution",
|
|
384
|
+
reason: "allocation_failed",
|
|
385
|
+
});
|
|
319
386
|
}
|
|
320
|
-
return { shouldWarn: true };
|
|
321
387
|
}
|
|
322
|
-
|
|
388
|
+
if (policy.mode === "required") {
|
|
389
|
+
if (lastError) {
|
|
390
|
+
throw lastError instanceof ProxyResolutionError
|
|
391
|
+
? lastError
|
|
392
|
+
: new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `All proxy vendors [${chain.join(", ")}] failed for required proxy egress.`, { cause: lastError, vendorChain: chain });
|
|
393
|
+
}
|
|
394
|
+
if (blockedProtocol) {
|
|
395
|
+
throw new ProxyResolutionError("PROXY_PROTOCOL_UNSUPPORTED", `No proxy vendor in [${chain.join(", ")}] could serve a protocol supported by this transport (supports: ${transportProtocols.join(", ")}; vendor wanted "${blockedProtocol}"). Route this provider through the stealth transport.`, { protocol: blockedProtocol, vendorChain: chain });
|
|
396
|
+
}
|
|
397
|
+
throw new ProxyResolutionError("PROXY_REQUIRED", `Proxy egress is required but no vendor credentials are configured. Missing: ${chain
|
|
398
|
+
.map((vendor) => `${missingCredentialEnv(vendor)} (${vendor})`)
|
|
399
|
+
.join(", ")}.`, { vendorChain: chain });
|
|
400
|
+
}
|
|
401
|
+
return { shouldWarn: true };
|
|
402
|
+
}
|
|
403
|
+
/**
|
|
404
|
+
* Each vendor's default egress protocol, chosen from live KR benchmarks. HTTP
|
|
405
|
+
* CONNECT wins for nodemaven (socks5 adds ~500ms through the gateway) and ties
|
|
406
|
+
* for smartproxy, and is the only protocol ctx.http (Bun native fetch) supports.
|
|
407
|
+
* Override per call via ProxyResolutionOptions.protocol (harness/tests).
|
|
408
|
+
*/
|
|
409
|
+
const VENDOR_DEFAULT_PROTOCOL = {
|
|
410
|
+
smartproxy: "http",
|
|
411
|
+
nodemaven: NODEMAVEN_DEFAULT_PROTOCOL,
|
|
412
|
+
};
|
|
413
|
+
/**
|
|
414
|
+
* Guard the No-MITM invariant: a resolved proxy URL must use a tunnelling scheme
|
|
415
|
+
* (http CONNECT or socks5) so the client TLS handshake reaches the origin
|
|
416
|
+
* end-to-end. Anything else would intercept TLS and break fingerprinting.
|
|
417
|
+
*/
|
|
418
|
+
export function assertTunnelingScheme(url) {
|
|
419
|
+
let scheme;
|
|
323
420
|
try {
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
421
|
+
scheme = new URL(url).protocol.replace(/:$/, "").toLowerCase();
|
|
422
|
+
}
|
|
423
|
+
catch {
|
|
424
|
+
throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `Malformed proxy URL: ${url}`);
|
|
425
|
+
}
|
|
426
|
+
if (scheme !== "http" && scheme !== "socks5") {
|
|
427
|
+
throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `Resolved proxy scheme "${scheme}" is not a tunnelling scheme (expected http or socks5). Refusing to route TLS through a non-tunnelling proxy.`);
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
async function resolveWithVendor(vendor, policy, options, context) {
|
|
431
|
+
if (vendor === "nodemaven") {
|
|
432
|
+
const startedAt = Date.now();
|
|
433
|
+
const synthesized = synthesizeNodemavenProxy({
|
|
434
|
+
policy,
|
|
435
|
+
affinityKey: options.affinityKey,
|
|
436
|
+
protocol: context.protocol,
|
|
437
|
+
poolIndex: context.poolIndex,
|
|
438
|
+
refreshEpoch: context.refreshEpoch,
|
|
439
|
+
country: resolveSmartproxyCountry(policy),
|
|
440
|
+
});
|
|
441
|
+
options.telemetry?.recordProxyResolution({
|
|
442
|
+
provider: "nodemaven",
|
|
443
|
+
protocol: synthesized.protocol,
|
|
444
|
+
cacheStatus: "disabled",
|
|
445
|
+
cacheHit: false,
|
|
446
|
+
resolutionMs: Math.max(0, Date.now() - startedAt),
|
|
447
|
+
attempts: 1,
|
|
448
|
+
});
|
|
449
|
+
assertTunnelingScheme(synthesized.url);
|
|
327
450
|
return {
|
|
328
451
|
shouldWarn: false,
|
|
329
|
-
url:
|
|
330
|
-
source: "
|
|
452
|
+
url: synthesized.url,
|
|
453
|
+
source: "nodemaven-gateway",
|
|
454
|
+
protocol: synthesized.protocol,
|
|
331
455
|
diagnostics: {
|
|
332
|
-
...
|
|
333
|
-
|
|
334
|
-
poolIndex,
|
|
456
|
+
...synthesized.diagnostics,
|
|
457
|
+
poolIndex: context.poolIndex,
|
|
335
458
|
},
|
|
336
459
|
};
|
|
337
460
|
}
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
throw error instanceof ProxyResolutionError
|
|
344
|
-
? error
|
|
345
|
-
: new ProxyResolutionError("PROXY_ALLOCATION_FAILED", "Smartproxy allocator failed for required proxy egress.", { cause: error });
|
|
346
|
-
}
|
|
347
|
-
return { shouldWarn: true };
|
|
461
|
+
// smartproxy allocation-style vendor.
|
|
462
|
+
const appKey = process.env[SMARTPROXY_APP_KEY_ENV]?.trim();
|
|
463
|
+
if (!appKey) {
|
|
464
|
+
// Guarded by vendorHasCredentials; treated as a vendor-internal failure.
|
|
465
|
+
throw new ProxyResolutionError("PROXY_ALLOCATION_FAILED", `${SMARTPROXY_APP_KEY_ENV} is not configured.`, { vendor: "smartproxy" });
|
|
348
466
|
}
|
|
467
|
+
const lifetimeMinutes = resolveSmartproxyLifetime(policy);
|
|
468
|
+
const allocated = await allocateSmartproxy(policy, appKey, lifetimeMinutes, options.affinityKey, context.protocol);
|
|
469
|
+
options.telemetry?.recordProxyResolution({ ...allocated.telemetry, protocol: context.protocol });
|
|
470
|
+
const poolIndex = selectProxyPoolIndex(allocated.pool.urls.length, context.poolIndex);
|
|
471
|
+
const url = allocated.pool.urls[poolIndex];
|
|
472
|
+
if (url)
|
|
473
|
+
assertTunnelingScheme(url);
|
|
474
|
+
return {
|
|
475
|
+
shouldWarn: false,
|
|
476
|
+
url,
|
|
477
|
+
source: "smartproxy-allocator",
|
|
478
|
+
protocol: context.protocol,
|
|
479
|
+
diagnostics: {
|
|
480
|
+
...allocated.pool.diagnostics,
|
|
481
|
+
poolSize: allocated.pool.urls.length,
|
|
482
|
+
poolIndex,
|
|
483
|
+
},
|
|
484
|
+
};
|
|
349
485
|
}
|
|
350
486
|
function resolvePolicy(options) {
|
|
351
487
|
if (options.proxyPolicy) {
|
|
@@ -357,19 +493,174 @@ function resolvePolicy(options) {
|
|
|
357
493
|
}
|
|
358
494
|
return undefined;
|
|
359
495
|
}
|
|
360
|
-
function
|
|
361
|
-
return
|
|
362
|
-
|
|
363
|
-
|
|
496
|
+
function isRegistryVendor(name) {
|
|
497
|
+
return name === "smartproxy" || name === "nodemaven";
|
|
498
|
+
}
|
|
499
|
+
/**
|
|
500
|
+
* Ordered list of SDK-native proxy vendors declared by the policy. `providers`
|
|
501
|
+
* takes precedence over the legacy singular `provider`; the platform default
|
|
502
|
+
* env is the final fallback. Non-registry names (decodo/custom) are dropped so
|
|
503
|
+
* an all-static chain falls through to the legacy env-URL path unchanged.
|
|
504
|
+
*/
|
|
505
|
+
export function resolveVendorChain(policy) {
|
|
506
|
+
const declared = policy.providers?.length
|
|
507
|
+
? policy.providers
|
|
508
|
+
: [policy.provider ?? envDefaultProvider()];
|
|
509
|
+
const chain = [];
|
|
510
|
+
for (const name of declared) {
|
|
511
|
+
if (isRegistryVendor(name) && !chain.includes(name)) {
|
|
512
|
+
chain.push(name);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
return chain;
|
|
516
|
+
}
|
|
517
|
+
function envDefaultProvider() {
|
|
518
|
+
const raw = process.env[DEFAULT_PROXY_PROVIDER_ENV]?.trim().toLowerCase();
|
|
519
|
+
return raw ?? undefined;
|
|
520
|
+
}
|
|
521
|
+
function vendorHasCredentials(vendor) {
|
|
522
|
+
if (vendor === "nodemaven")
|
|
523
|
+
return hasNodemavenCredentials();
|
|
524
|
+
return Boolean(process.env[SMARTPROXY_APP_KEY_ENV]?.trim());
|
|
525
|
+
}
|
|
526
|
+
function missingCredentialEnv(vendor) {
|
|
527
|
+
return vendor === "nodemaven" ? "APIFUSE__PROXY__NODEMAVEN_USERNAME" : SMARTPROXY_APP_KEY_ENV;
|
|
528
|
+
}
|
|
529
|
+
function vendorPoolSize(vendor, policy) {
|
|
530
|
+
return vendor === "nodemaven" ? nodemavenPoolSize(policy) : resolveSmartproxyPoolSize(policy);
|
|
531
|
+
}
|
|
532
|
+
/**
|
|
533
|
+
* Total attempt span across a policy's vendor chain — the sum of each vendor's
|
|
534
|
+
* pool size. Transports use this so successive attempts rotate a vendor's pool
|
|
535
|
+
* and then fail over to the next vendor via the flat attempt index. With one
|
|
536
|
+
* vendor this equals that vendor's pool size (today's behaviour).
|
|
537
|
+
*/
|
|
538
|
+
export function resolvePolicyProxyPoolSpan(policy) {
|
|
539
|
+
const chain = resolveVendorChain(policy);
|
|
540
|
+
if (chain.length === 0)
|
|
541
|
+
return resolveSmartproxyPoolSize(policy);
|
|
542
|
+
return chain.reduce((sum, vendor) => sum + vendorPoolSize(vendor, policy), 0);
|
|
543
|
+
}
|
|
544
|
+
function vendorMaxPoolSize(vendor) {
|
|
545
|
+
return vendor === "nodemaven" ? NODEMAVEN_MAX_POOL_SIZE : SMARTPROXY_MAX_POOL_SIZE;
|
|
546
|
+
}
|
|
547
|
+
/**
|
|
548
|
+
* Absolute upper bound on a chain's attempt span — the sum of each vendor's
|
|
549
|
+
* *maximum* pool size. Unlike `resolvePolicyProxyPoolSpan` (the configured
|
|
550
|
+
* span), this backstop is independent of `session.poolSize`, so it never
|
|
551
|
+
* truncates a legitimately large pool below the point where the flat attempt
|
|
552
|
+
* index would cross into the next vendor (e.g. a 50-slot NodeMaven pool).
|
|
553
|
+
*/
|
|
554
|
+
export function maxPolicyProxyPoolSpan(policy) {
|
|
555
|
+
const chain = resolveVendorChain(policy);
|
|
556
|
+
if (chain.length === 0)
|
|
557
|
+
return SMARTPROXY_MAX_POOL_SIZE;
|
|
558
|
+
return chain.reduce((sum, vendor) => sum + vendorMaxPoolSize(vendor), 0);
|
|
559
|
+
}
|
|
560
|
+
const UNSAFE_TRANSPORT_RETRY_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE", "TRACE"]);
|
|
561
|
+
/**
|
|
562
|
+
* Transport-retry attempt cap for a policy-managed request. A transport failure
|
|
563
|
+
* rotates the flat attempt index onto the *next* endpoint (and, once the index
|
|
564
|
+
* passes the primary vendor's pool span, the *next vendor*), so the cap must be
|
|
565
|
+
* the chain's full pool span for failover to reach the fallback vendor — the
|
|
566
|
+
* per-endpoint retry budget (default 3) never gets there.
|
|
567
|
+
*
|
|
568
|
+
* The span only widens beyond the caller's retry budget when ALL hold:
|
|
569
|
+
* - the request is policy-allocator managed (not a caller-supplied proxy URL);
|
|
570
|
+
* - the caller did NOT pin an explicit retry policy — `HttpRetryOptions.attempts`
|
|
571
|
+
* is the documented total-attempt ceiling and must be honoured verbatim;
|
|
572
|
+
* - the method is safe/idempotent — an unsafe request must never be duplicated
|
|
573
|
+
* across the pool even if some framework default would allow it;
|
|
574
|
+
* - the policy resolves a non-empty *registry* vendor chain (smartproxy /
|
|
575
|
+
* nodemaven). Static vendors (custom / decodo) and credential-less policies
|
|
576
|
+
* resolve no allocator pool, so every attempt would hit the same endpoint
|
|
577
|
+
* with no possible crossover — they keep the retry budget.
|
|
578
|
+
*
|
|
579
|
+
* The widened cap is bounded by the chain's true maximum span (sum of each
|
|
580
|
+
* vendor's max pool size), so a large NodeMaven pool (≤50) stays reachable and
|
|
581
|
+
* a pathological chain can never spin unbounded.
|
|
582
|
+
*/
|
|
583
|
+
/**
|
|
584
|
+
* True when a policy request is in *implicit chain-rotation* mode: successive
|
|
585
|
+
* transport attempts rotate the flat index across the concatenated vendor pool
|
|
586
|
+
* spans (and, past the primary vendor's span, into the fallback vendor). This is
|
|
587
|
+
* the ONLY mode in which the transport loop widens its attempt cap AND
|
|
588
|
+
* de-duplicates repeated endpoints — the two behaviours must share one predicate
|
|
589
|
+
* so they never diverge. It holds when ALL of the widening conditions hold:
|
|
590
|
+
* - the request is policy-allocator managed (not a caller-supplied proxy URL);
|
|
591
|
+
* - the caller did NOT pin an explicit retry policy — its `attempts` ceiling is
|
|
592
|
+
* the documented contract and must be honoured verbatim against whatever
|
|
593
|
+
* endpoint each attempt resolves (even a repeated one), so no de-duplication;
|
|
594
|
+
* - the method is safe/idempotent — an unsafe request is never duplicated;
|
|
595
|
+
* - the policy resolves a non-empty registry vendor chain (smartproxy /
|
|
596
|
+
* nodemaven). Static vendors (custom / decodo) resolve the same URL every
|
|
597
|
+
* attempt, so there is nothing to rotate or de-duplicate.
|
|
598
|
+
*/
|
|
599
|
+
export function policyRotatesTransportVendorChain(input) {
|
|
600
|
+
if (!input.usesPolicyAllocator || !input.policy || input.explicitRetry) {
|
|
601
|
+
return false;
|
|
602
|
+
}
|
|
603
|
+
if (UNSAFE_TRANSPORT_RETRY_METHODS.has(input.method.toUpperCase())) {
|
|
604
|
+
return false;
|
|
605
|
+
}
|
|
606
|
+
return resolveVendorChain(input.policy).length > 0;
|
|
607
|
+
}
|
|
608
|
+
export function resolvePolicyTransportAttemptCap(input) {
|
|
609
|
+
const budget = Math.max(1, Math.floor(input.retryAttempts));
|
|
610
|
+
if (!policyRotatesTransportVendorChain({
|
|
611
|
+
policy: input.policy,
|
|
612
|
+
usesPolicyAllocator: input.usesPolicyAllocator,
|
|
613
|
+
explicitRetry: input.explicitRetry,
|
|
614
|
+
method: input.method,
|
|
615
|
+
})) {
|
|
616
|
+
return budget;
|
|
617
|
+
}
|
|
618
|
+
const span = Math.min(maxPolicyProxyPoolSpan(input.policy), resolvePolicyProxyPoolSpan(input.policy));
|
|
619
|
+
return Math.max(budget, span);
|
|
620
|
+
}
|
|
621
|
+
/**
|
|
622
|
+
* A registry vendor chain (smartproxy/nodemaven) resolves a potentially
|
|
623
|
+
* *different* endpoint per flat attempt index, so a transport retry should
|
|
624
|
+
* advance across endpoints and de-duplicate once the chain stops yielding new
|
|
625
|
+
* ones. Static/custom/decodo policies (empty registry chain) resolve the *same*
|
|
626
|
+
* URL every attempt by design — retrying that same endpoint is intended, so the
|
|
627
|
+
* transport loop must not de-duplicate them.
|
|
628
|
+
*/
|
|
629
|
+
export function policyResolvesRegistryVendorChain(policy) {
|
|
630
|
+
return Boolean(policy) && resolveVendorChain(policy).length > 0;
|
|
631
|
+
}
|
|
632
|
+
/** Map a resolved proxy source label to the vendor that served it. */
|
|
633
|
+
export function vendorFromResolvedSource(source) {
|
|
634
|
+
if (source === "nodemaven-gateway")
|
|
635
|
+
return "nodemaven";
|
|
636
|
+
if (source === "smartproxy-allocator")
|
|
637
|
+
return "smartproxy";
|
|
638
|
+
return undefined;
|
|
639
|
+
}
|
|
640
|
+
function normalizeAttemptIndex(attempt) {
|
|
641
|
+
return Number.isFinite(attempt) ? Math.max(0, Math.floor(attempt)) : 0;
|
|
642
|
+
}
|
|
643
|
+
/**
|
|
644
|
+
* Map a flat attempt index into (vendorIndex, poolIndex) by concatenating each
|
|
645
|
+
* vendor's pool space in chain order. With a single vendor this reduces to
|
|
646
|
+
* `attempt % poolSize`, preserving today's behaviour exactly.
|
|
647
|
+
*/
|
|
648
|
+
export function mapFlatAttempt(flat, sizes) {
|
|
649
|
+
let cursor = flat;
|
|
650
|
+
for (let vendorIndex = 0; vendorIndex < sizes.length; vendorIndex++) {
|
|
651
|
+
const size = Math.max(1, sizes[vendorIndex] ?? 1);
|
|
652
|
+
if (cursor < size) {
|
|
653
|
+
return { vendorIndex, poolIndex: cursor };
|
|
654
|
+
}
|
|
655
|
+
cursor -= size;
|
|
656
|
+
}
|
|
657
|
+
return { vendorIndex: 0, poolIndex: 0 };
|
|
364
658
|
}
|
|
365
659
|
function resolveSmartproxyCountry(policy) {
|
|
366
|
-
return (policy.geo?.country ??
|
|
367
|
-
process.env[DEFAULT_PROXY_COUNTRY_ENV]?.trim().toUpperCase() ??
|
|
368
|
-
undefined);
|
|
660
|
+
return (policy.geo?.country ?? process.env[DEFAULT_PROXY_COUNTRY_ENV]?.trim().toUpperCase() ?? undefined);
|
|
369
661
|
}
|
|
370
662
|
function resolveSmartproxyLifetime(policy) {
|
|
371
|
-
const configuredLifetime = policy.session?.lifetimeMinutes ??
|
|
372
|
-
readPositiveNumberEnv(DEFAULT_PROXY_LIFETIME_ENV, 30);
|
|
663
|
+
const configuredLifetime = policy.session?.lifetimeMinutes ?? readPositiveNumberEnv(DEFAULT_PROXY_LIFETIME_ENV, 30);
|
|
373
664
|
return Math.min(SMARTPROXY_MAX_LIFETIME_MINUTES, Math.max(1, Math.floor(configuredLifetime)));
|
|
374
665
|
}
|
|
375
666
|
function readPositiveNumberEnv(name, fallback) {
|
|
@@ -389,26 +680,23 @@ function selectProxyPoolIndex(poolSize, attempt = 0) {
|
|
|
389
680
|
if (poolSize <= 1) {
|
|
390
681
|
return 0;
|
|
391
682
|
}
|
|
392
|
-
const normalizedAttempt = Number.isFinite(attempt)
|
|
393
|
-
? Math.max(0, Math.floor(attempt))
|
|
394
|
-
: 0;
|
|
683
|
+
const normalizedAttempt = Number.isFinite(attempt) ? Math.max(0, Math.floor(attempt)) : 0;
|
|
395
684
|
return normalizedAttempt % poolSize;
|
|
396
685
|
}
|
|
397
|
-
function buildSmartproxyCacheKey(policy, affinityKey, lifetimeMinutes) {
|
|
686
|
+
function buildSmartproxyCacheKey(policy, affinityKey, lifetimeMinutes, protocol) {
|
|
398
687
|
const poolSize = resolveSmartproxyPoolSize(policy);
|
|
399
688
|
return JSON.stringify({
|
|
400
689
|
provider: "smartproxy",
|
|
690
|
+
protocol,
|
|
401
691
|
country: resolveSmartproxyCountry(policy),
|
|
402
692
|
affinity: policy.session?.affinity ?? "request",
|
|
403
|
-
affinityKey: (policy.session?.affinity ?? "request") === "request"
|
|
404
|
-
? undefined
|
|
405
|
-
: affinityKey,
|
|
693
|
+
affinityKey: (policy.session?.affinity ?? "request") === "request" ? undefined : affinityKey,
|
|
406
694
|
lifetimeMinutes,
|
|
407
695
|
poolSize,
|
|
408
696
|
});
|
|
409
697
|
}
|
|
410
|
-
async function allocateSmartproxy(policy, appKey, lifetimeMinutes, affinityKey) {
|
|
411
|
-
const cacheKey = buildSmartproxyCacheKey(policy, affinityKey, lifetimeMinutes);
|
|
698
|
+
async function allocateSmartproxy(policy, appKey, lifetimeMinutes, affinityKey, protocol) {
|
|
699
|
+
const cacheKey = buildSmartproxyCacheKey(policy, affinityKey, lifetimeMinutes, protocol);
|
|
412
700
|
const startedAt = Date.now();
|
|
413
701
|
const now = startedAt;
|
|
414
702
|
const invalidatedUntil = invalidatedProxyKeys.get(cacheKey) ?? 0;
|
|
@@ -416,7 +704,7 @@ async function allocateSmartproxy(policy, appKey, lifetimeMinutes, affinityKey)
|
|
|
416
704
|
const cached = proxyCache.get(cacheKey);
|
|
417
705
|
if (!skipCached && cached && isFresh(cached, now)) {
|
|
418
706
|
if (shouldSoftRefresh(cached, now)) {
|
|
419
|
-
void refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes);
|
|
707
|
+
void refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, protocol);
|
|
420
708
|
return {
|
|
421
709
|
pool: cached,
|
|
422
710
|
telemetry: telemetryForPool(cached, "soft_stale_refresh", startedAt, {
|
|
@@ -444,7 +732,7 @@ async function allocateSmartproxy(policy, appKey, lifetimeMinutes, affinityKey)
|
|
|
444
732
|
}),
|
|
445
733
|
};
|
|
446
734
|
}
|
|
447
|
-
const promise = allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, startedAt).finally(() => {
|
|
735
|
+
const promise = allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, startedAt, protocol).finally(() => {
|
|
448
736
|
proxyInflight.delete(cacheKey);
|
|
449
737
|
});
|
|
450
738
|
proxyInflight.set(cacheKey, promise);
|
|
@@ -475,9 +763,9 @@ async function readSmartproxyRedisPool(cacheKey, startedAt) {
|
|
|
475
763
|
telemetry: telemetryForPool(pool, "redis_hit", startedAt, { redisReadMs }),
|
|
476
764
|
};
|
|
477
765
|
}
|
|
478
|
-
async function refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes) {
|
|
766
|
+
async function refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, protocol) {
|
|
479
767
|
try {
|
|
480
|
-
await allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, Date.now(), {
|
|
768
|
+
await allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, Date.now(), protocol, {
|
|
481
769
|
background: true,
|
|
482
770
|
});
|
|
483
771
|
}
|
|
@@ -485,10 +773,10 @@ async function refreshSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes)
|
|
|
485
773
|
// Soft refresh is opportunistic; current fresh pool remains usable.
|
|
486
774
|
}
|
|
487
775
|
}
|
|
488
|
-
async function allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, startedAt, options = {}) {
|
|
776
|
+
async function allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinutes, startedAt, protocol, options = {}) {
|
|
489
777
|
const redis = getProxyRedis();
|
|
490
778
|
if (!redis || !(await ensureRedisReady(redis))) {
|
|
491
|
-
return await allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, startedAt, { cacheStatus: "allocator" });
|
|
779
|
+
return await allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, startedAt, { cacheStatus: "allocator", protocol });
|
|
492
780
|
}
|
|
493
781
|
const poolKey = smartproxyRedisPoolKey(cacheKey);
|
|
494
782
|
const lockKey = smartproxyRedisLockKey(cacheKey);
|
|
@@ -499,11 +787,10 @@ async function allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinute
|
|
|
499
787
|
if (acquired === "OK") {
|
|
500
788
|
try {
|
|
501
789
|
return await allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, startedAt, {
|
|
502
|
-
cacheStatus: options.background
|
|
503
|
-
? "soft_stale_refresh"
|
|
504
|
-
: "allocator",
|
|
790
|
+
cacheStatus: options.background ? "soft_stale_refresh" : "allocator",
|
|
505
791
|
redis,
|
|
506
792
|
poolKey,
|
|
793
|
+
protocol,
|
|
507
794
|
});
|
|
508
795
|
}
|
|
509
796
|
finally {
|
|
@@ -526,8 +813,7 @@ async function allocateSmartproxyShared(cacheKey, policy, appKey, lifetimeMinute
|
|
|
526
813
|
if (typeof pttl === "number" && pttl <= 0) {
|
|
527
814
|
continue;
|
|
528
815
|
}
|
|
529
|
-
if (Date.now() - startedAt >
|
|
530
|
-
SMARTPROXY_LOCK_POLL_MAX_MS - SMARTPROXY_DEADLINE_MARGIN_MS) {
|
|
816
|
+
if (Date.now() - startedAt > SMARTPROXY_LOCK_POLL_MAX_MS - SMARTPROXY_DEADLINE_MARGIN_MS) {
|
|
531
817
|
break;
|
|
532
818
|
}
|
|
533
819
|
await sleep(Math.min(500, Math.max(50, typeof pttl === "number" ? pttl : 100)));
|
|
@@ -559,7 +845,7 @@ function sleep(ms, signal) {
|
|
|
559
845
|
});
|
|
560
846
|
}
|
|
561
847
|
function smartproxyAllocatorDeadlineMs() {
|
|
562
|
-
return
|
|
848
|
+
return smartproxyAllocatorDeadlineMsForTests ?? SMARTPROXY_ALLOCATOR_DEADLINE_MS;
|
|
563
849
|
}
|
|
564
850
|
function createDeadlineAbortController(deadlineAt) {
|
|
565
851
|
const controller = new AbortController();
|
|
@@ -617,7 +903,7 @@ async function readSmartproxyAllocatorBodyWithDeadline(response, signal) {
|
|
|
617
903
|
}
|
|
618
904
|
async function allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetimeMinutes, startedAt, options) {
|
|
619
905
|
const poolSize = resolveSmartproxyPoolSize(policy);
|
|
620
|
-
const allocatorUrl = buildSmartproxyAllocatorUrl(policy, appKey, lifetimeMinutes, poolSize);
|
|
906
|
+
const allocatorUrl = buildSmartproxyAllocatorUrl(policy, appKey, lifetimeMinutes, poolSize, options.protocol);
|
|
621
907
|
const allocatorStartedAt = Date.now();
|
|
622
908
|
const allocatorDeadlineAt = allocatorStartedAt + smartproxyAllocatorDeadlineMs();
|
|
623
909
|
let allocation;
|
|
@@ -627,7 +913,7 @@ async function allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetime
|
|
|
627
913
|
lastFailure = smartproxyAllocatorDeadlineFailure(attempt);
|
|
628
914
|
break;
|
|
629
915
|
}
|
|
630
|
-
const attemptResult = await fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, allocatorDeadlineAt);
|
|
916
|
+
const attemptResult = await fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, allocatorDeadlineAt, options.protocol);
|
|
631
917
|
if (attemptResult.ok) {
|
|
632
918
|
allocation = attemptResult;
|
|
633
919
|
break;
|
|
@@ -649,9 +935,7 @@ async function allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetime
|
|
|
649
935
|
}
|
|
650
936
|
}
|
|
651
937
|
const allocatorMs = Math.max(0, Date.now() - allocatorStartedAt);
|
|
652
|
-
const allocatorAttempts = allocation?.attempt ??
|
|
653
|
-
lastFailure?.attempt ??
|
|
654
|
-
SMARTPROXY_ALLOCATOR_MAX_ATTEMPTS;
|
|
938
|
+
const allocatorAttempts = allocation?.attempt ?? lastFailure?.attempt ?? SMARTPROXY_ALLOCATOR_MAX_ATTEMPTS;
|
|
655
939
|
const allocatorBodyClass = lastFailure?.bodyClass ?? allocation?.bodyClass ?? "usable_proxy_endpoints";
|
|
656
940
|
const allocatorStatus = lastFailure ? lastFailure.status : allocation?.status;
|
|
657
941
|
if (!allocation) {
|
|
@@ -712,7 +996,7 @@ async function allocateAndStoreSmartproxyPool(cacheKey, policy, appKey, lifetime
|
|
|
712
996
|
}),
|
|
713
997
|
};
|
|
714
998
|
}
|
|
715
|
-
async function fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, deadlineAt) {
|
|
999
|
+
async function fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, deadlineAt, protocol) {
|
|
716
1000
|
const { controller, dispose } = createDeadlineAbortController(deadlineAt);
|
|
717
1001
|
let response;
|
|
718
1002
|
try {
|
|
@@ -755,7 +1039,7 @@ async function fetchSmartproxyAllocatorAttempt(allocatorUrl, attempt, deadlineAt
|
|
|
755
1039
|
bodyClass: "http_error",
|
|
756
1040
|
};
|
|
757
1041
|
}
|
|
758
|
-
const urls = parseSmartproxyAllocatorProxies(body);
|
|
1042
|
+
const urls = parseSmartproxyAllocatorProxies(body, protocol);
|
|
759
1043
|
const bodyClass = classifySmartproxyAllocatorBody(body, urls);
|
|
760
1044
|
if (urls.length === 0) {
|
|
761
1045
|
return {
|
|
@@ -790,13 +1074,20 @@ function smartproxyAllocatorFailureMessage(failure) {
|
|
|
790
1074
|
}
|
|
791
1075
|
return "Smartproxy allocator response did not contain a usable proxy endpoint.";
|
|
792
1076
|
}
|
|
793
|
-
|
|
1077
|
+
// Smartproxy get-ip-v3 `protocol` param: 1 = HTTP. The SOCKS5 value ("2") is a
|
|
1078
|
+
// best-effort mapping pending live vendor confirmation; http is the default and
|
|
1079
|
+
// the only value exercised in production today.
|
|
1080
|
+
const SMARTPROXY_PROTOCOL_PARAM = {
|
|
1081
|
+
http: "1",
|
|
1082
|
+
socks5: "2",
|
|
1083
|
+
};
|
|
1084
|
+
function buildSmartproxyAllocatorUrl(policy, appKey, lifetimeMinutes, poolSize, protocol) {
|
|
794
1085
|
const params = new URLSearchParams({
|
|
795
1086
|
app_key: appKey,
|
|
796
1087
|
pt: "9",
|
|
797
1088
|
num: String(poolSize),
|
|
798
1089
|
life: String(lifetimeMinutes),
|
|
799
|
-
protocol:
|
|
1090
|
+
protocol: SMARTPROXY_PROTOCOL_PARAM[protocol],
|
|
800
1091
|
format: "txt",
|
|
801
1092
|
lb: "\\n",
|
|
802
1093
|
});
|
|
@@ -804,21 +1095,20 @@ function buildSmartproxyAllocatorUrl(policy, appKey, lifetimeMinutes, poolSize)
|
|
|
804
1095
|
if (country) {
|
|
805
1096
|
params.set("cc", country);
|
|
806
1097
|
}
|
|
807
|
-
|
|
1098
|
+
// www.smartproxy.org stopped serving the extraction API on 2026-07-21 (the
|
|
1099
|
+
// old path 404s into the marketing site); the API lives on the api host.
|
|
1100
|
+
return `https://api.smartproxy.org/web_v1/ip/get-ip-v3?${params.toString()}`;
|
|
808
1101
|
}
|
|
809
|
-
function parseSmartproxyAllocatorProxies(body) {
|
|
1102
|
+
function parseSmartproxyAllocatorProxies(body, protocol) {
|
|
1103
|
+
const scheme = protocol === "socks5" ? "socks5" : "http";
|
|
810
1104
|
const trimmed = body.trim();
|
|
811
1105
|
if (!trimmed) {
|
|
812
1106
|
return [];
|
|
813
1107
|
}
|
|
814
1108
|
try {
|
|
815
1109
|
const parsed = JSON.parse(trimmed);
|
|
816
|
-
const data = parsed && typeof parsed === "object" && "data" in parsed
|
|
817
|
-
|
|
818
|
-
: undefined;
|
|
819
|
-
const list = data && typeof data === "object" && "list" in data
|
|
820
|
-
? data.list
|
|
821
|
-
: undefined;
|
|
1110
|
+
const data = parsed && typeof parsed === "object" && "data" in parsed ? parsed.data : undefined;
|
|
1111
|
+
const list = data && typeof data === "object" && "list" in data ? data.list : undefined;
|
|
822
1112
|
if (Array.isArray(list)) {
|
|
823
1113
|
return list
|
|
824
1114
|
.map((item) => {
|
|
@@ -826,11 +1116,10 @@ function parseSmartproxyAllocatorProxies(body) {
|
|
|
826
1116
|
return null;
|
|
827
1117
|
}
|
|
828
1118
|
const ip = "ip" in item && typeof item.ip === "string" ? item.ip : "";
|
|
829
|
-
const port = "port" in item &&
|
|
830
|
-
(typeof item.port === "string" || typeof item.port === "number")
|
|
1119
|
+
const port = "port" in item && (typeof item.port === "string" || typeof item.port === "number")
|
|
831
1120
|
? item.port
|
|
832
1121
|
: "";
|
|
833
|
-
return ip && port ?
|
|
1122
|
+
return ip && port ? `${scheme}://${ip}:${port}` : null;
|
|
834
1123
|
})
|
|
835
1124
|
.filter((url) => url !== null);
|
|
836
1125
|
}
|
|
@@ -842,7 +1131,7 @@ function parseSmartproxyAllocatorProxies(body) {
|
|
|
842
1131
|
.split(/\r?\n/)
|
|
843
1132
|
.map((item) => item.trim())
|
|
844
1133
|
.filter((item) => /^\d{1,3}(?:\.\d{1,3}){3}:\d{2,5}$/.test(item))
|
|
845
|
-
.map((line) =>
|
|
1134
|
+
.map((line) => `${scheme}://${line}`);
|
|
846
1135
|
}
|
|
847
1136
|
function classifySmartproxyAllocatorBody(body, urls) {
|
|
848
1137
|
if (urls.length > 0) {
|
|
@@ -870,11 +1159,11 @@ function markSmartproxyCacheInvalidated(options = {}) {
|
|
|
870
1159
|
if (!policy || policy.mode === "disabled") {
|
|
871
1160
|
return undefined;
|
|
872
1161
|
}
|
|
873
|
-
if (
|
|
1162
|
+
if (!resolveVendorChain(policy).includes("smartproxy")) {
|
|
874
1163
|
return undefined;
|
|
875
1164
|
}
|
|
876
1165
|
const lifetimeMinutes = resolveSmartproxyLifetime(policy);
|
|
877
|
-
const cacheKey = buildSmartproxyCacheKey(policy, options.affinityKey, lifetimeMinutes);
|
|
1166
|
+
const cacheKey = buildSmartproxyCacheKey(policy, options.affinityKey, lifetimeMinutes, options.protocol ?? VENDOR_DEFAULT_PROTOCOL.smartproxy);
|
|
878
1167
|
invalidatedProxyKeys.set(cacheKey, Date.now() + SMARTPROXY_INVALIDATION_SKIP_REDIS_MS);
|
|
879
1168
|
proxyCache.delete(cacheKey);
|
|
880
1169
|
proxyInflight.delete(cacheKey);
|