@apifuse/provider-sdk 2.2.0-beta.5 → 2.2.0-beta.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +53 -0
- package/CHANGELOG.md +12 -0
- package/README.md +5 -1
- package/SUBMISSION.md +1 -1
- package/bin/apifuse-check.ts +26 -1
- package/bin/apifuse-pack-check.ts +14 -0
- package/bin/apifuse-submit-check.ts +193 -2
- package/bin/apifuse-sync-assets.ts +117 -0
- package/dist/auth-turn/index.d.ts +2 -2
- package/dist/cli/commands.d.ts +1 -1
- package/dist/cli/commands.js +8 -0
- package/dist/cli/create.d.ts +3 -0
- package/dist/cli/create.js +34 -35
- package/dist/cli/prompt-assets.d.ts +80 -0
- package/dist/cli/prompt-assets.js +743 -0
- package/dist/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/dist/config/loader.d.ts +79 -6
- package/dist/config/loader.js +272 -48
- package/dist/define.js +27 -3
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/runtime/executor.js +7 -0
- package/dist/runtime/http.js +3 -0
- package/dist/runtime/proxy-errors.js +6 -2
- package/dist/runtime/proxy-nodemaven.d.ts +34 -0
- package/dist/runtime/proxy-nodemaven.js +128 -0
- package/dist/runtime/proxy-telemetry.d.ts +2 -1
- package/dist/runtime/proxy-telemetry.js +39 -4
- package/dist/runtime/secrets.d.ts +27 -0
- package/dist/runtime/secrets.js +51 -0
- package/dist/runtime/stealth.js +20 -9
- package/dist/server/serve.d.ts +5 -0
- package/dist/server/serve.js +39 -0
- package/dist/server/types.d.ts +9 -9
- package/dist/types.d.ts +30 -1
- package/package.json +4 -3
- package/src/cli/commands.ts +10 -0
- package/src/cli/create.ts +42 -35
- package/src/cli/prompt-assets.ts +865 -0
- package/src/cli/templates/provider/AGENTS.md.tpl +17 -8
- package/src/config/loader.ts +405 -61
- package/src/define.ts +35 -3
- package/src/index.ts +5 -0
- package/src/runtime/executor.ts +8 -0
- package/src/runtime/http.ts +3 -0
- package/src/runtime/proxy-errors.ts +12 -4
- package/src/runtime/proxy-nodemaven.ts +178 -0
- package/src/runtime/proxy-telemetry.ts +56 -5
- package/src/runtime/secrets.ts +64 -0
- package/src/runtime/stealth.ts +26 -10
- package/src/server/serve.ts +53 -0
- package/src/types.ts +30 -1
- package/dist/cli/templates/provider/CLAUDE.md.tpl +0 -1
- package/src/cli/templates/provider/CLAUDE.md.tpl +0 -1
- /package/dist/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/dist/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/fixtures-and-recording/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/health-checks-and-fail-closed/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/normalization-standards/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/pagination-and-counts/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-contract-verification/SKILL.md.tpl +0 -0
- /package/src/cli/templates/provider/{skills → .agents/skills}/upstream-notes/README.md.tpl +0 -0
package/src/define.ts
CHANGED
|
@@ -78,7 +78,7 @@ const VALID_RUNTIMES = ["standard", "shared", "browser"] as const;
|
|
|
78
78
|
const VALID_AUTH_MODES = ["none", "platform-managed", "credentials", "oauth2"] as const;
|
|
79
79
|
const VALID_PROVIDER_ACCESS_VISIBILITIES = ["public", "early_access"] as const;
|
|
80
80
|
const VALID_PROVIDER_PROXY_MODES = ["disabled", "optional", "required"] as const;
|
|
81
|
-
const VALID_PROVIDER_PROXY_PROVIDERS = ["smartproxy", "decodo", "custom"] as const;
|
|
81
|
+
const VALID_PROVIDER_PROXY_PROVIDERS = ["smartproxy", "nodemaven", "decodo", "custom"] as const;
|
|
82
82
|
const VALID_PROVIDER_PROXY_AFFINITIES = [
|
|
83
83
|
"request",
|
|
84
84
|
"operation",
|
|
@@ -371,11 +371,24 @@ function validateProviderProxy(config: {
|
|
|
371
371
|
},
|
|
372
372
|
);
|
|
373
373
|
}
|
|
374
|
-
rejectUnknownFields(proxy, new Set(["mode", "provider", "geo", "session"]), "proxy");
|
|
374
|
+
rejectUnknownFields(proxy, new Set(["mode", "provider", "providers", "geo", "session"]), "proxy");
|
|
375
375
|
assertLiteralField(proxy.mode, "proxy.mode", VALID_PROVIDER_PROXY_MODES, config.id);
|
|
376
376
|
if (proxy.provider !== undefined) {
|
|
377
377
|
assertLiteralField(proxy.provider, "proxy.provider", VALID_PROVIDER_PROXY_PROVIDERS, config.id);
|
|
378
378
|
}
|
|
379
|
+
if (proxy.providers !== undefined) {
|
|
380
|
+
if (!Array.isArray(proxy.providers) || proxy.providers.length === 0) {
|
|
381
|
+
throw new ValidationError(
|
|
382
|
+
`Provider "${config.id}" has invalid proxy.providers: must be a non-empty array of proxy vendors.`,
|
|
383
|
+
{
|
|
384
|
+
fix: `Use proxy.providers: ["smartproxy", "nodemaven"] to declare an ordered fallback chain.`,
|
|
385
|
+
},
|
|
386
|
+
);
|
|
387
|
+
}
|
|
388
|
+
for (const vendor of proxy.providers) {
|
|
389
|
+
assertLiteralField(vendor, "proxy.providers[]", VALID_PROVIDER_PROXY_PROVIDERS, config.id);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
379
392
|
if (proxy.geo !== undefined) {
|
|
380
393
|
if (!proxy.geo || typeof proxy.geo !== "object" || Array.isArray(proxy.geo)) {
|
|
381
394
|
throw new ValidationError(
|
|
@@ -433,7 +446,16 @@ function validateProviderProxy(config: {
|
|
|
433
446
|
);
|
|
434
447
|
}
|
|
435
448
|
}
|
|
436
|
-
|
|
449
|
+
// Smartproxy uses a provider-declared secret; when it is a required-mode
|
|
450
|
+
// vendor (singular or in the chain) the app key must be declared so a missing
|
|
451
|
+
// credential fails at build/validation time, not during a live outage.
|
|
452
|
+
const vendorChain =
|
|
453
|
+
proxy.providers && proxy.providers.length > 0
|
|
454
|
+
? proxy.providers
|
|
455
|
+
: proxy.provider
|
|
456
|
+
? [proxy.provider]
|
|
457
|
+
: [];
|
|
458
|
+
if (proxy.mode === "required" && vendorChain.includes("smartproxy")) {
|
|
437
459
|
const hasSmartproxySecret = config.secrets?.some(
|
|
438
460
|
(secret) => secret.name === SMARTPROXY_APP_KEY_SECRET && secret.required !== false,
|
|
439
461
|
);
|
|
@@ -446,6 +468,16 @@ function validateProviderProxy(config: {
|
|
|
446
468
|
);
|
|
447
469
|
}
|
|
448
470
|
}
|
|
471
|
+
// `decodo`/`custom` are deprecated vendor values (string-union members, so the
|
|
472
|
+
// @deprecated symbol gate can't catch them — warn at validation time instead).
|
|
473
|
+
const deprecatedVendors = vendorChain.filter(
|
|
474
|
+
(vendor) => vendor === "decodo" || vendor === "custom",
|
|
475
|
+
);
|
|
476
|
+
if (deprecatedVendors.length > 0) {
|
|
477
|
+
console.warn(
|
|
478
|
+
`[provider-sdk] Provider "${config.id}" uses deprecated proxy vendor(s): ${deprecatedVendors.join(", ")}. Use "smartproxy"/"nodemaven", or the APIFUSE__PROXY__URL bring-your-own escape hatch.`,
|
|
479
|
+
);
|
|
480
|
+
}
|
|
449
481
|
}
|
|
450
482
|
|
|
451
483
|
function validateProviderStt(config: { id: string; stt?: ProviderSttConfig }): void {
|
package/src/index.ts
CHANGED
|
@@ -74,6 +74,11 @@ export {
|
|
|
74
74
|
} from "./runtime/instrumentation.js";
|
|
75
75
|
export { type PrevalidateResult, prevalidate } from "./runtime/prevalidate.js";
|
|
76
76
|
export { getProviderBaseUrl } from "./runtime/provider.js";
|
|
77
|
+
export {
|
|
78
|
+
assertRequiredSecretsPresent,
|
|
79
|
+
listMissingRequiredSecrets,
|
|
80
|
+
MISSING_SECRET_CODE,
|
|
81
|
+
} from "./runtime/secrets.js";
|
|
77
82
|
export {
|
|
78
83
|
createUnsupportedProviderRuntimeState,
|
|
79
84
|
UnsupportedProviderStateError,
|
package/src/runtime/executor.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { isSessionExpiredError, ProviderError, SessionExpiredError } from "../errors.js";
|
|
2
2
|
import { parseSchema } from "../schema.js";
|
|
3
3
|
import type { ProviderContext, ProviderDefinition } from "../types.js";
|
|
4
|
+
import { assertRequiredSecretsPresent } from "./secrets.js";
|
|
4
5
|
|
|
5
6
|
export function isStreamingOperation(provider: ProviderDefinition, operationId: string): boolean {
|
|
6
7
|
const kind = provider.operations[operationId]?.transport?.kind ?? "json";
|
|
@@ -34,6 +35,13 @@ export async function executeOperation(
|
|
|
34
35
|
});
|
|
35
36
|
}
|
|
36
37
|
|
|
38
|
+
// SDK-owned secret presence gate (single source of truth): declared
|
|
39
|
+
// `required: true` secrets are validated here, before input parsing and the
|
|
40
|
+
// handler, so every invocation path (serve /v1, self-test probes, perf,
|
|
41
|
+
// record) fails with the same structured MISSING_SECRET error instead of a
|
|
42
|
+
// handler-specific crash. Providers must not re-check presence locally.
|
|
43
|
+
assertRequiredSecretsPresent(provider, ctx.env);
|
|
44
|
+
|
|
37
45
|
const validatedInput = await parseSchema(
|
|
38
46
|
operation.input,
|
|
39
47
|
input,
|
package/src/runtime/http.ts
CHANGED
|
@@ -256,6 +256,9 @@ async function resolveNativeProxy(
|
|
|
256
256
|
baseProxyAttempt: clientOptions.proxyAttempt,
|
|
257
257
|
retryAttemptOffset: proxyAttemptOffset,
|
|
258
258
|
}),
|
|
259
|
+
// Bun's native fetch proxy option tunnels HTTP CONNECT only; SOCKS5 is not
|
|
260
|
+
// supported here, so a socks5 policy fails loudly rather than downgrading.
|
|
261
|
+
transportProtocols: ["http"],
|
|
259
262
|
telemetry: clientOptions.telemetry,
|
|
260
263
|
});
|
|
261
264
|
if (resolvedProxy.shouldWarn) {
|
|
@@ -19,10 +19,18 @@ const PROXY_AUTH_IP_DENIED_PATTERN =
|
|
|
19
19
|
/\b(?:source|egress|client)\s+ip\b.{0,120}\b(?:deny|denied|unauthori[sz]ed|not\s+authori[sz]ed|white\s*list|allow\s*list)\b|\b(?:white\s*list|allow\s*list)\b.{0,120}\b(?:source|egress|client)\s+ip\b/i;
|
|
20
20
|
const PROXY_EDGE_AUTH_REJECTED_PATTERN =
|
|
21
21
|
/\bauth\s+ip\s+err\b|\bproxy\b.{0,120}\bauth(?:entication)?\b.{0,120}\b(?:reject(?:ed)?|fail(?:ed)?|invalid|den(?:y|ied)|unauthori[sz]ed)\b|\bauth(?:entication)?\b.{0,120}\b(?:reject(?:ed)?|fail(?:ed)?|invalid|den(?:y|ied)|unauthori[sz]ed)\b.{0,120}\bproxy\b/i;
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
22
|
+
// Vendor host tokens that can appear in upstream error strings. Adding a proxy
|
|
23
|
+
// vendor updates every classifier below in one place. `proxy` is the generic
|
|
24
|
+
// fallback so vendor-agnostic messages still classify.
|
|
25
|
+
const PROXY_VENDOR_ALTERNATION = "smartproxy|nodemaven|proxy";
|
|
26
|
+
const PROXY_POOL_STALE_MESSAGE_PATTERN = new RegExp(
|
|
27
|
+
`\\bproxy\\b.{0,120}\\b(?:pool|lease|expired|unavailable|exhausted|non[\\s-]?200\\s+code:\\s*(?:509|512))\\b|\\bnon[\\s-]?200\\s+code:\\s*(?:509|512)\\b.{0,120}\\bproxy\\b|\\b(?:${PROXY_VENDOR_ALTERNATION})\\b.{0,120}\\b(?:509|512)\\b`,
|
|
28
|
+
"i",
|
|
29
|
+
);
|
|
30
|
+
const PROXY_EDGE_TLS_REJECTED_MESSAGE_PATTERN = new RegExp(
|
|
31
|
+
`\\b(?:${PROXY_VENDOR_ALTERNATION})\\b.{0,160}\\b(?:495|ssl|tls|cert(?:ificate)?|handshake|edge|connect|non[\\s-]?200)\\b|\\b(?:495|ssl|tls|cert(?:ificate)?|handshake|edge|connect|non[\\s-]?200)\\b.{0,160}\\b(?:${PROXY_VENDOR_ALTERNATION})\\b`,
|
|
32
|
+
"i",
|
|
33
|
+
);
|
|
26
34
|
|
|
27
35
|
export function isProxyAuthIpDeniedMessage(message: string): boolean {
|
|
28
36
|
return PROXY_AUTH_IP_DENIED_PATTERN.test(message);
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
import type { ProviderProxyPolicy } from "../types.js";
|
|
4
|
+
|
|
5
|
+
export const NODEMAVEN_USERNAME_ENV = "APIFUSE__PROXY__NODEMAVEN_USERNAME";
|
|
6
|
+
export const NODEMAVEN_PASSWORD_ENV = "APIFUSE__PROXY__NODEMAVEN_PASSWORD";
|
|
7
|
+
export const NODEMAVEN_FILTER_ENV = "APIFUSE__PROXY__NODEMAVEN_FILTER";
|
|
8
|
+
|
|
9
|
+
export const NODEMAVEN_GATEWAY_HOST = "gate.nodemaven.com";
|
|
10
|
+
|
|
11
|
+
/** Both schemes tunnel bytes end-to-end, preserving the client TLS handshake. */
|
|
12
|
+
export type ProxyProtocol = "http" | "socks5";
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* NodeMaven's fastest protocol: HTTP CONNECT. Benchmarks (KR, cold + warm)
|
|
16
|
+
* showed socks5 through the gateway adds ~500ms per request over http, so
|
|
17
|
+
* NodeMaven never defaults to socks5.
|
|
18
|
+
*/
|
|
19
|
+
export const NODEMAVEN_DEFAULT_PROTOCOL: ProxyProtocol = "http";
|
|
20
|
+
|
|
21
|
+
/** NodeMaven gateway port ranges per protocol (docs: HTTP 8080-9080, SOCKS5 1080-2080). */
|
|
22
|
+
const NODEMAVEN_PORTS: Record<ProxyProtocol, { min: number; max: number }> = {
|
|
23
|
+
http: { min: 8080, max: 9080 },
|
|
24
|
+
socks5: { min: 1080, max: 2080 },
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const NODEMAVEN_FILTERS = new Set(["medium", "high"]);
|
|
28
|
+
const DEFAULT_NODEMAVEN_FILTER = "medium";
|
|
29
|
+
const DEFAULT_NODEMAVEN_POOL_SIZE = 20;
|
|
30
|
+
const NODEMAVEN_MAX_POOL_SIZE = 50;
|
|
31
|
+
/** NodeMaven sticky sessions persist up to 24h server-side, keyed by the sid. */
|
|
32
|
+
const NODEMAVEN_MAX_LIFETIME_MINUTES = 1440;
|
|
33
|
+
const SID_LENGTH = 10;
|
|
34
|
+
|
|
35
|
+
export function hasNodemavenCredentials(): boolean {
|
|
36
|
+
return Boolean(readNodemavenUsername() && readNodemavenPassword());
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function readNodemavenUsername(): string | undefined {
|
|
40
|
+
return process.env[NODEMAVEN_USERNAME_ENV]?.trim() || undefined;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function readNodemavenPassword(): string | undefined {
|
|
44
|
+
return process.env[NODEMAVEN_PASSWORD_ENV]?.trim() || undefined;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function resolveNodemavenFilter(): string {
|
|
48
|
+
const raw = process.env[NODEMAVEN_FILTER_ENV]?.trim().toLowerCase();
|
|
49
|
+
if (!raw) return DEFAULT_NODEMAVEN_FILTER;
|
|
50
|
+
if (!NODEMAVEN_FILTERS.has(raw)) {
|
|
51
|
+
throw new Error(`${NODEMAVEN_FILTER_ENV} must be "medium" or "high"`);
|
|
52
|
+
}
|
|
53
|
+
return raw;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function nodemavenPoolSize(policy: ProviderProxyPolicy): number {
|
|
57
|
+
return Math.min(
|
|
58
|
+
NODEMAVEN_MAX_POOL_SIZE,
|
|
59
|
+
Math.max(1, Math.floor(policy.session?.poolSize ?? DEFAULT_NODEMAVEN_POOL_SIZE)),
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function nodemavenLifetimeMinutes(policy: ProviderProxyPolicy): number {
|
|
64
|
+
const configured = policy.session?.lifetimeMinutes;
|
|
65
|
+
if (typeof configured !== "number" || !Number.isFinite(configured) || configured <= 0) {
|
|
66
|
+
return NODEMAVEN_MAX_LIFETIME_MINUTES;
|
|
67
|
+
}
|
|
68
|
+
return Math.min(NODEMAVEN_MAX_LIFETIME_MINUTES, Math.max(1, Math.floor(configured)));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** NodeMaven username tokens accept `[a-z0-9]`; slugify geo values to that set. */
|
|
72
|
+
function slugifyGeo(value: string | undefined): string | undefined {
|
|
73
|
+
if (!value) return undefined;
|
|
74
|
+
const slug = value
|
|
75
|
+
.trim()
|
|
76
|
+
.toLowerCase()
|
|
77
|
+
.replace(/[^a-z0-9]+/g, "");
|
|
78
|
+
return slug || undefined;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function isStickyAffinity(policy: ProviderProxyPolicy): boolean {
|
|
82
|
+
return (policy.session?.affinity ?? "request") !== "request";
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* A sticky sid is deterministic from the affinity key so every process serving
|
|
87
|
+
* the same connection derives the same egress IP without shared storage. A
|
|
88
|
+
* rotating sid is random per call (a fresh egress IP per request).
|
|
89
|
+
*/
|
|
90
|
+
function deriveSid(
|
|
91
|
+
policy: ProviderProxyPolicy,
|
|
92
|
+
affinityKey: string | undefined,
|
|
93
|
+
poolIndex: number,
|
|
94
|
+
refreshEpoch: number,
|
|
95
|
+
): string {
|
|
96
|
+
if (!isStickyAffinity(policy) || !affinityKey) {
|
|
97
|
+
return randomBytes(SID_LENGTH).toString("hex").slice(0, SID_LENGTH);
|
|
98
|
+
}
|
|
99
|
+
const digest = createHash("sha256")
|
|
100
|
+
.update(`${affinityKey}:${refreshEpoch}:${poolIndex}`)
|
|
101
|
+
.digest("hex");
|
|
102
|
+
// hex digits are a subset of the allowed [a-z0-9] sid charset.
|
|
103
|
+
return digest.slice(0, SID_LENGTH);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function selectPort(protocol: ProxyProtocol, sid: string, poolIndex: number): number {
|
|
107
|
+
const { min, max } = NODEMAVEN_PORTS[protocol];
|
|
108
|
+
const span = max - min + 1;
|
|
109
|
+
const hashInt = Number.parseInt(
|
|
110
|
+
createHash("sha256").update(`${sid}:${poolIndex}`).digest("hex").slice(0, 8),
|
|
111
|
+
16,
|
|
112
|
+
);
|
|
113
|
+
return min + (hashInt % span);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export type NodemavenSynthesisInput = {
|
|
117
|
+
policy: ProviderProxyPolicy;
|
|
118
|
+
affinityKey: string | undefined;
|
|
119
|
+
protocol: ProxyProtocol;
|
|
120
|
+
poolIndex: number;
|
|
121
|
+
refreshEpoch: number;
|
|
122
|
+
/** ISO 3166-1 alpha-2, already resolved by the caller (falls back to env). */
|
|
123
|
+
country?: string;
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
export type NodemavenSynthesis = {
|
|
127
|
+
url: string;
|
|
128
|
+
protocol: ProxyProtocol;
|
|
129
|
+
diagnostics: Record<string, string | number | boolean>;
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Synthesize a NodeMaven gateway proxy URL locally from static credentials.
|
|
134
|
+
* There is no allocation API — geo/session are encoded in the username.
|
|
135
|
+
*/
|
|
136
|
+
export function synthesizeNodemavenProxy(input: NodemavenSynthesisInput): NodemavenSynthesis {
|
|
137
|
+
const username = readNodemavenUsername();
|
|
138
|
+
const password = readNodemavenPassword();
|
|
139
|
+
if (!username || !password) {
|
|
140
|
+
throw new Error(
|
|
141
|
+
`NodeMaven credentials missing: set ${NODEMAVEN_USERNAME_ENV} and ${NODEMAVEN_PASSWORD_ENV}.`,
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const filter = resolveNodemavenFilter();
|
|
146
|
+
const sid = deriveSid(input.policy, input.affinityKey, input.poolIndex, input.refreshEpoch);
|
|
147
|
+
const port = selectPort(input.protocol, sid, input.poolIndex);
|
|
148
|
+
const lifetimeMinutes = nodemavenLifetimeMinutes(input.policy);
|
|
149
|
+
|
|
150
|
+
const country = slugifyGeo(input.country ?? input.policy.geo?.country);
|
|
151
|
+
const region = slugifyGeo(input.policy.geo?.subdivision);
|
|
152
|
+
const city = slugifyGeo(input.policy.geo?.city);
|
|
153
|
+
|
|
154
|
+
const tokens = [username];
|
|
155
|
+
if (country) tokens.push("country", country);
|
|
156
|
+
if (region) tokens.push("region", region);
|
|
157
|
+
if (city) tokens.push("city", city);
|
|
158
|
+
tokens.push("sid", sid);
|
|
159
|
+
tokens.push("filter", filter);
|
|
160
|
+
tokens.push("ipv4", "true");
|
|
161
|
+
const proxyUsername = tokens.join("-");
|
|
162
|
+
|
|
163
|
+
// Username tokens are [a-z0-9-] only, which survive URL encoding unchanged.
|
|
164
|
+
const url = `${input.protocol}://${proxyUsername}:${encodeURIComponent(password)}@${NODEMAVEN_GATEWAY_HOST}:${port}`;
|
|
165
|
+
|
|
166
|
+
return {
|
|
167
|
+
url,
|
|
168
|
+
protocol: input.protocol,
|
|
169
|
+
diagnostics: {
|
|
170
|
+
vendor: "nodemaven",
|
|
171
|
+
protocol: input.protocol,
|
|
172
|
+
sticky: isStickyAffinity(input.policy),
|
|
173
|
+
filter,
|
|
174
|
+
lifetimeMinutes,
|
|
175
|
+
...(country ? { country } : {}),
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
}
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import type {
|
|
2
2
|
ProxyAttemptTelemetryEvent,
|
|
3
3
|
ProxyCacheStatus,
|
|
4
|
+
ProxyProtocol,
|
|
4
5
|
ProxyResolutionTelemetryEvent,
|
|
5
6
|
ProxyTelemetrySink,
|
|
7
|
+
ProxyVendorFailoverTelemetryEvent,
|
|
8
|
+
ProxyVendorName,
|
|
6
9
|
SmartproxyAllocatorBodyClass,
|
|
7
10
|
} from "../config/loader.js";
|
|
8
11
|
|
|
@@ -11,7 +14,8 @@ export const PROVIDER_TELEMETRY_HEADER = "X-ApiFuse-Provider-Telemetry";
|
|
|
11
14
|
type ProviderTelemetryHeader = {
|
|
12
15
|
v: 1;
|
|
13
16
|
proxy?: {
|
|
14
|
-
provider:
|
|
17
|
+
provider: ProxyVendorName;
|
|
18
|
+
protocol?: ProxyProtocol;
|
|
15
19
|
cacheStatus: ProxyCacheStatus;
|
|
16
20
|
cacheHit: boolean;
|
|
17
21
|
resolutionMs: number;
|
|
@@ -27,6 +31,10 @@ type ProviderTelemetryHeader = {
|
|
|
27
31
|
attempts: number;
|
|
28
32
|
refreshes?: number;
|
|
29
33
|
attemptSamples?: CompactProxyAttemptSample[];
|
|
34
|
+
/** Distinct vendors attempted across the resolution chain, in order seen. */
|
|
35
|
+
vendors?: ProxyVendorName[];
|
|
36
|
+
/** Cross-vendor failover events (bounded). */
|
|
37
|
+
failovers?: CompactVendorFailoverSample[];
|
|
30
38
|
};
|
|
31
39
|
};
|
|
32
40
|
|
|
@@ -41,8 +49,17 @@ type CompactProxyAttemptSample = {
|
|
|
41
49
|
d?: number;
|
|
42
50
|
};
|
|
43
51
|
|
|
52
|
+
type CompactVendorFailoverSample = {
|
|
53
|
+
v: ProxyVendorName;
|
|
54
|
+
nx?: ProxyVendorName;
|
|
55
|
+
p: "resolution" | "transport";
|
|
56
|
+
r: ProxyVendorFailoverTelemetryEvent["reason"];
|
|
57
|
+
a?: number;
|
|
58
|
+
};
|
|
59
|
+
|
|
44
60
|
const MAX_HEADER_BYTES = 4_096;
|
|
45
61
|
const MAX_PROXY_ATTEMPT_SAMPLES = 24;
|
|
62
|
+
const MAX_PROXY_FAILOVER_SAMPLES = 12;
|
|
46
63
|
|
|
47
64
|
const CACHE_STATUS_SEVERITY: Record<ProxyCacheStatus, number> = {
|
|
48
65
|
disabled: 0,
|
|
@@ -76,10 +93,12 @@ function encodeBase64Url(value: string): string {
|
|
|
76
93
|
export class ProxyTelemetryCollector implements ProxyTelemetrySink {
|
|
77
94
|
#events: ProxyResolutionTelemetryEvent[] = [];
|
|
78
95
|
#attempts: ProxyAttemptTelemetryEvent[] = [];
|
|
96
|
+
#failovers: ProxyVendorFailoverTelemetryEvent[] = [];
|
|
79
97
|
|
|
80
98
|
recordProxyResolution(event: ProxyResolutionTelemetryEvent): void {
|
|
81
99
|
this.#events.push({
|
|
82
|
-
provider:
|
|
100
|
+
provider: event.provider,
|
|
101
|
+
...(event.protocol ? { protocol: event.protocol } : {}),
|
|
83
102
|
cacheStatus: event.cacheStatus,
|
|
84
103
|
cacheHit: event.cacheHit,
|
|
85
104
|
resolutionMs: Math.max(0, Math.floor(event.resolutionMs)),
|
|
@@ -112,10 +131,21 @@ export class ProxyTelemetryCollector implements ProxyTelemetrySink {
|
|
|
112
131
|
});
|
|
113
132
|
}
|
|
114
133
|
|
|
134
|
+
recordProxyVendorFailover(event: ProxyVendorFailoverTelemetryEvent): void {
|
|
135
|
+
if (this.#failovers.length >= MAX_PROXY_FAILOVER_SAMPLES) return;
|
|
136
|
+
this.#failovers.push({
|
|
137
|
+
vendor: event.vendor,
|
|
138
|
+
...(event.nextVendor ? { nextVendor: event.nextVendor } : {}),
|
|
139
|
+
phase: event.phase,
|
|
140
|
+
reason: event.reason,
|
|
141
|
+
...(event.attempt === undefined ? {} : { attempt: Math.max(0, Math.floor(event.attempt)) }),
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
|
|
115
145
|
recordProxyAttempt(event: ProxyAttemptTelemetryEvent): void {
|
|
116
146
|
if (this.#attempts.length >= MAX_PROXY_ATTEMPT_SAMPLES) return;
|
|
117
147
|
this.#attempts.push({
|
|
118
|
-
provider:
|
|
148
|
+
provider: event.provider,
|
|
119
149
|
attempt: Math.max(1, Math.floor(event.attempt || 1)),
|
|
120
150
|
...(event.poolIndex === undefined
|
|
121
151
|
? {}
|
|
@@ -134,9 +164,17 @@ export class ProxyTelemetryCollector implements ProxyTelemetrySink {
|
|
|
134
164
|
const [first, ...rest] = this.#events;
|
|
135
165
|
if (!first) return undefined;
|
|
136
166
|
|
|
167
|
+
// The serving vendor/protocol is the last recorded resolution (a failed
|
|
168
|
+
// vendor records first, the vendor that served records last).
|
|
169
|
+
const serving = this.#events[this.#events.length - 1] ?? first;
|
|
170
|
+
const vendors: ProxyVendorName[] = [];
|
|
171
|
+
for (const event of this.#events) {
|
|
172
|
+
if (!vendors.includes(event.provider)) vendors.push(event.provider);
|
|
173
|
+
}
|
|
174
|
+
|
|
137
175
|
const aggregate = rest.reduce<ProxyResolutionTelemetryEvent>(
|
|
138
176
|
(acc, event) => ({
|
|
139
|
-
provider:
|
|
177
|
+
provider: event.provider,
|
|
140
178
|
cacheStatus: worseStatus(acc.cacheStatus, event.cacheStatus),
|
|
141
179
|
cacheHit: acc.cacheHit && event.cacheHit,
|
|
142
180
|
resolutionMs: acc.resolutionMs + event.resolutionMs,
|
|
@@ -157,7 +195,8 @@ export class ProxyTelemetryCollector implements ProxyTelemetrySink {
|
|
|
157
195
|
const payload: ProviderTelemetryHeader = {
|
|
158
196
|
v: 1,
|
|
159
197
|
proxy: {
|
|
160
|
-
provider:
|
|
198
|
+
provider: serving.provider,
|
|
199
|
+
...(serving.protocol ? { protocol: serving.protocol } : {}),
|
|
161
200
|
cacheStatus: aggregate.cacheStatus,
|
|
162
201
|
cacheHit: aggregate.cacheHit,
|
|
163
202
|
resolutionMs: aggregate.resolutionMs,
|
|
@@ -194,6 +233,18 @@ export class ProxyTelemetryCollector implements ProxyTelemetrySink {
|
|
|
194
233
|
})),
|
|
195
234
|
}
|
|
196
235
|
: {}),
|
|
236
|
+
...(vendors.length > 1 ? { vendors } : {}),
|
|
237
|
+
...(this.#failovers.length > 0
|
|
238
|
+
? {
|
|
239
|
+
failovers: this.#failovers.map((failover) => ({
|
|
240
|
+
v: failover.vendor,
|
|
241
|
+
...(failover.nextVendor ? { nx: failover.nextVendor } : {}),
|
|
242
|
+
p: failover.phase,
|
|
243
|
+
r: failover.reason,
|
|
244
|
+
...(failover.attempt === undefined ? {} : { a: failover.attempt }),
|
|
245
|
+
})),
|
|
246
|
+
}
|
|
247
|
+
: {}),
|
|
197
248
|
},
|
|
198
249
|
};
|
|
199
250
|
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { ProviderSecretError } from "../errors.js";
|
|
2
|
+
import type { EnvContext, ProviderDefinition } from "../types.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Canonical error code for a declared-but-unprovisioned provider secret.
|
|
6
|
+
*
|
|
7
|
+
* The SDK is the single source of truth for env/secret presence validation:
|
|
8
|
+
* providers declare secrets in `defineProvider({ secrets: [...] })` and the
|
|
9
|
+
* runtime enforces presence before any handler or auth-flow code runs.
|
|
10
|
+
* Provider-local presence guards (requireServiceKey/requireApiKey style) are a
|
|
11
|
+
* deprecated antipattern — see the `sdk-owned-secret-presence` submit-check
|
|
12
|
+
* rule.
|
|
13
|
+
*/
|
|
14
|
+
export const MISSING_SECRET_CODE = "MISSING_SECRET";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Names of declared `required: true` secrets whose env values are unset or
|
|
18
|
+
* whitespace-only. Whitespace-only values count as missing for parity with the
|
|
19
|
+
* `.trim()` guards well-built providers used before the SDK owned this check —
|
|
20
|
+
* a blank value provisioned by a broken secret pipeline must not pass the gate.
|
|
21
|
+
*/
|
|
22
|
+
export function listMissingRequiredSecrets(
|
|
23
|
+
provider: ProviderDefinition,
|
|
24
|
+
env: EnvContext,
|
|
25
|
+
): string[] {
|
|
26
|
+
const missing: string[] = [];
|
|
27
|
+
for (const secret of provider.secrets ?? []) {
|
|
28
|
+
if (secret.required !== true) {
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
const value = env.get(secret.name);
|
|
32
|
+
if (value === undefined || value.trim() === "") {
|
|
33
|
+
missing.push(secret.name);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return missing;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Throws the canonical structured missing-secret error when any declared
|
|
41
|
+
* `required: true` secret is absent. All missing names are reported in a
|
|
42
|
+
* single error so operators can provision the full set in one pass instead of
|
|
43
|
+
* discovering them one deploy at a time (the 2026-07-22 unprovisioned-secret
|
|
44
|
+
* incident failure mode).
|
|
45
|
+
*/
|
|
46
|
+
export function assertRequiredSecretsPresent(
|
|
47
|
+
provider: ProviderDefinition,
|
|
48
|
+
env: EnvContext,
|
|
49
|
+
): void {
|
|
50
|
+
const missing = listMissingRequiredSecrets(provider, env);
|
|
51
|
+
if (missing.length === 0) {
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
const names = missing.join(", ");
|
|
55
|
+
throw new ProviderSecretError(
|
|
56
|
+
`Missing required provider secret${missing.length > 1 ? "s" : ""}: ${names}`,
|
|
57
|
+
{
|
|
58
|
+
code: MISSING_SECRET_CODE,
|
|
59
|
+
category: "credential_unavailable",
|
|
60
|
+
retryable: false,
|
|
61
|
+
fix: `Provision ${names} in the provider environment (e.g. Doppler). Declared in defineProvider({ secrets: [...] }).`,
|
|
62
|
+
},
|
|
63
|
+
);
|
|
64
|
+
}
|
package/src/runtime/stealth.ts
CHANGED
|
@@ -2,13 +2,15 @@ import { createHash } from "node:crypto";
|
|
|
2
2
|
import type { Browser, ImpitOptions, ImpitResponse, RequestInit } from "impit";
|
|
3
3
|
import { Impit } from "impit";
|
|
4
4
|
|
|
5
|
-
import type { ProxyResolutionOptions } from "../config/loader.js";
|
|
5
|
+
import type { ProxyResolutionOptions, ProxyVendorName } from "../config/loader.js";
|
|
6
6
|
import {
|
|
7
7
|
DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
8
8
|
invalidateProxyResolutionCacheAsync,
|
|
9
9
|
ProxyResolutionError,
|
|
10
|
+
resolvePolicyProxyPoolSpan,
|
|
10
11
|
resolveProxyConfigAsync,
|
|
11
12
|
SMARTPROXY_MAX_POOL_SIZE,
|
|
13
|
+
vendorFromResolvedSource,
|
|
12
14
|
} from "../config/loader.js";
|
|
13
15
|
import { SDKError, TransportError } from "../errors.js";
|
|
14
16
|
import { getStealthProfile } from "../stealth/profiles.js";
|
|
@@ -51,7 +53,11 @@ const DEFAULT_PROFILE = "chrome-146";
|
|
|
51
53
|
const MISSING_PROXY_WARNING =
|
|
52
54
|
"[provider-sdk] Provider requested proxy routing, but no proxy URL was configured. Continuing without proxy.";
|
|
53
55
|
|
|
54
|
-
|
|
56
|
+
/**
|
|
57
|
+
* Upper bound on attempts across a multi-vendor chain, so a two-vendor chain can
|
|
58
|
+
* exhaust each vendor's pool before failing over and finally throwing.
|
|
59
|
+
*/
|
|
60
|
+
const MAX_POLICY_PROXY_TOTAL_ATTEMPTS = SMARTPROXY_MAX_POOL_SIZE * 2;
|
|
55
61
|
const MAX_POLICY_PROXY_POOL_REFRESHES = 1;
|
|
56
62
|
const PROXY_CONNECT_FAILURE_CODE = "proxy_connect_failed";
|
|
57
63
|
const PROXY_CONNECT_FAILURE_BODY_PATTERN =
|
|
@@ -426,6 +432,7 @@ type ResolvedAttemptProxy = {
|
|
|
426
432
|
url?: string;
|
|
427
433
|
poolIndex?: number;
|
|
428
434
|
proxyHash?: string;
|
|
435
|
+
vendor?: ProxyVendorName;
|
|
429
436
|
};
|
|
430
437
|
|
|
431
438
|
function proxyPoolIndexFromDiagnostics(
|
|
@@ -615,6 +622,7 @@ function createSessionFetcher(
|
|
|
615
622
|
async function resolveRequestProxy(
|
|
616
623
|
options?: StealthFetchOptions,
|
|
617
624
|
proxyAttempt?: number,
|
|
625
|
+
refreshEpoch?: number,
|
|
618
626
|
): Promise<ResolvedAttemptProxy> {
|
|
619
627
|
const resolvedProxy = await resolveProxyConfigAsync({
|
|
620
628
|
proxy: options?.proxy ?? clientOptions.proxy,
|
|
@@ -626,6 +634,10 @@ function createSessionFetcher(
|
|
|
626
634
|
proxyAttemptOffset: options?.proxyAttemptOffset,
|
|
627
635
|
retryAttemptOffset: proxyAttempt,
|
|
628
636
|
}),
|
|
637
|
+
// The impit stealth transport tunnels both HTTP CONNECT and SOCKS5,
|
|
638
|
+
// preserving the client TLS fingerprint end-to-end.
|
|
639
|
+
transportProtocols: ["http", "socks5"],
|
|
640
|
+
...(refreshEpoch === undefined ? {} : { proxyRefreshEpoch: refreshEpoch }),
|
|
629
641
|
telemetry: clientOptions.telemetry,
|
|
630
642
|
});
|
|
631
643
|
|
|
@@ -638,6 +650,7 @@ function createSessionFetcher(
|
|
|
638
650
|
url: resolvedProxy.url,
|
|
639
651
|
poolIndex: proxyPoolIndexFromDiagnostics(resolvedProxy.diagnostics),
|
|
640
652
|
proxyHash: proxyEndpointHash(resolvedProxy.url),
|
|
653
|
+
vendor: vendorFromResolvedSource(resolvedProxy.source),
|
|
641
654
|
};
|
|
642
655
|
}
|
|
643
656
|
|
|
@@ -662,15 +675,18 @@ function createSessionFetcher(
|
|
|
662
675
|
const hasPolicyProxy = isPolicyManagedProxy(clientOptions);
|
|
663
676
|
const usesPolicyAllocator = hasPolicyProxy && !options.proxy && !clientOptions.proxy;
|
|
664
677
|
const retryAttemptCap = Math.max(1, stealthRetryOptions?.attempts ?? 1);
|
|
678
|
+
// Span the whole vendor chain: successive attempts rotate one vendor's
|
|
679
|
+
// pool, then fail over to the next vendor via the flat attempt index.
|
|
680
|
+
const policyProxy =
|
|
681
|
+
clientOptions.proxyPolicy ??
|
|
682
|
+
(typeof clientOptions.upstream?.proxy === "object"
|
|
683
|
+
? clientOptions.upstream.proxy
|
|
684
|
+
: undefined);
|
|
665
685
|
const policyProxyAttemptCap = Math.max(
|
|
666
686
|
1,
|
|
667
687
|
Math.min(
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
(typeof clientOptions.upstream?.proxy === "object"
|
|
671
|
-
? clientOptions.upstream.proxy.session?.poolSize
|
|
672
|
-
: undefined) ??
|
|
673
|
-
DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
688
|
+
MAX_POLICY_PROXY_TOTAL_ATTEMPTS,
|
|
689
|
+
policyProxy ? resolvePolicyProxyPoolSpan(policyProxy) : DEFAULT_SMARTPROXY_POOL_SIZE,
|
|
674
690
|
),
|
|
675
691
|
);
|
|
676
692
|
const maxAttempts = usesPolicyAllocator ? policyProxyAttemptCap : retryAttemptCap;
|
|
@@ -698,7 +714,7 @@ function createSessionFetcher(
|
|
|
698
714
|
if (attemptRecorded || !proxy) return;
|
|
699
715
|
attemptRecorded = true;
|
|
700
716
|
clientOptions.telemetry?.recordProxyAttempt?.({
|
|
701
|
-
provider: "smartproxy",
|
|
717
|
+
provider: attemptProxy?.vendor ?? "smartproxy",
|
|
702
718
|
attempt: attempt + 1,
|
|
703
719
|
...(attemptProxy?.poolIndex === undefined
|
|
704
720
|
? {}
|
|
@@ -712,7 +728,7 @@ function createSessionFetcher(
|
|
|
712
728
|
};
|
|
713
729
|
try {
|
|
714
730
|
assertNoUnsupportedFingerprintOverrides(options);
|
|
715
|
-
attemptProxy = await resolveRequestProxy(options, attempt);
|
|
731
|
+
attemptProxy = await resolveRequestProxy(options, attempt, refreshAttempt);
|
|
716
732
|
proxy = attemptProxy.url;
|
|
717
733
|
if (proxy && usesPolicyAllocator) {
|
|
718
734
|
if (attemptedProxies.has(proxy)) {
|