@apifuse/provider-sdk 2.2.0-beta.8 → 2.2.0-beta.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/dist/config/loader.d.ts +68 -0
- package/dist/config/loader.js +89 -1
- package/dist/define.js +40 -9
- package/dist/runtime/choice.js +20 -2
- package/dist/runtime/http.js +112 -9
- package/dist/runtime/proxy-nodemaven.d.ts +1 -0
- package/dist/runtime/proxy-nodemaven.js +1 -1
- package/dist/runtime/state.js +12 -1
- package/dist/runtime/stealth.js +47 -16
- package/dist/server/serve.js +11 -0
- package/package.json +1 -1
- package/src/config/loader.ts +109 -0
- package/src/define.ts +45 -14
- package/src/runtime/choice.ts +25 -4
- package/src/runtime/http.ts +131 -11
- package/src/runtime/proxy-nodemaven.ts +1 -1
- package/src/runtime/state.ts +11 -1
- package/src/runtime/stealth.ts +49 -19
- package/src/server/serve.ts +11 -0
package/dist/server/serve.js
CHANGED
|
@@ -312,11 +312,22 @@ function toErrorResponse(error, requestId) {
|
|
|
312
312
|
},
|
|
313
313
|
};
|
|
314
314
|
}
|
|
315
|
+
// A masked internal error MUST NOT be advertised as retryable: without an
|
|
316
|
+
// explicit retryable:false the hub (bori provider-backed engine) defaults 5xx
|
|
317
|
+
// to retryable:true, which turns a deterministic pre-upstream crash into an
|
|
318
|
+
// infinite START->CONTINUE->restart loop (2026-07-22 catchtable reserve RCA).
|
|
319
|
+
// We still refuse to leak message/stack — only the error class name (or the
|
|
320
|
+
// primitive type for non-Error throwables) is surfaced for ops triage.
|
|
315
321
|
return {
|
|
316
322
|
error: {
|
|
317
323
|
code: "internal_error",
|
|
318
324
|
message: "Internal error",
|
|
319
325
|
...(requestId ? { requestId } : {}),
|
|
326
|
+
details: {
|
|
327
|
+
retryable: false,
|
|
328
|
+
category: "internal_error",
|
|
329
|
+
errorClass: error instanceof Error ? error.name : typeof error,
|
|
330
|
+
},
|
|
320
331
|
},
|
|
321
332
|
};
|
|
322
333
|
}
|
package/package.json
CHANGED
package/src/config/loader.ts
CHANGED
|
@@ -7,6 +7,7 @@ import { Redis } from "ioredis";
|
|
|
7
7
|
import type { ProviderProxyPolicy, ProviderProxyProvider, TraceConfig } from "../types.js";
|
|
8
8
|
import {
|
|
9
9
|
NODEMAVEN_DEFAULT_PROTOCOL,
|
|
10
|
+
NODEMAVEN_MAX_POOL_SIZE,
|
|
10
11
|
type ProxyProtocol,
|
|
11
12
|
hasNodemavenCredentials,
|
|
12
13
|
nodemavenPoolSize,
|
|
@@ -822,6 +823,114 @@ export function resolvePolicyProxyPoolSpan(policy: ProviderProxyPolicy): number
|
|
|
822
823
|
return chain.reduce((sum, vendor) => sum + vendorPoolSize(vendor, policy), 0);
|
|
823
824
|
}
|
|
824
825
|
|
|
826
|
+
function vendorMaxPoolSize(vendor: ProxyVendorName): number {
|
|
827
|
+
return vendor === "nodemaven" ? NODEMAVEN_MAX_POOL_SIZE : SMARTPROXY_MAX_POOL_SIZE;
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
/**
|
|
831
|
+
* Absolute upper bound on a chain's attempt span — the sum of each vendor's
|
|
832
|
+
* *maximum* pool size. Unlike `resolvePolicyProxyPoolSpan` (the configured
|
|
833
|
+
* span), this backstop is independent of `session.poolSize`, so it never
|
|
834
|
+
* truncates a legitimately large pool below the point where the flat attempt
|
|
835
|
+
* index would cross into the next vendor (e.g. a 50-slot NodeMaven pool).
|
|
836
|
+
*/
|
|
837
|
+
export function maxPolicyProxyPoolSpan(policy: ProviderProxyPolicy): number {
|
|
838
|
+
const chain = resolveVendorChain(policy);
|
|
839
|
+
if (chain.length === 0) return SMARTPROXY_MAX_POOL_SIZE;
|
|
840
|
+
return chain.reduce((sum, vendor) => sum + vendorMaxPoolSize(vendor), 0);
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
const UNSAFE_TRANSPORT_RETRY_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE", "TRACE"]);
|
|
844
|
+
|
|
845
|
+
/**
|
|
846
|
+
* Transport-retry attempt cap for a policy-managed request. A transport failure
|
|
847
|
+
* rotates the flat attempt index onto the *next* endpoint (and, once the index
|
|
848
|
+
* passes the primary vendor's pool span, the *next vendor*), so the cap must be
|
|
849
|
+
* the chain's full pool span for failover to reach the fallback vendor — the
|
|
850
|
+
* per-endpoint retry budget (default 3) never gets there.
|
|
851
|
+
*
|
|
852
|
+
* The span only widens beyond the caller's retry budget when ALL hold:
|
|
853
|
+
* - the request is policy-allocator managed (not a caller-supplied proxy URL);
|
|
854
|
+
* - the caller did NOT pin an explicit retry policy — `HttpRetryOptions.attempts`
|
|
855
|
+
* is the documented total-attempt ceiling and must be honoured verbatim;
|
|
856
|
+
* - the method is safe/idempotent — an unsafe request must never be duplicated
|
|
857
|
+
* across the pool even if some framework default would allow it;
|
|
858
|
+
* - the policy resolves a non-empty *registry* vendor chain (smartproxy /
|
|
859
|
+
* nodemaven). Static vendors (custom / decodo) and credential-less policies
|
|
860
|
+
* resolve no allocator pool, so every attempt would hit the same endpoint
|
|
861
|
+
* with no possible crossover — they keep the retry budget.
|
|
862
|
+
*
|
|
863
|
+
* The widened cap is bounded by the chain's true maximum span (sum of each
|
|
864
|
+
* vendor's max pool size), so a large NodeMaven pool (≤50) stays reachable and
|
|
865
|
+
* a pathological chain can never spin unbounded.
|
|
866
|
+
*/
|
|
867
|
+
/**
|
|
868
|
+
* True when a policy request is in *implicit chain-rotation* mode: successive
|
|
869
|
+
* transport attempts rotate the flat index across the concatenated vendor pool
|
|
870
|
+
* spans (and, past the primary vendor's span, into the fallback vendor). This is
|
|
871
|
+
* the ONLY mode in which the transport loop widens its attempt cap AND
|
|
872
|
+
* de-duplicates repeated endpoints — the two behaviours must share one predicate
|
|
873
|
+
* so they never diverge. It holds when ALL of the widening conditions hold:
|
|
874
|
+
* - the request is policy-allocator managed (not a caller-supplied proxy URL);
|
|
875
|
+
* - the caller did NOT pin an explicit retry policy — its `attempts` ceiling is
|
|
876
|
+
* the documented contract and must be honoured verbatim against whatever
|
|
877
|
+
* endpoint each attempt resolves (even a repeated one), so no de-duplication;
|
|
878
|
+
* - the method is safe/idempotent — an unsafe request is never duplicated;
|
|
879
|
+
* - the policy resolves a non-empty registry vendor chain (smartproxy /
|
|
880
|
+
* nodemaven). Static vendors (custom / decodo) resolve the same URL every
|
|
881
|
+
* attempt, so there is nothing to rotate or de-duplicate.
|
|
882
|
+
*/
|
|
883
|
+
export function policyRotatesTransportVendorChain(input: {
|
|
884
|
+
policy: ProviderProxyPolicy | undefined;
|
|
885
|
+
usesPolicyAllocator: boolean;
|
|
886
|
+
explicitRetry: boolean;
|
|
887
|
+
method: string;
|
|
888
|
+
}): boolean {
|
|
889
|
+
if (!input.usesPolicyAllocator || !input.policy || input.explicitRetry) {
|
|
890
|
+
return false;
|
|
891
|
+
}
|
|
892
|
+
if (UNSAFE_TRANSPORT_RETRY_METHODS.has(input.method.toUpperCase())) {
|
|
893
|
+
return false;
|
|
894
|
+
}
|
|
895
|
+
return resolveVendorChain(input.policy).length > 0;
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
export function resolvePolicyTransportAttemptCap(input: {
|
|
899
|
+
policy: ProviderProxyPolicy | undefined;
|
|
900
|
+
usesPolicyAllocator: boolean;
|
|
901
|
+
retryAttempts: number;
|
|
902
|
+
explicitRetry: boolean;
|
|
903
|
+
method: string;
|
|
904
|
+
}): number {
|
|
905
|
+
const budget = Math.max(1, Math.floor(input.retryAttempts));
|
|
906
|
+
if (
|
|
907
|
+
!policyRotatesTransportVendorChain({
|
|
908
|
+
policy: input.policy,
|
|
909
|
+
usesPolicyAllocator: input.usesPolicyAllocator,
|
|
910
|
+
explicitRetry: input.explicitRetry,
|
|
911
|
+
method: input.method,
|
|
912
|
+
})
|
|
913
|
+
) {
|
|
914
|
+
return budget;
|
|
915
|
+
}
|
|
916
|
+
const span = Math.min(maxPolicyProxyPoolSpan(input.policy as ProviderProxyPolicy), resolvePolicyProxyPoolSpan(input.policy as ProviderProxyPolicy));
|
|
917
|
+
return Math.max(budget, span);
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
/**
|
|
921
|
+
* A registry vendor chain (smartproxy/nodemaven) resolves a potentially
|
|
922
|
+
* *different* endpoint per flat attempt index, so a transport retry should
|
|
923
|
+
* advance across endpoints and de-duplicate once the chain stops yielding new
|
|
924
|
+
* ones. Static/custom/decodo policies (empty registry chain) resolve the *same*
|
|
925
|
+
* URL every attempt by design — retrying that same endpoint is intended, so the
|
|
926
|
+
* transport loop must not de-duplicate them.
|
|
927
|
+
*/
|
|
928
|
+
export function policyResolvesRegistryVendorChain(
|
|
929
|
+
policy: ProviderProxyPolicy | undefined,
|
|
930
|
+
): boolean {
|
|
931
|
+
return Boolean(policy) && resolveVendorChain(policy as ProviderProxyPolicy).length > 0;
|
|
932
|
+
}
|
|
933
|
+
|
|
825
934
|
/** Map a resolved proxy source label to the vendor that served it. */
|
|
826
935
|
export function vendorFromResolvedSource(
|
|
827
936
|
source: ResolvedProxyConfig["source"],
|
package/src/define.ts
CHANGED
|
@@ -25,6 +25,7 @@ import type {
|
|
|
25
25
|
ProviderDeploymentOverrides,
|
|
26
26
|
ProviderHealthMonitorConfig,
|
|
27
27
|
ProviderProxyConfig,
|
|
28
|
+
ProviderProxyProvider,
|
|
28
29
|
ProviderPublicProfile,
|
|
29
30
|
ProviderReviewed,
|
|
30
31
|
ProviderSecretDeclaration,
|
|
@@ -87,6 +88,20 @@ const VALID_PROVIDER_PROXY_AFFINITIES = [
|
|
|
87
88
|
] as const;
|
|
88
89
|
const VALID_PROVIDER_STT_MODES = ["optional", "required"] as const;
|
|
89
90
|
const SMARTPROXY_APP_KEY_SECRET = "APIFUSE__PROXY__SMARTPROXY_APP_KEY";
|
|
91
|
+
const NODEMAVEN_USERNAME_SECRET = "APIFUSE__PROXY__NODEMAVEN_USERNAME";
|
|
92
|
+
const NODEMAVEN_PASSWORD_SECRET = "APIFUSE__PROXY__NODEMAVEN_PASSWORD";
|
|
93
|
+
// Per-vendor provider-declared credential secrets. A required-mode chain must
|
|
94
|
+
// declare every secret of every credentialed vendor it names, so a missing
|
|
95
|
+
// credential fails at build/validation time rather than during a live outage: a
|
|
96
|
+
// declared-but-uncredentialed fallback leg is a silently dead SPOF, which is
|
|
97
|
+
// exactly the failure class the multi-vendor chain exists to remove. Vendors
|
|
98
|
+
// absent from this map (e.g. `custom`/`decodo`, whose credentials come from the
|
|
99
|
+
// `APIFUSE__PROXY__URL` bring-your-own escape hatch, not provider secrets) impose
|
|
100
|
+
// no declaration requirement.
|
|
101
|
+
const VENDOR_REQUIRED_SECRETS: Partial<Record<ProviderProxyProvider, readonly string[]>> = {
|
|
102
|
+
smartproxy: [SMARTPROXY_APP_KEY_SECRET],
|
|
103
|
+
nodemaven: [NODEMAVEN_USERNAME_SECRET, NODEMAVEN_PASSWORD_SECRET],
|
|
104
|
+
};
|
|
90
105
|
const RESERVED_OPERATION_IDS = new Set(["auth", "health"]);
|
|
91
106
|
const MCP_TOOL_NAME_REGEX = /^[A-Za-z][A-Za-z0-9_]{0,127}$/;
|
|
92
107
|
const VALID_OPERATION_RISK_CLASSES = ["read", "write", "destructive", "external-send"] as const;
|
|
@@ -446,26 +461,42 @@ function validateProviderProxy(config: {
|
|
|
446
461
|
);
|
|
447
462
|
}
|
|
448
463
|
}
|
|
449
|
-
//
|
|
450
|
-
//
|
|
451
|
-
//
|
|
464
|
+
// Every credentialed vendor in a required-mode chain must declare its
|
|
465
|
+
// provider secret(s) so a missing credential fails at build/validation time,
|
|
466
|
+
// not during a live outage. This covers the fallback legs too (not just the
|
|
467
|
+
// first vendor): a declared-but-uncredentialed nodemaven fallback would leave
|
|
468
|
+
// the chain silently down to a single vendor, reintroducing the SPOF the chain
|
|
469
|
+
// removes.
|
|
452
470
|
const vendorChain =
|
|
453
471
|
proxy.providers && proxy.providers.length > 0
|
|
454
472
|
? proxy.providers
|
|
455
473
|
: proxy.provider
|
|
456
474
|
? [proxy.provider]
|
|
457
475
|
: [];
|
|
458
|
-
if (proxy.mode === "required"
|
|
459
|
-
const
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
476
|
+
if (proxy.mode === "required") {
|
|
477
|
+
for (const vendor of vendorChain) {
|
|
478
|
+
const requiredSecrets = VENDOR_REQUIRED_SECRETS[vendor];
|
|
479
|
+
if (!requiredSecrets) continue;
|
|
480
|
+
for (const secretName of requiredSecrets) {
|
|
481
|
+
// Match the canonical runtime gate (assertRequiredSecretsPresent /
|
|
482
|
+
// listMissingRequiredSecrets), which enforces only `required === true`
|
|
483
|
+
// declarations. A declaration that omits `required` (defaulting to
|
|
484
|
+
// optional) is skipped at runtime, so accepting it here would pass
|
|
485
|
+
// validation while leaving the credential unenforced until proxy
|
|
486
|
+
// resolution during a live request — the fail-open gap this check exists
|
|
487
|
+
// to close.
|
|
488
|
+
const declared = config.secrets?.some(
|
|
489
|
+
(secret) => secret.name === secretName && secret.required === true,
|
|
490
|
+
);
|
|
491
|
+
if (!declared) {
|
|
492
|
+
throw new ValidationError(
|
|
493
|
+
`Provider "${config.id}" requires ${vendor} egress but does not declare ${secretName}.`,
|
|
494
|
+
{
|
|
495
|
+
fix: `Add secrets: [{ name: "${secretName}", required: true }] to the provider (every vendor in a required proxy chain must declare its credential secrets).`,
|
|
496
|
+
},
|
|
497
|
+
);
|
|
498
|
+
}
|
|
499
|
+
}
|
|
469
500
|
}
|
|
470
501
|
}
|
|
471
502
|
// `decodo`/`custom` are deprecated vendor values (string-union members, so the
|
package/src/runtime/choice.ts
CHANGED
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
ProviderChoiceTokenError,
|
|
12
12
|
type ProviderChoiceTokenPayload,
|
|
13
13
|
} from "../choice-token.js";
|
|
14
|
-
import { ProviderError } from "../errors.js";
|
|
14
|
+
import { isProviderError, ProviderError } from "../errors.js";
|
|
15
15
|
import type {
|
|
16
16
|
CredentialContext,
|
|
17
17
|
EnvContext,
|
|
@@ -23,6 +23,7 @@ import type {
|
|
|
23
23
|
ProviderRequestContext,
|
|
24
24
|
ProviderRuntimeState,
|
|
25
25
|
ProviderStateDurationString,
|
|
26
|
+
StateValue,
|
|
26
27
|
} from "../types.js";
|
|
27
28
|
|
|
28
29
|
export const PROVIDER_RUNTIME_CHOICE_TOKEN_MASTER_SECRET_ENV =
|
|
@@ -359,9 +360,29 @@ async function parseServerStoredChoice(options: {
|
|
|
359
360
|
storage,
|
|
360
361
|
contextState: options.contextState,
|
|
361
362
|
});
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
)
|
|
363
|
+
// Reading a server-stored choice back deserializes a persisted value. A
|
|
364
|
+
// corrupt/undecodable value would otherwise surface as a raw JSON.parse
|
|
365
|
+
// SyntaxError (or another unexpected throwable) that escapes the choice error
|
|
366
|
+
// taxonomy, gets masked as internal_error 500, and is treated as retryable by
|
|
367
|
+
// the hub -> reservation restart loop (2026-07-22 catchtable RCA, candidate A).
|
|
368
|
+
// Convert any non-branded throwable into a branded invalid_payload so it maps
|
|
369
|
+
// to a clean, non-retryable 400. Branded ProviderChoiceTokenError and genuine
|
|
370
|
+
// ProviderError (e.g. Redis-unavailable / state-unavailable) pass through so
|
|
371
|
+
// their category/retryable semantics are preserved.
|
|
372
|
+
let record: StateValue<ProviderChoiceTokenPayload> | null;
|
|
373
|
+
try {
|
|
374
|
+
record = await namespace.get<ProviderChoiceTokenPayload>(
|
|
375
|
+
optionsStateKey(options.handle.state_id),
|
|
376
|
+
);
|
|
377
|
+
} catch (error) {
|
|
378
|
+
if (error instanceof ProviderChoiceTokenError || isProviderError(error)) {
|
|
379
|
+
throw error;
|
|
380
|
+
}
|
|
381
|
+
throw new ProviderChoiceTokenError(
|
|
382
|
+
"invalid_payload",
|
|
383
|
+
"Provider choice token state payload could not be decoded.",
|
|
384
|
+
);
|
|
385
|
+
}
|
|
365
386
|
if (!record) {
|
|
366
387
|
throw new ProviderChoiceTokenError(
|
|
367
388
|
"invalid_payload",
|
package/src/runtime/http.ts
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
import type { ProxyResolutionOptions } from "../config/loader.js";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
policyRotatesTransportVendorChain,
|
|
4
|
+
resolvePolicyTransportAttemptCap,
|
|
5
|
+
resolveProxyConfigAsync,
|
|
6
|
+
} from "../config/loader.js";
|
|
3
7
|
import { ProviderError, TransportError } from "../errors.js";
|
|
4
8
|
import { parseSseStream, readableBytes, readableLines, readableTextChunks } from "../stream.js";
|
|
5
9
|
import type {
|
|
@@ -8,6 +12,7 @@ import type {
|
|
|
8
12
|
HttpResponse,
|
|
9
13
|
HttpRetrySummary,
|
|
10
14
|
HttpStreamResponse,
|
|
15
|
+
ProviderProxyPolicy,
|
|
11
16
|
RequestOptions,
|
|
12
17
|
RequestWithMethodOptions,
|
|
13
18
|
SseMessage,
|
|
@@ -41,16 +46,28 @@ type HttpStatusOutcome = {
|
|
|
41
46
|
proxyUsed: boolean;
|
|
42
47
|
};
|
|
43
48
|
|
|
44
|
-
|
|
49
|
+
/**
|
|
50
|
+
* Sentinel returned when a policy-allocator attempt resolved an endpoint that a
|
|
51
|
+
* prior attempt already tried (an under-filled pool repeats endpoints via the
|
|
52
|
+
* modulo mapping before the flat offset crosses into the next vendor). The retry
|
|
53
|
+
* loop advances to the next offset rather than re-issuing the request — but does
|
|
54
|
+
* NOT treat it as chain exhaustion, so the offset still walks into the fallback
|
|
55
|
+
* vendor's span.
|
|
56
|
+
*/
|
|
57
|
+
type NativeHttpSkipOutcome = { kind: "dedupe-skip" };
|
|
58
|
+
|
|
59
|
+
type NativeHttpAttemptOutcome = HttpResponse | HttpStatusOutcome | NativeHttpSkipOutcome;
|
|
45
60
|
|
|
46
61
|
type NativeHttpAttemptError = TransportError & { proxyUsed?: boolean };
|
|
47
62
|
|
|
48
|
-
function isHttpStatusOutcome(
|
|
49
|
-
outcome: HttpResponse | HttpStatusOutcome,
|
|
50
|
-
): outcome is HttpStatusOutcome {
|
|
63
|
+
function isHttpStatusOutcome(outcome: NativeHttpAttemptOutcome): outcome is HttpStatusOutcome {
|
|
51
64
|
return "kind" in outcome && outcome.kind === "http-status";
|
|
52
65
|
}
|
|
53
66
|
|
|
67
|
+
function isDedupeSkipOutcome(outcome: NativeHttpAttemptOutcome): outcome is NativeHttpSkipOutcome {
|
|
68
|
+
return "kind" in outcome && outcome.kind === "dedupe-skip";
|
|
69
|
+
}
|
|
70
|
+
|
|
54
71
|
async function sleep(ms: number): Promise<void> {
|
|
55
72
|
if (ms <= 0) return;
|
|
56
73
|
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
@@ -297,6 +314,7 @@ async function fetchNativeHttp(
|
|
|
297
314
|
warn: (message: string) => void,
|
|
298
315
|
statusRetryCodes?: readonly number[],
|
|
299
316
|
proxyAttemptOffset = 0,
|
|
317
|
+
dedupe?: { attempted: Set<string> },
|
|
300
318
|
): Promise<NativeHttpAttemptOutcome> {
|
|
301
319
|
const requestUrl = appendQueryParams(resolveHttpUrl(baseUrl, url), options.params);
|
|
302
320
|
const controller = options.timeout ? new AbortController() : undefined;
|
|
@@ -306,7 +324,20 @@ async function fetchNativeHttp(
|
|
|
306
324
|
|
|
307
325
|
let proxy: string | undefined;
|
|
308
326
|
try {
|
|
327
|
+
// Resolve inside the try (and after the timeout is armed) so allocator
|
|
328
|
+
// failures are branded as TransportErrors and count against the request
|
|
329
|
+
// deadline, exactly as an inline resolve would.
|
|
309
330
|
proxy = await resolveNativeProxy(options, clientOptions, warn, proxyAttemptOffset);
|
|
331
|
+
// For a registry allocator chain, skip an endpoint a prior attempt already
|
|
332
|
+
// tried rather than re-issuing the same request. Returning the sentinel
|
|
333
|
+
// (instead of breaking) lets the loop keep advancing the flat offset until
|
|
334
|
+
// it crosses into the fallback vendor's pool span.
|
|
335
|
+
if (dedupe && proxy) {
|
|
336
|
+
if (dedupe.attempted.has(proxy)) {
|
|
337
|
+
return { kind: "dedupe-skip" };
|
|
338
|
+
}
|
|
339
|
+
dedupe.attempted.add(proxy);
|
|
340
|
+
}
|
|
310
341
|
const requestInit: NativeFetchInit = {
|
|
311
342
|
headers: options.headers,
|
|
312
343
|
method,
|
|
@@ -452,6 +483,58 @@ export function createHttpClient(
|
|
|
452
483
|
? { ...headersOptions, throwOnHttpError: false }
|
|
453
484
|
: headersOptions;
|
|
454
485
|
|
|
486
|
+
// Span the whole vendor chain on transport failures. Like ctx.stealth, a
|
|
487
|
+
// policy-managed proxy resolves a *different* endpoint/vendor per attempt
|
|
488
|
+
// (the flat proxyAttemptOffset rotates across the concatenated vendor pool
|
|
489
|
+
// spans), so a transport failure should advance to the next endpoint —
|
|
490
|
+
// potentially crossing into the fallback vendor — rather than stopping at
|
|
491
|
+
// the per-endpoint retry budget and stranding the request on the primary
|
|
492
|
+
// vendor. resolvePolicyTransportAttemptCap widens the cap to the chain span
|
|
493
|
+
// only for implicit, safe-method allocator requests; explicit retry
|
|
494
|
+
// policies (their documented `attempts` ceiling), unsafe methods, and
|
|
495
|
+
// static/non-registry vendors keep the retry budget. Status-code retries
|
|
496
|
+
// stay bounded by the retry budget regardless; only transport rotation gets
|
|
497
|
+
// the full span.
|
|
498
|
+
const policyProxy: ProviderProxyPolicy | undefined = (() => {
|
|
499
|
+
const policy = clientOptions.proxyPolicy ?? clientOptions.upstream?.proxy;
|
|
500
|
+
return policy && typeof policy === "object" ? policy : undefined;
|
|
501
|
+
})();
|
|
502
|
+
const usesPolicyAllocator = Boolean(policyProxy) && !options.proxy && !clientOptions.proxy;
|
|
503
|
+
const transportAttemptCap = retryOptions
|
|
504
|
+
? resolvePolicyTransportAttemptCap({
|
|
505
|
+
policy: policyProxy,
|
|
506
|
+
usesPolicyAllocator,
|
|
507
|
+
retryAttempts: retryOptions.attempts,
|
|
508
|
+
explicitRetry,
|
|
509
|
+
method: methodName,
|
|
510
|
+
})
|
|
511
|
+
: 1;
|
|
512
|
+
|
|
513
|
+
// Track resolved endpoints across a policy-allocator chain. Successive
|
|
514
|
+
// attempts rotate the flat offset across the concatenated vendor pool
|
|
515
|
+
// spans, but an under-filled allocation (fewer live endpoints than the
|
|
516
|
+
// configured pool size) makes the modulo mapping repeat endpoints before
|
|
517
|
+
// the offset reaches the next vendor. Rather than re-hammering an
|
|
518
|
+
// already-tried endpoint under backoff, fetchNativeHttp returns a skip
|
|
519
|
+
// sentinel for a duplicate; the loop then advances the flat offset without
|
|
520
|
+
// issuing the request, so it keeps walking toward — and into — the fallback
|
|
521
|
+
// vendor's pool span instead of stalling on the primary vendor.
|
|
522
|
+
// De-duplication is gated on the SAME predicate that widens the attempt cap
|
|
523
|
+
// (implicit, safe-method, registry-chain rotation). It must NOT engage for
|
|
524
|
+
// an explicit retry policy: there the caller's `attempts` count is the
|
|
525
|
+
// contract and each attempt must issue against whatever endpoint it resolves
|
|
526
|
+
// — even a repeat — instead of being silently skipped (which would collapse
|
|
527
|
+
// a `poolSize: 1` + `attempts: 3` request to a single fetch).
|
|
528
|
+
const dedupeAllocatorEndpoints = policyRotatesTransportVendorChain({
|
|
529
|
+
policy: policyProxy,
|
|
530
|
+
usesPolicyAllocator,
|
|
531
|
+
explicitRetry,
|
|
532
|
+
method: methodName,
|
|
533
|
+
});
|
|
534
|
+
const dedupeContext = dedupeAllocatorEndpoints
|
|
535
|
+
? { attempted: new Set<string>() }
|
|
536
|
+
: undefined;
|
|
537
|
+
|
|
455
538
|
const executeOnce = (proxyAttemptOffset = 0): Promise<NativeHttpAttemptOutcome> =>
|
|
456
539
|
fetchNativeHttp(
|
|
457
540
|
baseUrl,
|
|
@@ -462,24 +545,52 @@ export function createHttpClient(
|
|
|
462
545
|
warnOnce,
|
|
463
546
|
statusRetryEnabled ? retryOptions?.statusCodes : undefined,
|
|
464
547
|
proxyAttemptOffset,
|
|
548
|
+
dedupeContext,
|
|
465
549
|
);
|
|
466
550
|
|
|
467
551
|
if (!retryEnabled || !retryOptions) {
|
|
468
552
|
const outcome = await executeOnce();
|
|
553
|
+
if (isDedupeSkipOutcome(outcome)) {
|
|
554
|
+
// Single-shot path never de-duplicates (dedupeContext is undefined),
|
|
555
|
+
// but keep the union total.
|
|
556
|
+
throw new TransportError("HTTP request produced no terminal result", {
|
|
557
|
+
code: "retry_exhausted",
|
|
558
|
+
});
|
|
559
|
+
}
|
|
469
560
|
if (isHttpStatusOutcome(outcome)) {
|
|
470
561
|
throw toUpstreamHttpError(outcome.status);
|
|
471
562
|
}
|
|
472
563
|
return outcome;
|
|
473
564
|
}
|
|
474
565
|
|
|
566
|
+
let lastError: unknown;
|
|
475
567
|
let lastErrorCode: string | undefined;
|
|
476
568
|
let lastStatus: number | undefined;
|
|
477
|
-
|
|
569
|
+
// `attempt` walks the flat proxy offset across the full chain span; `issued`
|
|
570
|
+
// counts requests that were actually sent (skipped duplicate offsets do not
|
|
571
|
+
// increment it). Retry summaries and the status-retry budget must reflect
|
|
572
|
+
// issued requests, not the raw offset, so they stay accurate when partial
|
|
573
|
+
// allocations skip offsets.
|
|
574
|
+
let issued = 0;
|
|
575
|
+
for (let attempt = 1; attempt <= transportAttemptCap; attempt += 1) {
|
|
576
|
+
// Whether this offset actually issued a request (vs. a skipped duplicate),
|
|
577
|
+
// so the catch counts a thrown *transport* failure once without
|
|
578
|
+
// double-counting a status outcome that already incremented before it
|
|
579
|
+
// re-threw as an upstream HTTP error.
|
|
580
|
+
let issuedThisAttempt = false;
|
|
478
581
|
try {
|
|
479
582
|
const outcome = await executeOnce(attempt - 1);
|
|
583
|
+
if (isDedupeSkipOutcome(outcome)) {
|
|
584
|
+
// Duplicate endpoint from a partial allocation: advance the flat
|
|
585
|
+
// offset without issuing the request (no backoff, not a failure) so
|
|
586
|
+
// the loop keeps rotating toward the fallback vendor.
|
|
587
|
+
continue;
|
|
588
|
+
}
|
|
589
|
+
issued += 1;
|
|
590
|
+
issuedThisAttempt = true;
|
|
480
591
|
if (isHttpStatusOutcome(outcome)) {
|
|
481
592
|
lastStatus = outcome.status;
|
|
482
|
-
if (outcome.retryable &&
|
|
593
|
+
if (outcome.retryable && issued < retryOptions.attempts) {
|
|
483
594
|
await sleep(computeProxyTransportRetryDelayMs(retryOptions, attempt, outcome.headers));
|
|
484
595
|
continue;
|
|
485
596
|
}
|
|
@@ -491,10 +602,10 @@ export function createHttpClient(
|
|
|
491
602
|
throw toUpstreamHttpError(response.status);
|
|
492
603
|
}
|
|
493
604
|
|
|
494
|
-
if (
|
|
605
|
+
if (issued > 1) {
|
|
495
606
|
const summary: HttpRetrySummary = {
|
|
496
|
-
attempts:
|
|
497
|
-
retries:
|
|
607
|
+
attempts: issued,
|
|
608
|
+
retries: issued - 1,
|
|
498
609
|
...(retryOptions.preset ? { preset: retryOptions.preset } : {}),
|
|
499
610
|
transport: "native",
|
|
500
611
|
...(lastErrorCode ? { lastErrorCode } : {}),
|
|
@@ -504,11 +615,13 @@ export function createHttpClient(
|
|
|
504
615
|
}
|
|
505
616
|
return response;
|
|
506
617
|
} catch (error) {
|
|
618
|
+
if (!issuedThisAttempt) issued += 1;
|
|
619
|
+
lastError = error;
|
|
507
620
|
lastErrorCode = proxyTransportRetryErrorCode(error);
|
|
508
621
|
lastStatus = proxyTransportRetryErrorStatus(error);
|
|
509
622
|
const proxyUsed = Boolean((error as NativeHttpAttemptError).proxyUsed);
|
|
510
623
|
if (
|
|
511
|
-
attempt <
|
|
624
|
+
attempt < transportAttemptCap &&
|
|
512
625
|
shouldRetryProxyTransportAttempt({
|
|
513
626
|
error,
|
|
514
627
|
explicitRetry,
|
|
@@ -524,6 +637,13 @@ export function createHttpClient(
|
|
|
524
637
|
}
|
|
525
638
|
}
|
|
526
639
|
|
|
640
|
+
// Reached when the attempt cap is consumed without a terminal outcome —
|
|
641
|
+
// e.g. the final offsets of a partial allocation all resolved to
|
|
642
|
+
// already-tried endpoints and were skipped. Surface the last real transport
|
|
643
|
+
// failure rather than a synthetic exhaustion error.
|
|
644
|
+
if (lastError !== undefined) {
|
|
645
|
+
throw lastError;
|
|
646
|
+
}
|
|
527
647
|
throw new TransportError("HTTP retry exhausted without a terminal result", {
|
|
528
648
|
code: "retry_exhausted",
|
|
529
649
|
});
|
|
@@ -27,7 +27,7 @@ const NODEMAVEN_PORTS: Record<ProxyProtocol, { min: number; max: number }> = {
|
|
|
27
27
|
const NODEMAVEN_FILTERS = new Set(["medium", "high"]);
|
|
28
28
|
const DEFAULT_NODEMAVEN_FILTER = "medium";
|
|
29
29
|
const DEFAULT_NODEMAVEN_POOL_SIZE = 20;
|
|
30
|
-
const NODEMAVEN_MAX_POOL_SIZE = 50;
|
|
30
|
+
export const NODEMAVEN_MAX_POOL_SIZE = 50;
|
|
31
31
|
/** NodeMaven sticky sessions persist up to 24h server-side, keyed by the sid. */
|
|
32
32
|
const NODEMAVEN_MAX_LIFETIME_MINUTES = 1440;
|
|
33
33
|
const SID_LENGTH = 10;
|
package/src/runtime/state.ts
CHANGED
|
@@ -115,7 +115,17 @@ function envelopeFromJson(
|
|
|
115
115
|
// biome-ignore lint/suspicious/noExplicitAny: state envelopes deserialize caller-owned generic values.
|
|
116
116
|
): StateValue<any> | null {
|
|
117
117
|
if (!raw) return null;
|
|
118
|
-
|
|
118
|
+
// A corrupt/undecodable persisted envelope must be treated as absent rather
|
|
119
|
+
// than throwing a raw JSON.parse SyntaxError: an uncaught SyntaxError escapes
|
|
120
|
+
// the provider error taxonomy, is masked as internal_error 500, and is then
|
|
121
|
+
// retried by the hub (2026-07-22 catchtable reserve RCA, candidate A). Returning
|
|
122
|
+
// null also keeps list() from aborting the whole scan on a single bad entry.
|
|
123
|
+
let parsed: unknown;
|
|
124
|
+
try {
|
|
125
|
+
parsed = JSON.parse(raw);
|
|
126
|
+
} catch {
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
119
129
|
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
120
130
|
return null;
|
|
121
131
|
}
|