@apifuse/provider-sdk 2.2.0-beta.57 → 2.2.0-beta.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AUTHORING.md +9 -0
- package/CHANGELOG.md +8 -0
- package/README.md +22 -5
- package/SUBMISSION.md +2 -1
- package/bin/apifuse-pack-types.ts +49 -0
- package/bin/apifuse-record.ts +6 -0
- package/dist/cli/templates/provider/README.md.tpl +19 -5
- package/dist/config/loader.d.ts +7 -0
- package/dist/config/loader.js +30 -6
- package/dist/declaration-validation.d.ts +2 -1
- package/dist/declaration-validation.js +39 -0
- package/dist/define.d.ts +5 -4
- package/dist/define.js +25 -1
- package/dist/engine.d.ts +16 -0
- package/dist/engine.js +58 -1
- package/dist/error-resolution.js +13 -0
- package/dist/index.d.ts +6 -4
- package/dist/index.js +4 -2
- package/dist/lint.js +3 -3
- package/dist/provider.d.ts +3 -3
- package/dist/provider.js +2 -2
- package/dist/runtime/bounded-expiring-map.d.ts +21 -0
- package/dist/runtime/bounded-expiring-map.js +64 -0
- package/dist/runtime/cache.d.ts +2 -0
- package/dist/runtime/cache.js +8 -1
- package/dist/runtime/egress-lease.d.ts +63 -0
- package/dist/runtime/egress-lease.js +233 -0
- package/dist/runtime/http.js +1 -0
- package/dist/runtime/proxy-retry-policy.d.ts +4 -1
- package/dist/runtime/proxy-retry-policy.js +11 -2
- package/dist/runtime/request-telemetry.d.ts +7 -9
- package/dist/runtime/resolver-config.d.ts +7 -0
- package/dist/runtime/resolver-config.js +8 -0
- package/dist/runtime/resolver-public.d.ts +3 -1
- package/dist/runtime/resolver-public.js +1 -1
- package/dist/runtime/resolver-shared.d.ts +9 -0
- package/dist/runtime/resolver-shared.js +57 -0
- package/dist/runtime/resolver-telemetry.d.ts +134 -0
- package/dist/runtime/resolver-telemetry.js +204 -0
- package/dist/runtime/resolver-usage.d.ts +24 -0
- package/dist/runtime/resolver-usage.js +60 -0
- package/dist/runtime/resolver-vendors/bindings.d.ts +5 -0
- package/dist/runtime/resolver-vendors/bindings.js +5 -0
- package/dist/runtime/resolver-vendors/capsolver.d.ts +2 -2
- package/dist/runtime/resolver-vendors/capsolver.js +21 -9
- package/dist/runtime/resolver-vendors/hypersolutions.d.ts +27 -0
- package/dist/runtime/resolver-vendors/hypersolutions.js +369 -0
- package/dist/runtime/resolver-vendors/twocaptcha.d.ts +2 -2
- package/dist/runtime/resolver-vendors/twocaptcha.js +26 -10
- package/dist/runtime/resolver-vendors/types.d.ts +35 -5
- package/dist/runtime/resolver-vendors/types.js +20 -10
- package/dist/runtime/resolver.d.ts +5 -1
- package/dist/runtime/resolver.js +397 -109
- package/dist/runtime/stealth-akamai-sbsd.d.ts +54 -0
- package/dist/runtime/stealth-akamai-sbsd.js +142 -0
- package/dist/runtime/stealth.js +547 -53
- package/dist/schema.d.ts +9 -0
- package/dist/schema.js +16 -0
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +1 -0
- package/dist/server/self-test.d.ts +4 -0
- package/dist/server/self-test.js +10 -1
- package/dist/server/serve-implementation.d.ts +7 -1
- package/dist/server/serve-implementation.js +181 -53
- package/dist/server/types.d.ts +6 -0
- package/dist/server/types.js +14 -0
- package/dist/stealth/profiles.d.ts +7 -1
- package/dist/stealth/profiles.js +9 -0
- package/dist/testing/run.js +6 -0
- package/dist/types.d.ts +115 -12
- package/package.json +1 -1
- package/src/cli/templates/provider/README.md.tpl +19 -5
- package/src/config/loader.ts +34 -6
- package/src/declaration-validation.ts +47 -1
- package/src/define.ts +46 -3
- package/src/engine.ts +76 -1
- package/src/error-resolution.ts +13 -0
- package/src/index.ts +32 -1
- package/src/lint.ts +3 -3
- package/src/provider.ts +9 -0
- package/src/runtime/bounded-expiring-map.ts +67 -0
- package/src/runtime/cache.ts +10 -1
- package/src/runtime/egress-lease.ts +336 -0
- package/src/runtime/http.ts +1 -0
- package/src/runtime/proxy-retry-policy.ts +12 -1
- package/src/runtime/request-telemetry.ts +5 -7
- package/src/runtime/resolver-config.ts +12 -0
- package/src/runtime/resolver-public.ts +21 -0
- package/src/runtime/resolver-shared.ts +75 -0
- package/src/runtime/resolver-telemetry.ts +362 -0
- package/src/runtime/resolver-usage.ts +96 -0
- package/src/runtime/resolver-vendors/bindings.ts +5 -0
- package/src/runtime/resolver-vendors/capsolver.ts +22 -3
- package/src/runtime/resolver-vendors/hypersolutions.ts +480 -0
- package/src/runtime/resolver-vendors/twocaptcha.ts +26 -12
- package/src/runtime/resolver-vendors/types.ts +60 -12
- package/src/runtime/resolver.ts +465 -124
- package/src/runtime/stealth-akamai-sbsd.ts +207 -0
- package/src/runtime/stealth.ts +727 -95
- package/src/schema.ts +17 -0
- package/src/server/index.ts +21 -0
- package/src/server/self-test.ts +13 -0
- package/src/server/serve-implementation.ts +268 -48
- package/src/server/types.ts +14 -0
- package/src/stealth/profiles.ts +10 -0
- package/src/testing/run.ts +6 -0
- package/src/types.ts +130 -22
package/dist/stealth/profiles.js
CHANGED
|
@@ -194,6 +194,15 @@ export function getStealthProfile(selection = {}) {
|
|
|
194
194
|
h2Settings: profile.h2Settings ? { ...profile.h2Settings } : undefined,
|
|
195
195
|
};
|
|
196
196
|
}
|
|
197
|
+
/**
|
|
198
|
+
* The transport owns the browser version (Chromium follows the pinned wreq-js build), so a
|
|
199
|
+
* resolver `clientProfile` such as `safari17_0` or `chrome_149` only has to name the same
|
|
200
|
+
* browser family as the stealth session it runs on.
|
|
201
|
+
*/
|
|
202
|
+
export function resolverClientProfileFamily(clientProfile) {
|
|
203
|
+
const family = /^[a-z]+/u.exec(clientProfile.trim().toLowerCase())?.[0];
|
|
204
|
+
return family === "chrome" || family === "firefox" || family === "safari" ? family : undefined;
|
|
205
|
+
}
|
|
197
206
|
export function listStealthProfiles() {
|
|
198
207
|
return SUPPORTED_STEALTH_PROFILES.map((profile) => ({ ...profile }));
|
|
199
208
|
}
|
package/dist/testing/run.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import { ProviderError } from "../errors.js";
|
|
2
3
|
import { createProviderCache } from "../runtime/cache.js";
|
|
3
4
|
import { createTestProviderChoiceContext } from "../runtime/choice.js";
|
|
4
5
|
import { createMemoryProviderRuntimeState } from "../runtime/state.js";
|
|
@@ -274,6 +275,11 @@ function createUpstreamContext(provider, operationName, upstreamStub) {
|
|
|
274
275
|
};
|
|
275
276
|
const createStealthSession = () => ({
|
|
276
277
|
fetch: stealthCall,
|
|
278
|
+
async replayChallenged() {
|
|
279
|
+
throw new ProviderError("Recorded transports cannot replay an engine-owned challenge", {
|
|
280
|
+
code: "REPLAY_SESSION_MISMATCH",
|
|
281
|
+
});
|
|
282
|
+
},
|
|
277
283
|
cookies: {
|
|
278
284
|
...emptyCookieJar,
|
|
279
285
|
has: () => false,
|
package/dist/types.d.ts
CHANGED
|
@@ -219,7 +219,7 @@ export interface ProviderSttConfig {
|
|
|
219
219
|
*
|
|
220
220
|
* `ProviderResolverConfig.vendors` overrides the SDK policy when declared.
|
|
221
221
|
*/
|
|
222
|
-
export type ProviderResolverVendor = "browser" | "capsolver" | "capmonster" | "2captcha" | "custom";
|
|
222
|
+
export type ProviderResolverVendor = "browser" | "capsolver" | "capmonster" | "2captcha" | "hypersolutions" | "custom";
|
|
223
223
|
/**
|
|
224
224
|
* Token-family kinds resolve to `{ form: "token" }`. Cookie-family kinds resolve
|
|
225
225
|
* to `{ form: "cookies" }`; network-identity binding is defined per kind. `aws_waf`
|
|
@@ -275,6 +275,22 @@ export type ProviderChallenge = {
|
|
|
275
275
|
readonly bmsz?: string;
|
|
276
276
|
/** Bot Manager major version when known ("3" measured on zozo.jp). */
|
|
277
277
|
readonly version?: string;
|
|
278
|
+
} | {
|
|
279
|
+
readonly kind: "akamai_sbsd";
|
|
280
|
+
readonly pageUrl: string;
|
|
281
|
+
/**
|
|
282
|
+
* Discovered SBSD script URL. `?v=&t=` is a hard challenge and `?v=`
|
|
283
|
+
* is passive or a remembered script. A later challenge token never
|
|
284
|
+
* changes this URL.
|
|
285
|
+
*/
|
|
286
|
+
readonly scriptUrl: string;
|
|
287
|
+
readonly stateCookieName: "sbsd_o" | "bm_so";
|
|
288
|
+
/**
|
|
289
|
+
* Token returned by a later `cpr_chlge` response. When present with a
|
|
290
|
+
* remembered v-only script, it selects index 0 and is appended only to
|
|
291
|
+
* the payload POST URL.
|
|
292
|
+
*/
|
|
293
|
+
readonly challengeToken?: string;
|
|
278
294
|
};
|
|
279
295
|
export type ProviderChallengeKind = ProviderChallenge["kind"];
|
|
280
296
|
/**
|
|
@@ -282,7 +298,9 @@ export type ProviderChallengeKind = ProviderChallenge["kind"];
|
|
|
282
298
|
* per challenge kind: `aws_waf` was measured portable across residential leases
|
|
283
299
|
* on buyee, while `cf_clearance` is unmeasured here and treated as scoped to the
|
|
284
300
|
* identity that produced it. The provider attaches the returned cookies to its
|
|
285
|
-
* own requests.
|
|
301
|
+
* own requests. `cookie_state` solutions carry no values: SBSD state lives only
|
|
302
|
+
* in the engine-owned bound jar, and a 2xx payload POST is not proof of a solve;
|
|
303
|
+
* only the next protected GET can verify it (the safe refetch is Phase 2).
|
|
286
304
|
*/
|
|
287
305
|
export type ChallengeSolution = {
|
|
288
306
|
readonly form: "token";
|
|
@@ -298,8 +316,28 @@ export type ChallengeSolution = {
|
|
|
298
316
|
* the upstream cookie's expiry. `expires` takes precedence when both are present.
|
|
299
317
|
*/
|
|
300
318
|
readonly sdkEstimatedExpires?: number;
|
|
319
|
+
} | {
|
|
320
|
+
readonly form: "cookie_state";
|
|
321
|
+
readonly kind: "akamai_sbsd";
|
|
322
|
+
readonly outcome: "payload_accepted";
|
|
323
|
+
/** Always false until a later protected GET verifies the bound jar (Phase 2). */
|
|
324
|
+
readonly verified: false;
|
|
325
|
+
/** Name only; the identity-bound cookie value remains in the engine-owned jar. */
|
|
326
|
+
readonly stateCookieName: "sbsd_o" | "bm_so";
|
|
327
|
+
/** Epoch seconds observed on the updated state cookie, when supplied upstream. */
|
|
328
|
+
readonly expires?: number;
|
|
301
329
|
};
|
|
302
|
-
|
|
330
|
+
/**
|
|
331
|
+
* `akamai_sbsd` declarations require a transport-owned client profile because the
|
|
332
|
+
* solve runs on the provider's admitted session. Other kinds may omit it.
|
|
333
|
+
*/
|
|
334
|
+
export type ProviderResolverConfig = {
|
|
335
|
+
/** Optional ordered override for the SDK-owned vendor fallback chain. */
|
|
336
|
+
readonly vendors?: readonly ProviderResolverVendor[];
|
|
337
|
+
/** Challenge kinds this provider is permitted to request. */
|
|
338
|
+
readonly kinds: readonly Exclude<ProviderChallengeKind, "akamai_sbsd">[];
|
|
339
|
+
readonly clientProfile?: string;
|
|
340
|
+
} | {
|
|
303
341
|
/** Optional ordered override for the SDK-owned vendor fallback chain. */
|
|
304
342
|
readonly vendors?: readonly ProviderResolverVendor[];
|
|
305
343
|
/** Challenge kinds this provider is permitted to request. */
|
|
@@ -310,8 +348,8 @@ export interface ProviderResolverConfig {
|
|
|
310
348
|
* challenge is served, while a Safari profile is admitted. Provider-declared
|
|
311
349
|
* because only the provider knows its upstream's admission rule.
|
|
312
350
|
*/
|
|
313
|
-
readonly clientProfile
|
|
314
|
-
}
|
|
351
|
+
readonly clientProfile: string;
|
|
352
|
+
};
|
|
315
353
|
export type SttAudioInput = {
|
|
316
354
|
kind: "base64";
|
|
317
355
|
data: string;
|
|
@@ -774,6 +812,13 @@ export type StealthProfileSelection = {
|
|
|
774
812
|
browser: "safari";
|
|
775
813
|
os?: "macos" | "ios";
|
|
776
814
|
};
|
|
815
|
+
/** Provider-owned opt-ins layered onto the transport-owned browser/OS profile. */
|
|
816
|
+
export type ProviderStealthConfig = StealthProfileSelection & {
|
|
817
|
+
readonly challengeDetection?: {
|
|
818
|
+
/** Classify SBSD responses even when no resolver is declared (detect/report mode). */
|
|
819
|
+
readonly akamaiSbsd?: boolean;
|
|
820
|
+
};
|
|
821
|
+
};
|
|
777
822
|
export type BrowserEngine = "playwright-stealth" | "nodriver" | "selenium-uc";
|
|
778
823
|
export interface BrowserOptions {
|
|
779
824
|
headless?: boolean;
|
|
@@ -1027,9 +1072,25 @@ export interface HttpRedirectPolicy {
|
|
|
1027
1072
|
export type HttpRedirectFailureReason = Exclude<RedirectRunReason, "completed">;
|
|
1028
1073
|
export type HttpMethod = "HEAD" | "head" | "GET" | "get" | "POST" | "post" | "PUT" | "put" | "DELETE" | "delete" | "OPTIONS" | "options" | "TRACE" | "trace" | "PATCH" | "patch";
|
|
1029
1074
|
export interface StealthFetchOptions extends Omit<RequestOptions, "redirectPolicy" | "headers"> {
|
|
1075
|
+
/**
|
|
1076
|
+
* Defaults to true. Set to false when callers need to inspect upstream
|
|
1077
|
+
* non-2xx bodies themselves instead of converting them to TransportError.
|
|
1078
|
+
*
|
|
1079
|
+
* Documented exception: a non-2xx response that an opted-in SDK challenge
|
|
1080
|
+
* detector classified (`response.challenge` is set) is returned, not thrown,
|
|
1081
|
+
* whatever this flag says, so the classification stays observable. Callers
|
|
1082
|
+
* that need the failure semantics check `response.challenge` first.
|
|
1083
|
+
*/
|
|
1084
|
+
throwOnHttpError?: boolean;
|
|
1030
1085
|
/**
|
|
1031
1086
|
* Request headers. Array values and case-insensitive duplicate names are
|
|
1032
1087
|
* combined in caller order using `", "`, matching Chrome's Fetch behavior.
|
|
1088
|
+
*
|
|
1089
|
+
* The stealth transport owns `host`, `connection`, `user-agent`, `sec-ch-ua`,
|
|
1090
|
+
* `sec-ch-ua-mobile`, `sec-ch-ua-platform`, `accept-encoding`, and every
|
|
1091
|
+
* `sec-fetch-*` header. Supplying any of them throws
|
|
1092
|
+
* `STEALTH_HEADER_OVERRIDE_UNSUPPORTED` at request time; declare the request
|
|
1093
|
+
* shape with `stealth.requestClass` instead.
|
|
1033
1094
|
*/
|
|
1034
1095
|
headers?: Record<string, string | string[]>;
|
|
1035
1096
|
method?: HttpMethod;
|
|
@@ -1049,13 +1110,26 @@ export interface StealthFetchOptions extends Omit<RequestOptions, "redirectPolic
|
|
|
1049
1110
|
*/
|
|
1050
1111
|
proxyAttemptOffset?: number;
|
|
1051
1112
|
/**
|
|
1052
|
-
* SDK-specific stealth controls
|
|
1053
|
-
* referrer, and content
|
|
1113
|
+
* SDK-specific stealth controls, including a per-request browser and/or OS
|
|
1114
|
+
* override. Standard HTTP metadata such as language, referrer, and content
|
|
1115
|
+
* type belongs in `headers`.
|
|
1054
1116
|
*/
|
|
1055
1117
|
stealth?: StealthProfileSelection & {
|
|
1056
|
-
/**
|
|
1057
|
-
|
|
1118
|
+
/**
|
|
1119
|
+
* Declare the Chrome request class when it cannot be inferred from the
|
|
1120
|
+
* method (`POST` infers `post`, everything else `navigation`). This is the
|
|
1121
|
+
* replacement for hand-set `Sec-Fetch-*` headers, which the transport
|
|
1122
|
+
* rejects; it drives `Sec-Fetch-Mode`/`Dest`/`User`, `Accept`, `Priority`,
|
|
1123
|
+
* and `Upgrade-Insecure-Requests`.
|
|
1124
|
+
*/
|
|
1058
1125
|
requestClass?: "navigation" | "xhr" | "post";
|
|
1126
|
+
/**
|
|
1127
|
+
* Whether a `navigation` request carries a user gesture and therefore
|
|
1128
|
+
* `Sec-Fetch-User: ?1` (default `true`). Set `false` for script-driven
|
|
1129
|
+
* navigations such as `location.replace`, which Chrome sends without it.
|
|
1130
|
+
* Ignored for the `xhr` and `post` request classes.
|
|
1131
|
+
*/
|
|
1132
|
+
userActivation?: boolean;
|
|
1059
1133
|
/**
|
|
1060
1134
|
* Use only for proxy products that terminate CONNECT with a private CA
|
|
1061
1135
|
* instead of tunneling the origin certificate chain.
|
|
@@ -1110,6 +1184,8 @@ export interface DeclarativeStealthResponse {
|
|
|
1110
1184
|
headers: Record<string, string>;
|
|
1111
1185
|
rawHeaders: [string, string][];
|
|
1112
1186
|
body: string;
|
|
1187
|
+
/** Present only when an opted-in SDK detector returned an unresolved challenge response. */
|
|
1188
|
+
challenge?: StealthChallengeClassification;
|
|
1113
1189
|
httpVersion?: string;
|
|
1114
1190
|
tlsInfo?: {
|
|
1115
1191
|
protocol?: string;
|
|
@@ -1122,6 +1198,20 @@ export interface DeclarativeStealthResponse {
|
|
|
1122
1198
|
bytes(): Promise<Uint8Array>;
|
|
1123
1199
|
}
|
|
1124
1200
|
export type StealthResponse = DeclarativeStealthResponse;
|
|
1201
|
+
/** Why a detected challenge response was returned to the caller instead of being solved and refetched. */
|
|
1202
|
+
export type StealthChallengeClassification = {
|
|
1203
|
+
readonly challenge: Extract<ProviderChallenge, {
|
|
1204
|
+
readonly kind: "akamai_sbsd";
|
|
1205
|
+
}>;
|
|
1206
|
+
/**
|
|
1207
|
+
* `resolver_unavailable`: detect-only provider or no solver configured.
|
|
1208
|
+
* `replay_required`: the request is not a plain GET, so it is never solved or replayed
|
|
1209
|
+
* automatically. `solve_failed`: a concurrent request on the same session owned the
|
|
1210
|
+
* solve and it rejected (that owner receives the error). `challenge_persisted`: the
|
|
1211
|
+
* single refetch after a solve was challenged again.
|
|
1212
|
+
*/
|
|
1213
|
+
readonly outcome: "resolver_unavailable" | "replay_required" | "solve_failed" | "challenge_persisted";
|
|
1214
|
+
};
|
|
1125
1215
|
export type RequestWithMethodOptions = RequestOptions & {
|
|
1126
1216
|
method?: string;
|
|
1127
1217
|
body?: unknown;
|
|
@@ -1152,6 +1242,12 @@ export interface StealthRedirectRunResult {
|
|
|
1152
1242
|
}
|
|
1153
1243
|
export interface StealthSession {
|
|
1154
1244
|
fetch(url: string, options?: StealthFetchOptions): Promise<StealthResponse>;
|
|
1245
|
+
/**
|
|
1246
|
+
* Explicitly solve and replay one unsafe request previously classified as
|
|
1247
|
+
* `replay_required`. The original method, URL, headers, and body bytes are
|
|
1248
|
+
* retained by this session and may be replayed exactly once.
|
|
1249
|
+
*/
|
|
1250
|
+
replayChallenged(response: StealthResponse): Promise<StealthResponse>;
|
|
1155
1251
|
cookies: StealthSessionCookies;
|
|
1156
1252
|
redirects: {
|
|
1157
1253
|
run(options: StealthRedirectRunOptions): Promise<StealthRedirectRunResult>;
|
|
@@ -1860,8 +1956,9 @@ export interface ProviderRuntimeState {
|
|
|
1860
1956
|
/**
|
|
1861
1957
|
* The operation context exposed for one provider declaration. Capability
|
|
1862
1958
|
* bindings are present only when their corresponding declaration is present;
|
|
1863
|
-
*
|
|
1864
|
-
*
|
|
1959
|
+
* platform-managed auth itself declares the credential binding. Trace and
|
|
1960
|
+
* request remain ambient runtime bindings. Omitting the type parameter
|
|
1961
|
+
* preserves the legacy full context shape for existing annotations.
|
|
1865
1962
|
*/
|
|
1866
1963
|
export type ProviderContext<TConfig = Record<string, unknown>> = {
|
|
1867
1964
|
request?: ProviderRequestContext;
|
|
@@ -1870,6 +1967,12 @@ export type ProviderContext<TConfig = Record<string, unknown>> = {
|
|
|
1870
1967
|
env: EnvContext;
|
|
1871
1968
|
} : Record<never, never>) & ("credential" extends keyof TConfig ? {
|
|
1872
1969
|
credential: CredentialContext;
|
|
1970
|
+
} : TConfig extends {
|
|
1971
|
+
auth: {
|
|
1972
|
+
mode: "platform-managed";
|
|
1973
|
+
};
|
|
1974
|
+
} ? {
|
|
1975
|
+
credential: CredentialContext;
|
|
1873
1976
|
} : Record<never, never>) & ("http" extends keyof TConfig ? {
|
|
1874
1977
|
http: HttpClient;
|
|
1875
1978
|
} : Record<never, never>) & ("files" extends keyof TConfig ? string extends keyof TConfig ? {
|
|
@@ -2047,7 +2150,7 @@ export interface ProviderDefinition<TContext = ProviderContext> {
|
|
|
2047
2150
|
http?: Record<string, never> | true;
|
|
2048
2151
|
allowedHosts?: string[];
|
|
2049
2152
|
native?: NativeProviderConfig;
|
|
2050
|
-
stealth?:
|
|
2153
|
+
stealth?: ProviderStealthConfig;
|
|
2051
2154
|
proxy?: ProviderProxyConfig;
|
|
2052
2155
|
ocr?: ProviderOcrConfig;
|
|
2053
2156
|
stt?: ProviderSttConfig;
|
package/package.json
CHANGED
|
@@ -97,9 +97,21 @@ The `POST /v1/{operation}` body is a request envelope:
|
|
|
97
97
|
- `requestId` is required and can be any unique local debugging string.
|
|
98
98
|
- `input` contains the operation input shape.
|
|
99
99
|
- `headers` is optional.
|
|
100
|
-
- `
|
|
101
|
-
|
|
102
|
-
|
|
100
|
+
- `connectionId` is optional connection identity only and does not include
|
|
101
|
+
credentials. The gateway sends it for `optional` connection mode, and only
|
|
102
|
+
when the caller supplied a connection that passed authorization. It is
|
|
103
|
+
exposed as `ctx.request.connectionId`.
|
|
104
|
+
- `connection` is optional credential-bearing connection data. The gateway
|
|
105
|
+
sends it for `required` connection mode; omit it for no-auth/public (`none`
|
|
106
|
+
mode) operations. For local debugging, pass
|
|
107
|
+
`{ "id", "mode", "secrets", "metadata", "externalRef" }` with local-only
|
|
108
|
+
secret values.
|
|
109
|
+
|
|
110
|
+
The gateway sends only `connection` for `required` mode, only `connectionId`
|
|
111
|
+
for `optional` mode (and nothing when the caller passed no connection), and
|
|
112
|
+
neither field for `none` mode. If a malformed or manually constructed envelope
|
|
113
|
+
contains both, a non-empty nested `connection.id` takes precedence over the
|
|
114
|
+
top-level `connectionId`; an empty string is treated as absent.
|
|
103
115
|
|
|
104
116
|
Structured errors return an `error` object with `code`, `message`,
|
|
105
117
|
`requestId`, and optional `details`; validation failures include field paths in
|
|
@@ -109,8 +121,10 @@ Structured errors return an `error` object with `code`, `message`,
|
|
|
109
121
|
|
|
110
122
|
- `invalid_request`: include `requestId` and `input`; omit `connection` for
|
|
111
123
|
public/no-auth operations and never send `connection: null`.
|
|
112
|
-
- Credentials: declare `credential.keys
|
|
113
|
-
|
|
124
|
+
- Credentials: declare `credential.keys` unless `auth.mode` is `platform-managed`;
|
|
125
|
+
that mode implies an unfiltered credential capability and forbids key declarations.
|
|
126
|
+
Pass local-only values through `connection.secrets` and read them with
|
|
127
|
+
`ctx.credential`.
|
|
114
128
|
- Auth flow: call `/auth/start`, then `/auth/continue` with the same `flowId`;
|
|
115
129
|
carry returned `contextPatch` values into the next request's `context`.
|
|
116
130
|
- Stealth/browser runtime: keep access-sensitive operations on `ctx.stealth.fetch()` with
|
package/src/config/loader.ts
CHANGED
|
@@ -6,6 +6,7 @@ import path from "node:path";
|
|
|
6
6
|
import type { Redis } from "ioredis";
|
|
7
7
|
|
|
8
8
|
import type { ProviderProxyPolicy, ProviderProxyProvider, TraceConfig } from "../types.js";
|
|
9
|
+
import { BoundedExpiringMap } from "../runtime/bounded-expiring-map.js";
|
|
9
10
|
import {
|
|
10
11
|
NODEMAVEN_DEFAULT_PROTOCOL,
|
|
11
12
|
NODEMAVEN_FILTER_ENV,
|
|
@@ -223,7 +224,13 @@ type ProxyRedisClient = Pick<
|
|
|
223
224
|
"connect" | "del" | "eval" | "get" | "on" | "pttl" | "set" | "status"
|
|
224
225
|
>;
|
|
225
226
|
|
|
226
|
-
|
|
227
|
+
// Keyed per (policy, affinity). Connection affinities are minted by callers, so
|
|
228
|
+
// without a cap a long-lived process accumulates one pool per affinity it ever
|
|
229
|
+
// saw. Evicting a still-fresh pool forces a fresh allocation (a different
|
|
230
|
+
// egress endpoint) for that affinity, so the cap is generous and every
|
|
231
|
+
// capacity eviction is reported as `diagnostics.poolCacheEvictions`.
|
|
232
|
+
const SMARTPROXY_POOL_CACHE_MAX_ENTRIES = 10_000;
|
|
233
|
+
let proxyCache = createSmartproxyPoolCache(SMARTPROXY_POOL_CACHE_MAX_ENTRIES);
|
|
227
234
|
const require = createRequire(import.meta.url);
|
|
228
235
|
const proxyInflight = new Map<string, Promise<SmartproxyAllocationResult>>();
|
|
229
236
|
const invalidatedProxyKeys = new Map<string, number>();
|
|
@@ -277,6 +284,22 @@ export function __setSmartproxyAllocatorDeadlineMsForTests(deadlineMs: number |
|
|
|
277
284
|
smartproxyAllocatorDeadlineMsForTests = deadlineMs;
|
|
278
285
|
}
|
|
279
286
|
|
|
287
|
+
/** @internal Test-only hook for exercising the pool-cache capacity bound at a small size. */
|
|
288
|
+
export function __setSmartproxyPoolCacheMaxEntriesForTests(maxEntries: number | undefined): void {
|
|
289
|
+
proxyCache = createSmartproxyPoolCache(maxEntries ?? SMARTPROXY_POOL_CACHE_MAX_ENTRIES);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/** @internal Test-only view of the in-memory Smartproxy cache sizes. */
|
|
293
|
+
export function __getSmartproxyCacheSizesForTests(): { pools: number; tombstones: number } {
|
|
294
|
+
return { pools: proxyCache.size, tombstones: invalidatedProxyKeys.size };
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function createSmartproxyPoolCache(
|
|
298
|
+
maxEntries: number,
|
|
299
|
+
): BoundedExpiringMap<string, CachedProxyPool> {
|
|
300
|
+
return new BoundedExpiringMap(maxEntries, (pool) => pool.expiresAt);
|
|
301
|
+
}
|
|
302
|
+
|
|
280
303
|
function getProxyRedis(): ProxyRedisClient | undefined {
|
|
281
304
|
if (proxyRedisForTests) return proxyRedisForTests;
|
|
282
305
|
const redisUrl = redisUrlFromEnv();
|
|
@@ -760,6 +783,7 @@ export async function resolveWithVendor(
|
|
|
760
783
|
...allocated.pool.diagnostics,
|
|
761
784
|
poolSize: allocated.pool.urls.length,
|
|
762
785
|
poolIndex,
|
|
786
|
+
poolCacheEvictions: proxyCache.capacityEvictions,
|
|
763
787
|
},
|
|
764
788
|
};
|
|
765
789
|
}
|
|
@@ -1082,8 +1106,8 @@ async function allocateSmartproxy(
|
|
|
1082
1106
|
const now = startedAt;
|
|
1083
1107
|
const invalidatedUntil = invalidatedProxyKeys.get(cacheKey) ?? 0;
|
|
1084
1108
|
const skipCached = invalidatedUntil > now;
|
|
1085
|
-
const cached = proxyCache.get(cacheKey);
|
|
1086
|
-
if (
|
|
1109
|
+
const cached = skipCached ? undefined : proxyCache.get(cacheKey, now);
|
|
1110
|
+
if (cached) {
|
|
1087
1111
|
if (shouldSoftRefresh(cached, now)) {
|
|
1088
1112
|
void refreshSmartproxyPool(
|
|
1089
1113
|
cacheKey,
|
|
@@ -1158,7 +1182,7 @@ async function readSmartproxyRedisPool(
|
|
|
1158
1182
|
}
|
|
1159
1183
|
const now = Date.now();
|
|
1160
1184
|
if (!isFresh(pool, now)) return null;
|
|
1161
|
-
proxyCache.set(cacheKey, pool);
|
|
1185
|
+
proxyCache.set(cacheKey, pool, now);
|
|
1162
1186
|
return {
|
|
1163
1187
|
pool,
|
|
1164
1188
|
telemetry: telemetryForPool(pool, "redis_hit", startedAt, { redisReadMs }),
|
|
@@ -1482,7 +1506,7 @@ async function allocateAndStoreSmartproxyPool(
|
|
|
1482
1506
|
rawConnect: true,
|
|
1483
1507
|
},
|
|
1484
1508
|
};
|
|
1485
|
-
proxyCache.set(cacheKey, result);
|
|
1509
|
+
proxyCache.set(cacheKey, result, allocatedAt);
|
|
1486
1510
|
if (options.redis && options.poolKey) {
|
|
1487
1511
|
const redis = options.redis;
|
|
1488
1512
|
const poolKey = options.poolKey;
|
|
@@ -1744,7 +1768,11 @@ function markSmartproxyCacheInvalidated(options: ProxyResolutionOptions = {}): s
|
|
|
1744
1768
|
lifetimeMinutes,
|
|
1745
1769
|
options.protocol ?? VENDOR_DEFAULT_PROTOCOL.smartproxy,
|
|
1746
1770
|
);
|
|
1747
|
-
|
|
1771
|
+
const now = Date.now();
|
|
1772
|
+
for (const [key, invalidatedUntil] of invalidatedProxyKeys) {
|
|
1773
|
+
if (invalidatedUntil <= now) invalidatedProxyKeys.delete(key);
|
|
1774
|
+
}
|
|
1775
|
+
invalidatedProxyKeys.set(cacheKey, now + SMARTPROXY_INVALIDATION_SKIP_REDIS_MS);
|
|
1748
1776
|
proxyCache.delete(cacheKey);
|
|
1749
1777
|
proxyInflight.delete(cacheKey);
|
|
1750
1778
|
return cacheKey;
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { describeSchema } from "./contract-serialization.js";
|
|
2
2
|
import { ProviderError } from "./errors.js";
|
|
3
3
|
import { HealthScenarioSchema } from "./health-scenario.js";
|
|
4
|
+
import { kindRequiresClientProfile } from "./runtime/resolver-config.js";
|
|
5
|
+
import { resolverClientProfileFamily, resolveStealthProfileSelection } from "./stealth/profiles.js";
|
|
4
6
|
import type {
|
|
5
7
|
HealthJourneyDefinition,
|
|
6
8
|
ProviderDefinition,
|
|
@@ -22,6 +24,7 @@ export const DECLARATION_RULE_IDS = {
|
|
|
22
24
|
proxySmartproxyGeo: "proxy-smartproxy-country-only",
|
|
23
25
|
operationUpstreamProxy: "operation-upstream-proxy-unsupported",
|
|
24
26
|
pinnedWireFieldPathValid: "public-schema-pinned-wire-field-valid",
|
|
27
|
+
resolverClientProfileFamily: "resolver-client-profile-family",
|
|
25
28
|
} as const;
|
|
26
29
|
|
|
27
30
|
export type DeclarationRuleId =
|
|
@@ -57,11 +60,15 @@ export function validateFailClosedDeclaration(provider: ProviderDefinition): voi
|
|
|
57
60
|
validatePinnedWireFieldPaths(provider, violations);
|
|
58
61
|
validateSchemaDeclaration(provider, violations);
|
|
59
62
|
validateProxyDeclaration(provider, violations);
|
|
63
|
+
validateResolverClientProfileDeclaration(provider, violations);
|
|
60
64
|
validateOperationDeclaration(provider, violations);
|
|
61
65
|
if (violations.length > 0) throw declarationInvalidError(violations);
|
|
62
66
|
}
|
|
63
67
|
|
|
64
|
-
type ProviderDeclarationRulesInput = Pick<
|
|
68
|
+
type ProviderDeclarationRulesInput = Pick<
|
|
69
|
+
ProviderDefinition,
|
|
70
|
+
"healthJourneys" | "meta" | "proxy" | "resolver" | "stealth"
|
|
71
|
+
>;
|
|
65
72
|
type OperationDeclarationRulesInput = Pick<ProviderDefinition, "operations">;
|
|
66
73
|
|
|
67
74
|
/** Enforces fail-closed rules that only depend on the provider declaration. */
|
|
@@ -89,6 +96,7 @@ function collectProviderDeclarationViolations(
|
|
|
89
96
|
validateHealthDeclaration(provider, violations);
|
|
90
97
|
validatePinnedWireFieldPaths(provider, violations);
|
|
91
98
|
validateProxyDeclaration(provider, violations);
|
|
99
|
+
validateResolverClientProfileDeclaration(provider, violations);
|
|
92
100
|
}
|
|
93
101
|
|
|
94
102
|
function validatePinnedWireFieldPaths(
|
|
@@ -307,6 +315,44 @@ function validateProxyDeclaration(
|
|
|
307
315
|
}
|
|
308
316
|
}
|
|
309
317
|
|
|
318
|
+
/**
|
|
319
|
+
* An akamai_sbsd solve runs on the provider's own stealth session, so the declared
|
|
320
|
+
* resolver.clientProfile must name the browser family that session emulates. The
|
|
321
|
+
* stealth runtime re-checks per request (a fetch may override the browser); this
|
|
322
|
+
* rule fails the declaration instead of the first challenged request.
|
|
323
|
+
*/
|
|
324
|
+
function validateResolverClientProfileDeclaration(
|
|
325
|
+
provider: ProviderDeclarationRulesInput,
|
|
326
|
+
violations: DeclarationViolation[],
|
|
327
|
+
): void {
|
|
328
|
+
const resolver = provider.resolver;
|
|
329
|
+
if (
|
|
330
|
+
!resolver ||
|
|
331
|
+
!Array.isArray(resolver.kinds) ||
|
|
332
|
+
!resolver.kinds.some(kindRequiresClientProfile) ||
|
|
333
|
+
typeof resolver.clientProfile !== "string"
|
|
334
|
+
) {
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
let sessionBrowser: string;
|
|
338
|
+
try {
|
|
339
|
+
sessionBrowser = resolveStealthProfileSelection(provider.stealth).browser;
|
|
340
|
+
} catch {
|
|
341
|
+
// An unsupported stealth selection is reported by the stealth runtime itself.
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
const family = resolverClientProfileFamily(resolver.clientProfile);
|
|
345
|
+
if (family === sessionBrowser) return;
|
|
346
|
+
violations.push({
|
|
347
|
+
ruleId: DECLARATION_RULE_IDS.resolverClientProfileFamily,
|
|
348
|
+
path: "resolver.clientProfile",
|
|
349
|
+
message: family
|
|
350
|
+
? `"${resolver.clientProfile}" names ${family}, but the stealth session browser is ${sessionBrowser}.`
|
|
351
|
+
: `"${resolver.clientProfile}" does not name a supported browser family (chrome, firefox, safari).`,
|
|
352
|
+
fix: `Declare stealth.browser as the family resolver.clientProfile names, or set resolver.clientProfile to the ${sessionBrowser} profile the SBSD session runs on.`,
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
|
|
310
356
|
function declaredProxyVendors(policy: ProviderProxyPolicy): ProviderProxyProvider[] {
|
|
311
357
|
const vendors = [...(policy.providers ?? [])];
|
|
312
358
|
if (policy.provider !== undefined) vendors.push(policy.provider);
|
package/src/define.ts
CHANGED
|
@@ -6,8 +6,15 @@ import {
|
|
|
6
6
|
} from "./declaration-validation.js";
|
|
7
7
|
import { SDK_RUNTIME_OWNED_ERROR_CODES } from "./error-resolution.js";
|
|
8
8
|
import { ProviderError, ValidationError } from "./errors.js";
|
|
9
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
engineOwnedResolverCredentialTarget,
|
|
11
|
+
isEngineOwnedEnvName,
|
|
12
|
+
isEngineOwnedProxyCredentialName,
|
|
13
|
+
isEngineOwnedRuntimeEnvName,
|
|
14
|
+
isEngineOwnedTelemetryEnvName,
|
|
15
|
+
} from "./engine.js";
|
|
10
16
|
import { HealthScenarioSchema } from "./health-scenario.js";
|
|
17
|
+
import { kindRequiresClientProfile } from "./runtime/resolver-config.js";
|
|
11
18
|
import {
|
|
12
19
|
NativeEgressPolicyValidationError,
|
|
13
20
|
validateNativeProviderConfig,
|
|
@@ -49,9 +56,9 @@ import type {
|
|
|
49
56
|
ProviderSecretDeclaration,
|
|
50
57
|
ProviderStreamEvent,
|
|
51
58
|
ProviderSttConfig,
|
|
59
|
+
ProviderStealthConfig,
|
|
52
60
|
SchemaLike,
|
|
53
61
|
SmsOtpMatcherDefinition,
|
|
54
|
-
StealthProfileSelection,
|
|
55
62
|
} from "./types.js";
|
|
56
63
|
import {
|
|
57
64
|
HEALTH_CHECK_DEGRADED_THRESHOLD_MS_MAX,
|
|
@@ -151,6 +158,7 @@ export const VALID_PROVIDER_RESOLVER_VENDORS = exhaustiveLiteralArray<ProviderRe
|
|
|
151
158
|
"capsolver",
|
|
152
159
|
"capmonster",
|
|
153
160
|
"2captcha",
|
|
161
|
+
"hypersolutions",
|
|
154
162
|
"custom",
|
|
155
163
|
] as const);
|
|
156
164
|
export const VALID_PROVIDER_CHALLENGE_KINDS = exhaustiveLiteralArray<ProviderChallengeKind>()([
|
|
@@ -162,6 +170,7 @@ export const VALID_PROVIDER_CHALLENGE_KINDS = exhaustiveLiteralArray<ProviderCha
|
|
|
162
170
|
"aws_waf",
|
|
163
171
|
"akamai_sec_cpt",
|
|
164
172
|
"akamai_sensor",
|
|
173
|
+
"akamai_sbsd",
|
|
165
174
|
] as const);
|
|
166
175
|
const RESERVED_OPERATION_IDS = new Set(["auth", "health"]);
|
|
167
176
|
const VALID_OPERATION_RISK_CLASSES = ["read", "write", "destructive", "external-send"] as const;
|
|
@@ -584,7 +593,7 @@ export interface ProviderDeclaration {
|
|
|
584
593
|
/** Declares upstream host policy; this does not add a `ctx.allowedHosts` member. */
|
|
585
594
|
allowedHosts?: string[];
|
|
586
595
|
native?: NativeProviderConfig;
|
|
587
|
-
stealth?:
|
|
596
|
+
stealth?: ProviderStealthConfig;
|
|
588
597
|
/** Declares proxy policy; this is provider intent and does not add a `ctx.proxy` member. */
|
|
589
598
|
proxy?: ProviderProxyConfig;
|
|
590
599
|
ocr?: ProviderOcrConfig;
|
|
@@ -600,6 +609,7 @@ export interface ProviderDeclaration {
|
|
|
600
609
|
secrets?: ProviderSecretDeclaration[];
|
|
601
610
|
/** Declares the environment capability binding. A bare object states use without configuration. */
|
|
602
611
|
env?: Record<string, never> | true;
|
|
612
|
+
/** Declares credential keys; platform-managed auth declares this capability without key filtering. */
|
|
603
613
|
credential?: CredentialDeclaration;
|
|
604
614
|
/** Declares provider context metadata; this does not add a `ctx.context` member. */
|
|
605
615
|
context?: ContextDeclaration;
|
|
@@ -938,6 +948,15 @@ function validateProviderProxy(config: {
|
|
|
938
948
|
secrets?: ProviderSecretDeclaration[];
|
|
939
949
|
}): void {
|
|
940
950
|
for (const secret of config.secrets ?? []) {
|
|
951
|
+
const engineName = engineOwnedResolverCredentialTarget(secret.name);
|
|
952
|
+
if (engineName !== undefined) {
|
|
953
|
+
throw new ValidationError(
|
|
954
|
+
`Provider "${config.id}" cannot declare engine-owned resolver credential "${secret.name}"`,
|
|
955
|
+
{
|
|
956
|
+
fix: `Remove "${secret.name}" from provider secrets; configure the solver key only on the provider engine as ${engineName}.`,
|
|
957
|
+
},
|
|
958
|
+
);
|
|
959
|
+
}
|
|
941
960
|
if (isEngineOwnedTelemetryEnvName(secret.name)) {
|
|
942
961
|
throw new ValidationError(
|
|
943
962
|
`Provider "${config.id}" cannot declare engine-owned telemetry variable "${secret.name}"`,
|
|
@@ -946,6 +965,22 @@ function validateProviderProxy(config: {
|
|
|
946
965
|
},
|
|
947
966
|
);
|
|
948
967
|
}
|
|
968
|
+
if (isEngineOwnedRuntimeEnvName(secret.name)) {
|
|
969
|
+
throw new ValidationError(
|
|
970
|
+
`Provider "${config.id}" cannot declare engine-owned runtime variable "${secret.name}"`,
|
|
971
|
+
{
|
|
972
|
+
fix: `Remove "${secret.name}" from provider secrets; the provider engine reads it from its own environment.`,
|
|
973
|
+
},
|
|
974
|
+
);
|
|
975
|
+
}
|
|
976
|
+
if (isEngineOwnedEnvName(secret.name) && !isEngineOwnedProxyCredentialName(secret.name)) {
|
|
977
|
+
throw new ValidationError(
|
|
978
|
+
`Provider "${config.id}" cannot declare engine-owned control variable "${secret.name}"`,
|
|
979
|
+
{
|
|
980
|
+
fix: `Remove "${secret.name}" from provider secrets; configure it only on the provider engine.`,
|
|
981
|
+
},
|
|
982
|
+
);
|
|
983
|
+
}
|
|
949
984
|
if (!isEngineOwnedProxyCredentialName(secret.name)) continue;
|
|
950
985
|
throw new ValidationError(
|
|
951
986
|
`Provider "${config.id}" cannot declare engine-owned proxy credential "${secret.name}"`,
|
|
@@ -1162,6 +1197,14 @@ function validateProviderResolver(config: { id: string; resolver?: ProviderResol
|
|
|
1162
1197
|
},
|
|
1163
1198
|
);
|
|
1164
1199
|
}
|
|
1200
|
+
if (resolver.kinds.some(kindRequiresClientProfile) && resolver.clientProfile === undefined) {
|
|
1201
|
+
throw new ValidationError(
|
|
1202
|
+
`Provider "${config.id}" must declare resolver.clientProfile for akamai_sbsd.`,
|
|
1203
|
+
{
|
|
1204
|
+
fix: `Set resolver.clientProfile for provider "${config.id}" to the transport-owned profile used by its Akamai session.`,
|
|
1205
|
+
},
|
|
1206
|
+
);
|
|
1207
|
+
}
|
|
1165
1208
|
}
|
|
1166
1209
|
|
|
1167
1210
|
function validateResolverLiteralArray<TValue extends string>(
|