@apifuse/provider-sdk 2.2.0-beta.23 → 2.2.0-beta.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/bin/apifuse-dev.ts +2 -0
- package/bin/apifuse-pack-types.ts +234 -38
- package/bin/apifuse-perf.ts +15 -12
- package/bin/apifuse-record.ts +2 -0
- package/bin/apifuse-submit-check.ts +15 -2
- package/dist/config/loader.d.ts +8 -19
- package/dist/config/loader.js +28 -86
- package/dist/define.d.ts +4 -1
- package/dist/define.js +64 -6
- package/dist/index.d.ts +4 -3
- package/dist/index.js +2 -1
- package/dist/provider.d.ts +1 -1
- package/dist/runtime/auth-flow.js +2 -0
- package/dist/runtime/browser.js +50 -0
- package/dist/runtime/cache.d.ts +1 -0
- package/dist/runtime/cache.js +169 -15
- package/dist/runtime/http.js +0 -1
- package/dist/runtime/instrumentation.js +26 -1
- package/dist/runtime/resolver-vendors/bindings.d.ts +8 -0
- package/dist/runtime/resolver-vendors/bindings.js +15 -0
- package/dist/runtime/resolver-vendors/browser.d.ts +24 -0
- package/dist/runtime/resolver-vendors/browser.js +287 -0
- package/dist/runtime/resolver-vendors/types.d.ts +42 -0
- package/dist/runtime/resolver-vendors/types.js +57 -0
- package/dist/runtime/resolver.d.ts +39 -0
- package/dist/runtime/resolver.js +414 -0
- package/dist/runtime/state.d.ts +3 -0
- package/dist/runtime/state.js +245 -141
- package/dist/runtime/stealth.js +3 -6
- package/dist/server/serve.d.ts +4 -1
- package/dist/server/serve.js +35 -8
- package/dist/testing/run.js +7 -0
- package/dist/types.d.ts +115 -7
- package/package.json +1 -1
- package/src/config/loader.ts +35 -111
- package/src/define.ts +105 -8
- package/src/index.ts +21 -1
- package/src/provider.ts +1 -0
- package/src/runtime/auth-flow.ts +2 -0
- package/src/runtime/browser.ts +69 -0
- package/src/runtime/cache.ts +189 -14
- package/src/runtime/http.ts +0 -1
- package/src/runtime/instrumentation.ts +36 -2
- package/src/runtime/resolver-vendors/bindings.ts +31 -0
- package/src/runtime/resolver-vendors/browser.ts +420 -0
- package/src/runtime/resolver-vendors/types.ts +113 -0
- package/src/runtime/resolver.ts +668 -0
- package/src/runtime/state.ts +323 -166
- package/src/runtime/stealth.ts +3 -6
- package/src/server/serve.ts +73 -5
- package/src/testing/run.ts +8 -0
- package/src/types.ts +133 -7
package/dist/server/serve.js
CHANGED
|
@@ -21,6 +21,7 @@ import { getProviderBaseUrl } from "../runtime/provider.js";
|
|
|
21
21
|
import { createOcrClientFromEnv } from "../runtime/ocr.js";
|
|
22
22
|
import { PROXY_AUTH_IP_DENIED_CODE, PROXY_EDGE_AUTH_REJECTED_CODE, PROXY_POOL_EXHAUSTED_CODE, } from "../runtime/proxy-errors.js";
|
|
23
23
|
import { PROVIDER_TELEMETRY_HEADER, ProxyTelemetryCollector } from "../runtime/proxy-telemetry.js";
|
|
24
|
+
import { bindResolverSignal, createResolverClientFromEnv } from "../runtime/resolver.js";
|
|
24
25
|
import { assertRequiredSecretsPresent, listMissingRequiredSecrets, MISSING_SECRET_CODE, } from "../runtime/secrets.js";
|
|
25
26
|
import { createProviderRuntimeStateFromEnv, createUnsupportedProviderRuntimeState, } from "../runtime/state.js";
|
|
26
27
|
import { createStealthClient } from "../runtime/stealth.js";
|
|
@@ -149,6 +150,13 @@ export function resolveProviderProxyAffinityKey(provider, request, operationId)
|
|
|
149
150
|
}
|
|
150
151
|
return connectionKey ?? provider.id;
|
|
151
152
|
}
|
|
153
|
+
export function resolveProviderResolverIdentityScope(provider, affinityKey, contextId) {
|
|
154
|
+
return JSON.stringify({
|
|
155
|
+
proxy: provider.proxy ?? null,
|
|
156
|
+
affinityKey,
|
|
157
|
+
contextId,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
152
160
|
function resolveOperationConnectionId(request) {
|
|
153
161
|
return request.connection?.id ?? request.connectionId;
|
|
154
162
|
}
|
|
@@ -161,7 +169,7 @@ function resolveNativeProxyPolicy(provider) {
|
|
|
161
169
|
return { mode: "disabled" };
|
|
162
170
|
return undefined;
|
|
163
171
|
}
|
|
164
|
-
function createProviderContext(provider, request, operationId, options = {}, state = createUnsupportedProviderRuntimeState(), proxyTelemetry) {
|
|
172
|
+
function createProviderContext(provider, request, operationId, options = {}, state = createUnsupportedProviderRuntimeState(), proxyTelemetry, signal) {
|
|
165
173
|
const baseUrl = getProviderBaseUrl(provider);
|
|
166
174
|
const stealthBaseUrl = getProviderStealthBaseUrl(provider);
|
|
167
175
|
const stealthProfile = getProviderStealthProfile(provider);
|
|
@@ -170,6 +178,7 @@ function createProviderContext(provider, request, operationId, options = {}, sta
|
|
|
170
178
|
affinityKey: resolveProviderProxyAffinityKey(provider, request, operationId),
|
|
171
179
|
telemetry: proxyTelemetry,
|
|
172
180
|
};
|
|
181
|
+
const resolverIdentityScope = resolveProviderResolverIdentityScope(provider, proxyClientOptions.affinityKey, request.requestId);
|
|
173
182
|
let wrappedContext;
|
|
174
183
|
const stealthClientOptions = {
|
|
175
184
|
upstream: proxyClientOptions.upstream,
|
|
@@ -190,6 +199,8 @@ function createProviderContext(provider, request, operationId, options = {}, sta
|
|
|
190
199
|
connectionId: resolveOperationConnectionId(request),
|
|
191
200
|
headers: request.headers ?? {},
|
|
192
201
|
};
|
|
202
|
+
const requestState = state.forConnection(requestContext.connectionId);
|
|
203
|
+
const cache = createProviderCache({ providerId: provider.id });
|
|
193
204
|
const context = wrapWithInstrumentation({
|
|
194
205
|
env,
|
|
195
206
|
credential,
|
|
@@ -202,8 +213,8 @@ function createProviderContext(provider, request, operationId, options = {}, sta
|
|
|
202
213
|
retryResponseMeta.set(wrappedContext, summary);
|
|
203
214
|
},
|
|
204
215
|
}),
|
|
205
|
-
cache
|
|
206
|
-
state,
|
|
216
|
+
cache,
|
|
217
|
+
state: requestState,
|
|
207
218
|
stealth: stealthBaseUrl
|
|
208
219
|
? stealthProfile
|
|
209
220
|
? createStealthClient(stealthBaseUrl, stealthProfile.name, stealthClientOptions)
|
|
@@ -235,12 +246,18 @@ function createProviderContext(provider, request, operationId, options = {}, sta
|
|
|
235
246
|
auth: createAuthStub(),
|
|
236
247
|
ocr: options.ocr ?? createOcrClientFromEnv(provider.ocr),
|
|
237
248
|
stt: options.stt ?? createSttClientFromEnv(provider.stt),
|
|
249
|
+
resolver: bindResolverSignal(options.resolver ??
|
|
250
|
+
createResolverClientFromEnv(provider.resolver, undefined, {
|
|
251
|
+
allowedHosts: provider.allowedHosts,
|
|
252
|
+
cache,
|
|
253
|
+
identityScope: resolverIdentityScope,
|
|
254
|
+
}), signal),
|
|
238
255
|
choice: createProviderChoiceContext({
|
|
239
256
|
providerId: provider.id,
|
|
240
257
|
env,
|
|
241
258
|
request: requestContext,
|
|
242
259
|
credential,
|
|
243
|
-
state,
|
|
260
|
+
state: requestState,
|
|
244
261
|
}),
|
|
245
262
|
});
|
|
246
263
|
wrappedContext = context;
|
|
@@ -284,6 +301,7 @@ function createAuthFlowContext(provider, request, options = {}, signal) {
|
|
|
284
301
|
request.providerId ??
|
|
285
302
|
provider.id,
|
|
286
303
|
};
|
|
304
|
+
const resolverIdentityScope = resolveProviderResolverIdentityScope(provider, proxyClientOptions.affinityKey, request.requestId);
|
|
287
305
|
const stealthClientOptions = {
|
|
288
306
|
upstream: proxyClientOptions.upstream,
|
|
289
307
|
affinityKey: proxyClientOptions.affinityKey,
|
|
@@ -296,6 +314,7 @@ function createAuthFlowContext(provider, request, options = {}, signal) {
|
|
|
296
314
|
values: request.connection.secrets,
|
|
297
315
|
})
|
|
298
316
|
: undefined;
|
|
317
|
+
const cache = createProviderCache({ providerId: provider.id });
|
|
299
318
|
return {
|
|
300
319
|
context: {
|
|
301
320
|
flowId: request.flowId,
|
|
@@ -331,6 +350,12 @@ function createAuthFlowContext(provider, request, options = {}, signal) {
|
|
|
331
350
|
context: flowContextStore.context,
|
|
332
351
|
ocr: options.ocr ?? createOcrClientFromEnv(provider.ocr),
|
|
333
352
|
stt: options.stt ?? createSttClientFromEnv(provider.stt),
|
|
353
|
+
resolver: bindResolverSignal(options.resolver ??
|
|
354
|
+
createResolverClientFromEnv(provider.resolver, undefined, {
|
|
355
|
+
allowedHosts: provider.allowedHosts,
|
|
356
|
+
cache,
|
|
357
|
+
identityScope: resolverIdentityScope,
|
|
358
|
+
}), signal),
|
|
334
359
|
auth: createAuthFlowHelpers({ signal }),
|
|
335
360
|
},
|
|
336
361
|
getPatch: flowContextStore.getPatch,
|
|
@@ -1048,8 +1073,8 @@ function withAuthRequestHeaders(request, headers) {
|
|
|
1048
1073
|
},
|
|
1049
1074
|
};
|
|
1050
1075
|
}
|
|
1051
|
-
async function handleOperation(provider, request, operationId, options = {}, state = createUnsupportedProviderRuntimeState(), proxyTelemetry) {
|
|
1052
|
-
const ctx = createProviderContext(provider, request, operationId, options, state, proxyTelemetry);
|
|
1076
|
+
async function handleOperation(provider, request, operationId, options = {}, state = createUnsupportedProviderRuntimeState(), proxyTelemetry, signal) {
|
|
1077
|
+
const ctx = createProviderContext(provider, request, operationId, options, state, proxyTelemetry, signal);
|
|
1053
1078
|
const operation = provider.operations[operationId];
|
|
1054
1079
|
const streaming = operation?.transport?.kind && operation.transport.kind !== "json";
|
|
1055
1080
|
let cleanupCalled = false;
|
|
@@ -1077,6 +1102,7 @@ async function handleOperation(provider, request, operationId, options = {}, sta
|
|
|
1077
1102
|
operationId,
|
|
1078
1103
|
ctx,
|
|
1079
1104
|
request,
|
|
1105
|
+
signal,
|
|
1080
1106
|
})
|
|
1081
1107
|
: await executeOperation(provider, operationId, ctx, request.input);
|
|
1082
1108
|
if (streaming && operation) {
|
|
@@ -1396,7 +1422,7 @@ export function createServerApp(provider, options = {}) {
|
|
|
1396
1422
|
}
|
|
1397
1423
|
const request = operationRequestFromForwardingEnvelope(envelope);
|
|
1398
1424
|
operationId = envelope.operationId;
|
|
1399
|
-
const ctx = createProviderContext(provider, request, operationId, options, state);
|
|
1425
|
+
const ctx = createProviderContext(provider, request, operationId, options, state, undefined, signal);
|
|
1400
1426
|
if (deadlineAtMs !== undefined && deadlineAtMs <= Date.now()) {
|
|
1401
1427
|
throw new StatefulRoutingDeadlineError(envelope.requestId, envelope.deadlineAt);
|
|
1402
1428
|
}
|
|
@@ -1435,7 +1461,7 @@ export function createServerApp(provider, options = {}) {
|
|
|
1435
1461
|
const body = OperationRequestSchema.parse(rawBody);
|
|
1436
1462
|
const requestHeaders = Object.fromEntries(c.req.raw.headers.entries());
|
|
1437
1463
|
body.headers = { ...requestHeaders, ...body.headers };
|
|
1438
|
-
const response = await handleOperation(provider, body, operation, options, state, proxyTelemetry);
|
|
1464
|
+
const response = await handleOperation(provider, body, operation, options, state, proxyTelemetry, c.req.raw.signal);
|
|
1439
1465
|
if (response instanceof Response) {
|
|
1440
1466
|
logProviderSuccess(logger, provider, "operation", operation, body.requestId, response.status, finishRequestCost(requestCost));
|
|
1441
1467
|
return responseWithProviderTelemetry(response, proxyTelemetry);
|
|
@@ -1613,6 +1639,7 @@ export async function serve(provider, options = {}) {
|
|
|
1613
1639
|
logger: options.logger,
|
|
1614
1640
|
ocr: options.ocr,
|
|
1615
1641
|
stt: options.stt,
|
|
1642
|
+
resolver: options.resolver,
|
|
1616
1643
|
state: options.state,
|
|
1617
1644
|
allowMemoryStateFallback: options.allowMemoryStateFallback,
|
|
1618
1645
|
operationExecutor: options.operationExecutor,
|
package/dist/testing/run.js
CHANGED
|
@@ -248,6 +248,7 @@ function createUpstreamContext(provider, operationName, upstreamStub) {
|
|
|
248
248
|
},
|
|
249
249
|
}),
|
|
250
250
|
close: async () => { },
|
|
251
|
+
cookies: async () => (await browserAction("cookies")).data,
|
|
251
252
|
fill: async (selector, textValue) => {
|
|
252
253
|
await browserAction("fill", { selector, text: textValue });
|
|
253
254
|
},
|
|
@@ -400,6 +401,9 @@ function createUpstreamContext(provider, operationName, upstreamStub) {
|
|
|
400
401
|
auth: { requestField: async (name) => unsupported(`ctx.auth.requestField(${name})`) },
|
|
401
402
|
ocr: createUnsupportedOcrClient("Standard test upstream context does not support ctx.ocr.recognize"),
|
|
402
403
|
stt: createUnsupportedSttClient("Standard test upstream context does not support ctx.stt.transcribe"),
|
|
404
|
+
resolver: {
|
|
405
|
+
solve: async () => unsupported("ctx.resolver.solve"),
|
|
406
|
+
},
|
|
403
407
|
choice: createTestProviderChoiceContext({
|
|
404
408
|
providerId: `standard-test-${operationName}`,
|
|
405
409
|
request,
|
|
@@ -507,6 +511,9 @@ export function createSnapshotContext(rawFixture) {
|
|
|
507
511
|
},
|
|
508
512
|
ocr: createUnsupportedOcrClient("Standard test snapshot context does not support ctx.ocr.recognize"),
|
|
509
513
|
stt: createUnsupportedSttClient("Standard test snapshot context does not support ctx.stt.transcribe"),
|
|
514
|
+
resolver: {
|
|
515
|
+
solve: async () => unsupported("ctx.resolver.solve"),
|
|
516
|
+
},
|
|
510
517
|
choice: createTestProviderChoiceContext({
|
|
511
518
|
providerId: "standard-test",
|
|
512
519
|
request,
|
package/dist/types.d.ts
CHANGED
|
@@ -251,6 +251,79 @@ export type ProviderSttMode = "optional" | "required";
|
|
|
251
251
|
export interface ProviderSttConfig {
|
|
252
252
|
mode: ProviderSttMode;
|
|
253
253
|
}
|
|
254
|
+
/**
|
|
255
|
+
* `browser` is the in-house CDP pool (`apps/cdp-pool`, reached through
|
|
256
|
+
* `createBrowserClient`) and is a first-class vendor rather than an escape hatch:
|
|
257
|
+
* for fingerprint-family kinds, it was measured faster than a paid vendor
|
|
258
|
+
* (4.5 s vs 17.5 s) at zero marginal cost.
|
|
259
|
+
*
|
|
260
|
+
* `2captcha` is the vendor already carrying production traffic in
|
|
261
|
+
* `apifuse-provider-tabelog`.
|
|
262
|
+
*
|
|
263
|
+
* Union order is documentation only; the effective fallback order is whatever
|
|
264
|
+
* `ProviderResolverConfig.vendors` declares.
|
|
265
|
+
*/
|
|
266
|
+
export type ProviderResolverVendor = "browser" | "capsolver" | "capmonster" | "2captcha" | "custom";
|
|
267
|
+
/**
|
|
268
|
+
* Token-family kinds resolve to `{ form: "token" }`. Cookie-family kinds resolve
|
|
269
|
+
* to `{ form: "cookies" }`; network-identity binding is defined per kind. `aws_waf`
|
|
270
|
+
* was measured portable across residential leases on buyee, while `cf_clearance`
|
|
271
|
+
* remains unmeasured here and is treated as identity-scoped because it is widely
|
|
272
|
+
* described as IP-bound.
|
|
273
|
+
*/
|
|
274
|
+
export type ProviderChallenge = {
|
|
275
|
+
readonly kind: "turnstile";
|
|
276
|
+
readonly siteKey: string;
|
|
277
|
+
readonly pageUrl: string;
|
|
278
|
+
readonly action?: string;
|
|
279
|
+
readonly cdata?: string;
|
|
280
|
+
} | {
|
|
281
|
+
readonly kind: "recaptcha_v2";
|
|
282
|
+
readonly siteKey: string;
|
|
283
|
+
readonly pageUrl: string;
|
|
284
|
+
} | {
|
|
285
|
+
readonly kind: "recaptcha_v3";
|
|
286
|
+
readonly siteKey: string;
|
|
287
|
+
readonly pageUrl: string;
|
|
288
|
+
readonly action: string;
|
|
289
|
+
readonly minScore?: number;
|
|
290
|
+
} | {
|
|
291
|
+
readonly kind: "hcaptcha";
|
|
292
|
+
readonly siteKey: string;
|
|
293
|
+
readonly pageUrl: string;
|
|
294
|
+
} | {
|
|
295
|
+
readonly kind: "cloudflare_interstitial";
|
|
296
|
+
readonly pageUrl: string;
|
|
297
|
+
readonly blockedHtml?: string;
|
|
298
|
+
} | {
|
|
299
|
+
readonly kind: "aws_waf";
|
|
300
|
+
readonly pageUrl: string;
|
|
301
|
+
readonly captchaScript?: string;
|
|
302
|
+
readonly context?: string;
|
|
303
|
+
readonly iv?: string;
|
|
304
|
+
};
|
|
305
|
+
export type ProviderChallengeKind = ProviderChallenge["kind"];
|
|
306
|
+
/**
|
|
307
|
+
* Token solutions carry no network-identity binding. Cookie-solution binding is
|
|
308
|
+
* per challenge kind: `aws_waf` was measured portable across residential leases
|
|
309
|
+
* on buyee, while `cf_clearance` is unmeasured here and treated as scoped to the
|
|
310
|
+
* identity that produced it. The provider attaches the returned cookies to its
|
|
311
|
+
* own requests.
|
|
312
|
+
*/
|
|
313
|
+
export type ChallengeSolution = {
|
|
314
|
+
readonly form: "token";
|
|
315
|
+
readonly token: string;
|
|
316
|
+
} | {
|
|
317
|
+
readonly form: "cookies";
|
|
318
|
+
readonly cookies: Readonly<Record<string, string>>;
|
|
319
|
+
readonly userAgent: string;
|
|
320
|
+
};
|
|
321
|
+
export interface ProviderResolverConfig {
|
|
322
|
+
/** Ordered vendor fallback chain, tried first to last. */
|
|
323
|
+
readonly vendors: readonly ProviderResolverVendor[];
|
|
324
|
+
/** Challenge kinds this provider is permitted to request. */
|
|
325
|
+
readonly kinds: readonly ProviderChallengeKind[];
|
|
326
|
+
}
|
|
254
327
|
export type SttAudioInput = {
|
|
255
328
|
kind: "base64";
|
|
256
329
|
data: string;
|
|
@@ -315,6 +388,9 @@ export interface SttContext {
|
|
|
315
388
|
transcribe(request: SttTranscribeRequest): Promise<SttTranscript>;
|
|
316
389
|
extractVerificationCode(text: string, options?: SttVerificationCodeOptions): VerificationCodeExtractionResult;
|
|
317
390
|
}
|
|
391
|
+
export interface ResolverContext {
|
|
392
|
+
solve(challenge: ProviderChallenge, signal?: AbortSignal): Promise<ChallengeSolution>;
|
|
393
|
+
}
|
|
318
394
|
export interface HealthJourneySchedule {
|
|
319
395
|
kind: "interval";
|
|
320
396
|
/** ISO 8601 duration, for example PT8H. */
|
|
@@ -716,10 +792,12 @@ export type ProviderProxyMode = "disabled" | "optional" | "required";
|
|
|
716
792
|
* - `decodo` — **decodo.com**, the *gateway* proxy that was named "Smartproxy"
|
|
717
793
|
* (smartproxy.com) before its 2025 rebrand to Decodo. Sticky sessions via
|
|
718
794
|
* username params. A different company from `smartproxy` above.
|
|
719
|
-
* **@deprecated** — unused; no managed adapter.
|
|
720
|
-
*
|
|
721
|
-
*
|
|
722
|
-
*
|
|
795
|
+
* **@deprecated** — unused; no managed adapter. Declare a
|
|
796
|
+
* `ProviderProxyPolicy` using `smartproxy` or `nodemaven` instead; the
|
|
797
|
+
* `smartproxy` allocator requires `APIFUSE__PROXY__SMARTPROXY_APP_KEY`.
|
|
798
|
+
* - `custom` — **@deprecated** static proxy marker with no managed adapter.
|
|
799
|
+
* Use `ProviderProxyPolicy` with `smartproxy`/`nodemaven`; the `smartproxy`
|
|
800
|
+
* allocator requires `APIFUSE__PROXY__SMARTPROXY_APP_KEY`.
|
|
723
801
|
*/
|
|
724
802
|
export type ProviderProxyProvider = "smartproxy" | "nodemaven" | "decodo" | "custom";
|
|
725
803
|
export type ProviderProxySessionAffinity = "request" | "operation" | "auth-flow" | "connection";
|
|
@@ -1254,9 +1332,9 @@ export interface NativeProviderConfig {
|
|
|
1254
1332
|
}
|
|
1255
1333
|
export interface ProviderCacheKeyOptions {
|
|
1256
1334
|
/**
|
|
1257
|
-
* Additional field names
|
|
1258
|
-
*
|
|
1259
|
-
* cookie, token, password, and secret.
|
|
1335
|
+
* Additional field names whose values are hashed in stable key material. The
|
|
1336
|
+
* SDK always hashes values under known secret-bearing names such as serviceKey,
|
|
1337
|
+
* authorization, cookie, token, password, and secret.
|
|
1260
1338
|
*/
|
|
1261
1339
|
redactFields?: string[];
|
|
1262
1340
|
}
|
|
@@ -1329,6 +1407,17 @@ export interface BrowserFrame {
|
|
|
1329
1407
|
evaluate<T>(fn: string | (() => T)): Promise<T>;
|
|
1330
1408
|
locator(selector: string): BrowserLocator;
|
|
1331
1409
|
}
|
|
1410
|
+
export interface BrowserCookie {
|
|
1411
|
+
readonly name: string;
|
|
1412
|
+
readonly value: string;
|
|
1413
|
+
readonly domain: string;
|
|
1414
|
+
readonly path: string;
|
|
1415
|
+
/** Unix seconds. Absent for a session cookie. */
|
|
1416
|
+
readonly expires?: number;
|
|
1417
|
+
readonly httpOnly: boolean;
|
|
1418
|
+
readonly secure: boolean;
|
|
1419
|
+
readonly sameSite?: "Strict" | "Lax" | "None";
|
|
1420
|
+
}
|
|
1332
1421
|
export type BrowserResourceMethod = "GET" | "HEAD";
|
|
1333
1422
|
export type BrowserResourceRequest = {
|
|
1334
1423
|
readonly url: string;
|
|
@@ -1357,6 +1446,11 @@ export type BrowserResourcePolicy = {
|
|
|
1357
1446
|
};
|
|
1358
1447
|
export interface BrowserPage extends BrowserFrame {
|
|
1359
1448
|
close(): Promise<void>;
|
|
1449
|
+
/**
|
|
1450
|
+
* Reads the browser context's cookie jar, including httpOnly cookies.
|
|
1451
|
+
* Cookie expiry values are Unix seconds and are absent for session cookies.
|
|
1452
|
+
*/
|
|
1453
|
+
cookies(): Promise<readonly BrowserCookie[]>;
|
|
1360
1454
|
fill(selector: string, text: string): Promise<void>;
|
|
1361
1455
|
goto(url: string): Promise<void>;
|
|
1362
1456
|
pageId?: string;
|
|
@@ -1593,6 +1687,7 @@ export interface FlowContext {
|
|
|
1593
1687
|
context: ContextScratchpad;
|
|
1594
1688
|
ocr: OcrContext;
|
|
1595
1689
|
stt: SttContext;
|
|
1690
|
+
resolver: ResolverContext;
|
|
1596
1691
|
auth: AuthFlowTerminalContext;
|
|
1597
1692
|
}
|
|
1598
1693
|
export interface AuthTurn {
|
|
@@ -1624,7 +1719,13 @@ export interface AuthFlowDefinition {
|
|
|
1624
1719
|
refresh?: AuthFlowInputHandler;
|
|
1625
1720
|
}
|
|
1626
1721
|
export type ProviderStateDurationString = `${number}${"ms" | "s" | "m" | "h" | "d"}` | `PT${string}`;
|
|
1722
|
+
export type StateNamespaceScope = "connection" | "provider";
|
|
1627
1723
|
export interface StateNamespaceOptions {
|
|
1724
|
+
/**
|
|
1725
|
+
* State isolation boundary. Connection scope is the default; provider scope
|
|
1726
|
+
* must be selected explicitly for provider-wide coordination state.
|
|
1727
|
+
*/
|
|
1728
|
+
scope?: StateNamespaceScope;
|
|
1628
1729
|
/** Default TTL used when a write omits ttl. Required to avoid unbounded state. */
|
|
1629
1730
|
defaultTtl: ProviderStateDurationString;
|
|
1630
1731
|
/** Maximum allowed TTL; writes are rejected when they exceed this policy. */
|
|
@@ -1666,6 +1767,11 @@ export interface ProviderStateNamespace {
|
|
|
1666
1767
|
increment(key: string, field: string, delta?: number, options?: StateWriteOptions): Promise<StateValue<Record<string, unknown>>>;
|
|
1667
1768
|
}
|
|
1668
1769
|
export interface ProviderRuntimeState {
|
|
1770
|
+
/**
|
|
1771
|
+
* Returns an immutable view bound to one request connection. An unresolved
|
|
1772
|
+
* connection uses an isolated reserved sentinel, never the provider-global scope.
|
|
1773
|
+
*/
|
|
1774
|
+
forConnection(connectionId: string | undefined): ProviderRuntimeState;
|
|
1669
1775
|
namespace(name: string, options: StateNamespaceOptions): ProviderStateNamespace;
|
|
1670
1776
|
}
|
|
1671
1777
|
export interface ProviderContext {
|
|
@@ -1685,6 +1791,7 @@ export interface ProviderContext {
|
|
|
1685
1791
|
auth: AuthContext;
|
|
1686
1792
|
ocr: OcrContext;
|
|
1687
1793
|
stt: SttContext;
|
|
1794
|
+
resolver: ResolverContext;
|
|
1688
1795
|
choice: ProviderChoiceContext;
|
|
1689
1796
|
}
|
|
1690
1797
|
export interface ProxiedOAuthConfig {
|
|
@@ -1837,6 +1944,7 @@ export interface ProviderDefinition {
|
|
|
1837
1944
|
proxy?: ProviderProxyConfig;
|
|
1838
1945
|
ocr?: ProviderOcrConfig;
|
|
1839
1946
|
stt?: ProviderSttConfig;
|
|
1947
|
+
resolver?: ProviderResolverConfig;
|
|
1840
1948
|
browser?: {
|
|
1841
1949
|
engine: BrowserEngine;
|
|
1842
1950
|
};
|
package/package.json
CHANGED
package/src/config/loader.ts
CHANGED
|
@@ -19,7 +19,7 @@ import {
|
|
|
19
19
|
|
|
20
20
|
export type { ProxyProtocol } from "../runtime/proxy-nodemaven.js";
|
|
21
21
|
|
|
22
|
-
/** Proxy vendors
|
|
22
|
+
/** Proxy vendors with SDK-managed resolution. */
|
|
23
23
|
export type ProxyVendorName = "smartproxy" | "nodemaven";
|
|
24
24
|
|
|
25
25
|
// "smartproxy" here is api.smartproxy.org — a residential proxy with an IP
|
|
@@ -37,15 +37,6 @@ export const PROVIDER_CACHE_REDIS_URL_ENV = "APIFUSE__PROVIDER__CACHE_REDIS_URL"
|
|
|
37
37
|
export const PROVIDER_STATE_REDIS_URL_ENV = "APIFUSE__PROVIDER__STATE_REDIS_URL";
|
|
38
38
|
export const REDIS_URL_ENV = "APIFUSE__REDIS__URL";
|
|
39
39
|
|
|
40
|
-
export type ProxyOptions = {
|
|
41
|
-
url: string;
|
|
42
|
-
};
|
|
43
|
-
|
|
44
|
-
export type ProxyConfig = Partial<ProxyOptions> & {
|
|
45
|
-
provider?: string;
|
|
46
|
-
apiKey?: string;
|
|
47
|
-
};
|
|
48
|
-
|
|
49
40
|
export type BrowserConfig = {
|
|
50
41
|
executablePath?: string;
|
|
51
42
|
headless?: boolean;
|
|
@@ -57,7 +48,6 @@ export type SessionConfig = {
|
|
|
57
48
|
};
|
|
58
49
|
|
|
59
50
|
export type ApiFuseConfig = {
|
|
60
|
-
proxy?: ProxyConfig;
|
|
61
51
|
browser?: BrowserConfig;
|
|
62
52
|
session?: SessionConfig;
|
|
63
53
|
trace?: TraceConfig;
|
|
@@ -67,7 +57,6 @@ export type ApiFuseConfig = {
|
|
|
67
57
|
export type ProxyResolutionOptions = {
|
|
68
58
|
proxy?: string;
|
|
69
59
|
upstream?: { proxy?: boolean | ProviderProxyPolicy };
|
|
70
|
-
apifuseConfig?: Pick<ApiFuseConfig, "proxy">;
|
|
71
60
|
proxyPolicy?: ProviderProxyPolicy;
|
|
72
61
|
affinityKey?: string;
|
|
73
62
|
/** Zero-based proxy-pool attempt index used by SDK transports for failover. */
|
|
@@ -444,75 +433,7 @@ function serializeSmartproxyPool(pool: CachedProxyPool): string {
|
|
|
444
433
|
|
|
445
434
|
function normalizeProxyUrl(url?: string): string | undefined {
|
|
446
435
|
const normalized = url?.trim();
|
|
447
|
-
return normalized
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
function readPositiveIntegerEnv(name: string): string | undefined {
|
|
451
|
-
const raw = process.env[name]?.trim();
|
|
452
|
-
if (!raw) return undefined;
|
|
453
|
-
if (!/^[1-9]\d*$/.test(raw)) {
|
|
454
|
-
throw new Error(`${name} must be a positive integer`);
|
|
455
|
-
}
|
|
456
|
-
return raw;
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
function applyStickyProxySession(proxyUrl: string): string {
|
|
460
|
-
let parsed: URL;
|
|
461
|
-
try {
|
|
462
|
-
parsed = new URL(proxyUrl);
|
|
463
|
-
} catch {
|
|
464
|
-
return proxyUrl;
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
if (!parsed.hostname || !parsed.username || !parsed.password) {
|
|
468
|
-
return proxyUrl;
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
// This rewrites sticky-session usernames for a bring-your-own *gateway* URL
|
|
472
|
-
// (APIFUSE__PROXY__URL). The `smartproxy` host here means a smartproxy.com /
|
|
473
|
-
// Decodo-family gateway that authenticates by username — NOT the
|
|
474
|
-
// api.smartproxy.org allocation vendor, whose endpoints are raw ip:port with
|
|
475
|
-
// no credentials and therefore return early above.
|
|
476
|
-
const host = parsed.hostname.toLowerCase();
|
|
477
|
-
if (!host.includes("smartproxy") && !host.includes("decodo")) {
|
|
478
|
-
return proxyUrl;
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
const username = decodeURIComponent(parsed.username);
|
|
482
|
-
const sessionId = process.env.APIFUSE__PROXY__SESSION_ID?.trim() || "apifuse-shared";
|
|
483
|
-
const sessionDuration = readPositiveIntegerEnv("APIFUSE__PROXY__SESSION_DURATION");
|
|
484
|
-
const stickyUsername = host.includes("smartproxy")
|
|
485
|
-
? buildSmartproxyUsername(username, sessionId, sessionDuration)
|
|
486
|
-
: buildDecodoUsername(username, sessionId, sessionDuration ?? "60");
|
|
487
|
-
|
|
488
|
-
parsed.username = stickyUsername;
|
|
489
|
-
return parsed.toString();
|
|
490
|
-
}
|
|
491
|
-
|
|
492
|
-
function buildSmartproxyUsername(
|
|
493
|
-
username: string,
|
|
494
|
-
sessionId: string,
|
|
495
|
-
sessionDuration?: string,
|
|
496
|
-
): string {
|
|
497
|
-
const parts = username.split("_");
|
|
498
|
-
const configuredLife = parts.find((part) => part.startsWith("life-"))?.slice("life-".length);
|
|
499
|
-
const baseUsername = parts
|
|
500
|
-
.filter((part) => !part.startsWith("session-") && !part.startsWith("life-"))
|
|
501
|
-
.join("_");
|
|
502
|
-
return `${baseUsername}_session-${sessionId}_life-${sessionDuration ?? configuredLife ?? "60"}`;
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
function buildDecodoUsername(username: string, sessionId: string, sessionDuration: string): string {
|
|
506
|
-
const withoutSticky = username.replace(/-session-.+-sessionduration-\d+$/, "");
|
|
507
|
-
const baseUsername = withoutSticky.startsWith("user-") ? withoutSticky : `user-${withoutSticky}`;
|
|
508
|
-
return `${baseUsername}-session-${sessionId}-sessionduration-${sessionDuration}`;
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
function syncProxyEnv(config: ApiFuseConfig): void {
|
|
512
|
-
const configProxyUrl = normalizeProxyUrl(config.proxy?.url);
|
|
513
|
-
if (!process.env.APIFUSE__PROXY__URL && configProxyUrl) {
|
|
514
|
-
process.env.APIFUSE__PROXY__URL = configProxyUrl;
|
|
515
|
-
}
|
|
436
|
+
return normalized || undefined;
|
|
516
437
|
}
|
|
517
438
|
|
|
518
439
|
export function resolveProxyConfig(options: ProxyResolutionOptions = {}): ResolvedProxyConfig {
|
|
@@ -532,16 +453,6 @@ export function resolveProxyConfig(options: ProxyResolutionOptions = {}): Resolv
|
|
|
532
453
|
return { shouldWarn: false };
|
|
533
454
|
}
|
|
534
455
|
|
|
535
|
-
const envProxyUrl = normalizeProxyUrl(process.env.APIFUSE__PROXY__URL);
|
|
536
|
-
if (envProxyUrl) {
|
|
537
|
-
return { shouldWarn: false, url: envProxyUrl };
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
const configuredProxyUrl = normalizeProxyUrl(options.apifuseConfig?.proxy?.url);
|
|
541
|
-
if (configuredProxyUrl) {
|
|
542
|
-
return { shouldWarn: false, url: configuredProxyUrl };
|
|
543
|
-
}
|
|
544
|
-
|
|
545
456
|
return { shouldWarn: true };
|
|
546
457
|
}
|
|
547
458
|
|
|
@@ -563,7 +474,22 @@ export async function resolveProxyConfigAsync(
|
|
|
563
474
|
|
|
564
475
|
const chain = resolveVendorChain(policy);
|
|
565
476
|
if (chain.length === 0) {
|
|
566
|
-
|
|
477
|
+
const declared = declaredVendorChain(policy);
|
|
478
|
+
const deprecated = declared.filter((vendor) => vendor === "decodo" || vendor === "custom");
|
|
479
|
+
if (policy.mode === "required") {
|
|
480
|
+
const providerIds =
|
|
481
|
+
declared.length > 0 ? declared.map((vendor) => `"${vendor}"`).join(", ") : "none";
|
|
482
|
+
const deprecatedDetail =
|
|
483
|
+
deprecated.length > 0
|
|
484
|
+
? ` Deprecated vendor(s): ${deprecated.map((vendor) => `"${vendor}"`).join(", ")}.`
|
|
485
|
+
: "";
|
|
486
|
+
throw new ProxyResolutionError(
|
|
487
|
+
"PROXY_REQUIRED",
|
|
488
|
+
`Required proxy policy has no SDK-managed adapter for provider id(s): ${providerIds}.${deprecatedDetail} Use "smartproxy" or "nodemaven".`,
|
|
489
|
+
);
|
|
490
|
+
}
|
|
491
|
+
// Deprecated decodo/custom providers have no SDK-managed adapter. Optional
|
|
492
|
+
// policies preserve the warning-only behavior and may continue directly.
|
|
567
493
|
return resolveProxyConfig({
|
|
568
494
|
...options,
|
|
569
495
|
upstream: { proxy: true },
|
|
@@ -842,18 +768,22 @@ function isRegistryVendor(name: string | undefined): name is ProxyVendorName {
|
|
|
842
768
|
return name === "smartproxy" || name === "nodemaven";
|
|
843
769
|
}
|
|
844
770
|
|
|
771
|
+
function declaredVendorChain(policy: ProviderProxyPolicy): ProviderProxyProvider[] {
|
|
772
|
+
const declared = policy.providers?.length
|
|
773
|
+
? policy.providers
|
|
774
|
+
: [policy.provider ?? envDefaultProvider()];
|
|
775
|
+
return declared.filter((vendor): vendor is ProviderProxyProvider => vendor !== undefined);
|
|
776
|
+
}
|
|
777
|
+
|
|
845
778
|
/**
|
|
846
779
|
* Ordered list of SDK-native proxy vendors declared by the policy. `providers`
|
|
847
780
|
* takes precedence over the legacy singular `provider`; the platform default
|
|
848
781
|
* env is the final fallback. Non-registry names (decodo/custom) are dropped so
|
|
849
|
-
* an all-
|
|
782
|
+
* an all-deprecated chain has no managed adapter.
|
|
850
783
|
*/
|
|
851
784
|
export function resolveVendorChain(policy: ProviderProxyPolicy): ProxyVendorName[] {
|
|
852
|
-
const declared: (ProviderProxyProvider | undefined)[] = policy.providers?.length
|
|
853
|
-
? policy.providers
|
|
854
|
-
: [policy.provider ?? envDefaultProvider()];
|
|
855
785
|
const chain: ProxyVendorName[] = [];
|
|
856
|
-
for (const name of
|
|
786
|
+
for (const name of declaredVendorChain(policy)) {
|
|
857
787
|
if (isRegistryVendor(name) && !chain.includes(name)) {
|
|
858
788
|
chain.push(name);
|
|
859
789
|
}
|
|
@@ -924,9 +854,8 @@ const UNSAFE_TRANSPORT_RETRY_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"
|
|
|
924
854
|
* - the method is safe/idempotent — an unsafe request must never be duplicated
|
|
925
855
|
* across the pool even if some framework default would allow it;
|
|
926
856
|
* - the policy resolves a non-empty *registry* vendor chain (smartproxy /
|
|
927
|
-
* nodemaven).
|
|
928
|
-
*
|
|
929
|
-
* with no possible crossover — they keep the retry budget.
|
|
857
|
+
* nodemaven). Deprecated vendors (custom / decodo) resolve no managed pool,
|
|
858
|
+
* so there is no possible endpoint crossover — they keep the retry budget.
|
|
930
859
|
*
|
|
931
860
|
* The widened cap is bounded by the chain's true maximum span (sum of each
|
|
932
861
|
* vendor's max pool size), so a large NodeMaven pool (≤50) stays reachable and
|
|
@@ -945,8 +874,8 @@ const UNSAFE_TRANSPORT_RETRY_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"
|
|
|
945
874
|
* endpoint each attempt resolves (even a repeated one), so no de-duplication;
|
|
946
875
|
* - the method is safe/idempotent — an unsafe request is never duplicated;
|
|
947
876
|
* - the policy resolves a non-empty registry vendor chain (smartproxy /
|
|
948
|
-
* nodemaven).
|
|
949
|
-
*
|
|
877
|
+
* nodemaven). Deprecated vendors (custom / decodo) resolve no managed
|
|
878
|
+
* endpoint, so there is nothing to rotate or de-duplicate.
|
|
950
879
|
*/
|
|
951
880
|
export function policyRotatesTransportVendorChain(input: {
|
|
952
881
|
policy: ProviderProxyPolicy | undefined;
|
|
@@ -989,9 +918,8 @@ export function resolvePolicyTransportAttemptCap(input: {
|
|
|
989
918
|
* A registry vendor chain (smartproxy/nodemaven) resolves a potentially
|
|
990
919
|
* *different* endpoint per flat attempt index, so a transport retry should
|
|
991
920
|
* advance across endpoints and de-duplicate once the chain stops yielding new
|
|
992
|
-
* ones.
|
|
993
|
-
*
|
|
994
|
-
* transport loop must not de-duplicate them.
|
|
921
|
+
* ones. Deprecated custom/decodo policies have an empty registry chain and no
|
|
922
|
+
* managed endpoint, so the transport loop has nothing to rotate or de-duplicate.
|
|
995
923
|
*/
|
|
996
924
|
export function policyResolvesRegistryVendorChain(
|
|
997
925
|
policy: ProviderProxyPolicy | undefined,
|
|
@@ -1828,17 +1756,13 @@ export async function loadApiFuseConfig(dir: string = process.cwd()): Promise<Ap
|
|
|
1828
1756
|
const tsPath = path.resolve(dir, "apifuse.config.ts");
|
|
1829
1757
|
if (existsSync(tsPath)) {
|
|
1830
1758
|
const config = await importConfig(tsPath);
|
|
1831
|
-
|
|
1832
|
-
syncProxyEnv(resolvedConfig);
|
|
1833
|
-
return resolvedConfig;
|
|
1759
|
+
return config ?? {};
|
|
1834
1760
|
}
|
|
1835
1761
|
|
|
1836
1762
|
const jsPath = path.resolve(dir, "apifuse.config.js");
|
|
1837
1763
|
if (existsSync(jsPath)) {
|
|
1838
1764
|
const config = await importConfig(jsPath);
|
|
1839
|
-
|
|
1840
|
-
syncProxyEnv(resolvedConfig);
|
|
1841
|
-
return resolvedConfig;
|
|
1765
|
+
return config ?? {};
|
|
1842
1766
|
}
|
|
1843
1767
|
|
|
1844
1768
|
return {};
|