@apifuse/provider-sdk 2.2.0-beta.28 → 2.2.0-beta.29

package/CHANGELOG.md

@@ -1,5 +1,9 @@
 # @apifuse/provider-sdk Changelog
 
+## 2.2.0-beta.29
+
+- Release candidate for main commit b7243459ab12403e11d7b5b93292f09b734b0a96.
+
 ## 2.2.0-beta.28
 
 - Release candidate for main commit 2936f1891d4e2326502f0585081f8a58060d6dd7.

package/bin/apifuse-dev.ts

@@ -2,7 +2,6 @@
 
 import { existsSync } from "node:fs";
 import { dirname, relative, resolve } from "node:path";
-import type { ProviderDefinition } from "../src/index.js";
 import {
 	createCredentialContext,
 	createEnvContext,
@@ -13,12 +12,16 @@ import {
 	createUnsupportedResolverClient,
 	createSttClientFromEnv,
 	PROVIDER_RUNTIME_CHOICE_TOKEN_MASTER_SECRET_ENV,
+	type ProviderDefinition,
 	ProviderError,
+	type ProviderProxyPolicy,
 } from "../src/index.js";
 import { createBrowserClient } from "../src/runtime/browser.js";
+import { createResolverClientFromEnv } from "../src/runtime/resolver.js";
 import { createMemoryProviderRuntimeState } from "../src/runtime/state.js";
 import { createStealthClient } from "../src/runtime/stealth.js";
 import { createTraceContext } from "../src/runtime/trace.js";
+import { getStealthProfile } from "../src/stealth/profiles.js";
 import type { BrowserClient, ProviderContext } from "../src/types.js";
 
 const HELP_TEXT = `Usage: apifuse dev [path]
@@ -82,6 +85,11 @@ export function createProviderContext(provider: ProviderDefinition): {
 	]);
 	const credential = createCredentialContext();
 	const state = createMemoryProviderRuntimeState();
+	const cache = createProviderCache({ providerId: provider.id });
+	const proxyPolicy = resolveNativeProxyPolicy(provider);
+	const stealthProfile = provider.stealth?.profile
+		? getStealthProfile(provider.stealth.profile)
+		: undefined;
 	const ctx: ProviderContext = {
 		env,
 		credential,
@@ -94,13 +102,27 @@ export function createProviderContext(provider: ProviderDefinition): {
 					})
 				: createUnsupportedBrowserStub(),
 		http: createHttpClient(),
-		cache: createProviderCache({ providerId: provider.id }),
+		cache,
 		state,
 		trace: createTraceContext(),
 		stealth: createStealthClient("http://localhost"),
 		ocr: createOcrClientFromEnv(provider.ocr),
 		stt: createSttClientFromEnv(provider.stt),
-		resolver: createUnsupportedResolverClient("Resolver is not available in apifuse dev"),
+		resolver: provider.resolver
+			? createResolverClientFromEnv(provider.resolver, undefined, {
+					allowedHosts: provider.allowedHosts,
+					cache,
+					...(proxyPolicy
+						? {
+								proxyIntent: {
+									mode: proxyPolicy.mode,
+									upstream: { proxy: provider.proxy },
+									...(stealthProfile ? { userAgent: stealthProfile.userAgent } : {}),
+								},
+							}
+						: {}),
+				})
+			: createUnsupportedResolverClient("Provider does not declare resolver capability"),
 		choice: createProviderChoiceContext({
 			providerId: provider.id,
 			env,
@@ -112,6 +134,13 @@ export function createProviderContext(provider: ProviderDefinition): {
 	return { ctx };
 }
 
+function resolveNativeProxyPolicy(provider: ProviderDefinition): ProviderProxyPolicy | undefined {
+	if (typeof provider.proxy === "object") return provider.proxy;
+	if (provider.proxy === true) return { mode: "optional" };
+	if (provider.proxy === false) return { mode: "disabled" };
+	return undefined;
+}
+
 function normalizeArgs(argv: string[]): string[] {
 	return argv[0] === "dev" ? argv.slice(1) : argv;
 }

package/bin/apifuse-pack-types.ts

@@ -242,6 +242,28 @@ const NEGATIVE_CONTROLS = [
 			"",
 		].join("\n"),
 	},
+	{
+		filename: "negative-control-resolver-default-user-agent-test-seam.ts",
+		expectedCode: "TS2305",
+		description: "the resolver subpath does not expose its default user-agent test seam",
+		source: [
+			'import { swapResolverDefaultUserAgentForTests } from "@apifuse/provider-sdk/runtime/resolver";',
+			"",
+			"export const mustNotCompile = swapResolverDefaultUserAgentForTests;",
+			"",
+		].join("\n"),
+	},
+	{
+		filename: "negative-control-cache-reset-test-seam.ts",
+		expectedCode: "TS2305",
+		description: "the package root does not expose its cache-reset test seam",
+		source: [
+			'import { resetProviderCacheForTests } from "@apifuse/provider-sdk";',
+			"",
+			"export const mustNotCompile = resetProviderCacheForTests;",
+			"",
+		].join("\n"),
+	},
 	{
 		filename: "negative-control-resolver-runtime-allowed-hosts.ts",
 		expectedCode: "TS2322",
@@ -366,7 +388,7 @@ function setUpFixtureConsumer(consumerDir: string, tarballPath: string): void {
 			'import { extractProviderContract } from "@apifuse/provider-sdk/contract";',
 			'import { AUTH_TURN_SCHEMA } from "@apifuse/provider-sdk/auth-turn";',
 			'import { serve } from "@apifuse/provider-sdk/server";',
-			'import { runStandardTests } from "@apifuse/provider-sdk/testing";',
+			'import { resetProviderCacheForTests, runStandardTests } from "@apifuse/provider-sdk/testing";',
 			"",
 			"// ProviderError must keep its inherited Error members under nodenext.",
 			"// When dist d.ts imports fail to resolve, the class type degrades and",
@@ -451,6 +473,7 @@ function setUpFixtureConsumer(consumerDir: string, tarballPath: string): void {
 			"	extractProviderContract,",
 			"	AUTH_TURN_SCHEMA,",
 			"	serve,",
+			"	resetProviderCacheForTests,",
 			"	runStandardTests,",
 			"};",
 			"",

package/bin/apifuse-record.ts

@@ -19,6 +19,7 @@ import {
 	type ProviderContext,
 	type ProviderDefinition,
 	ProviderError,
+	type ProviderProxyPolicy,
 	type RequestOptions,
 	type StealthClient,
 	TransportError,
@@ -31,13 +32,12 @@ import {
 	sanitizeDiagnosticText,
 	sanitizeFixtureString,
 } from "../src/fixture-sanitization.js";
-import { createMemoryProviderRuntimeState } from "../src/runtime/state.js";
-import { createStealthClient } from "../src/runtime/stealth.js";
+import { createResolverClientFromEnv } from "../src/runtime/resolver.js";
 import {
-	REDACTED_QUERY_VALUE,
 	isSensitiveKey,
 	normalizeSensitiveParams,
 	parseHttpRequestInvocation,
+	REDACTED_QUERY_VALUE,
 	redactSensitiveError,
 	redactSensitiveText,
 	redactUrlQueryParams,
@@ -45,7 +45,10 @@ import {
 	requestOptionsFromHttpInvocation,
 	serializeRequestUrl,
 } from "../src/runtime/request-options.js";
+import { createMemoryProviderRuntimeState } from "../src/runtime/state.js";
+import { createStealthClient } from "../src/runtime/stealth.js";
 import { parseSchema } from "../src/schema.js";
+import { getStealthProfile } from "../src/stealth/profiles.js";
 import {
 	captureStreamEvidence,
 	createStreamCaptureEnvelope,
@@ -422,7 +425,18 @@ function resolveOperationBaseUrl(provider: ProviderRuntime, operationName: strin
 	return baseUrl;
 }
 
-function createCaptureContext(provider: ProviderRuntime, baseUrl: string, sanitize: boolean) {
+function resolveNativeProxyPolicy(provider: ProviderDefinition): ProviderProxyPolicy | undefined {
+	if (typeof provider.proxy === "object") return provider.proxy;
+	if (provider.proxy === true) return { mode: "optional" };
+	if (provider.proxy === false) return { mode: "disabled" };
+	return undefined;
+}
+
+export function createCaptureContext(
+	provider: ProviderRuntime,
+	baseUrl: string,
+	sanitize: boolean,
+) {
 	let nextCaptureOrder = 0;
 	let nextStreamOrdinal = 0;
 	let capturedRaw: JsonValue | undefined;
@@ -506,12 +520,17 @@ function createCaptureContext(provider: ProviderRuntime, baseUrl: string, saniti
 		getScopes: () => [],
 	};
 	const state = createMemoryProviderRuntimeState();
+	const cache = createBypassProviderCache({ providerId: provider.id });
+	const proxyPolicy = resolveNativeProxyPolicy(provider);
+	const stealthProfile = provider.stealth?.profile
+		? getStealthProfile(provider.stealth.profile)
+		: undefined;
 	const ctx: ProviderContext = {
 		env,
 		credential,
 		request: { headers: {} },
 		http,
-		cache: createBypassProviderCache({ providerId: provider.id }),
+		cache,
 		state,
 		stealth,
 		browser: {
@@ -540,7 +559,21 @@ function createCaptureContext(provider: ProviderRuntime, baseUrl: string, saniti
 		},
 		ocr: createOcrClientFromEnv(provider.ocr),
 		stt: createSttClientFromEnv(provider.stt),
-		resolver: createUnsupportedResolverClient("Resolver is not available in apifuse record"),
+		resolver: provider.resolver
+			? createResolverClientFromEnv(provider.resolver, undefined, {
+					allowedHosts: provider.allowedHosts,
+					cache,
+					...(proxyPolicy
+						? {
+								proxyIntent: {
+									mode: proxyPolicy.mode,
+									upstream: { proxy: provider.proxy },
+									...(stealthProfile ? { userAgent: stealthProfile.userAgent } : {}),
+								},
+							}
+						: {}),
+				})
+			: createUnsupportedResolverClient("Provider does not declare resolver capability"),
 		choice: createProviderChoiceContext({
 			providerId: provider.id,
 			env,

package/dist/auth.d.ts

@@ -66,6 +66,20 @@ export interface DefineCredentialsAuthOptions<TFields extends CredentialsAuthFie
     /** Extra auth-flow context keys used by custom login/challenge code. */
     contextKeys?: readonly string[];
     login(ctx: FlowContext, input: CredentialsAuthInput<TFields>): CredentialsAuthLoginResult<TCredentialKeys, keyof TChallenges & string> | Promise<CredentialsAuthLoginResult<TCredentialKeys, keyof TChallenges & string>>;
+    /**
+     * Optional re-mint of an expired session from the stored credential, wired
+     * to `auth.flow.refresh`.
+     *
+     * Credential-auth upstreams routinely invalidate a session well before the
+     * `expiresAt` the provider advertised, which leaves every operation failing
+     * with a reauth error until a human repeats the whole interactive login.
+     * Implement this to re-establish the session from what is already stored on
+     * the connection; the result is resolved exactly like `login`, so it may
+     * also raise a challenge when the upstream demands one. Omit it when the
+     * upstream has no non-interactive path and re-authentication genuinely
+     * requires the user.
+     */
+    refresh?(ctx: FlowContext, input: Partial<CredentialsAuthInput<TFields>>): CredentialsAuthLoginResult<TCredentialKeys, keyof TChallenges & string> | Promise<CredentialsAuthLoginResult<TCredentialKeys, keyof TChallenges & string>>;
 }
 export interface DefinedCredentialsAuth {
     auth: AuthConfig;

package/dist/auth.js

@@ -199,6 +199,23 @@ function normalizeInput(fields, input) {
     }
     return result;
 }
+/**
+ * Refresh variant of {@link normalizeInput}. `login` coerces every declared
+ * field to a string because the interactive turn has already enforced that they
+ * are present; refresh runs with no user present, so an absent field is omitted
+ * rather than turned into an empty string. That keeps "the user did not supply
+ * this" distinguishable from "the user supplied an empty value".
+ */
+function normalizePartialInput(fields, input) {
+    const result = {};
+    for (const name of Object.keys(fields)) {
+        const value = input?.[name];
+        if (typeof value === "string") {
+            result[name] = value;
+        }
+    }
+    return result;
+}
 function assertCredentialKeys(credentialKeys, credential) {
     const missing = credentialKeys.filter((key) => {
         const value = credential[key];
@@ -413,6 +430,27 @@ export function defineCredentialsAuth(options) {
                     }
                     return await pollPendingChallenge(ctx, options.credentialKeys, challenges, pending, completeTurnId);
                 },
+                // Only advertise refresh when the provider implements it: the
+                // protocol treats the hook's presence as "this connection can be
+                // re-established without the user", and exposing a stub that
+                // cannot actually re-mint would turn a recoverable expiry into a
+                // silent failure.
+                ...(options.refresh
+                    ? {
+                        refresh: async (ctx, rawInput) => {
+                            // A pending challenge belongs to the interactive flow that
+                            // raised it; finish it there rather than restarting.
+                            const pending = getPendingChallenge(ctx);
+                            if (pending) {
+                                return await continuePendingChallenge(ctx, options.credentialKeys, challenges, pending, rawInput, completeTurnId);
+                            }
+                            // Refresh runs without user input, so fields are optional
+                            // here — unlike `continue`, missing ones are not a retry.
+                            const result = await options.refresh(ctx, normalizePartialInput(options.fields, rawInput));
+                            return await resolveAuthResult(ctx, options.credentialKeys, challenges, result, completeTurnId);
+                        },
+                    }
+                    : {}),
             },
         },
         credential: {

package/dist/config/loader.d.ts

@@ -61,8 +61,10 @@ export type ProxyResolutionOptions = {
 };
 export type ProxyCacheStatus = "memory_hit" | "redis_hit" | "allocator" | "soft_stale_refresh" | "lock_wait" | "redis_error" | "redis_corrupt" | "disabled";
 export type SmartproxyAllocatorBodyClass = "network_error" | "http_error" | "empty" | "json_without_proxies" | "text_without_proxies" | "usable_proxy_endpoints";
+export type ProxyUserAgentSource = "declared" | "defaulted";
 export type ProxyResolutionTelemetryEvent = {
     provider: ProxyVendorName;
+    userAgentSource?: ProxyUserAgentSource;
     protocol?: ProxyProtocol;
     cacheStatus: ProxyCacheStatus;
     cacheHit: boolean;

package/dist/index.d.ts

@@ -16,7 +16,7 @@ export * from "./recipes/gov-api.js";
 export * from "./recipes/rest-api.js";
 export { createFlowContext, createScratchpad } from "./runtime/auth-flow.js";
 export type { BrowserClientOptions } from "./runtime/browser.js";
-export { APIFUSE__CACHE__KEY_PEPPER_ENV, createBypassProviderCache, createProviderCache, type ProviderCacheOptions, resetProviderCacheForTests, } from "./runtime/cache.js";
+export { APIFUSE__CACHE__KEY_PEPPER_ENV, createBypassProviderCache, createProviderCache, type ProviderCacheOptions, } from "./runtime/cache.js";
 export { type CreateProviderChoiceContextOptions, createProviderChoiceContext, createTestProviderChoiceContext, PROVIDER_RUNTIME_CHOICE_TOKEN_MASTER_SECRET_ENV, } from "./runtime/choice.js";
 export { type CreateCredentialContextOptions, createCredentialContext, } from "./runtime/credential.js";
 export { createEnvContext } from "./runtime/env.js";

package/dist/index.js

@@ -14,7 +14,7 @@ export { lintOperation, lintProvider, } from "./lint.js";
 export * from "./recipes/gov-api.js";
 export * from "./recipes/rest-api.js";
 export { createFlowContext, createScratchpad } from "./runtime/auth-flow.js";
-export { APIFUSE__CACHE__KEY_PEPPER_ENV, createBypassProviderCache, createProviderCache, resetProviderCacheForTests, } from "./runtime/cache.js";
+export { APIFUSE__CACHE__KEY_PEPPER_ENV, createBypassProviderCache, createProviderCache, } from "./runtime/cache.js";
 export { createProviderChoiceContext, createTestProviderChoiceContext, PROVIDER_RUNTIME_CHOICE_TOKEN_MASTER_SECRET_ENV, } from "./runtime/choice.js";
 export { createCredentialContext, } from "./runtime/credential.js";
 export { createEnvContext } from "./runtime/env.js";

package/dist/runtime/browser.js

@@ -141,7 +141,13 @@ function formatExpression(fn) {
 }
 function toLaunchOptions(options) {
     return {
-        args: options.extraArgs,
+        // `extraArgs` is optional, but playwright-extra's stealth evasions mutate
+        // `options.args` unguarded (navigator.webdriver does
+        // `options.args.findIndex(...)` in beforeLaunch). Forwarding `undefined`
+        // therefore crashes every stealth launch that omits extraArgs with
+        // "TypeError: undefined is not an object (evaluating 'options.args.findIndex')".
+        // Always hand the launcher a concrete array.
+        args: options.extraArgs ?? [],
         executablePath: options.executablePath,
         headless: options.headless ?? true,
         proxy: options.proxy ? { server: options.proxy } : undefined,
@@ -684,6 +690,17 @@ function getCdpExecutionContext(params) {
         id: typeof contextId === "number" ? contextId : undefined,
     };
 }
+function getCdpDestroyedExecutionContextId(params) {
+    if (!isRecord(params)) {
+        return undefined;
+    }
+    return typeof params.executionContextId === "number"
+        ? params.executionContextId
+        : undefined;
+}
+function isMissingExecutionContextError(error) {
+    return error instanceof Error && /\b(?:cannot|failed to) find context\b/i.test(error.message);
+}
 class CdpBrowserLocator {
     frame;
     selector;
@@ -809,7 +826,19 @@ class CdpPoolBrowserPage {
     async evaluateInFrame(frameId, fn) {
         await this.initialize();
         const contextId = await this.getFrameExecutionContextId(frameId);
-        return await this.evaluateWithContext(fn, contextId);
+        try {
+            return await this.evaluateWithContext(fn, contextId);
+        }
+        catch (error) {
+            if (!isMissingExecutionContextError(error)) {
+                throw error;
+            }
+            if (this.frameExecutionContexts.get(frameId) === contextId) {
+                this.frameExecutionContexts.delete(frameId);
+            }
+            const refreshedContextId = await this.getFrameExecutionContextId(frameId);
+            return await this.evaluateWithContext(fn, refreshedContextId);
+        }
     }
     async waitForSelectorInFrame(frameId, selector, options) {
         const timeout = options?.timeout ?? DEFAULT_WAIT_TIMEOUT_MS;
@@ -1010,6 +1039,20 @@ class CdpPoolBrowserPage {
                 this.frameExecutionContexts.set(context.frameId, context.id);
             }
         });
+        this.pageClient.on("Runtime.executionContextDestroyed", (params) => {
+            const destroyedContextId = getCdpDestroyedExecutionContextId(params);
+            if (destroyedContextId === undefined) {
+                return;
+            }
+            for (const [frameId, contextId] of this.frameExecutionContexts) {
+                if (contextId === destroyedContextId) {
+                    this.frameExecutionContexts.delete(frameId);
+                }
+            }
+        });
+        this.pageClient.on("Runtime.executionContextsCleared", () => {
+            this.frameExecutionContexts.clear();
+        });
         await this.pageClient.send("Page.enable");
         await this.pageClient.send("Runtime.enable");
         this.initialized = true;

package/dist/runtime/proxy-telemetry.js

@@ -33,6 +33,7 @@ export class ProxyTelemetryCollector {
     recordProxyResolution(event) {
         this.#events.push({
             provider: event.provider,
+            ...(event.userAgentSource ? { userAgentSource: event.userAgentSource } : {}),
             ...(event.protocol ? { protocol: event.protocol } : {}),
             cacheStatus: event.cacheStatus,
             cacheHit: event.cacheHit,
@@ -99,6 +100,7 @@ export class ProxyTelemetryCollector {
         }
         const aggregate = rest.reduce((acc, event) => ({
             provider: event.provider,
+            userAgentSource: event.userAgentSource ?? acc.userAgentSource,
             cacheStatus: worseStatus(acc.cacheStatus, event.cacheStatus),
             cacheHit: acc.cacheHit && event.cacheHit,
             resolutionMs: acc.resolutionMs + event.resolutionMs,
@@ -118,6 +120,7 @@ export class ProxyTelemetryCollector {
             v: 1,
             proxy: {
                 provider: serving.provider,
+                ...(aggregate.userAgentSource ? { userAgentSource: aggregate.userAgentSource } : {}),
                 ...(serving.protocol ? { protocol: serving.protocol } : {}),
                 cacheStatus: aggregate.cacheStatus,
                 cacheHit: aggregate.cacheHit,

package/dist/runtime/resolver-public.d.ts

@@ -0,0 +1 @@
+export { APIFUSE__CDP_POOL__URL, APIFUSE__RESOLVER__2CAPTCHA__API_KEY, APIFUSE__RESOLVER__CAPMONSTER__API_KEY, APIFUSE__RESOLVER__CAPSOLVER__API_KEY, APIFUSE__RESOLVER__TIMEOUT_MS, bindResolverSignal, createResolverClient, createResolverClientFromEnv, createUnsupportedResolverClient, DEFAULT_RESOLVER_TIMEOUT_MS, invalidateResolverSolution, RESOLVER_ADAPTER_REGISTRY, RESOLVER_INSTRUMENTATION_METADATA, type ResolverAdapterFactory, type ResolverInstrumentationMetadata, type ResolverRuntimeOptions, } from "./resolver.js";

package/dist/runtime/resolver-public.js

@@ -0,0 +1 @@
+export { APIFUSE__CDP_POOL__URL, APIFUSE__RESOLVER__2CAPTCHA__API_KEY, APIFUSE__RESOLVER__CAPMONSTER__API_KEY, APIFUSE__RESOLVER__CAPSOLVER__API_KEY, APIFUSE__RESOLVER__TIMEOUT_MS, bindResolverSignal, createResolverClient, createResolverClientFromEnv, createUnsupportedResolverClient, DEFAULT_RESOLVER_TIMEOUT_MS, invalidateResolverSolution, RESOLVER_ADAPTER_REGISTRY, RESOLVER_INSTRUMENTATION_METADATA, } from "./resolver.js";

package/dist/runtime/resolver-vendors/browser.js

@@ -204,8 +204,9 @@ export function createBrowserResolverVendorAdapter(options) {
             });
         },
         async solve(challenge, identity, callerSignal, traceRecorder) {
-            void identity;
-            if (!options.cdpUrl?.trim()) {
+            const cdpUrl = options.cdpUrl?.trim();
+            const proxyUrl = identity?.proxyUrl;
+            if (!cdpUrl && proxyUrl === undefined) {
                 throw new ResolverVendorUnavailableError(BROWSER_VENDOR_ID, "missing_credentials");
             }
             if (!isSupportedKind(challenge.kind)) {
@@ -214,6 +215,14 @@ export function createBrowserResolverVendorAdapter(options) {
             assertResolverHostAllowed(challenge.pageUrl, options.allowedHosts);
             const challengeKind = challenge.kind;
             callerSignal.throwIfAborted();
+            if (proxyUrl !== undefined && proxyUrl.length === 0) {
+                throw new ResolverVendorUnavailableError(BROWSER_VENDOR_ID, "missing_proxy_identity");
+            }
+            if (cdpUrl && proxyUrl !== undefined) {
+                // The current pool acquire protocol cannot bind a proxy to its browser context.
+                // Refuse so the chain can choose a vendor that can honor the resolved identity.
+                throw new ResolverVendorUnavailableError(BROWSER_VENDOR_ID, "not_implemented");
+            }
             const solveController = new AbortController();
             const onCallerAbort = () => solveController.abort(abortReason(callerSignal));
             callerSignal.addEventListener("abort", onCallerAbort, { once: true });
@@ -223,8 +232,9 @@ export function createBrowserResolverVendorAdapter(options) {
             try {
                 client = createClient({
                     allowedHosts: [...options.allowedHosts],
-                    cdpUrl: options.cdpUrl.trim(),
-                    requireCdpPool: true,
+                    cdpUrl: cdpUrl ?? "",
+                    ...(proxyUrl === undefined ? {} : { proxy: proxyUrl }),
+                    requireCdpPool: cdpUrl !== undefined,
                 });
                 const contextOperation = client.withIsolatedContext(async (page) => {
                     handlerEntered = true;