@apifuse/provider-sdk 2.2.0-beta.11 → 2.2.0-beta.13

package/AUTHORING.md

@@ -106,6 +106,90 @@ description:
 
 Use `defineOperation()` when an operation is large enough to live beside helper functions or in a separate module. It preserves the same type inference as inline `defineProvider()` operations and can be placed directly in the provider `operations` map. `defineProvider()` accepts Zod and Standard Schema v1-compatible schemas. If config validation fails, the SDK names the field to fix, for example `runtime`, `auth.mode`, `operations.<id>.handler`, or `operations.<id>.fixtures.response`.
 
+### Replay-safe fixtures
+
+Keep public operation schemas strict: date fields should accept absolute dates,
+not relative tokens. Inside `fixtures.request` only, the SDK resolves `+Nd` and
+`+Nd:YYYYMMDD` (1–365 days ahead) before import-time schema validation and
+stores the resolved request in provider metadata. Health-check case inputs use
+the same resolver when a probe runs. The default calendar is **KST**, including
+the 15:00–23:59 UTC window when KST is already on the next day.
+
+`fixtures.recordedAt` is the KST `YYYY-MM-DD` date when the response evidence
+was captured. It must be a real, non-future calendar date. Response date fields
+are expected to align with `recordedAt`, not with the newly resolved request;
+this permits stable recorded evidence alongside a replay-safe request.
+
+```ts
+const FlightInput = z.object({
+  departureDate: z.string().date(), // public calls remain absolute-date only
+});
+
+const searchFlights = {
+  input: FlightInput,
+  output: FlightOutput,
+  async handler(ctx, input) {
+    return fetchAndNormalizeFlights(ctx, input);
+  },
+  fixtures: {
+    request: { departureDate: "+45d" },
+    response: recordedFlightResponse, // dates reflect the capture below
+    recordedAt: "2026-07-15",
+  },
+  healthCheckUnsupported: { reason: "Upstream search is cost-bearing." },
+};
+```
+
+For code that explicitly calls the shared resolver, omit the third argument to
+use KST or pass `"UTC"` deliberately:
+
+```ts
+import { resolveHealthCheckInputDateTokens } from "@apifuse/provider-sdk/server";
+
+const kstInput = resolveHealthCheckInputDateTokens({ date: "+45d" });
+const utcInput = resolveHealthCheckInputDateTokens({ date: "+45d" }, new Date(), "UTC");
+```
+
+Do not re-resolve a health assertion's dates in UTC when its case input used the
+default KST calendar.
+
+### Real-handler E2E in standard tests
+
+`runStandardTests(provider)` validates declarations and fixtures but reports a
+per-operation warning because it has no handler E2E coverage. Opt in with an
+`upstreamStub`: the runner calls each fixture-backed real handler with its
+already-resolved fixture request, routes ProviderContext upstream transports to
+the stub, and validates the result against the output schema. It never compares
+the result to the recorded response because that evidence belongs to
+`recordedAt`.
+
+```ts
+import { runStandardTests } from "@apifuse/provider-sdk/testing";
+import provider from "../index.js";
+
+runStandardTests(provider, {
+  upstreamStub: ({ transport, method, url }) => {
+    if (
+      transport === "http" &&
+      method === "GET" &&
+      url === "https://api.example.test/flights"
+    ) {
+      return Response.json({ flights: [{ id: "fixture-flight" }] });
+    }
+    return undefined; // fails the test: live-network passthrough is forbidden
+  },
+});
+```
+
+The stub also identifies `stealth`, `browser`, and `native` interactions. Return
+a Web `Response` or `{ status, headers, body }`; an unmatched call fails with
+the operation, transport, and method named in the error. Browser handlers expose
+method-level calls such as `goto`, `evaluate`, and `locator.click`, so provide a
+canned result for each method the handler uses. Native connections similarly
+identify `connectTcp`/`connectTls` and subsequent `write` calls. Direct global
+`fetch` or socket usage is outside this ProviderContext seam and should not be
+used by provider handlers.
+
 ### Health assertion context
 
 `healthCheck.cases[].assertions` receives a `HealthCheckAssertionContext` with
@@ -362,6 +446,44 @@ Note the asymmetry: the gate treats whitespace-only values as missing, but
 `ctx.env.get()` still returns the raw value to handlers — trim at the point of
 use if the upstream is whitespace-sensitive.
 
+### Credentials forced into query parameters
+
+Prefer an authorization header or request body whenever the upstream supports
+one. When the upstream requires a credential in the URL query (for example
+`serviceKey`, `confmKey`, or `crtfc_key`), use `sensitiveParams`:
+
+```ts
+const response = await ctx.http.get("/openapi/lookup", {
+	params: { pageNo: 1, numOfRows: 100 },
+	sensitiveParams: {
+		serviceKey: ctx.env.get("APIFUSE__PROVIDER__EXAMPLE__SERVICE_KEY")!,
+	},
+});
+```
+
+`sensitiveParams` is merged into the outgoing query like `params`, while its
+values are redacted from SDK transport errors, traces, and `apifuse record`
+fixtures. Do not put query credentials in `params`, and do not hand-build a URL
+containing a key; those paths cannot declare which query values are secret.
+
+#### Residual risks
+
+Redaction is unconditional in structural positions (declared query keys and
+exact scalar fixture/error fields) for values of every length. In unstructured
+free text, values of four or more characters are replaced as substrings; shorter
+values are replaced only at token boundaries to avoid corrupting unrelated text
+(for example, a secret `api` must not rewrite `rapid`). The residual risk is that
+a sub-four-character secret embedded directly inside a larger alphanumeric token
+can remain in free text. Prefer a higher-entropy credential, or a header/body
+credential channel, whenever the upstream permits it. An empty
+`sensitiveParams: {}` is treated exactly as if the option were omitted.
+
+For `session.redirects.run()`, returned hop URLs are diagnostic metadata and
+therefore keep declared query values and common response-only credential keys
+redacted. If a login flow must consume a rotated credential from `Location`,
+inspect it inside `stopWhen`; that callback receives the real hop while callback
+failures are sanitized before propagation.
+
 ### Public local debugging checklist
 
 - Operation smoke requests use the provider server envelope:
@@ -517,6 +639,122 @@ const credentialsAuth = defineCredentialsAuth({
   `bunx playwright install chromium`, or set
   `APIFUSE__CDP_POOL__URL` for remote browser debugging.
 
+### Limiting stealth response bodies
+
+Set `maxBodyBytes` on `ctx.stealth.fetch()` or `session.redirects.run()` when an
+upstream response has a known safe maximum. The limit is opt-in and counts
+decoded bytes as impit streams them. It applies to every redirect hop, uses a
+parseable `Content-Length` for an early rejection, and still enforces the limit
+incrementally when the header is absent or inaccurate. Exceeding the limit
+aborts the response and throws a non-retryable `TransportError` with code
+`response_too_large`.
+
+Pass the limit to the transport instead of checking `Content-Length` in provider
+code:
+
+```ts
+const response = await ctx.stealth.fetch("/api/search", {
+  params: { query: input.query },
+  maxBodyBytes: 2 * 1024 * 1024,
+})
+```
+
+### Persisting stealth session cookies
+
+Persist `session.cookies.serialize()` as JSON when an authenticated session must
+survive a restart or move to another replica. The returned
+`StealthCookieStoreV1` has an explicit version and retains every cookie together
+with its Domain, Path, Secure, expiry, host-only, and other cookie attributes.
+Restore it with `session.cookies.deserialize()`. Unsupported future versions
+fail explicitly instead of being accepted as a partial cookie jar.
+
+Credential values are strings, so stringify the store at the credential
+boundary and parse it when rebuilding the session:
+
+```ts
+// After login (including any redirects across sibling hosts):
+const result = await session.redirects.run({ url: loginUrl });
+return {
+  credential: {
+    cookieStore: JSON.stringify(result.cookieStore),
+  },
+};
+
+// In a later operation or replica:
+const persisted = ctx.credential.get("cookieStore");
+if (persisted) {
+  session.cookies.deserialize(JSON.parse(persisted));
+}
+```
+
+`snapshot()` and `restore()` remain only for backward compatibility with flat
+`Record<string, string>` credentials. `snapshot()` enumerates cookies across all
+hosts and paths, but the flat shape is inherently lossy: duplicate names
+collapse and Domain, Path, Secure, expiry, and host-only attributes cannot be
+represented. `restore()` therefore recreates host-only `Path=/` cookies on the
+session base origin. Do not use the flat form for new persistence code. Cookie
+headers remain origin-filtered: use `toHeader(url)` for a particular request and
+never build a request header from serialized or snapshotted persistence data.
+
+### Recording and replaying streaming responses
+
+`apifuse record` passes responses returned by `ctx.http.stream()` directly to the operation
+handler while incrementally capturing a bounded preview. If the handler returns or cancels its
+reader before EOF, the recorder drains the retained upstream reader before writing a JSON evidence
+record to `__fixtures__/raw.json`. The record contains the status, success
+flag, `content-type`/`content-length`/`content-disposition` headers when present, the
+full body SHA-256 and byte count, and a base64 preview up to the configured stream preview limit.
+Textual previews are decoded and passed through the fixture sanitizer before base64 encoding.
+Classification uses both the declared content type and the preview bytes, so missing or incorrect
+content-type headers do not bypass sanitization. PEM private-key blocks and long high-entropy
+tokens in otherwise unstructured text are redacted as well. If the full preview is not valid UTF-8,
+the entire lossy-decoded preview is scanned and matching decodable byte windows are sanitized. Only a
+magic-number-confirmed binary preview with no textual-secret pattern anywhere in the preview bypasses
+sanitization; other undecodable data fails closed.
+Sanitized previews carry `preview_sanitized: true`, plus a
+`preview_redaction_reason` when capture had to fail closed. The original hash and byte count always
+describe upstream bytes, not a sanitized preview.
+
+Each record includes query-free request provenance (`method`, `path`, and a one-based stream call
+ordinal). Provenance never stores the origin, URL userinfo, query, or fragment. Every retained path
+segment is scrubbed before persistence: credential-key segments, values following those keys,
+known token shapes, and long high-entropy opaque segments become `[REDACTED]`. If an operation
+opens multiple streams, the recorder finalizes every retained reader and
+writes all evidence records in stream call order. Stream invocations use a tagged capture envelope
+whose items distinguish stream evidence from ordinary JSON responses. Evidence-only snapshot replay
+consumes that exact call order and fails immediately when evidence is exhausted or a call kind is
+reordered. When request provenance is present, replay also rejects method or path changes (relative
+URLs are resolved against the recorded path prefix); ordinals remain diagnostic and are not matched.
+Appended fixtures replay a stream envelope only when it is the latest invocation. SSE
+recording remains unsupported and fails explicitly instead of retaining an unrelated earlier
+response.
+
+#### Residual risks
+
+- Credential-path sanitization decodes each URL path segment once. Double-encoded separators or
+  values such as `%252F` are not decoded recursively, so they can conceal a credential-shaped
+  segment from the recorder. This single-pass policy keeps path handling deterministic and avoids
+  interpreting ambiguous or intentionally layered encodings differently from the upstream. Never
+  place credentials in URL paths, and review recorded provenance before committing fixtures.
+- Primitive strings embedded in prose are redacted only when they match the current PEM,
+  credential-assignment, known-token, or entropy heuristics. Other secret formats can remain because
+  blanket redaction of ordinary strings would destroy useful fixture content and create broad false
+  positives. Keep secrets under credential-named structured fields where possible and manually
+  inspect sanitized fixture text before committing it.
+
+Stream fixture replay in `runStandardTests(..., { snapshot: true })` is evidence-only:
+`ctx.http.stream()` returns a usable stream containing exactly the recorded preview,
+not a fabricated full body. The replay response also carries runtime metadata
+`evidence_only: true`, `body_sha256`, `body_bytes`, and the optional preview sanitization fields
+for assertions about the original capture. Do not assert that the replay body hashes to
+`body_sha256` when `body_bytes`
+exceeds the decoded preview length or `preview_sanitized` is present; use the metadata for
+full-body integrity and limit body-content assertions to the preview.
+
+Golden snapshot suites can set `requireSnapshot: true` so a missing committed snapshot fails instead
+of being created implicitly. Regenerate intentional changes with
+`bun test --update-snapshots`; review and commit the resulting `transform.snap.json` file.
+
 ### Running the pre-submission report
 
 ```bash

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @apifuse/provider-sdk Changelog
 
+## 2.2.0-beta.13
+
+- Release candidate for main commit 75e840d0614aea3b99a1e5cef4f93f8cdccf0507.
+
+## 2.2.0-beta.12
+
+- Release candidate for main commit c66789c4745c72fc94ad3c10b3e0d7e5ed83fd25.
+
 ## 2.2.0-beta.11
 
 - Release candidate for main commit f6f739bd5265afe714bbace9900edc2695fcf826.
@@ -74,8 +82,14 @@
 
 ## Unreleased
 
+- Add an opt-in native connection idle read timeout with a typed error, independently from TCP/SOCKS/TLS establishment deadlines.
+- Add opt-in `maxBodyBytes` enforcement to stealth fetches and redirect hops, aborting oversized decoded response streams with `response_too_large`.
+- Resolve relative date tokens in fixture requests before input-schema validation, add KST capture-date `fixtures.recordedAt` metadata, and support explicit KST/UTC calendars in the shared health-input resolver.
+- Add opt-in `runStandardTests(provider, { upstreamStub })` real-handler E2E coverage with strict offline transport stubs, output-schema validation, and per-operation warnings when handler E2E is not enabled.
+- Export native-network and request-file TypeScript contracts from the package root and `./provider`, including typed native provider declarations and optional runtime capabilities on provider/auth contexts.
 - Add `arrayBuffer()` and `bytes()` to `HttpResponse` so `ctx.http` consumers can read binary-safe upstream bodies; internal response handling is now byte-first.
 - Preserve identity-only operation `connectionId` values in `ProviderContext` without requiring credential material.
+- Accept and validate `proxy.session.drainLeadSeconds` in `defineProvider`, so providers can actually declare the native sticky-expiry drain lead time the type surface already exposed; a non-positive value, or one that meets or exceeds the sticky lifetime, is rejected at define time.
 
 ## 2.1.0-beta.15
 

package/README.md

@@ -138,14 +138,56 @@ the bad request path; provider/runtime failures include `code`, `message`, and
 - **Stealth-sensitive providers**: use `ctx.http` for normal JSON/REST calls and
   `ctx.stealth.fetch()` when you need browser-like session or cookie control.
   `ctx.stealth.fetch()` uses the impit-backed browser stealth transport and
-  accepts request controls for `params`, `proxy`, `timeout`, `profile`,
-  `redirect`, `throwOnHttpError`, and `stealth.insecureSkipVerify`. For login
+  accepts request controls for `params`, `sensitiveParams`, `proxy`, `timeout`, `profile`,
+  `maxBodyBytes`, `redirect`, `throwOnHttpError`, and
+  `stealth.insecureSkipVerify`. For login
   flows that must inspect intermediate `Location`/`Set-Cookie` headers, create
   a session with `ctx.stealth.createSession()` and use `session.redirects.run()`;
   inspect accumulated cookies through `session.cookies`. Select an SDK stealth
   `profile` such as `chrome-146`; do not tune JA3, HTTP/2 SETTINGS, or
   pseudo-header order in provider code. Chrome/Firefox-style profiles are
   supported; use `ctx.browser` when Safari-specific behavior is required.
+- **Query-parameter credentials**: when an upstream requires a credential in
+  its URL query, pass it through `sensitiveParams`, not `params` and never a
+  hand-built URL. It is sent as a normal query parameter while the SDK redacts
+  its value from transport errors, traces, and recorded fixtures:
+
+  ```ts
+  const response = await ctx.http.get("/openapi/service", {
+    params: { page: 1 },
+    sensitiveParams: { serviceKey: ctx.env.get("APIFUSE__PROVIDER__EXAMPLE__API_KEY")! },
+  });
+  ```
+
+  Use this only when the upstream offers no header or body credential channel.
+  Declared query-key positions and exact scalar diagnostics are always redacted.
+  Free-text values of four or more characters are redacted as substrings; shorter
+  values require token boundaries so low-entropy values do not corrupt unrelated
+  words or timestamps. Consequently, a sub-four-character secret embedded in a
+  larger alphanumeric token can remain in free text; prefer higher-entropy or
+  non-query credentials when possible. `sensitiveParams: {}` is equivalent to
+  omitting the option. Redirect results structurally redact declared keys and
+  common response-only credential keys from hop URLs; use
+  `redirects.run({ stopWhen })` to inspect a real intermediate `Location` during
+  the run when a login flow needs a rotated value.
+- **Proxy URLs for non-stealth consumers**: use `resolveProxy()` when a
+  provider-owned client outside `ctx.stealth` needs the provider's proxy, such
+  as a CAPTCHA solver that must use matching egress. Pass the provider proxy
+  policy used by the provider and consume the returned `url`; the SDK owns
+  vendor selection, allocation, failover, and URL formats. Never call proxy
+  allocator APIs or hardcode proxy vendor hostnames in provider code.
+
+  ```ts
+  import { resolveProxy } from "@apifuse/provider-sdk"
+
+  const resolvedProxy = await resolveProxy({
+    proxyPolicy,
+    affinityKey: connectionId,
+  })
+
+  if (!resolvedProxy.url) throw new Error("This login requires proxy egress")
+  const captchaTask = { proxy: resolvedProxy.url }
+  ```
 - **Browser providers**: for TypeScript Providers use `runtime: "browser"` plus
   `browser.engine: "playwright-stealth"`; `nodriver` is a Python-runtime path.
   Install local browser assets with `bunx playwright install chromium` when

package/bin/apifuse-pack-smoke.ts

@@ -59,6 +59,20 @@ try {
 	);
 
 	run("bun", ["install"], consumerDir);
+	run(
+		"bun",
+		[
+			"--eval",
+			[
+				'import { resolveProxy } from "@apifuse/provider-sdk";',
+				'if (typeof resolveProxy !== "function") throw new Error("resolveProxy is not exported");',
+				'const resolved = await resolveProxy({ proxy: "http://127.0.0.1:8080" });',
+				'if (resolved.url !== "http://127.0.0.1:8080") throw new Error("resolveProxy returned the wrong URL");',
+				'console.log("packed root resolveProxy export OK");',
+			].join("\n"),
+		],
+		consumerDir,
+	);
 
 	const cliBin = join(consumerDir, "node_modules", ".bin", "apifuse");
 	if (!existsSync(cliBin)) {

package/bin/apifuse-pack-types.ts

@@ -123,8 +123,11 @@ function setUpFixtureConsumer(consumerDir: string, tarballPath: string): void {
 	writeFileSync(
 		join(consumerDir, "consumer.ts"),
 		[
-			'import { ProviderError, SessionExpiredError, z } from "@apifuse/provider-sdk";',
+			'import { ProviderError, resolveProxy, SessionExpiredError, z } from "@apifuse/provider-sdk";',
+			'import type { NativeNetworkClient, NativeNetworkConnection, NativeProviderConfig, NativeProviderContext, NativeTcpEgressGrant, ProviderContext, ProviderFileRef, ProviderFilesContext, ProviderResolvedFile } from "@apifuse/provider-sdk";',
+			'import type { ProxyProtocol, ProxyResolutionOptions, ProxyResolutionSource, ProxyVendorName, RequestOptions, ResolvedProxyConfig } from "@apifuse/provider-sdk";',
 			'import { defineCredentialsAuth } from "@apifuse/provider-sdk/provider";',
+			'import type { NativeNetworkClient as ProviderEntryNativeNetworkClient, ProviderFilesContext as ProviderEntryFilesContext } from "@apifuse/provider-sdk/provider";',
 			'import { extractProviderContract } from "@apifuse/provider-sdk/contract";',
 			'import { AUTH_TURN_SCHEMA } from "@apifuse/provider-sdk/auth-turn";',
 			'import { serve } from "@apifuse/provider-sdk/server";',
@@ -143,6 +146,25 @@ function setUpFixtureConsumer(consumerDir: string, tarballPath: string): void {
 			"// Re-exported zod must keep .refine() callback parameter inference.",
 			"const refined = z.object({ shopId: z.string() }).refine((value) => value.shopId.length > 0);",
 			'const refinedString = z.string().refine((value) => value.startsWith("tabelog:"));',
+			'const proxyOptions: ProxyResolutionOptions = { proxyPolicy: { mode: "disabled" } };',
+			'const proxyProtocol: ProxyProtocol = "http";',
+			"const proxyResult: Promise<ResolvedProxyConfig> = resolveProxy(proxyOptions);",
+			'const proxySource: ProxyResolutionSource = "smartproxy-allocator";',
+			'const proxyVendor: ProxyVendorName = "smartproxy";',
+			'const requestFile: ProviderFileRef = { type: "request_file", id: "photo", filename: "photo.jpg", mime_type: "image/jpeg", size: 4 };',
+			'const resolvedFile: ProviderResolvedFile = { type: "request_file", id: requestFile.id, filename: requestFile.filename, size: requestFile.size, sha256: requestFile.sha256, mimeType: requestFile.mime_type, arrayBuffer: async () => new ArrayBuffer(0), bytes: async () => new Uint8Array(), stream: () => new ReadableStream<Uint8Array>() };',
+			"const files: ProviderFilesContext = { has: () => true, resolve: async () => resolvedFile };",
+			"const providerEntryFiles: ProviderEntryFilesContext = files;",
+			"const connection: NativeNetworkConnection = { read: async () => null, write: async () => {}, close: async () => {} };",
+			"const network: NativeNetworkClient = { connectTcp: async () => connection, connectTls: async () => connection, grantTcpEgress: () => ({ revoke() {} }) };",
+			"const providerEntryNetwork: ProviderEntryNativeNetworkClient = network;",
+			"const nativeContext: NativeProviderContext = { network };",
+			'const grant: NativeTcpEgressGrant = network.grantTcpEgress({ sourceHost: "booking-loco.kakao.com", sourcePort: 443, host: "loco.kakao.com", port: 5228, tls: "disabled" });',
+			'const nativeConfig: NativeProviderConfig = { network: { tcp: [{ host: "booking-loco.kakao.com", ports: [443], tls: "required" }] } };',
+			"const providerContext = undefined as unknown as ProviderContext;",
+			"const optionalFiles: ProviderFilesContext | undefined = providerContext.files;",
+			"const optionalNative: NativeProviderContext | undefined = providerContext.native;",
+			'const queryCredentialOptions: RequestOptions = { sensitiveParams: { serviceKey: "type-test-key" } };',
 			"",
 			"export const witnesses = {",
 			"	inheritedName,",
@@ -152,6 +174,23 @@ function setUpFixtureConsumer(consumerDir: string, tarballPath: string): void {
 			"	sessionExpired,",
 			"	refined,",
 			"	refinedString,",
+			"	proxyResult,",
+			"	proxyProtocol,",
+			"	proxySource,",
+			"	proxyVendor,",
+			"	requestFile,",
+			"	resolvedFile,",
+			"	files,",
+			"	providerEntryFiles,",
+			"	connection,",
+			"	network,",
+			"	providerEntryNetwork,",
+			"	nativeContext,",
+			"	grant,",
+			"	nativeConfig,",
+			"	optionalFiles,",
+			"	optionalNative,",
+			"	queryCredentialOptions,",
 			"	defineCredentialsAuth,",
 			"	extractProviderContract,",
 			"	AUTH_TURN_SCHEMA,",