@apifuse/provider-sdk 2.2.0-beta.12 → 2.2.0-beta.13

package/dist/testing/index.d.ts

@@ -1,2 +1,2 @@
 export { describeTransform, snapshotTransform, toMatchShape } from "./helpers.js";
-export { runStandardTests } from "./run.js";
+export { runStandardTests, type StandardTestsManifest, type StandardTestsOptions, type StandardTestsResult, type StandardTestsUpstreamCall, type StandardTestsUpstreamResponse, type StandardTestsUpstreamStub, } from "./run.js";

package/dist/testing/index.js

@@ -1,2 +1,2 @@
 export { describeTransform, snapshotTransform, toMatchShape } from "./helpers.js";
-export { runStandardTests } from "./run.js";
+export { runStandardTests, } from "./run.js";

package/dist/testing/run.d.ts

@@ -1,4 +1,4 @@
-import type { AuthMode, ProviderDefinition } from "../types.js";
+import type { AuthMode, ProviderContext, ProviderDefinition } from "../types.js";
 export interface StandardTestsManifest {
     id?: string;
     displayName?: string;
@@ -22,7 +22,34 @@ export interface StandardTestsOptions {
     validateAuthMode?: boolean;
     /** Override inferred __fixtures__ directory for tests generated outside providers/<id>. */
     fixtureDir?: string;
+    /** Opt in to real-handler E2E with strict, offline canned upstream responses. */
+    upstreamStub?: StandardTestsUpstreamStub;
+    /** Require a committed snapshot; missing files fail unless --update-snapshots is passed. */
+    requireSnapshot?: boolean;
 }
+export interface StandardTestsUpstreamCall {
+    /** Operation whose real handler initiated the call. */
+    operationName: string;
+    /** ProviderContext transport surface used by the handler. */
+    transport: "http" | "stealth" | "browser" | "native";
+    method: string;
+    url?: string;
+    body?: unknown;
+    options?: unknown;
+}
+export interface StandardTestsUpstreamResponse {
+    status?: number;
+    headers?: Readonly<Record<string, string>>;
+    /** JSON-compatible values are encoded as JSON; strings and bytes are preserved. */
+    body?: unknown;
+}
+export type StandardTestsUpstreamStub = (call: StandardTestsUpstreamCall) => Response | StandardTestsUpstreamResponse | undefined | Promise<Response | StandardTestsUpstreamResponse | undefined>;
+export interface StandardTestsResult {
+    warnings: readonly string[];
+}
+export declare function createSnapshotContext(rawFixture: unknown): ProviderContext;
+/** Internal execution seam exported for focused SDK tests; use runStandardTests as public API. */
+export declare function executeStandardTestHandler(provider: ProviderDefinition, operationName: string, upstreamStub: StandardTestsUpstreamStub): Promise<unknown>;
 /**
  * Run standard SDK tests for a provider in one line.
  *
@@ -31,4 +58,7 @@ export interface StandardTestsOptions {
  * import { runStandardTests } from "@apifuse/provider-sdk/testing";
  * runStandardTests(myProvider, rawFixture, manifest, { snapshot: true });
  */
-export declare function runStandardTests(provider: ProviderDefinition, rawFixture?: unknown, manifest?: StandardTestsManifest, options?: StandardTestsOptions): void;
+export declare function runStandardTests(provider: ProviderDefinition, options: StandardTestsOptions & {
+    upstreamStub: StandardTestsUpstreamStub;
+}): StandardTestsResult;
+export declare function runStandardTests(provider: ProviderDefinition, rawFixture?: unknown, manifest?: StandardTestsManifest, options?: StandardTestsOptions): StandardTestsResult;

package/dist/testing/run.js

@@ -4,12 +4,15 @@ import { createTestProviderChoiceContext } from "../runtime/choice.js";
 import { createMemoryProviderRuntimeState } from "../runtime/state.js";
 import { createUnsupportedSttClient } from "../runtime/stt.js";
 import { safeParseSchemaSync } from "../schema.js";
+import { requestPathForFixture } from "../fixture-sanitization.js";
+import { findStreamCaptureGroup, replayStreamEvidence } from "../stream-evidence.js";
 // Mirrors CONNECTOR_ID_REGEX in ../define.ts, which defineProvider() enforces.
 // A single lowercase segment (no hyphen) is a valid id, so the trailing group
 // is optional (`*`), matching providers like `kakaomap`, `kstartup`, `triple`.
 const CONNECTOR_ID_REGEX = /^[a-z][a-z0-9]*(-[a-z][a-z0-9]*)*$/;
 const VALID_AUTH_MODES = ["none", "platform-managed", "credentials", "oauth2"];
 const UPDATE_SNAPSHOT_ARGS = new Set(["-u", "--update-snapshots"]);
+const snapshotCaptureStates = new WeakMap();
 function isFixtureEnvelope(value) {
     return (value !== null &&
         typeof value === "object" &&
@@ -74,10 +77,331 @@ function jsonResponse(data) {
         bytes: async () => bodyBytes.slice(0),
     };
 }
+function headersToRecord(headers) {
+    return Object.fromEntries(headers.entries());
+}
+async function normalizeUpstreamResponse(response) {
+    if (response instanceof Response) {
+        const bytes = new Uint8Array(await response.arrayBuffer());
+        const text = new TextDecoder().decode(bytes);
+        const headers = headersToRecord(response.headers);
+        let data = text;
+        if (response.headers.get("content-type")?.includes("application/json")) {
+            data = text.length > 0 ? JSON.parse(text) : null;
+        }
+        return { status: response.status, headers, data, text, bytes };
+    }
+    const headers = { ...(response.headers ?? {}) };
+    const body = response.body ?? null;
+    let bytes;
+    let text;
+    let data;
+    if (body instanceof Uint8Array) {
+        bytes = body.slice(0);
+        text = new TextDecoder().decode(bytes);
+        data = text;
+    }
+    else if (body instanceof ArrayBuffer) {
+        bytes = new Uint8Array(body.slice(0));
+        text = new TextDecoder().decode(bytes);
+        data = text;
+    }
+    else if (typeof body === "string") {
+        text = body;
+        bytes = new TextEncoder().encode(text);
+        data = body;
+    }
+    else {
+        text = JSON.stringify(body);
+        bytes = new TextEncoder().encode(text);
+        data = body;
+        if (!Object.keys(headers).some((name) => name.toLowerCase() === "content-type")) {
+            headers["content-type"] = "application/json";
+        }
+    }
+    return { status: response.status ?? 200, headers, data, text, bytes };
+}
+function toHttpResponse(response) {
+    return {
+        status: response.status,
+        ok: response.status >= 200 && response.status < 300,
+        headers: response.headers,
+        data: response.data,
+        json: async () => JSON.parse(response.text),
+        text: async () => response.text,
+        arrayBuffer: async () => response.bytes.slice(0).buffer,
+        bytes: async () => response.bytes.slice(0),
+    };
+}
+const emptyCookieJar = {
+    get: () => undefined,
+    getAll: () => ({}),
+    toString: () => "",
+};
+function emptyCookieStore() {
+    return {
+        version: 1,
+        jar: { cookies: [] },
+    };
+}
+function toStealthResponse(response, url) {
+    return {
+        status: response.status,
+        ok: response.status >= 200 && response.status < 300,
+        url,
+        redirected: false,
+        headers: response.headers,
+        rawHeaders: Object.entries(response.headers),
+        body: response.text,
+        cookies: emptyCookieJar,
+        json: async () => JSON.parse(response.text),
+        arrayBuffer: async () => response.bytes.slice(0).buffer,
+        bytes: async () => response.bytes.slice(0),
+    };
+}
+function streamFromBytes(bytes) {
+    return new ReadableStream({
+        start(controller) {
+            controller.enqueue(bytes.slice(0));
+            controller.close();
+        },
+    });
+}
+async function* singleBytes(bytes) {
+    yield bytes.slice(0);
+}
+async function* singleText(textValue) {
+    yield textValue;
+}
+function createUpstreamContext(provider, operationName, upstreamStub) {
+    const credential = {
+        mode: "none",
+        get: () => undefined,
+        getAll: () => ({}),
+        getAccessToken: () => undefined,
+        getScopes: () => [],
+    };
+    const request = { headers: {} };
+    const state = createMemoryProviderRuntimeState();
+    const dispatch = async (call) => {
+        const canned = await upstreamStub({ operationName, ...call });
+        if (canned === undefined) {
+            throw new Error(`Unmatched upstream call for operation "${operationName}": ${call.transport}.${call.method}${call.url ? ` ${call.url}` : ""}. Add a canned response to upstreamStub; live network passthrough is disabled.`);
+        }
+        return normalizeUpstreamResponse(canned);
+    };
+    const httpCall = async (method, url, body, options) => toHttpResponse(await dispatch({ transport: "http", method, url, body, options }));
+    const stealthCall = async (url, options) => toStealthResponse(await dispatch({
+        transport: "stealth",
+        method: options?.method?.toUpperCase() ?? "GET",
+        url,
+        body: options?.body,
+        options,
+    }), url);
+    const createBrowserPage = () => {
+        let currentUrl = "about:blank";
+        let currentResponse;
+        const browserAction = async (method, body) => {
+            currentResponse = await dispatch({
+                transport: "browser",
+                method,
+                url: currentUrl,
+                body,
+            });
+            return currentResponse;
+        };
+        const page = {
+            id: `standard-test-${operationName}`,
+            url: async () => currentUrl,
+            title: async () => currentResponse?.text.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1] ?? "",
+            content: async () => currentResponse?.text ?? "",
+            evaluate: async (fn) => (await browserAction("evaluate", typeof fn === "string" ? fn : String(fn))).data,
+            locator: (selector) => ({
+                click: async () => {
+                    await browserAction("locator.click", { selector });
+                },
+                fill: async (textValue) => {
+                    await browserAction("locator.fill", { selector, text: textValue });
+                },
+                textContent: async () => {
+                    const value = (await browserAction("locator.textContent", { selector })).data;
+                    return value === null || value === undefined ? null : String(value);
+                },
+                waitFor: async (options) => {
+                    await browserAction("locator.waitFor", { selector, options });
+                },
+            }),
+            close: async () => { },
+            fill: async (selector, textValue) => {
+                await browserAction("fill", { selector, text: textValue });
+            },
+            goto: async (url) => {
+                currentUrl = url;
+                await browserAction("goto");
+            },
+            screenshot: async (options) => Buffer.from((await browserAction("screenshot", options)).bytes),
+            click: async (selector) => {
+                await browserAction("click", { selector });
+            },
+            type: async (selector, textValue) => {
+                await browserAction("type", { selector, text: textValue });
+            },
+            waitForSelector: async (selector, options) => {
+                await browserAction("waitForSelector", { selector, options });
+            },
+            frames: async () => [page],
+            withResourcePolicy: async (_policy, run) => run(),
+        };
+        return page;
+    };
+    const createStealthSession = () => ({
+        fetch: stealthCall,
+        cookies: {
+            ...emptyCookieJar,
+            has: () => false,
+            setFromCookieStrings: () => { },
+            toHeader: () => "",
+            snapshot: () => ({}),
+            restore: () => { },
+            serialize: emptyCookieStore,
+            deserialize: () => { },
+            clear: () => { },
+        },
+        redirects: {
+            run: async (options) => {
+                const final = await stealthCall(options.url, options);
+                return {
+                    final,
+                    hops: [],
+                    reason: "completed",
+                    cookies: {},
+                    cookieStore: emptyCookieStore(),
+                };
+            },
+        },
+        close: () => { },
+    });
+    return {
+        env: { get: () => undefined },
+        credential,
+        request,
+        http: {
+            request: (url, options) => httpCall(options?.method?.toUpperCase() ?? "GET", url, options?.body, options),
+            get: (url, options) => httpCall("GET", url, undefined, options),
+            post: (url, body, options) => httpCall("POST", url, body, options),
+            put: (url, body, options) => httpCall("PUT", url, body, options),
+            delete: (url, options) => httpCall("DELETE", url, undefined, options),
+            stream: async (url, options) => {
+                const response = await dispatch({
+                    transport: "http",
+                    method: options?.method?.toUpperCase() ?? "GET",
+                    url,
+                    body: options?.body,
+                    options,
+                });
+                return {
+                    status: response.status,
+                    ok: response.status >= 200 && response.status < 300,
+                    headers: response.headers,
+                    body: streamFromBytes(response.bytes),
+                    bytes: () => singleBytes(response.bytes),
+                    textChunks: () => singleText(response.text),
+                    lines: () => singleText(response.text),
+                };
+            },
+            sse: async (url, options) => {
+                const response = await dispatch({
+                    transport: "http",
+                    method: options?.method?.toUpperCase() ?? "GET",
+                    url,
+                    body: options?.body,
+                    options,
+                });
+                async function* messages() {
+                    for (const block of response.text.split(/\r?\n\r?\n/)) {
+                        const data = block
+                            .split(/\r?\n/)
+                            .filter((line) => line.startsWith("data:"))
+                            .map((line) => line.slice(5).trimStart())
+                            .join("\n");
+                        if (!data)
+                            continue;
+                        yield { event: "message", data, json: () => JSON.parse(data) };
+                    }
+                }
+                return messages();
+            },
+        },
+        cache: createProviderCache({ providerId: `standard-test-${operationName}` }),
+        state,
+        stealth: {
+            fetch: stealthCall,
+            createSession: createStealthSession,
+        },
+        browser: {
+            engine: "playwright-stealth",
+            newPage: async () => createBrowserPage(),
+            rawPage: async () => createBrowserPage(),
+            withIsolatedContext: async (handler) => handler(createBrowserPage()),
+            solveChallenge: async (challenge) => (await dispatch({
+                transport: "browser",
+                method: "solveChallenge",
+                body: challenge,
+            })).data,
+        },
+        ...(provider.native
+            ? {
+                native: {
+                    network: {
+                        connectTcp: async (options) => createNativeConnection(await dispatch({
+                            transport: "native",
+                            method: "connectTcp",
+                            url: `tcp://${options.host}:${options.port}`,
+                            options,
+                        }), dispatch, `tcp://${options.host}:${options.port}`),
+                        connectTls: async (options) => createNativeConnection(await dispatch({
+                            transport: "native",
+                            method: "connectTls",
+                            url: `tls://${options.host}:${options.port}`,
+                            options,
+                        }), dispatch, `tls://${options.host}:${options.port}`),
+                        grantTcpEgress: () => ({ revoke: () => { } }),
+                    },
+                },
+            }
+            : {}),
+        trace: { span: async (_name, fn) => fn() },
+        auth: { requestField: async (name) => unsupported(`ctx.auth.requestField(${name})`) },
+        stt: createUnsupportedSttClient("Standard test upstream context does not support ctx.stt.transcribe"),
+        choice: createTestProviderChoiceContext({
+            providerId: `standard-test-${operationName}`,
+            request,
+            credential,
+            state,
+        }),
+    };
+}
+function createNativeConnection(initialResponse, dispatch, url) {
+    let unread = initialResponse.bytes.slice(0);
+    return {
+        read: async () => {
+            if (unread.byteLength === 0)
+                return null;
+            const bytes = unread;
+            unread = new Uint8Array();
+            return bytes;
+        },
+        write: async (body) => {
+            const response = await dispatch({ transport: "native", method: "write", url, body });
+            unread = response.bytes.slice(0);
+        },
+        close: async () => { },
+    };
+}
 function unsupported(name) {
     throw new Error(`Standard test snapshot context does not support ${name}`);
 }
-function createSnapshotContext(rawFixture) {
+export function createSnapshotContext(rawFixture) {
     const credential = {
         mode: "none",
         get: () => undefined,
@@ -87,17 +411,52 @@ function createSnapshotContext(rawFixture) {
     };
     const request = { headers: {} };
     const state = createMemoryProviderRuntimeState();
-    return {
+    const streamCaptureGroup = findStreamCaptureGroup(rawFixture);
+    let nextCaptureItem = 0;
+    const replayJsonResponse = () => {
+        if (!streamCaptureGroup)
+            return jsonResponse(rawFixture);
+        const item = streamCaptureGroup.items[nextCaptureItem];
+        if (!item) {
+            throw new Error(`Stream fixture exhausted: no recorded response exists for ordinary HTTP call ${nextCaptureItem + 1}.`);
+        }
+        if (item.kind !== "response") {
+            throw new Error(`Stream fixture call-order mismatch: expected a stream call at position ${nextCaptureItem + 1}, received an ordinary HTTP call.`);
+        }
+        nextCaptureItem += 1;
+        return jsonResponse(item.value);
+    };
+    const context = {
         env: { get: () => undefined },
         credential,
         request,
         http: {
-            request: async () => jsonResponse(rawFixture),
-            get: async () => jsonResponse(rawFixture),
-            post: async () => jsonResponse(rawFixture),
-            put: async () => jsonResponse(rawFixture),
-            delete: async () => jsonResponse(rawFixture),
-            stream: async () => unsupported("ctx.http.stream"),
+            request: async () => replayJsonResponse(),
+            get: async () => replayJsonResponse(),
+            post: async () => replayJsonResponse(),
+            put: async () => replayJsonResponse(),
+            delete: async () => replayJsonResponse(),
+            stream: async (...args) => {
+                if (!streamCaptureGroup)
+                    return unsupported("ctx.http.stream");
+                const item = streamCaptureGroup.items[nextCaptureItem];
+                if (!item) {
+                    throw new Error(`Stream fixture exhausted: no recorded evidence exists for stream call ${nextCaptureItem + 1}.`);
+                }
+                if (item.kind !== "stream") {
+                    throw new Error(`Stream fixture call-order mismatch: expected an ordinary HTTP call at position ${nextCaptureItem + 1}, received a stream call.`);
+                }
+                if (item.evidence.request) {
+                    const expected = item.evidence.request;
+                    const actualMethod = (args[1]?.method ?? "GET").toUpperCase();
+                    const actualPath = replayRequestPath(args[0], expected.path);
+                    if (actualMethod !== expected.method || actualPath !== expected.path) {
+                        throw new Error(`Stream fixture request mismatch: expected ${expected.method} ${expected.path}, received ${actualMethod} ${actualPath}.`);
+                    }
+                }
+                nextCaptureItem += 1;
+                return replayStreamEvidence(item.evidence);
+            },
             sse: async () => unsupported("ctx.http.sse"),
         },
         cache: createProviderCache({ providerId: "standard-test" }),
@@ -127,15 +486,39 @@ function createSnapshotContext(rawFixture) {
             state,
         }),
     };
+    snapshotCaptureStates.set(context, {
+        assertConsumed() {
+            if (streamCaptureGroup && nextCaptureItem !== streamCaptureGroup.items.length) {
+                throw new Error(`Stream fixture has ${streamCaptureGroup.items.length - nextCaptureItem} unconsumed capture item${streamCaptureGroup.items.length - nextCaptureItem === 1 ? "" : "s"} after handler completion.`);
+            }
+        },
+        consumed: () => nextCaptureItem,
+    });
+    return context;
+}
+function replayRequestPath(requestUrl, expectedPath) {
+    if (/^[a-z][a-z\d+.-]*:\/\//i.test(requestUrl) || requestUrl.startsWith("/")) {
+        return requestPathForFixture(requestUrl);
+    }
+    return requestPathForFixture(new URL(requestUrl, `https://fixture.invalid${expectedPath}`).toString());
 }
 async function transformSnapshotOutput(provider, rawFixture) {
     const entries = Object.entries(provider.operations);
-    const context = createSnapshotContext(rawFixture);
+    const captureStates = [];
     const outputs = await Promise.all(entries.map(async ([operationName, operation]) => {
+        const context = createSnapshotContext(rawFixture);
         const request = operation.fixtures?.request ?? {};
         const output = await operation.handler(context, request);
+        const captureState = snapshotCaptureStates.get(context);
+        if (captureState)
+            captureStates.push(captureState);
+        if (captureState?.consumed())
+            captureState.assertConsumed();
         return [operationName, output];
     }));
+    if (findStreamCaptureGroup(rawFixture) && !captureStates.some((state) => state.consumed() > 0)) {
+        captureStates[0]?.assertConsumed();
+    }
     if (outputs.length === 1) {
         return outputs[0]?.[1];
     }
@@ -180,16 +563,51 @@ function expectSchemaFixture(operationName, fieldName, fixture, result) {
 function parseSchemaFixture(operationName, fieldName, schema, fixture) {
     expectSchemaFixture(operationName, fieldName, fixture, safeParseSchemaSync(schema, fixture, `operations.${operationName}.fixtures.${fieldName}`));
 }
-/**
- * Run standard SDK tests for a provider in one line.
- *
- * Usage:
- * import { myProvider } from "../index";
- * import { runStandardTests } from "@apifuse/provider-sdk/testing";
- * runStandardTests(myProvider, rawFixture, manifest, { snapshot: true });
- */
-export function runStandardTests(provider, rawFixture, manifest, options = {}) {
+async function materializeHandlerOutput(output) {
+    if (!(output instanceof Response))
+        return output;
+    const contentType = output.headers.get("content-type") ?? "";
+    if (contentType.includes("application/json"))
+        return output.json();
+    return output.text();
+}
+/** Internal execution seam exported for focused SDK tests; use runStandardTests as public API. */
+export async function executeStandardTestHandler(provider, operationName, upstreamStub) {
+    const operation = provider.operations[operationName];
+    if (!operation)
+        throw new Error(`Unknown operation "${operationName}".`);
+    if (operation.fixtures?.request === undefined) {
+        throw new Error(`Operation "${operationName}" has no fixtures.request for handler E2E execution.`);
+    }
+    const context = createUpstreamContext(provider, operationName, upstreamStub);
+    const output = await materializeHandlerOutput(await operation.handler(context, operation.fixtures.request));
+    const result = safeParseSchemaSync(operation.output, output, `operations.${operationName}.handler.output`);
+    if (!result.success) {
+        throw new Error([
+            `Handler output for operation "${operationName}" failed schema validation.`,
+            formatJsonDiff({ valid: false, value: output, error: result.error }, { valid: true, value: output }),
+        ].join("\n"));
+    }
+    return output;
+}
+function isOptionsShortcut(value) {
+    return (value !== null &&
+        typeof value === "object" &&
+        !Array.isArray(value) &&
+        Object.hasOwn(value, "upstreamStub"));
+}
+export function runStandardTests(provider, rawFixtureOrOptions, manifest, legacyOptions) {
+    const shortcut = manifest === undefined && legacyOptions === undefined && isOptionsShortcut(rawFixtureOrOptions);
+    const rawFixture = shortcut ? undefined : rawFixtureOrOptions;
+    const options = shortcut ? rawFixtureOrOptions : (legacyOptions ?? {});
     const operations = Object.entries(provider.operations);
+    const warnings = options.upstreamStub
+        ? operations
+            .filter(([, operation]) => operation.fixtures?.request === undefined)
+            .map(([operationName]) => `[provider-sdk] Operation "${provider.id}.${operationName}" has no fixtures.request, so runStandardTests cannot invoke its handler E2E.`)
+        : operations.map(([operationName]) => `[provider-sdk] Operation "${provider.id}.${operationName}" has no handler E2E coverage in runStandardTests; configure upstreamStub to invoke the real handler.`);
+    for (const warning of warnings)
+        console.warn(warning);
     const assertFixtureValidation = () => {
         expect(rawFixture).toBeDefined();
         expect(isJsonCompatible(rawFixture)).toBe(true);
@@ -289,12 +707,26 @@ export function runStandardTests(provider, rawFixture, manifest, options = {}) {
                 const actual = await transformSnapshotOutput(provider, rawFixture);
                 const serialized = stableStringify(actual);
                 const snapshotFile = Bun.file(snapshotPath);
-                if (shouldUpdateSnapshots() || !(await snapshotFile.exists())) {
+                const snapshotExists = await snapshotFile.exists();
+                if (!snapshotExists && options.requireSnapshot && !shouldUpdateSnapshots()) {
+                    throw new Error(`Required golden snapshot is missing: ${snapshotPath}. Regenerate it with bun test --update-snapshots.`);
+                }
+                if (shouldUpdateSnapshots() || !snapshotExists) {
                     await Bun.write(snapshotPath, serialized);
                 }
                 const expected = JSON.parse(await Bun.file(snapshotPath).text());
                 expect(actual).toEqual(expected);
             });
         }
+        if (options.upstreamStub) {
+            for (const [operationName, operation] of operations) {
+                if (operation.fixtures?.request === undefined)
+                    continue;
+                it(`invokes the real ${operationName} handler with canned upstream responses`, async () => {
+                    await executeStandardTestHandler(provider, operationName, options.upstreamStub);
+                });
+            }
+        }
     });
+    return { warnings };
 }