@cogitator-ai/test-utils 0.2.20 → 0.3.0

package/dist/conformance/fake-provider.d.ts

@@ -0,0 +1,59 @@
+/** A request the fake provider received. */
+export interface FakeProviderRequest {
+    method: string;
+    /** Path with the query string, e.g. `/v1/chat/completions` */
+    path: string;
+    /** Header names in lower case */
+    headers: Record<string, string>;
+    /** The body parsed as JSON, or the raw text when it is not JSON */
+    body: unknown;
+}
+/** How the fake provider answers one request. */
+export type FakeProviderReply = {
+    type: 'json';
+    status?: number;
+    body: unknown;
+    headers?: Record<string, string>;
+} | {
+    type: 'stream';
+    status?: number;
+    contentType: string;
+    /** Written one by one, each flushed on its own */
+    chunks: readonly (string | Uint8Array)[];
+    headers?: Record<string, string>;
+    /** Keep the response open after the chunks, until the client goes away */
+    hang?: boolean;
+}
+/** Never answer: the request stays open until the client goes away */
+ | {
+    type: 'hang';
+};
+export type FakeProviderHandler = (request: FakeProviderRequest) => FakeProviderReply;
+/**
+ * An HTTP server standing in for a model provider. Each request is answered by the handler set
+ * with `reply`, so a backend pointed at `url` talks to it through its real SDK and wire format.
+ */
+export interface FakeProvider {
+    /** The base URL, e.g. `http://127.0.0.1:53111` */
+    readonly url: string;
+    /** Every request received so far, oldest first */
+    readonly requests: readonly FakeProviderRequest[];
+    /** Sets how the next requests are answered */
+    reply(handler: FakeProviderHandler): void;
+    /** Resolves once a request has arrived, with that request */
+    nextRequest(): Promise<FakeProviderRequest>;
+    /** Resolves once the connection of the latest request has closed */
+    latestClosed(): Promise<void>;
+    /** Stops the server and drops open connections */
+    close(): Promise<void>;
+}
+export interface FakeProviderOptions {
+    /**
+     * Speak HTTP/2 without TLS (h2c with prior knowledge) instead of HTTP/1.1, for clients that
+     * only speak HTTP/2, such as the AWS SDK's Bedrock Runtime client.
+     */
+    http2?: boolean;
+}
+/** Starts a fake provider on a free local port. */
+export declare function startFakeProvider(options?: FakeProviderOptions): Promise<FakeProvider>;
+//# sourceMappingURL=fake-provider.d.ts.map

package/dist/conformance/fake-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fake-provider.d.ts","sourceRoot":"","sources":["../../src/conformance/fake-provider.ts"],"names":[],"mappings":"AAQA,4CAA4C;AAC5C,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,IAAI,EAAE,MAAM,CAAC;IACb,iCAAiC;IACjC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,mEAAmE;IACnE,IAAI,EAAE,OAAO,CAAC;CACf;AAED,iDAAiD;AACjD,MAAM,MAAM,iBAAiB,GACzB;IACE,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,GACD;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,MAAM,EAAE,SAAS,CAAC,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,0EAA0E;IAC1E,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AACH,sEAAsE;GACpE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAErB,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,EAAE,mBAAmB,KAAK,iBAAiB,CAAC;AAEtF;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,kDAAkD;IAClD,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,QAAQ,CAAC,QAAQ,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAClD,8CAA8C;IAC9C,KAAK,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI,CAAC;IAC1C,6DAA6D;IAC7D,WAAW,IAAI,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAC5C,oEAAoE;IACpE,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,kDAAkD;IAClD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAeD,mDAAmD;AACnD,wBAAsB,iBAAiB,CAAC,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,YAAY,CAAC,CAwEhG"}

package/dist/conformance/fake-provider.js

@@ -0,0 +1,134 @@
+import { createServer } from 'node:http';
+import { createServer as createHttp2Server, } from 'node:http2';
+/** Starts a fake provider on a free local port. */
+export async function startFakeProvider(options = {}) {
+    const requests = [];
+    const waiting = [];
+    const sessions = new Set();
+    let latestClosed = Promise.resolve();
+    let handler = () => ({
+        type: 'json',
+        status: 500,
+        body: { error: { message: 'The fake provider has no reply set' } },
+    });
+    const onRequest = (req, res) => {
+        const closed = new Promise((resolve) => res.once('close', () => resolve()));
+        latestClosed = closed;
+        void readBody(req).then((text) => {
+            const request = {
+                method: req.method ?? 'GET',
+                path: req.url ?? '/',
+                headers: flatHeaders(req),
+                body: parseBody(text),
+            };
+            requests.push(request);
+            waiting.splice(0).forEach((resolve) => resolve(request));
+            let reply;
+            try {
+                reply = handler(request);
+            }
+            catch (error) {
+                reply = {
+                    type: 'json',
+                    status: 500,
+                    body: {
+                        error: {
+                            message: `The fake provider's handler failed: ${error instanceof Error ? error.message : String(error)}`,
+                        },
+                    },
+                };
+            }
+            answer(res, closed, reply);
+        });
+    };
+    const server = options.http2 ? createHttp2Server(onRequest) : createServer(onRequest);
+    if (options.http2) {
+        server.on('session', (session) => {
+            sessions.add(session);
+            session.once('close', () => sessions.delete(session));
+        });
+    }
+    await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
+    const { port } = server.address();
+    return {
+        url: `http://127.0.0.1:${port}`,
+        requests,
+        reply(next) {
+            handler = next;
+        },
+        nextRequest() {
+            return new Promise((resolve) => waiting.push(resolve));
+        },
+        latestClosed() {
+            return latestClosed;
+        },
+        close() {
+            if ('closeAllConnections' in server)
+                server.closeAllConnections();
+            for (const session of sessions)
+                session.destroy();
+            return new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
+        },
+    };
+}
+function answer(res, closed, reply) {
+    switch (reply.type) {
+        case 'hang':
+            return;
+        case 'json':
+            res.writeHead(reply.status ?? 200, { 'content-type': 'application/json', ...reply.headers });
+            res.end(JSON.stringify(reply.body));
+            return;
+        case 'stream': {
+            res.writeHead(reply.status ?? 200, {
+                'content-type': reply.contentType,
+                'cache-control': 'no-cache',
+                ...reply.headers,
+            });
+            res.flushHeaders?.();
+            let gone = false;
+            void closed.then(() => {
+                gone = true;
+            });
+            void writeChunks(res, reply.chunks, () => gone).then(() => {
+                if (!reply.hang && !gone)
+                    res.end();
+            });
+        }
+    }
+}
+async function writeChunks(res, chunks, gone) {
+    for (const chunk of chunks) {
+        if (gone())
+            return;
+        res.write(chunk);
+        await new Promise((resolve) => setImmediate(resolve));
+    }
+}
+function readBody(req) {
+    return new Promise((resolve) => {
+        const parts = [];
+        req.on('data', (part) => parts.push(part));
+        req.on('end', () => resolve(Buffer.concat(parts).toString('utf8')));
+        req.on('error', () => resolve(''));
+    });
+}
+function parseBody(text) {
+    if (!text)
+        return undefined;
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return text;
+    }
+}
+function flatHeaders(req) {
+    const headers = {};
+    for (const [name, value] of Object.entries(req.headers)) {
+        if (value !== undefined && !name.startsWith(':')) {
+            headers[name] = Array.isArray(value) ? value.join(', ') : value;
+        }
+    }
+    return headers;
+}

package/dist/conformance/index.d.ts

@@ -0,0 +1,9 @@
+export { startFakeProvider, type FakeProvider, type FakeProviderHandler, type FakeProviderOptions, type FakeProviderReply, type FakeProviderRequest, } from './fake-provider.js';
+export { backendConformanceCases, conformanceRequest, CONFORMANCE_FIXTURE, type BackendConformanceCase, type BackendConformanceTarget, type ConformanceMode, type ConformanceScenario, type ProviderWire, } from './scenarios.js';
+export { openAIChatWire, openAIResponsesWire } from './wires/openai.js';
+export { anthropicWire } from './wires/anthropic.js';
+export { geminiWire } from './wires/gemini.js';
+export { ollamaWire } from './wires/ollama.js';
+export { bedrockConverseWire } from './wires/bedrock.js';
+export { sse, ndjson, awsEventStreamMessage } from './wires/encoding.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/conformance/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/conformance/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,GACzB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,mBAAmB,EACnB,KAAK,sBAAsB,EAC3B,KAAK,wBAAwB,EAC7B,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,YAAY,GAClB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/conformance/index.js

@@ -0,0 +1,8 @@
+export { startFakeProvider, } from './fake-provider.js';
+export { backendConformanceCases, conformanceRequest, CONFORMANCE_FIXTURE, } from './scenarios.js';
+export { openAIChatWire, openAIResponsesWire } from './wires/openai.js';
+export { anthropicWire } from './wires/anthropic.js';
+export { geminiWire } from './wires/gemini.js';
+export { ollamaWire } from './wires/ollama.js';
+export { bedrockConverseWire } from './wires/bedrock.js';
+export { sse, ndjson, awsEventStreamMessage } from './wires/encoding.js';

package/dist/conformance/scenarios.d.ts

@@ -0,0 +1,95 @@
+import { type ChatRequest, type LLMBackend } from '@cogitator-ai/types';
+import { type FakeProviderReply, type FakeProviderRequest } from './fake-provider.js';
+/** Whether a case calls `chat` or reads `chatStream`. */
+export type ConformanceMode = 'chat' | 'stream';
+/**
+ * What the provider does in a case. Each {@link ProviderWire} answers it the way its provider
+ * would, and every backend must turn that answer into the same result.
+ *
+ * - `text`: answers {@link CONFORMANCE_FIXTURE}`.text`
+ * - `tool-call`: calls `get_weather` and reports a tool stop
+ * - `tool-call-with-stop`: calls the tool but reports a plain stop, as OpenAI-compatible servers
+ *   do for a forced tool
+ * - `truncated-tool-call`: runs out of output tokens halfway through the call's JSON
+ * - `content-filter`: stops the answer for its safety policy
+ * - `stream-error`: streams the start of the answer, then reports that it is overloaded
+ * - `abort`: never finishes, until the caller aborts
+ * - `rate-limit`, `quota`, `auth`, `overloaded`, `server-error`, `context-overflow`: fails the
+ *   request with that error, the way its API reports it
+ */
+export type ConformanceScenario = 'text' | 'tool-call' | 'tool-call-with-stop' | 'truncated-tool-call' | 'content-filter' | 'stream-error' | 'abort' | 'rate-limit' | 'quota' | 'auth' | 'overloaded' | 'server-error' | 'context-overflow';
+/** The data every wire answers with, so each backend can be held to the same result. */
+export declare const CONFORMANCE_FIXTURE: {
+    readonly text: "Hello there";
+    /** The text in the chunks a stream sends it in */
+    readonly textChunks: readonly ["Hello", " there"];
+    /** The text a stream sends before it fails, in `stream-error` */
+    readonly partialText: "Hel";
+    readonly toolCall: {
+        readonly id: "call_weather_1";
+        readonly name: "get_weather";
+        readonly arguments: {
+            readonly city: "Paris";
+        };
+    };
+    /** The JSON of the call's arguments where the output tokens ran out, in `truncated-tool-call` */
+    readonly truncatedArguments: "{\"city\": \"Par";
+    readonly usage: {
+        readonly inputTokens: 12;
+        readonly outputTokens: 5;
+    };
+    /** Seconds the provider asks to wait, in `rate-limit` */
+    readonly retryAfterSeconds: 7;
+    /** What the provider says in its errors */
+    readonly errorMessage: "Conformance check error";
+};
+/**
+ * A provider's wire format: how it answers each scenario over HTTP. `undefined` means the
+ * provider has no such case (Ollama has no content filter), and the case is skipped.
+ */
+export interface ProviderWire {
+    readonly name: string;
+    /** The path every request of `mode` must go to, when the wire checks it */
+    path?(mode: ConformanceMode): RegExp;
+    reply(scenario: ConformanceScenario, mode: ConformanceMode, request: FakeProviderRequest): FakeProviderReply | undefined;
+}
+/** A backend to hold to the conformance contract. */
+export interface BackendConformanceTarget {
+    /** Name in the case titles */
+    name: string;
+    wire: ProviderWire;
+    /** The backend under test, sending its requests to `providerUrl` */
+    createBackend(providerUrl: string): LLMBackend;
+    /** The model the requests name */
+    model: string;
+    /** Headers `createBackend` sets the backend to send; every request must carry them */
+    headers?: Record<string, string>;
+    /** Scenarios to leave out, with the reason */
+    skip?: Partial<Record<ConformanceScenario, string>>;
+    /** The fake provider speaks HTTP/2, for clients that only speak it (Bedrock Runtime) */
+    http2?: boolean;
+}
+export interface BackendConformanceCase {
+    /** `scenario (mode)` */
+    name: string;
+    scenario: ConformanceScenario;
+    mode: ConformanceMode;
+    /** Why the case does not apply, when it does not */
+    skip?: string;
+    /** Runs the case against its own fake provider; throws an `AssertionError` when the backend breaks the contract */
+    run(): Promise<void>;
+}
+/**
+ * The conformance cases of `target`: every scenario in both modes (`stream-error` in streams
+ * only), each against a fresh fake provider. Framework-agnostic, so any test runner can run them:
+ *
+ * ```ts
+ * for (const c of backendConformanceCases(target)) {
+ *   (c.skip ? it.skip : it)(c.name, () => c.run());
+ * }
+ * ```
+ */
+export declare function backendConformanceCases(target: BackendConformanceTarget): BackendConformanceCase[];
+/** The request every case sends: a question and the one tool the model may call. */
+export declare function conformanceRequest(model: string, signal?: AbortSignal): ChatRequest;
+//# sourceMappingURL=scenarios.d.ts.map

package/dist/conformance/scenarios.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenarios.d.ts","sourceRoot":"","sources":["../../src/conformance/scenarios.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,KAAK,WAAW,EAGhB,KAAK,UAAU,EAGhB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAGL,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACzB,MAAM,iBAAiB,CAAC;AAEzB,yDAAyD;AACzD,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEhD;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,mBAAmB,GAC3B,MAAM,GACN,WAAW,GACX,qBAAqB,GACrB,qBAAqB,GACrB,gBAAgB,GAChB,cAAc,GACd,OAAO,GACP,YAAY,GACZ,OAAO,GACP,MAAM,GACN,YAAY,GACZ,cAAc,GACd,kBAAkB,CAAC;AAEvB,wFAAwF;AACxF,eAAO,MAAM,mBAAmB;;IAE9B,kDAAkD;;IAElD,iEAAiE;;;;;;;;;IAGjE,iGAAiG;;;;;;IAGjG,yDAAyD;;IAEzD,2CAA2C;;CAEnC,CAAC;AAEX;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,2EAA2E;IAC3E,IAAI,CAAC,CAAC,IAAI,EAAE,eAAe,GAAG,MAAM,CAAC;IACrC,KAAK,CACH,QAAQ,EAAE,mBAAmB,EAC7B,IAAI,EAAE,eAAe,EACrB,OAAO,EAAE,mBAAmB,GAC3B,iBAAiB,GAAG,SAAS,CAAC;CAClC;AAED,qDAAqD;AACrD,MAAM,WAAW,wBAAwB;IACvC,8BAA8B;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,YAAY,CAAC;IACnB,oEAAoE;IACpE,aAAa,CAAC,WAAW,EAAE,MAAM,GAAG,UAAU,CAAC;IAC/C,kCAAkC;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,sFAAsF;IACtF,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,8CAA8C;IAC9C,IAAI,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,wFAAwF;IACxF,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,sBAAsB;IACrC,wBAAwB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,IAAI,EAAE,eAAe,CAAC;IACtB,oDAAoD;IACpD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mHAAmH;IACnH,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACtB;AAgDD;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,wBAAwB,GAC/B,sBAAsB,EAAE,CAmB1B;AAOD,oFAAoF;AACpF,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,WAAW,CAqBnF"}

package/dist/conformance/scenarios.js

@@ -0,0 +1,287 @@
+import assert from 'node:assert/strict';
+import { CogitatorError, ErrorCode, } from '@cogitator-ai/types';
+import { startFakeProvider, } from './fake-provider.js';
+/** The data every wire answers with, so each backend can be held to the same result. */
+export const CONFORMANCE_FIXTURE = {
+    text: 'Hello there',
+    /** The text in the chunks a stream sends it in */
+    textChunks: ['Hello', ' there'],
+    /** The text a stream sends before it fails, in `stream-error` */
+    partialText: 'Hel',
+    toolCall: { id: 'call_weather_1', name: 'get_weather', arguments: { city: 'Paris' } },
+    /** The JSON of the call's arguments where the output tokens ran out, in `truncated-tool-call` */
+    truncatedArguments: '{"city": "Par',
+    usage: { inputTokens: 12, outputTokens: 5 },
+    /** Seconds the provider asks to wait, in `rate-limit` */
+    retryAfterSeconds: 7,
+    /** What the provider says in its errors */
+    errorMessage: 'Conformance check error',
+};
+const SCENARIOS = [
+    'text',
+    'tool-call',
+    'tool-call-with-stop',
+    'truncated-tool-call',
+    'content-filter',
+    'stream-error',
+    'abort',
+    'rate-limit',
+    'quota',
+    'auth',
+    'overloaded',
+    'server-error',
+    'context-overflow',
+];
+const MODES = ['chat', 'stream'];
+const ERROR_EXPECTATIONS = {
+    'rate-limit': {
+        code: ErrorCode.LLM_RATE_LIMITED,
+        kind: 'rate_limit',
+        retryable: true,
+        retryAfter: CONFORMANCE_FIXTURE.retryAfterSeconds * 1000,
+    },
+    quota: { code: ErrorCode.LLM_QUOTA_EXCEEDED, kind: 'quota', retryable: false },
+    auth: { code: ErrorCode.LLM_UNAVAILABLE, kind: 'authentication', retryable: false },
+    overloaded: { code: ErrorCode.LLM_UNAVAILABLE, kind: 'overloaded', retryable: true },
+    'server-error': { code: ErrorCode.LLM_UNAVAILABLE, kind: 'server_error', retryable: true },
+    'context-overflow': {
+        code: ErrorCode.LLM_CONTEXT_LENGTH_EXCEEDED,
+        kind: 'context_overflow',
+        retryable: false,
+    },
+};
+/** How long an aborted call may take to give up, and its connection to close. */
+const ABORT_DEADLINE_MS = 2000;
+/**
+ * The conformance cases of `target`: every scenario in both modes (`stream-error` in streams
+ * only), each against a fresh fake provider. Framework-agnostic, so any test runner can run them:
+ *
+ * ```ts
+ * for (const c of backendConformanceCases(target)) {
+ *   (c.skip ? it.skip : it)(c.name, () => c.run());
+ * }
+ * ```
+ */
+export function backendConformanceCases(target) {
+    return SCENARIOS.flatMap((scenario) => MODES.filter((mode) => scenario !== 'stream-error' || mode === 'stream').map((mode) => {
+        const skip = target.skip?.[scenario] ??
+            (applies(target.wire, scenario, mode)
+                ? undefined
+                : `${target.wire.name} has no such case`);
+        return {
+            name: `${scenario} (${mode})`,
+            scenario,
+            mode,
+            ...(skip && { skip }),
+            run: () => runCase(target, scenario, mode),
+        };
+    }));
+}
+function applies(wire, scenario, mode) {
+    const probe = { method: 'POST', path: '/', headers: {}, body: undefined };
+    return wire.reply(scenario, mode, probe) !== undefined;
+}
+/** The request every case sends: a question and the one tool the model may call. */
+export function conformanceRequest(model, signal) {
+    return {
+        model,
+        messages: [
+            { role: 'system', content: 'You report the weather.' },
+            { role: 'user', content: 'What is the weather in Paris?' },
+        ],
+        tools: [
+            {
+                name: CONFORMANCE_FIXTURE.toolCall.name,
+                description: 'Current weather in a city',
+                parameters: {
+                    type: 'object',
+                    properties: { city: { type: 'string' } },
+                    required: ['city'],
+                },
+            },
+        ],
+        maxTokens: 64,
+        ...(signal && { signal }),
+    };
+}
+async function runCase(target, scenario, mode) {
+    const provider = await startFakeProvider({ http2: target.http2 });
+    provider.reply((request) => {
+        const reply = target.wire.reply(scenario, mode, request);
+        if (!reply)
+            throw new Error(`${target.wire.name} has no reply for ${scenario} (${mode})`);
+        return reply;
+    });
+    const label = `${target.name}: ${scenario} (${mode})`;
+    try {
+        const backend = target.createBackend(provider.url);
+        if (scenario === 'abort') {
+            await checkAbort(backend, target, provider, mode, label);
+        }
+        else {
+            const outcome = await callBackend(backend, conformanceRequest(target.model), mode);
+            checkOutcome(scenario, mode, outcome, label);
+        }
+        checkRequests(target, provider, scenario, mode, label);
+    }
+    finally {
+        await provider.close();
+    }
+}
+async function callBackend(backend, request, mode) {
+    if (mode === 'chat') {
+        try {
+            const response = await backend.chat(request);
+            return {
+                content: response.content,
+                toolCalls: response.toolCalls ?? [],
+                finishReason: response.finishReason,
+                usage: response.usage,
+            };
+        }
+        catch (error) {
+            return { content: '', toolCalls: [], error };
+        }
+    }
+    const outcome = { content: '', toolCalls: [] };
+    try {
+        for await (const chunk of backend.chatStream(request))
+            addChunk(outcome, chunk);
+    }
+    catch (error) {
+        outcome.error = error;
+    }
+    return outcome;
+}
+function addChunk(outcome, chunk) {
+    outcome.content += chunk.delta.content ?? '';
+    outcome.toolCalls.push(...(chunk.delta.toolCalls ?? []));
+    if (chunk.finishReason)
+        outcome.finishReason = chunk.finishReason;
+    if (chunk.usage)
+        outcome.usage = chunk.usage;
+}
+function checkOutcome(scenario, mode, outcome, label) {
+    const expected = ERROR_EXPECTATIONS[scenario];
+    if (expected) {
+        checkError(outcome.error, expected, label);
+        return;
+    }
+    if (scenario === 'stream-error') {
+        assert.equal(outcome.content, CONFORMANCE_FIXTURE.partialText, `${label}: the text streamed before the error is passed on`);
+        checkError(outcome.error, { code: ErrorCode.LLM_UNAVAILABLE, kind: 'overloaded', retryable: true }, label);
+        return;
+    }
+    if (outcome.error !== undefined) {
+        assert.fail(`${label}: the call failed: ${describe(outcome.error)}`);
+    }
+    switch (scenario) {
+        case 'text':
+            assert.equal(outcome.content, CONFORMANCE_FIXTURE.text, `${label}: content`);
+            assert.equal(outcome.finishReason, 'stop', `${label}: finishReason`);
+            assert.equal(outcome.toolCalls.length, 0, `${label}: no tool calls`);
+            assert.deepEqual(outcome.usage && {
+                inputTokens: outcome.usage.inputTokens,
+                outputTokens: outcome.usage.outputTokens,
+            }, CONFORMANCE_FIXTURE.usage, `${label}: usage${mode === 'stream' ? ' of the last chunk that reports it' : ''}`);
+            return;
+        case 'tool-call':
+        case 'tool-call-with-stop': {
+            assert.equal(outcome.finishReason, 'tool_calls', `${label}: a tool turn`);
+            assert.equal(outcome.toolCalls.length, 1, `${label}: one tool call`);
+            const [call] = outcome.toolCalls;
+            assert.equal(call.name, CONFORMANCE_FIXTURE.toolCall.name, `${label}: tool name`);
+            assert.deepEqual(call.arguments, CONFORMANCE_FIXTURE.toolCall.arguments, `${label}: tool arguments`);
+            assert.equal(call.argumentsError, undefined, `${label}: arguments read without error`);
+            assert.ok(call.id, `${label}: the call has an id`);
+            return;
+        }
+        case 'truncated-tool-call':
+            assert.equal(outcome.finishReason, 'length', `${label}: cut at the token limit`);
+            assert.equal(outcome.toolCalls.length, 0, `${label}: a cut-off call never runs`);
+            return;
+        case 'content-filter':
+            assert.ok(outcome.finishReason === 'content_filter' || outcome.finishReason === 'refusal', `${label}: finishReason is content_filter or refusal, got ${String(outcome.finishReason)}`);
+            assert.equal(outcome.toolCalls.length, 0, `${label}: a filtered turn runs no tools`);
+            return;
+        default:
+            assert.fail(`${label}: no check for this scenario`);
+    }
+}
+function checkError(error, expected, label) {
+    assert.ok(error !== undefined, `${label}: the call fails`);
+    assert.ok(error instanceof CogitatorError, `${label}: fails with a CogitatorError, got ${describe(error)}`);
+    assert.equal(error.code, expected.code, `${label}: code (${error.message})`);
+    assert.equal(error.details?.kind, expected.kind, `${label}: details.kind (${error.message})`);
+    assert.equal(error.retryable, expected.retryable, `${label}: retryable (${error.message})`);
+    if (expected.retryAfter !== undefined) {
+        assert.equal(error.retryAfter, expected.retryAfter, `${label}: retryAfter in milliseconds`);
+    }
+}
+/**
+ * An abort while the provider is still answering: the call gives up at once with the reason the
+ * caller aborted with, not a retryable provider error a retry or fallback layer would act on,
+ * and the connection to the provider closes, so it stops generating (and billing) the answer.
+ */
+async function checkAbort(backend, target, provider, mode, label) {
+    const controller = new AbortController();
+    const reason = new Error('Caller aborted the conformance call');
+    const request = conformanceRequest(target.model, controller.signal);
+    const arrived = provider.nextRequest();
+    const settled = (async () => {
+        try {
+            if (mode === 'chat') {
+                await backend.chat(request);
+            }
+            else {
+                for await (const chunk of backend.chatStream(request)) {
+                    if (chunk.delta.content)
+                        controller.abort(reason);
+                }
+            }
+            return undefined;
+        }
+        catch (error) {
+            return error;
+        }
+    })();
+    await within(arrived, `${label}: the request reaches the provider`);
+    if (mode === 'chat')
+        controller.abort(reason);
+    const error = await within(settled, `${label}: the call gives up after the abort`);
+    assert.ok(error !== undefined, `${label}: the aborted call fails`);
+    assert.equal(error, reason, `${label}: fails with the abort reason, got ${describe(error)}`);
+    await within(provider.latestClosed(), `${label}: the connection to the provider closes`);
+}
+function checkRequests(target, provider, scenario, mode, label) {
+    assert.ok(provider.requests.length > 0, `${label}: the backend called the provider`);
+    const expected = ERROR_EXPECTATIONS[scenario];
+    if (expected && !expected.retryable) {
+        assert.equal(provider.requests.length, 1, `${label}: an error that is not retryable is not retried, by the backend or its SDK`);
+    }
+    const path = target.wire.path?.(mode);
+    if (path) {
+        for (const request of provider.requests) {
+            assert.match(request.path, path, `${label}: the request goes to the ${mode} endpoint`);
+        }
+    }
+    for (const [name, value] of Object.entries(target.headers ?? {})) {
+        for (const request of provider.requests) {
+            assert.equal(request.headers[name.toLowerCase()], value, `${label}: request ${request.path} carries the ${name} header`);
+        }
+    }
+}
+function within(promise, what) {
+    let timer;
+    const deadline = new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new assert.AssertionError({ message: `${what} within ${ABORT_DEADLINE_MS}ms` })), ABORT_DEADLINE_MS);
+    });
+    return Promise.race([promise, deadline]).finally(() => clearTimeout(timer));
+}
+function describe(error) {
+    if (error instanceof CogitatorError)
+        return `${error.name} ${error.code}: ${error.message}`;
+    if (error instanceof Error)
+        return `${error.name}: ${error.message}`;
+    return String(error);
+}

package/dist/conformance/wires/anthropic.d.ts

@@ -0,0 +1,7 @@
+import { type ProviderWire } from '../scenarios.js';
+/**
+ * The Anthropic Messages API at `/v1/messages`. An error inside a stream that began is an
+ * `error` event, as the API sends an overload after the response started.
+ */
+export declare const anthropicWire: ProviderWire;
+//# sourceMappingURL=anthropic.d.ts.map

package/dist/conformance/wires/anthropic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"anthropic.d.ts","sourceRoot":"","sources":["../../../src/conformance/wires/anthropic.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,KAAK,YAAY,EAClB,MAAM,cAAc,CAAC;AAqJtB;;;GAGG;AACH,eAAO,MAAM,aAAa,EAAE,YA8B3B,CAAC"}