@dynamicagents/core 0.9.0

package/dist/testing/vcr-store.js

@@ -0,0 +1,191 @@
+/**
+ * Cassette load / match / write. **Node realm** (`node:fs`, `node:crypto`), no
+ * network and no `undici` — the recorder that drives this lives in `vcr.ts`.
+ *
+ * ## Why this replaced undici's `SnapshotAgent`
+ *
+ * `SnapshotAgent` keys every entry on a hash of method + URL + *every
+ * non-excluded request header*. In a Worker test those headers are runtime
+ * artifacts: a cassette recorded through the pool carries `user-agent: undici`,
+ * `cf-worker: vitest-pool-workers-runner-.example.com`, `sec-fetch-mode` and
+ * `accept-encoding`. Bump miniflare or workerd, any one of them changes, and
+ * every entry in every committed cassette misses at once — a whole suite going
+ * red with `No snapshot found` and nothing to point at.
+ *
+ * So the key here is **method + URL + request body**, and nothing else. Headers
+ * are still stored (minus secrets, see {@link CassetteOptions.excludeHeaders})
+ * because they are useful to read in a diff, but they cannot break a replay.
+ *
+ * The on-disk format owes `SnapshotAgent` nothing: a flat array of
+ * {@link CassetteEntry}. Cassettes it wrote are not readable, and deliberately
+ * so — its recorder keyed on `String(opts.body)`, and a Worker's POST body
+ * reaches a dispatcher as a `ReadableStream`, so every streamed request it ever
+ * captured stored the literal text `[object ReadableStream]` where the payload
+ * belonged. Those bodies are not recoverable from the file, and a reader that
+ * papered over it would have to match such entries on method + URL alone —
+ * quietly reintroducing the ambiguity this store exists to remove. Re-record
+ * instead.
+ */
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+/**
+ * Headers that describe how a body was framed on the wire, which is not how it
+ * is framed on replay: bodies are stored decoded and are handed back whole, so
+ * replaying `transfer-encoding: chunked` or a stale `content-length` mis-frames
+ * the response, and `content-encoding: gzip` makes the consumer try to inflate
+ * plaintext. Stripped on the way out, kept in the file.
+ */
+const HOP_BY_HOP = new Set([
+    "connection",
+    "content-encoding",
+    "content-length",
+    "keep-alive",
+    "proxy-authenticate",
+    "proxy-authorization",
+    "te",
+    "trailer",
+    "transfer-encoding",
+    "upgrade"
+]);
+/** `${METHOD}\n${url}\n${sha256(body)}` — the whole matching rule. */
+export function requestKey(method, url, body) {
+    const digest = createHash("sha256").update(body, "utf8").digest("base64url");
+    return `${method.toUpperCase()}\n${url}\n${digest}`;
+}
+function filterHeaders(headers, exclude) {
+    const kept = {};
+    for (const [name, value] of Object.entries(headers)) {
+        if (!exclude.has(name.toLowerCase()))
+            kept[name.toLowerCase()] = value;
+    }
+    return kept;
+}
+/**
+ * Turn a stored response into the pieces a `Response` is built from: body
+ * decoded from base64, headers flattened (a repeated header such as
+ * `set-cookie` becomes one entry per value) and {@link HOP_BY_HOP} dropped.
+ *
+ * Used on both paths, so a response served straight after recording is framed
+ * exactly as the same response will be on replay.
+ */
+export function replayable(response) {
+    const headers = [];
+    for (const [name, value] of Object.entries(response.headers)) {
+        if (HOP_BY_HOP.has(name.toLowerCase()))
+            continue;
+        for (const one of Array.isArray(value) ? value : [value]) {
+            headers.push([name, one]);
+        }
+    }
+    return {
+        status: response.statusCode,
+        headers,
+        body: Buffer.from(response.body, "base64")
+    };
+}
+/**
+ * One cassette file, loaded lazily and written only when it has changed.
+ *
+ * Playback is **read-only**: nothing here mutates on a replay, which is what
+ * stops a plain `npm test` from dirtying committed cassettes. The sequence
+ * counter ({@link #calls}) is in memory and dies with the process, deliberately
+ * — `SnapshotAgent` persisted its equivalent and re-saved on close, so every
+ * run left every cassette modified in git.
+ */
+export class Cassette {
+    #file;
+    #exclude;
+    /** Insertion-ordered, keyed by {@link requestKey}. */
+    #entries = new Map();
+    /** How many times each key has been replayed *this run*. Never persisted. */
+    #calls = new Map();
+    #dirty = false;
+    constructor(file, options = {}) {
+        this.#file = file;
+        this.#exclude = new Set((options.excludeHeaders ?? []).map((h) => h.toLowerCase()));
+    }
+    get file() {
+        return this.#file;
+    }
+    /** Every request in the file, as `METHOD url`, for error messages. */
+    get recorded() {
+        return [...this.#entries.values()].map((e) => `${e.request.method} ${e.request.url}`);
+    }
+    /**
+     * Read the file into memory, keying each entry by its own request rather than
+     * trusting anything stored alongside it.
+     *
+     * Two entries that key the same are a mistake rather than a merge: a request
+     * that was issued twice belongs in one entry with two `responses`, which is
+     * what {@link match} walks. Saying so here turns a hand-edit slip into a named
+     * error instead of a replay that silently serves the wrong response second.
+     */
+    load() {
+        if (!existsSync(this.#file))
+            return;
+        const raw = JSON.parse(readFileSync(this.#file, "utf8"));
+        if (!Array.isArray(raw)) {
+            throw new Error(`VCR cassette ${this.#file} is not a JSON array`);
+        }
+        for (const entry of raw) {
+            const { request, responses } = entry;
+            const key = requestKey(request.method, request.url, request.body ?? "");
+            if (this.#entries.has(key)) {
+                throw new Error(`VCR cassette ${path.basename(this.#file)} has two entries for ` +
+                    `${request.method} ${request.url} with the same body. Put the ` +
+                    `responses in one entry's \`responses\` array instead.`);
+            }
+            this.#entries.set(key, { request, responses: [...responses] });
+        }
+    }
+    /**
+     * The next recorded response for this request, or `null` if it was never
+     * recorded.
+     *
+     * Repeated identical requests walk `responses` in order and then **hold on
+     * the last one**, so a recording that captured a single response replays it
+     * for every call — which is what a poll loop needs when it runs a different
+     * number of times on replay than it did while recording.
+     */
+    match(method, url, body) {
+        const key = requestKey(method, url, body);
+        const entry = this.#entries.get(key);
+        if (!entry)
+            return null;
+        const call = this.#calls.get(key) ?? 0;
+        this.#calls.set(key, call + 1);
+        return replayable(entry.responses[Math.min(call, entry.responses.length - 1)]);
+    }
+    /** Append a live response. Repeats of a request extend its `responses`. */
+    record(request, response) {
+        const key = requestKey(request.method, request.url, request.body);
+        const stored = {
+            ...response,
+            headers: filterHeaders(response.headers, this.#exclude)
+        };
+        const entry = this.#entries.get(key);
+        if (entry) {
+            entry.responses.push(stored);
+        }
+        else {
+            this.#entries.set(key, {
+                request: {
+                    ...request,
+                    headers: filterHeaders(request.headers, this.#exclude)
+                },
+                responses: [stored]
+            });
+        }
+        this.#dirty = true;
+    }
+    /** Write if anything was recorded since the last flush. A no-op in playback. */
+    flush() {
+        if (!this.#dirty)
+            return;
+        mkdirSync(path.dirname(this.#file), { recursive: true });
+        const entries = [...this.#entries.values()];
+        writeFileSync(this.#file, `${JSON.stringify(entries, null, 2)}\n`);
+        this.#dirty = false;
+    }
+}

package/dist/testing/vcr.d.ts

@@ -0,0 +1,117 @@
+/**
+ * The **Node realm** half of the VCR harness: a Miniflare `outboundService`.
+ *
+ * ## Why `outboundService` and not `fetchMock`
+ *
+ * Miniflare 4 implemented `fetchMock` as one line of sugar over this same hook —
+ * `outboundService = (req) => fetch(req, { dispatcher: fetchMock })` — and
+ * Miniflare 5 dropped `fetchMock` entirely, keeping only `outboundService`.
+ * `@cloudflare/vitest-pool-workers` 0.20 removed it from its override options to
+ * match, saying so in as many words: *"`fetchMock`: you should use
+ * `outboundService` instead"*. Targeting the hook rather than the sugar is what
+ * lets one harness serve pool 0.18 through 0.20+.
+ *
+ * It also drops two constraints that had nothing to do with recording:
+ *
+ *  - **No `undici` peer.** `fetchMock` was validated with
+ *    `z.instanceof(MockAgent)` against *Miniflare's own* undici, so a second copy
+ *    in `node_modules` failed with `Input not instance of MockAgent` and the peer
+ *    range had to track Miniflare's exact pin. `outboundService` has no
+ *    `instanceof` check in either direction — a foreign `Response` is re-wrapped.
+ *  - **No silent no-op.** An unknown key in the `miniflare` options is ignored,
+ *    which is how a harness wired to `fetchMock` on pool 0.20 failed: every
+ *    request escaped to the real network and died as `internal error;
+ *    reference = …`, naming nothing. `setupRecording()` now proves the recorder
+ *    answered before a test runs — see {@link VCR_MARKER_HEADER}.
+ *
+ * ## How a request is routed
+ *
+ * Every outbound `fetch()` from workerd arrives here as a standard `Request`,
+ * with the true target URL already restored by Miniflare, and is dispatched on
+ * host alone:
+ *
+ *  - **The control channel** ({@link VCR_CONTROL_ORIGIN}) — specs run in workerd
+ *    and have no filesystem, so an in-band `fetch` is the only way to say which
+ *    cassette the current test is using. See `setupRecording` (vcr-spec.ts).
+ *  - **A {@link VcrOptions.handlers} host** — a consumer-supplied stub, for
+ *    things that are fixtures rather than recordings (a gatekeeper JWKS).
+ *  - **An {@link VcrOptions.allowNetworkHosts} host** — the real network, always.
+ *  - **Anything else, cassette active** — recorded or replayed against it.
+ *  - **Anything else, no cassette** — blocked. A test that was never wired for
+ *    recording cannot reach the network by accident.
+ */
+/** Minimal shape of the `Request` Miniflare hands an outbound service. */
+export interface VcrRequestHeaders {
+    forEach(callback: (value: string, name: string) => void): void;
+    get(name: string): string | null;
+}
+/** Minimal shape of the `Request` Miniflare hands an outbound service. */
+export interface VcrRequest {
+    readonly url: string;
+    readonly method: string;
+    readonly headers: VcrRequestHeaders;
+    arrayBuffer(): Promise<ArrayBuffer>;
+}
+/**
+ * Structural on purpose: core does not depend on `miniflare`, and this type
+ * assigns to `WorkerOptions["outboundService"]` under both Miniflare 4 and 5.
+ */
+export type VcrOutboundService = (request: VcrRequest) => Promise<Response>;
+export interface VcrOptions {
+    /** Absolute path of the cassettes directory. */
+    snapshotsDir: string;
+    /**
+     * true → every activated cassette captures live traffic, replacing whatever
+     * was there. Recording never loads the existing file first, so a recording run
+     * always reflects the live API rather than half of a stale one.
+     * false → replay, and no request ever leaves the machine.
+     */
+    record: boolean;
+    /** Hosts answered by a stub instead of a cassette (e.g. a gatekeeper JWKS). */
+    handlers?: Record<string, (request: VcrRequest) => Response | Promise<Response>>;
+    /** Hosts that always reach the real network, recorded or not. */
+    allowNetworkHosts?: string[];
+    /** Request *and* response headers never written to a cassette (API keys, auth). */
+    excludeHeaders?: string[];
+}
+export interface Vcr {
+    /** Hand this to `cloudflareTest({ miniflare: { outboundService } })`. */
+    readonly outboundService: VcrOutboundService;
+    /** Flush every cassette. Safe to call more than once. */
+    close(): Promise<void>;
+}
+/**
+ * Build the recorder and register it for {@link closeVcr}.
+ *
+ * ```ts
+ * // vitest.config.ts
+ * const vcr = createVcr({
+ *   snapshotsDir: path.resolve(import.meta.dirname, "test/snapshots"),
+ *   record: recordFromEnv(),
+ *   excludeHeaders: ["x-api-key", "authorization", "cookie", "set-cookie"]
+ * });
+ *
+ * export default defineConfig({
+ *   plugins: [cloudflareTest({ miniflare: { outboundService: vcr.outboundService } })],
+ *   test: { globalSetup: ["@dynamicagents/core/testing/vcr-global-setup"] }
+ * });
+ * ```
+ */
+export declare function createVcr(options: VcrOptions): Vcr;
+/**
+ * Whether this run records, from `RECORD`.
+ *
+ * Compared against `"1"` rather than tested for truthiness, because every
+ * non-empty string is truthy: `RECORD=0` and `RECORD=false` — the two things
+ * someone reaches for to turn recording *off* — would otherwise turn it on and
+ * overwrite committed cassettes with live traffic. The opposite mistake
+ * (`RECORD=true` not recording) fails loudly on the next assertion and costs
+ * nothing.
+ */
+export declare function recordFromEnv(): boolean;
+/**
+ * Flush every cassette. Cassettes are already written when each one is
+ * released, so this is a safety net for a run that ended without releasing —
+ * and a no-op if no recorder was created.
+ */
+export declare function closeVcr(): Promise<void>;

package/dist/testing/vcr.js

@@ -0,0 +1,275 @@
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { Cassette, replayable } from "./vcr-store.js";
+import { VCR_CONTROL_ORIGIN, VCR_MARKER_HEADER, CASSETTE_NAME_RE } from "./vcr-shared.js";
+const CONTROL_HOST = new URL(VCR_CONTROL_ORIGIN).host;
+/**
+ * Request headers dropped before a live call. `host` and `content-length` are
+ * recomputed by the outgoing fetch and a stale value corrupts it;
+ * `accept-encoding` is dropped so the body arrives as plaintext to store; the
+ * rest are hop-by-hop or Miniflare's own loopback bookkeeping.
+ */
+const SKIP_REQUEST_HEADERS = new Set([
+    "accept-encoding",
+    "connection",
+    "content-length",
+    "host",
+    "keep-alive",
+    "te",
+    "trailer",
+    "transfer-encoding",
+    "upgrade"
+]);
+function headerRecord(headers) {
+    const record = {};
+    headers.forEach((value, name) => {
+        record[name.toLowerCase()] = value;
+    });
+    return record;
+}
+/** `set-cookie` is the one header `forEach` folds together destructively. */
+function responseHeaderRecord(headers) {
+    const record = {};
+    headers.forEach((value, name) => {
+        record[name.toLowerCase()] = value;
+    });
+    const cookies = headers.getSetCookie?.() ?? [];
+    if (cookies.length > 0)
+        record["set-cookie"] = cookies;
+    return record;
+}
+/**
+ * The one place a `Response` is built, so recording and replaying a given
+ * cassette entry cannot disagree about it.
+ *
+ * `Response` rejects *any* non-null body on 204/205/304 — an empty `Uint8Array`
+ * throws just as a full one does — so a recorded 204 has to be handed back with
+ * a null body or it cannot be replayed at all.
+ */
+function toResponse({ status, headers, body }) {
+    const nullBody = status === 204 || status === 205 || status === 304;
+    return new Response(nullBody ? null : body, { status, headers });
+}
+function json(status, value) {
+    return new Response(JSON.stringify(value), {
+        status,
+        headers: {
+            "content-type": "application/json",
+            [VCR_MARKER_HEADER]: "1"
+        }
+    });
+}
+class Recorder {
+    #snapshotsDir;
+    #record;
+    #handlers;
+    #allowNetwork;
+    #excludeHeaders;
+    /** One cassette per name, created on first activation. */
+    #cassettes = new Map();
+    #active = null;
+    #activeName = null;
+    /** Requests that could not be served since the active cassette was set. */
+    #misses = [];
+    constructor(options) {
+        this.#snapshotsDir = options.snapshotsDir;
+        this.#record = options.record;
+        this.#handlers = options.handlers ?? {};
+        this.#allowNetwork = new Set(options.allowNetworkHosts ?? []);
+        this.#excludeHeaders = options.excludeHeaders ?? [];
+    }
+    handle = async (request) => {
+        const url = new URL(request.url);
+        if (url.host === CONTROL_HOST)
+            return this.#control(url);
+        const handler = this.#handlers[url.host];
+        if (handler)
+            return handler(request);
+        // Read once: the body is needed to key the cassette *and* to forward.
+        const bytes = Buffer.from(await request.arrayBuffer());
+        if (this.#allowNetwork.has(url.host))
+            return this.#live(request, bytes);
+        const body = bytes.toString("utf8");
+        const what = `${request.method} ${request.url}`;
+        if (this.#active === null) {
+            this.#misses.push(what);
+            return this.#blocked(`VCR blocked ${what}: no cassette is active for this test. ` +
+                `Call setupRecording() at the top of the spec, or add the host to ` +
+                `allowNetworkHosts / handlers.`);
+        }
+        if (this.#record)
+            return this.#recordOne(request, bytes);
+        const hit = this.#active.match(request.method, request.url, body);
+        if (hit)
+            return toResponse(hit);
+        this.#misses.push(what);
+        return this.#blocked(`VCR has no recording of ${what} in ${path.basename(this.#active.file)}. ` +
+            `Re-record with RECORD=1. The cassette holds: ` +
+            `${this.#active.recorded.join(", ") || "(nothing)"}.`);
+    };
+    /**
+     * A miss cannot be made to *reject* the Worker's `fetch()` — Miniflare catches
+     * whatever an outbound service throws and turns it into a 500 anyway. So the
+     * message goes in the body where it can be read, and the miss is also reported
+     * on `/release` so the test itself fails with it. That is the half the old
+     * harness lacked: `disableNetConnect()` rejected, and the rejection surfaced
+     * as `internal error; reference = …` naming nothing.
+     */
+    #blocked(message) {
+        return new Response(message, {
+            status: 500,
+            headers: { [VCR_MARKER_HEADER]: "1" }
+        });
+    }
+    #control(url) {
+        if (url.pathname === "/release") {
+            const misses = this.#misses;
+            // Write here rather than only at teardown, so a run that is interrupted
+            // still keeps every cassette whose test finished. A no-op in playback:
+            // `flush()` returns immediately unless something was recorded.
+            this.#active?.flush();
+            this.#active = null;
+            this.#activeName = null;
+            this.#misses = [];
+            return json(200, { misses });
+        }
+        if (url.pathname !== "/use")
+            return json(404, { error: "unknown" });
+        const name = url.searchParams.get("cassette") ?? "";
+        if (!CASSETTE_NAME_RE.test(name)) {
+            return json(400, { error: `invalid cassette name: ${name}` });
+        }
+        // Recorded specs are expected to run sequentially: a second cassette
+        // activating while one is held means two of them are sharing this recorder,
+        // which mis-records rather than failing, so say so instead.
+        if (this.#activeName !== null && this.#activeName !== name) {
+            return json(409, { error: `cassette ${this.#activeName} still active` });
+        }
+        const file = path.join(this.#snapshotsDir, name);
+        // Playback with no cassette on disk → 404, which the spec turns into a
+        // "record it" failure. Recording creates the file on first write.
+        if (!this.#record && !existsSync(file)) {
+            return json(404, { error: `no cassette ${name}` });
+        }
+        this.#active = this.#cassetteFor(name, file);
+        this.#activeName = name;
+        this.#misses = [];
+        return json(200, { cassette: name });
+    }
+    #cassetteFor(name, file) {
+        let cassette = this.#cassettes.get(name);
+        if (!cassette) {
+            cassette = new Cassette(file, { excludeHeaders: this.#excludeHeaders });
+            // Recording deliberately starts from nothing rather than topping up an
+            // existing file: a cassette that mixes today's traffic with last month's
+            // is stale in a way no assertion can see.
+            if (!this.#record)
+                cassette.load();
+            this.#cassettes.set(name, cassette);
+        }
+        return cassette;
+    }
+    /** The real call, with the headers the Worker sent minus the unforwardable. */
+    #live(request, body) {
+        const headers = [];
+        request.headers.forEach((value, name) => {
+            if (!SKIP_REQUEST_HEADERS.has(name.toLowerCase())) {
+                headers.push([name, value]);
+            }
+        });
+        return fetch(request.url, {
+            method: request.method,
+            headers,
+            body: body.byteLength > 0 ? body : undefined,
+            // Matches what Miniflare's own `fetch(req, { dispatcher })` sugar did, so
+            // a recording captures the same final response the Worker used to see.
+            redirect: "follow"
+        });
+    }
+    /**
+     * `requestBytes` is forwarded to the live endpoint **as received**. The
+     * cassette stores the utf-8 text of it, which is what the format has always
+     * held and what keys the entry — but a payload that is not valid utf-8 must
+     * not be re-encoded on its way to a real API, so the two are kept separate
+     * rather than the string being round-tripped back into bytes.
+     */
+    async #recordOne(request, requestBytes) {
+        const live = await this.#live(request, requestBytes);
+        const bytes = Buffer.from(await live.arrayBuffer());
+        const recordedRequest = {
+            method: request.method,
+            url: request.url,
+            headers: headerRecord(request.headers),
+            body: requestBytes.toString("utf8")
+        };
+        const recordedResponse = {
+            statusCode: live.status,
+            headers: responseHeaderRecord(live.headers),
+            body: bytes.toString("base64")
+        };
+        this.#active.record(recordedRequest, recordedResponse);
+        // Serve the response through the same path a replay takes, so a recording
+        // run and the replay of what it just wrote cannot disagree.
+        return toResponse(replayable(recordedResponse));
+    }
+    async close() {
+        for (const cassette of this.#cassettes.values())
+            cassette.flush();
+    }
+}
+/**
+ * Build the recorder and register it for {@link closeVcr}.
+ *
+ * ```ts
+ * // vitest.config.ts
+ * const vcr = createVcr({
+ *   snapshotsDir: path.resolve(import.meta.dirname, "test/snapshots"),
+ *   record: recordFromEnv(),
+ *   excludeHeaders: ["x-api-key", "authorization", "cookie", "set-cookie"]
+ * });
+ *
+ * export default defineConfig({
+ *   plugins: [cloudflareTest({ miniflare: { outboundService: vcr.outboundService } })],
+ *   test: { globalSetup: ["@dynamicagents/core/testing/vcr-global-setup"] }
+ * });
+ * ```
+ */
+export function createVcr(options) {
+    const recorder = new Recorder(options);
+    const vcr = {
+        outboundService: recorder.handle,
+        close: () => recorder.close()
+    };
+    globalThis[VCR_KEY] = vcr;
+    return vcr;
+}
+/**
+ * Whether this run records, from `RECORD`.
+ *
+ * Compared against `"1"` rather than tested for truthiness, because every
+ * non-empty string is truthy: `RECORD=0` and `RECORD=false` — the two things
+ * someone reaches for to turn recording *off* — would otherwise turn it on and
+ * overwrite committed cassettes with live traffic. The opposite mistake
+ * (`RECORD=true` not recording) fails loudly on the next assertion and costs
+ * nothing.
+ */
+export function recordFromEnv() {
+    return process.env.RECORD === "1";
+}
+/** globalThis slot so the Vitest globalSetup teardown can reach the recorder
+ *  created during config evaluation (same process, possibly a different module
+ *  realm — globalThis is the reliable channel). */
+const VCR_KEY = "__VCR_AGENT__";
+/**
+ * Flush every cassette. Cassettes are already written when each one is
+ * released, so this is a safety net for a run that ended without releasing —
+ * and a no-op if no recorder was created.
+ */
+export async function closeVcr() {
+    const g = globalThis;
+    const vcr = g[VCR_KEY];
+    if (!vcr)
+        return;
+    g[VCR_KEY] = undefined;
+    await vcr.close();
+}