@specific.dev/spectest 0.115.1 → 0.116.0

package/src/harness/live-fake.test.ts

@@ -0,0 +1,227 @@
+import { expect, test } from "bun:test";
+import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
+import { execFileSync } from "node:child_process";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+const moduleUrl = (file: string) => JSON.stringify(new URL(file, import.meta.url).href);
+
+// Isolate DNS/transport mocks, platform secrets and recorder globals. All egress
+// goes to a local stub; no real provider or credential is needed.
+test("whole-fake live HTTP uses the environment gate, brokers secrets and never captures", async () => {
+  const root = await mkdtemp(join(tmpdir(), "spectest-live-http-"));
+  try {
+    await mkdir(join(root, "spectest"));
+    await writeFile(join(root, "spectest/index.ts"), `
+      import { defineEnvironment, defineFake } from ${moduleUrl("../index.ts")};
+      export default defineEnvironment({ name: "live", services: {}, fakes: {
+        live: defineFake({ name: "live", hostnames: ["api.test"], handler: () => new Response(),
+          live: { inject: [{ headers: { authorization: "Bearer {{secret:TOKEN}}" } }],
+            sign: { type: "awsSigv4", accessKeyId: "ID", secretAccessKey: "KEY", sessionToken: "SESSION" } },
+          secretRefs: ["LEGACY", "TOKEN"],
+        }),
+      } }).project({});
+    `);
+    const script = `
+      import { strict as assert } from "node:assert";
+      import { EventEmitter } from "node:events";
+      import { test, mock } from "bun:test";
+      import * as dnsPacket from ${JSON.stringify(import.meta.resolve("dns-packet"))};
+      test("live HTTP", async () => {
+        let dns = 0, requests = [];
+        mock.module("node:dgram", () => ({ createSocket() {
+          dns++;
+          const socket = new EventEmitter(); socket.close = () => {};
+          socket.send = (query, _port, _host, done) => { done(); queueMicrotask(() => socket.emit("message", dnsPacket.encode({
+            type: "response", id: 1, answers: [{ type: "A", name: dnsPacket.decode(query).questions[0].name, ttl: 60, data: "127.0.0.1" }],
+          }))); }; return socket;
+        } }));
+        mock.module("node:https", () => ({ request(options, callback) {
+          requests.push(options);
+          const req = new EventEmitter(); let body = Buffer.alloc(0);
+          req.write = chunk => { body = Buffer.concat([body, chunk]); };
+          req.end = () => queueMicrotask(() => {
+            if (options.path === "/broken") { req.emit("error", new Error("upstream unavailable")); return; }
+            const res = new EventEmitter(); res.statusCode = options.path === "/empty" ? 204 : options.path === "/denied" ? 401 : 200;
+            res.headers = { "content-type": "application/octet-stream", "set-cookie": ["session=real; Path=/", "other=value; Expires=Wed, 21 Oct 2037 07:28:00 GMT"], "content-length": "999" };
+            callback(res); res.emit("data", body.length ? body : Buffer.from("real transcript")); res.emit("end");
+          }); return req;
+        } }));
+        const { defineFake } = await import(${moduleUrl("../index.ts")});
+        const { METHODS } = await import(${moduleUrl("../daemon.ts")});
+        const { dispatchFake } = await import(${moduleUrl("./fake-websocket.ts")});
+        const { setReplayEnvironmentMode } = await import(${moduleUrl("../replay-mode.ts")});
+        const { setRecordSecrets } = await import(${moduleUrl("../record-secrets.ts")});
+        const { startRecording, stopRecording, pauseRecording, resumeRecording } = await import(${moduleUrl("../recorder.ts")});
+        await METHODS.load({});
+        assert.deepEqual((await METHODS.recordSecretRefs({})).refs.sort(), ["ID", "KEY", "LEGACY", "SESSION", "TOKEN"]);
+        const state = { calls: 0 };
+        const def = defineFake({ name: "speech", hostnames: ["api.test", "speech.test"],
+          handler: () => { state.calls++; return new Response("scripted"); },
+          websocket: () => { state.calls++; return new Response("scripted websocket", { status: 403 }); },
+          helpers: () => ({ count: () => state.calls }),
+          live: { inject: [{ headers: { authorization: "Bearer {{secret:TOKEN}}" } }] },
+        });
+        const helpers = def.helpers({ name: def.name, state, ctx: {} });
+        const call = (host = "api.test", path = "/transcribe", options = {}) => dispatchFake(new Request("https://" + host + path, options), {}, def, state, {});
+        setRecordSecrets({ TOKEN: "real-key" });
+        async function checkScripted() {
+          assert.equal(await (await call()).text(), "scripted");
+          assert.equal((await call("speech.test", "/socket", { headers: { upgrade: "websocket" } })).status, 403);
+          assert.equal(dns, 0); assert.equal(requests.length, 0);
+        }
+        await checkScripted(); // baseline startup
+        startRecording(); await checkScripted(); pauseRecording(); await checkScripted();
+        resumeRecording(); stopRecording(); await checkScripted(); // between tests
+        setReplayEnvironmentMode("interactive");
+        const calls = helpers.count();
+        startRecording();
+        for (const host of def.hostnames) {
+          const response = await call(host, "/transcribe?lang=sv&word=a%20b", {
+            method: "POST", body: new Uint8Array([0, 128, 255]), headers: { authorization: "Bearer dummy" },
+          });
+          assert.deepEqual(new Uint8Array(await response.arrayBuffer()), new Uint8Array([0, 128, 255]));
+          assert.deepEqual(response.headers.getSetCookie(), ["session=real; Path=/", "other=value; Expires=Wed, 21 Oct 2037 07:28:00 GMT"]);
+          assert.equal(response.headers.get("content-length"), null);
+          const sent = requests.at(-1);
+          assert.equal(sent.host, "127.0.0.1"); assert.equal(sent.servername, host);
+          assert.equal(sent.headers.host, host); assert.equal(sent.headers.authorization, "Bearer real-key");
+          assert.equal(sent.path, "/transcribe?lang=sv&word=a%20b"); assert.equal(sent.method, "POST");
+        }
+        assert.equal(await (await call()).text(), "real transcript");
+        assert.deepEqual(stopRecording(), []);
+        const compressed = new Uint8Array([31, 139, 8, 0]);
+        await call("api.test", "/compressed", { method: "POST", body: compressed, headers: { "content-encoding": "gzip" } });
+        assert.equal(requests.at(-1).headers["content-encoding"], "gzip");
+        assert.equal(helpers.count(), calls);
+        assert.equal((await call("api.test", "/empty")).status, 204);
+        assert.equal((await call("api.test", "/denied")).status, 401);
+        assert.equal((await call("api.test", "/broken")).status, 599);
+        assert.equal(await (await call("api.test", "/transcribe", { method: "HEAD" })).text(), "");
+        let before = dns;
+        assert.equal((await call("outside.test")).status, 500); assert.equal(dns, before);
+        setRecordSecrets({});
+        assert.equal((await call()).status, 599); assert.equal(dns, before);
+        // Missing credentials never change routing or fall back to the fake.
+        assert.equal(helpers.count(), calls);
+        def.live = {};
+        assert.equal(await (await call()).text(), "real transcript");
+        def.live = { sign: { type: "awsSigv4", accessKeyId: "ID", secretAccessKey: "KEY" } };
+        def.hostnames = ["s3.eu-north-1.amazonaws.com"];
+        setRecordSecrets({ ID: "real-id", KEY: "real-signing-key" });
+        await call(def.hostnames[0]);
+        assert.ok(requests.at(-1).headers.authorization.startsWith("AWS4-HMAC-SHA256 Credential=real-id/"));
+        delete def.live;
+        before = dns; assert.equal(await (await call()).text(), "scripted"); assert.equal(dns, before);
+        def.live = {};
+        setReplayEnvironmentMode("test");
+        before = dns; await call(); assert.equal(dns, before);
+      });
+    `;
+    const file = join(root, "http.test.ts"); await writeFile(file, script);
+    const child = Bun.spawn([process.execPath, "test", file], {
+      env: { ...process.env, SPECTEST_APP_DIR: root }, stdout: "pipe", stderr: "pipe",
+    });
+    const [exit, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]);
+    expect(exit, stdout + stderr).toBe(0);
+  } finally { await rm(root, { recursive: true, force: true }); }
+}, 20_000);
+
+// Real TLS and ingress sockets. Only DNS and the upstream port are redirected.
+test("live WSS forwards binary/text without recording and cleans up both sides", async () => {
+  const root = await mkdtemp(join(tmpdir(), "spectest-live-ws-"));
+  try {
+    const key = join(root, "key.pem"), cert = join(root, "cert.pem");
+    execFileSync("openssl", ["req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout", key, "-out", cert,
+      "-days", "1", "-subj", "/CN=speech.test", "-addext", "subjectAltName=DNS:speech.test,DNS:api.test"], { stdio: "ignore" });
+    const script = `
+      import { test, mock } from "bun:test";
+      import { strict as assert } from "node:assert";
+      import { EventEmitter, once } from "node:events";
+      import { createServer } from "node:https";
+      import { readFileSync } from "node:fs";
+      import * as dnsPacket from ${JSON.stringify(import.meta.resolve("dns-packet"))};
+      import WS from ${JSON.stringify(import.meta.resolve("ws").replace(/wrapper\.mjs$/, "index.js"))};
+      test("live WSS", async () => {
+        mock.module("node:dgram", () => ({ createSocket() {
+          const socket = new EventEmitter(); socket.close = () => {};
+          socket.send = (_query, _port, _host, done) => { done(); queueMicrotask(() => socket.emit("message", dnsPacket.encode({
+            type: "response", id: 1, answers: [{ type: "A", name: "speech.test", ttl: 60, data: "127.0.0.1" }],
+          }))); }; return socket;
+        } }));
+        const upstream = createServer({ key: readFileSync(${JSON.stringify(key)}), cert: readFileSync(${JSON.stringify(cert)}) });
+        const wss = new WS.WebSocketServer({ server: upstream, handleProtocols: () => "voice" });
+        upstream.listen(0, "127.0.0.1"); await once(upstream, "listening");
+        const { forwardWebSocket } = await import(${moduleUrl("../fake-upstream-websocket.ts")});
+        mock.module(${moduleUrl("../fake-upstream-websocket.ts")}, () => ({ forwardWebSocket(args) {
+          assert.equal(args.observe, undefined); // no recorder, matcher or capture callback
+          assert.equal(args.url, "wss://" + args.host + "/live?lang=sv");
+          const url = new URL(args.url); url.port = String(upstream.address().port);
+          return forwardWebSocket({ ...args, url: url.href });
+        } }));
+        const { defineFake } = await import(${moduleUrl("../index.ts")});
+        const { setReplayEnvironmentMode } = await import(${moduleUrl("../replay-mode.ts")});
+        const { setRecordSecrets } = await import(${moduleUrl("../record-secrets.ts")});
+        const { startRecording, stopRecording } = await import(${moduleUrl("../recorder.ts")});
+        const { startIngressServer } = await import(${moduleUrl("./ingress-server.ts")});
+        const { dispatchFake } = await import(${moduleUrl("./fake-websocket.ts")});
+        setReplayEnvironmentMode("interactive"); setRecordSecrets({ TOKEN: "real-key" });
+        const state = { calls: 0 };
+        // Deliberately no scripted WebSocket handler: live routing still upgrades.
+        const def = defineFake({ name: "speech", hostnames: ["api.test", "speech.test"],
+          handler: () => { state.calls++; return new Response("scripted"); },
+          live: { inject: [{ headers: { authorization: "Bearer {{secret:TOKEN}}" } }] },
+        });
+        const ingress = await startIngressServer({ port: 0, hostname: "127.0.0.1", fetch: (req, server) => dispatchFake(req, server, def, state, {}) });
+        let connections = 0;
+        wss.on("connection", (socket, req) => {
+          connections++; assert.equal(req.headers.authorization, "Bearer real-key");
+          assert.equal(req.url, "/live?lang=sv");
+          socket.send("real transcript");
+          socket.on("message", (data, binary) => {
+            if (!binary && data.toString() === "finish") socket.close(4003, "provider done");
+            else socket.send(data, { binary });
+          });
+        });
+        try {
+          startRecording();
+          for (const host of def.hostnames) {
+            const client = new WS("ws://127.0.0.1:" + ingress.port + "/live?lang=sv", ["voice"], {
+              headers: { host, authorization: "Bearer dummy" },
+            });
+            const received = []; client.on("message", (data, binary) => received.push({ data: Buffer.from(data), binary }));
+            await once(client, "open"); assert.equal(client.protocol, "voice");
+            client.send('{"model":"test"}'); client.send(new Uint8Array([0, 128, 255]));
+            for (let i = 0; received.length < 3 && i < 200; i++) await Bun.sleep(5);
+            assert.deepEqual(received, [
+              { data: Buffer.from("real transcript"), binary: false },
+              { data: Buffer.from('{"model":"test"}'), binary: false },
+              { data: Buffer.from([0, 128, 255]), binary: true },
+            ]);
+            const closed = once(client, "close");
+            if (host === "api.test") client.close(4002, "client done"); else client.send("finish");
+            const [code, reason] = await closed;
+            assert.equal(code, host === "api.test" ? 4002 : 4003);
+            assert.equal(reason.toString(), host === "api.test" ? "client done" : "provider done");
+          }
+          assert.equal(connections, 2); assert.equal(state.calls, 0);
+          assert.deepEqual(stopRecording(), []);
+          for (let i = 0; wss.clients.size && i < 100; i++) await Bun.sleep(5);
+          assert.equal(wss.clients.size, 0);
+          setRecordSecrets({});
+          const response = await dispatchFake(new Request("https://speech.test/live", { headers: { upgrade: "websocket" } }), {}, def, state, {});
+          assert.equal(response.status, 500); assert.equal(connections, 2); assert.equal(state.calls, 0);
+          def.live.sign = { type: "awsSigv4", accessKeyId: "ID", secretAccessKey: "KEY" };
+          const signed = await dispatchFake(new Request("https://speech.test/live", { headers: { upgrade: "websocket" } }), {}, def, state, {});
+          assert.match(await signed.text(), /SigV4 WebSocket signing is not supported/);
+        } finally {
+          ingress.stop(true); for (const socket of wss.clients) socket.terminate(); wss.close(); upstream.close();
+        }
+      });
+    `;
+    const file = join(root, "ws.test.ts"); await writeFile(file, script);
+    const child = Bun.spawn([process.execPath, "test", file], { env: { ...process.env, NODE_EXTRA_CA_CERTS: cert }, stdout: "pipe", stderr: "pipe" });
+    const [exit, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]);
+    expect(exit, stdout + stderr).toBe(0);
+  } finally { await rm(root, { recursive: true, force: true }); }
+}, 20_000);

package/src/harness/live-fake.ts

@@ -0,0 +1,62 @@
+import type { FakeDefinition } from "../index.js";
+import {
+  brokerHeaders, forwardHttp, resolveExternalA, ruleMatches, STRIP_FORWARD_HEADERS,
+} from "../fake-upstream.js";
+import { forwardWebSocket } from "../fake-upstream-websocket.js";
+import { getWebSocketLifetime } from "./fake-websocket-lifetime.js";
+import { reportBackgroundFailure } from "./background-failures.js";
+import { isInteractiveEnvironment } from "../replay-mode.js";
+
+/** Select at request time, after the baseline has become interactive. */
+export function liveFakeHandlers(def: FakeDefinition): Pick<FakeDefinition, "handler" | "websocket"> {
+  function target(req: Request): string {
+    if (!isInteractiveEnvironment()) throw new Error("live forwarding requires an interactive environment");
+    const url = new URL(req.url);
+    if (!def.hostnames.some(host => host.toLowerCase() === url.hostname.toLowerCase())) {
+      throw new Error(`fake ${def.name}: upstream hostname is not declared on this fake`);
+    }
+    return url.hostname;
+  }
+
+  return {
+    async handler(req) {
+      const host = target(req);
+      const bytes = new Uint8Array(await req.arrayBuffer());
+      const result = await forwardHttp(req, host, def.live!, bytes, `fake(${def.name})`);
+      if (result instanceof Response) return result;
+      const { upstream } = result;
+      const headers = new Headers();
+      for (const [key, value] of Object.entries(upstream.headers)) {
+        if (key !== "set-cookie" && !STRIP_FORWARD_HEADERS.has(key)) headers.set(key, value);
+      }
+      for (const cookie of upstream.cookies ?? []) headers.append("set-cookie", cookie);
+      return new Response(req.method === "HEAD" || [204, 205, 304].includes(upstream.status)
+        ? null : upstream.bytes as unknown as BodyInit, { status: upstream.status, headers });
+    },
+    async websocket(req) {
+      const host = target(req);
+      const opts = def.live!;
+      if (opts.sign) throw new Error(`fake(${def.name}): SigV4 WebSocket signing is not supported`);
+      const url = new URL(req.url);
+      const rule = opts.inject?.find(rule => ruleMatches(rule.match, {
+        method: req.method, path: url.pathname, query: url.searchParams, headers: req.headers,
+      }));
+      const brokered = rule ? brokerHeaders(rule) : { headers: {}, missing: [] };
+      if (brokered.missing.length) throw new Error(`fake(${def.name}): missing WebSocket secrets ${brokered.missing.join(", ")}`);
+      const ips = await resolveExternalA(host);
+      if (!ips.length) throw new Error(`fake(${def.name}): could not resolve real upstream ${host}`);
+      const headers: Record<string, string> = {};
+      req.headers.forEach((value, key) => {
+        if (!STRIP_FORWARD_HEADERS.has(key) && !key.startsWith("sec-websocket-")) headers[key] = value;
+      });
+      Object.assign(headers, brokered.headers);
+      const { handlers } = await forwardWebSocket({
+        url: `wss://${host}${url.pathname}${url.search}`, ip: ips[0], host, headers,
+        protocols: (req.headers.get("sec-websocket-protocol") ?? "").split(",").map(p => p.trim()).filter(Boolean),
+        label: `fake(${def.name}) WebSocket`, lifetime: getWebSocketLifetime(req),
+        finished(_session, error) { if (error) reportBackgroundFailure(error); },
+      });
+      return handlers;
+    },
+  };
+}

package/src/index.ts

@@ -1,3 +1,5 @@
+import type { InjectRule, SignConfig } from "./fake-upstream.js";
+export type { InjectRule, InjectMatch, StringMatch, SignConfig, AwsSigV4Sign } from "./fake-upstream.js";
 export type { Phone, PhoneOptions, CallOptions, SpeechOptions } from "./phone.js";
 import type { Phone, PhoneOptions } from "./phone.js";
 import type { Android, AndroidApp } from "./android.js";
@@ -2140,6 +2142,15 @@ export interface TestSuite<
 // Fakes — in-daemon HTTP servers that masquerade as external APIs.
 // ──────────────────────────────────────────────────────────────────────────
 
+/** Forward the entire fake to real HTTPS/WSS upstreams in interactive environments.
+ * Tests and shared baselines always use the scripted handlers. No traffic is recorded. */
+export interface FakeLiveOptions {
+  /** Same header injection rules as replayFake, applied to HTTP and WebSocket handshakes. */
+  inject?: InjectRule[];
+  /** Re-sign HTTP requests using brokered AWS credentials. Not supported for WebSockets. */
+  sign?: SignConfig;
+}
+
 /**
  * A fake server hosted in the harness. Use this to stand in for
  * external HTTP APIs (auth providers, payment gateways, …) that you can't
@@ -2177,6 +2188,10 @@ export interface FakeDefinition<
    * other fakes.
    */
   hostnames: readonly string[];
+  /** Opt in to forwarding all HTTP/WebSocket traffic to the declared real hosts
+   * over HTTPS/WSS (port 443) in interactive environments. Errors never fall back
+   * to scripted handlers. Omit to keep the fake scripted in every environment. */
+  live?: FakeLiveOptions;
   /** TCP port the fake listens on. Default `80`. */
   port?: number;
   /** Preserve handler/interceptor CORS policy by default. "permissive"
@@ -2231,7 +2246,7 @@ export interface FakeDefinition<
    */
   helpers?: (args: { name: string; state: S; ctx: FakeContext }) => H | Promise<H>;
   /**
-   * Internal. Platform secret references this fake needs at *record* time
+   * Internal. Platform secret references this fake needs for upstream access
    * (set by {@link replayFake}). The daemon reports the union of these to
    * the control plane, which resolves each via its `SecretResolver` and
    * supplies them for the environment lifetime, including snapshots. Values
@@ -2270,6 +2285,9 @@ export function defineFake<
       );
     }
   }
+  if (opts.live !== undefined && (!opts.live || typeof opts.live !== "object" || Array.isArray(opts.live))) {
+    throw new Error(`defineFake(${opts.name}): \`live\` must be an options object`);
+  }
   validateCorsMode(opts.cors);
   if (typeof opts.handler !== "function") {
     throw new Error(`defineFake(${opts.name}): \`handler\` is required`);

package/src/record-secrets.ts

@@ -1,9 +1,9 @@
-// Environment-lifetime credentials for record-mode fakes. The control plane
+// Environment-lifetime credentials for live and record-mode fakes. The control plane
 // resolves declared references from the owning project's Secrets store before
 // service startup and refreshes them on warm starts and interactive forks.
 // They persist across evals, background requests and memory snapshots, just
 // like other environment state. Test forks inherit their parent's credentials;
-// replay mode still uses cassettes and never forwards upstream.
+// test mode still uses scripted handlers or cassettes and never forwards upstream.
 // Values stay out of project files, config hashes and recorded cassettes.
 
 const SECRETS = new Map<string, string>();
@@ -14,7 +14,7 @@ export function setRecordSecrets(secrets: Record<string, string>): void {
   for (const [ref, value] of Object.entries(secrets)) SECRETS.set(ref, value);
 }
 
-/** Resolve a declared project secret for an outbound recording request. */
+/** Resolve a declared project secret for an outbound live or recording request. */
 export function getRecordSecret(ref: string): string | undefined {
   return SECRETS.get(ref);
 }

package/src/replay-mode.ts

@@ -1,6 +1,6 @@
 // Environment state: travels with memory snapshots, independently of the
-// per-case event recorder. Shared/warm baselines must never record upstream.
-// The control plane enables recording only on an interactive environment,
+// per-case event recorder. Shared/warm baselines must never forward upstream.
+// The control plane enables upstream access only on an interactive environment,
 // after capturing/restoring the shared baseline and before interactive setup.
 let mode: "test" | "interactive" = "test";
 
@@ -12,3 +12,8 @@ export function resolveReplayMode(requested: "auto" | "record" | "replay" = "aut
   // Even an explicit record option cannot send test traffic to real providers.
   return mode === "test" || requested === "replay" ? "replay" : "record";
 }
+
+/** Shared gate for live and replay fakes. Credentials never enable upstream access. */
+export function isInteractiveEnvironment(): boolean {
+  return mode === "interactive";
+}