@specific.dev/spectest 0.115.0 → 0.116.0

package/src/fake-upstream-websocket.ts

@@ -0,0 +1,157 @@
+// Bidirectional forwarding with bounded opening buffers. Recording is an optional observer.
+import { createRequire } from "node:module";
+import { dirname, join } from "node:path";
+import type WS from "ws";
+import type { FakeWebSocket, FakeWebSocketHandlers } from "./fake-websocket.js";
+import { FakeWebSocketLifetime, bindWebSocketLifetime } from "./harness/fake-websocket-lifetime.js";
+
+// Bun's ws shim does not implement the external DNS lookup option.
+const require = createRequire(import.meta.url);
+const { WebSocket } = require(join(dirname(require.resolve("ws/package.json")), "index.js")) as typeof import("ws");
+export const MAX_BUFFER = 16 * 1024 * 1024;
+const MAX_PENDING = 1024;
+
+export function closeSocket(socket: FakeWebSocket | WS, code: number, reason: string): void {
+  // Reserved status codes describe transport closure and cannot go on the wire.
+  if (code === 1006 && socket.terminate) socket.terminate();
+  else if (code === 1005 || code === 1006) socket.close();
+  else socket.close(code, reason);
+}
+
+export interface WebSocketSession {
+  cancel(): void;
+}
+
+export async function forwardWebSocket(args: {
+  url: string;
+  ip: string;
+  host: string;
+  headers: Record<string, string>;
+  protocols: string[];
+  label: string;
+  lifetime?: FakeWebSocketLifetime;
+  finished(session: WebSocketSession, error?: Error): void;
+  observe?: {
+    open(protocol: string): void;
+    input(message: string | Uint8Array): void;
+    output(message: string | Uint8Array): void;
+    close(code: number, reason: string, initiator: "client" | "server"): void;
+  };
+}): Promise<{ handlers: FakeWebSocketHandlers; session: WebSocketSession }> {
+  const lifetime = args.lifetime ?? new FakeWebSocketLifetime();
+  if (lifetime.ended) throw new Error("WebSocket handshake aborted");
+  let detach = () => {};
+  const upstream = new WebSocket(args.url, args.protocols, {
+    headers: args.headers, maxPayload: MAX_BUFFER,
+    handshakeTimeout: 10_000,
+    lookup: ((_hostname: string, options: { all?: boolean }, callback: Function) => {
+      if (options.all) callback(null, [{ address: args.ip, family: 4 }]);
+      else callback(null, args.ip, 4);
+    }) as any,
+  });
+  let socket: FakeWebSocket | undefined;
+  let stopped = false;
+  let clientClosed = false;
+  let upstreamClosure: { code: number; reason: string } | undefined;
+  let queue: (string | Uint8Array)[] = [];
+  let queuedBytes = 0;
+  let openingTimer: ReturnType<typeof setTimeout> | undefined;
+  let opened = false;
+  let rejectOpen: (error: Error) => void = () => {};
+  const finish = (error?: Error) => {
+    if (stopped) return;
+    stopped = true;
+    clearTimeout(openingTimer);
+    detach();
+    queue = [];
+    upstream.terminate();
+    lifetime.end(error ? { kind: "failed", error } : { kind: "aborted" });
+    args.finished(session, error);
+  };
+  const fail = (error: unknown) => {
+    if (stopped) return;
+    const failure = new Error(`${args.label}: ${(error as Error).message}`);
+    rejectOpen(failure);
+    finish(failure);
+    socket?.close(1011, "WebSocket forwarding failed");
+  };
+  const session: WebSocketSession = {
+    cancel() { rejectOpen(new Error("WebSocket handshake aborted")); finish(); socket?.close(1001, "forwarding cleanup"); },
+  };
+  upstream.on("error", fail);
+  upstream.on("message", (data, binary) => {
+    if (stopped || clientClosed) return;
+    try {
+      const bytes = Array.isArray(data) ? Buffer.concat(data) : Buffer.from(data as ArrayBuffer);
+      const message = binary ? new Uint8Array(bytes) : bytes.toString();
+      args.observe?.output(message);
+      if (socket) socket.send(message);
+      else {
+        queuedBytes += bytes.length;
+        if (queue.length >= MAX_PENDING || queuedBytes > MAX_BUFFER) throw new Error("WebSocket opening buffer exceeded");
+        queue.push(message);
+      }
+    } catch (error) { fail(error); }
+  });
+  upstream.on("close", (code, reason) => {
+    if (stopped) return;
+    if (!opened) { fail(new Error("upstream closed before handshake completed")); return; }
+    try {
+      if (!clientClosed) {
+        upstreamClosure = { code, reason: reason.toString() };
+        args.observe?.close(code, reason.toString(), "server");
+      }
+      if (socket) { closeSocket(socket, code, reason.toString()); finish(); }
+      // If the upstream closes before downstream open, open() flushes the queue.
+    } catch (error) { fail(error); }
+  });
+  const opening = new Promise<void>((resolve, reject) => {
+    rejectOpen = reject;
+    upstream.once("open", () => {
+      try {
+        opened = true;
+        args.observe?.open(upstream.protocol);
+        openingTimer = setTimeout(() => fail(new Error("downstream did not open")), 10_000);
+        resolve();
+      } catch (error) { fail(error); }
+    });
+  });
+  const handlers: FakeWebSocketHandlers = {
+    open(ws) {
+      socket = ws;
+      clearTimeout(openingTimer);
+      if (stopped) { ws.close(1011, "WebSocket forwarding failed"); return; }
+      try {
+        for (const message of queue) ws.send(message);
+        queue = []; queuedBytes = 0;
+        if (upstreamClosure) { closeSocket(ws, upstreamClosure.code, upstreamClosure.reason); finish(); }
+      } catch (error) { fail(error); }
+    },
+    message(_ws, message) {
+      if (stopped) return;
+      try {
+        args.observe?.input(message);
+        const size = typeof message === "string" ? Buffer.byteLength(message) : message.byteLength;
+        if (size + upstream.bufferedAmount > MAX_BUFFER) throw new Error("WebSocket upstream buffer exceeded");
+        if (upstream.readyState !== WebSocket.OPEN) throw new Error("upstream is not open");
+        upstream.send(message);
+      } catch (error) { fail(error); throw error; }
+    },
+    close(_ws, code, reason) {
+      if (stopped || clientClosed) return;
+      try {
+        clientClosed = true;
+        args.observe?.close(code, reason, "client");
+        closeSocket(upstream, code, reason);
+        openingTimer = setTimeout(() => finish(), 1_000);
+      } catch (error) { fail(error); }
+    },
+  };
+  detach = lifetime.onEnd(event => {
+    if (event.kind === "closed" && socket) handlers.close!(socket, event.code, event.reason);
+    else session.cancel();
+  });
+  await opening;
+  handlers.protocol = upstream.protocol || undefined;
+  return { session, handlers: bindWebSocketLifetime(handlers, lifetime) };
+}

package/src/fake-upstream.ts

@@ -0,0 +1,465 @@
+// Shared upstream transport and credential brokering. No cassette state lives here.
+import { createSocket } from "node:dgram";
+import { request as httpsRequest } from "node:https";
+import { brotliDecompressSync, gunzipSync, inflateSync } from "node:zlib";
+import * as dnsPacket from "dns-packet";
+import { getRecordSecret } from "./record-secrets.js";
+import { inferSigV4Scope, signAwsV4Request, type SigV4Credentials } from "./aws-sigv4.js";
+
+/** Match one request dimension. A bare string is an exact match; the
+ *  object forms cover prefix and regex matching. */
+export type StringMatch =
+  | string
+  | { exact: string }
+  | { startsWith: string }
+  | { regex: string };
+
+/** Request matchers for a credential-injection rule. A rule applies only
+ *  when EVERY specified dimension matches; omit `match` to always apply. */
+export interface InjectMatch {
+  /** Match the request path (no query). */
+  path?: StringMatch;
+  /** HTTP method(s) — any one matching satisfies it. */
+  method?: string | string[];
+  /** Query-param matchers (ANDed); each key's value(s) must match. */
+  query?: Record<string, StringMatch>;
+  /** Header matchers (ANDed; header names case-insensitive). */
+  headers?: Record<string, StringMatch>;
+}
+
+/** One credential-brokering rule. When it matches a request, these headers
+ *  are SET on the egress forward — overwriting whatever the app sent (so app
+ *  code can't smuggle a different value past the broker). Header VALUES may
+ *  embed `{{secret:REF}}` tokens; each `REF` is resolved server-side from the
+ *  project's Secrets store and supplied for the environment lifetime. Values
+ *  never enter project files, the warm-cache hash, or a cassette (redaction
+ *  fails closed). */
+export interface InjectRule {
+  match?: InjectMatch;
+  headers: Record<string, string>;
+}
+
+/** SigV4 request-signing broker. Unlike `inject` (which SETs a static header
+ *  value), this RE-SIGNS the outbound request: the app signs with throwaway
+ *  dummy credentials, we strip that signature and re-sign the exact outbound
+ *  request with the real credentials — the only way to broker AWS auth, whose
+ *  signature is a keyed HMAC over the whole request, not a static token.
+ *
+ *  The three fields are **secret refs, resolved directly** from the project's
+ *  Secrets store (NOT `{{secret:}}` templates), with the same lifetime and
+ *  fail-closed redaction as `inject`. Region + service are inferred from the
+ *  incoming request (its credential scope, else the host); the app never
+ *  configures them. */
+export interface AwsSigV4Sign {
+  type: "awsSigv4";
+  /** Secret ref for the AWS access key id. */
+  accessKeyId: string;
+  /** Secret ref for the AWS secret access key. */
+  secretAccessKey: string;
+  /** Optional secret ref for a session token (temporary credentials). */
+  sessionToken?: string;
+}
+
+export type SignConfig = AwsSigV4Sign;
+
+/** One resolved secret value and the ref it came from (for the redaction
+ *  placeholder). */
+export interface ResolvedSecret {
+  ref: string;
+  value: string;
+}
+
+/** Hop-by-hop + body-framing headers never forwarded / never replayed
+ *  (we decompress on the way through, so the encoding/length no longer
+ *  describe the bytes the app receives). */
+export const STRIP_FORWARD_HEADERS = new Set([
+  "host",
+  "connection",
+  "keep-alive",
+  "proxy-authenticate",
+  "proxy-authorization",
+  "te",
+  "trailers",
+  "transfer-encoding",
+  "upgrade",
+  "content-length",
+  "content-encoding",
+  // We force `accept-encoding: identity` on the forward and decompress any
+  // residual encoding, so the app never sees a stale accept-encoding.
+  "accept-encoding",
+]);
+
+/** External resolver used to find the REAL upstream IP, bypassing the in-VM
+ *  resolver (which points the faked host back at the daemon). */
+const EXTERNAL_DNS = process.env.SPECTEST_UPSTREAM_DNS ?? "1.1.1.1";
+const EXTERNAL_DNS_PORT = Number(process.env.SPECTEST_UPSTREAM_PORT ?? "53");
+
+/** SigV4 puts per-request signing params (`X-Amz-Signature`, `X-Amz-Date`, …)
+ *  in the query for presigned URLs — they change every call and would poison
+ *  query matching, so they're dropped from both the match key and the forward
+ *  when signing is active. */
+export function isAmzParam(key: string): boolean {
+  return key.toLowerCase().startsWith("x-amz-");
+}
+
+function stripAmzParams(search: URLSearchParams): URLSearchParams {
+  const out = new URLSearchParams();
+  for (const [k, v] of search) if (!isAmzParam(k)) out.append(k, v);
+  return out;
+}
+
+/** `{{secret:REF}}` template token embedded in an injected header value. */
+const SECRET_TOKEN = /\{\{secret:([A-Za-z0-9_]+)\}\}/g;
+
+/** Every secret ref referenced by any inject rule's header templates plus the
+ *  SigV4 signing credentials (direct refs, not templates). */
+export function collectSecretRefs(rules: InjectRule[], sign: SignConfig | undefined): string[] {
+  const refs = new Set<string>();
+  for (const rule of rules) {
+    for (const value of Object.values(rule.headers)) {
+      for (const m of value.matchAll(SECRET_TOKEN)) refs.add(m[1]);
+    }
+  }
+  if (sign?.type === "awsSigv4") {
+    refs.add(sign.accessKeyId);
+    refs.add(sign.secretAccessKey);
+    if (sign.sessionToken) refs.add(sign.sessionToken);
+  }
+  return [...refs];
+}
+
+function matchString(m: StringMatch, value: string): boolean {
+  if (typeof m === "string") return value === m;
+  if ("exact" in m) return value === m.exact;
+  if ("startsWith" in m) return value.startsWith(m.startsWith);
+  if ("regex" in m) return new RegExp(m.regex).test(value);
+  return false;
+}
+
+interface IncomingRequest {
+  method: string;
+  path: string;
+  query: URLSearchParams;
+  headers: Headers;
+}
+
+/** True if every dimension the matcher specifies is satisfied (AND across
+ *  dimensions). No `match` → always true. */
+export function ruleMatches(match: InjectMatch | undefined, req: IncomingRequest): boolean {
+  if (!match) return true;
+  if (match.path !== undefined && !matchString(match.path, req.path)) return false;
+  if (match.method !== undefined) {
+    const methods = (Array.isArray(match.method) ? match.method : [match.method]).map((s) =>
+      s.toUpperCase(),
+    );
+    if (!methods.includes(req.method.toUpperCase())) return false;
+  }
+  if (match.query) {
+    for (const [k, m] of Object.entries(match.query)) {
+      if (!req.query.getAll(k).some((v) => matchString(m, v))) return false;
+    }
+  }
+  if (match.headers) {
+    for (const [k, m] of Object.entries(match.headers)) {
+      const v = req.headers.get(k.toLowerCase());
+      if (v === null || !matchString(m, v)) return false;
+    }
+  }
+  return true;
+}
+
+interface BrokeredHeaders {
+  /** Lowercased header → resolved value, to SET on the forward. */
+  headers: Record<string, string>;
+  /** Resolved secrets (for redaction). */
+  secrets: ResolvedSecret[];
+  /** Refs referenced but not supplied by the control plane. */
+  missing: string[];
+}
+
+/** Resolve a rule's header templates, substituting every `{{secret:REF}}`
+ *  with the environment credential. Collects resolved secrets (for redaction)
+ *  and any refs the control plane didn't supply (fail loud). */
+export function brokerHeaders(rule: InjectRule): BrokeredHeaders {
+  const headers: Record<string, string> = {};
+  const secrets: ResolvedSecret[] = [];
+  const missing: string[] = [];
+  const seen = new Set<string>();
+  for (const [name, template] of Object.entries(rule.headers)) {
+    headers[name.toLowerCase()] = template.replace(SECRET_TOKEN, (_full, ref: string) => {
+      const value = getRecordSecret(ref);
+      if (value === undefined) {
+        missing.push(ref);
+        return "";
+      }
+      if (!seen.has(ref)) {
+        seen.add(ref);
+        secrets.push({ ref, value });
+      }
+      return value;
+    });
+  }
+  return { headers, secrets, missing };
+}
+
+/** Resolve a host's A records against an EXTERNAL DNS server, bypassing the
+ *  in-VM resolver (which would answer with the daemon's own gateway for a
+ *  faked host). Mirrors spectest-resolver's own upstream-forward transport
+ *  (dns-packet over UDP) so it works identically in-VM. */
+export function resolveExternalA(host: string): Promise<string[]> {
+  const query = dnsPacket.encode({
+    type: "query",
+    id: 1,
+    flags: dnsPacket.RECURSION_DESIRED,
+    questions: [{ type: "A", name: host }],
+  });
+  return new Promise((resolve) => {
+    const sock = createSocket("udp4");
+    let done = false;
+    const finish = (ips: string[]): void => {
+      if (done) return;
+      done = true;
+      try {
+        sock.close();
+      } catch {
+        // ignore
+      }
+      resolve(ips);
+    };
+    sock.on("message", (msg) => {
+      try {
+        const res = dnsPacket.decode(msg);
+        const ips = (res.answers ?? []).flatMap((a) =>
+          a.type === "A" && typeof a.data === "string" && a.data.length > 0 ? [a.data] : [],
+        );
+        finish(ips);
+      } catch {
+        finish([]);
+      }
+    });
+    sock.on("error", () => finish([]));
+    sock.send(query, EXTERNAL_DNS_PORT, EXTERNAL_DNS, (err) => {
+      if (err) finish([]);
+    });
+    setTimeout(() => finish([]), 3_000);
+  });
+}
+
+interface UpstreamResponse {
+  status: number;
+  headers: Record<string, string>;
+  /** Keep cookie boundaries when returning an unrecorded response. */
+  cookies?: string[];
+  /** Decompressed body bytes. */
+  bytes: Uint8Array;
+}
+
+/** Open a real HTTPS connection to `ip` but with SNI + `Host` = `host`, so
+ *  the public certificate validates against the real hostname (the in-VM CA
+ *  is not involved on this leg). Decompresses the response so the stored and
+ *  returned bytes are the plain payload. */
+function forwardToRealUpstream(args: {
+  ip: string;
+  host: string;
+  method: string;
+  pathWithQuery: string;
+  headers: Record<string, string>;
+  body: Uint8Array;
+  signal?: AbortSignal;
+}): Promise<UpstreamResponse> {
+  return new Promise((resolve, reject) => {
+    const req = httpsRequest(
+      {
+        host: args.ip,
+        port: 443,
+        method: args.method,
+        signal: args.signal,
+        path: args.pathWithQuery,
+        servername: args.host,
+        headers: { ...args.headers, host: args.host, "accept-encoding": "identity" },
+      },
+      (res) => {
+        const chunks: Buffer[] = [];
+        res.on("data", (c: Buffer) => chunks.push(c));
+        res.on("end", () => {
+          let bytes: Uint8Array = Buffer.concat(chunks);
+          const enc = String(res.headers["content-encoding"] ?? "").toLowerCase();
+          try {
+            if (enc.includes("br")) bytes = brotliDecompressSync(bytes);
+            else if (enc.includes("gzip")) bytes = gunzipSync(bytes);
+            else if (enc.includes("deflate")) bytes = inflateSync(bytes);
+          } catch {
+            // Leave the bytes as-is if decompression fails.
+          }
+          const headers: Record<string, string> = {};
+          for (const [k, v] of Object.entries(res.headers)) {
+            if (typeof v === "string") headers[k.toLowerCase()] = v;
+            else if (Array.isArray(v)) headers[k.toLowerCase()] = v.join(", ");
+          }
+          resolve({ status: res.statusCode ?? 0, headers, bytes, cookies: res.headers["set-cookie"] });
+        });
+        res.on("error", reject);
+      },
+    );
+    req.on("error", reject);
+    if (args.body.length > 0) req.write(Buffer.from(args.body));
+    req.end();
+  });
+}
+
+/** Forward without capturing. Callers may separately record the result. */
+export async function forwardHttp(
+  req: Request,
+  host: string,
+  opts: { inject?: InjectRule[]; sign?: SignConfig },
+  bodyBytes: Uint8Array,
+  label: string,
+): Promise<Response | { upstream: UpstreamResponse; secrets: ResolvedSecret[] }> {
+  const url = new URL(req.url);
+  const reqLike = { method: req.method.toUpperCase(), path: url.pathname };
+  // Credential brokering (static header inject): first matching rule wins.
+  // Resolve its `{{secret:REF}}` tokens from the environment secret store.
+  const rule = (opts.inject ?? []).find((r) =>
+    ruleMatches(r.match, {
+      method: reqLike.method,
+      path: reqLike.path,
+      query: url.searchParams,
+      headers: req.headers,
+    }),
+  );
+  const brokered = rule ? brokerHeaders(rule) : { headers: {}, secrets: [], missing: [] };
+
+  // SigV4 signing: resolve the credential secret refs (direct refs, not
+  // `{{secret:}}` templates). Missing required refs fail loud alongside any
+  // missing inject refs.
+  let signCreds: SigV4Credentials | undefined;
+  const signMissing: string[] = [];
+  if (opts.sign?.type === "awsSigv4") {
+    const s = opts.sign;
+    const accessKeyId = getRecordSecret(s.accessKeyId);
+    const secretAccessKey = getRecordSecret(s.secretAccessKey);
+    const sessionToken = s.sessionToken ? getRecordSecret(s.sessionToken) : undefined;
+    if (accessKeyId === undefined) signMissing.push(s.accessKeyId);
+    if (secretAccessKey === undefined) signMissing.push(s.secretAccessKey);
+    if (s.sessionToken && sessionToken === undefined) signMissing.push(s.sessionToken);
+    if (accessKeyId !== undefined && secretAccessKey !== undefined) {
+      signCreds = { accessKeyId, secretAccessKey, sessionToken };
+    }
+  }
+
+  const missing = [...brokered.missing, ...signMissing];
+  if (missing.length > 0) {
+    const refs = [...new Set(missing)];
+    return new Response(
+      `${label}: secret(s) ${JSON.stringify(refs)} were not supplied ` +
+        `(configure them on the project's Secrets page, then start a new environment or fork).\n`,
+      { status: 599, headers: { "content-type": "text/plain" } },
+    );
+  }
+
+  // Resolve the REAL host's IP via an external resolver so we don't loop
+  // back into the daemon (the in-VM resolver answers our own gateway for
+  // this faked name). See the module header.
+  const ips = await resolveExternalA(host);
+  if (ips.length === 0) {
+    return new Response(
+      `${label}: could not resolve real upstream ${JSON.stringify(host)} ` +
+        `via external DNS ${EXTERNAL_DNS} (is the VM online?).\n`,
+      { status: 599, headers: { "content-type": "text/plain" } },
+    );
+  }
+
+  // Forward headers: drop hop-by-hop/framing, then SET the brokered
+  // headers — overwriting whatever the app sent (so the credential can't
+  // be smuggled or spoofed by app code). The real values only ever appear
+  // on this outbound wire.
+  const fwdHeaders: Record<string, string> = {};
+  req.headers.forEach((v, k) => {
+    if (k.toLowerCase() === "content-encoding" || !STRIP_FORWARD_HEADERS.has(k.toLowerCase())) fwdHeaders[k.toLowerCase()] = v;
+  });
+  for (const [k, v] of Object.entries(brokered.headers)) fwdHeaders[k] = v;
+
+  // SigV4 re-signing: strip the app's dummy signature and re-sign the exact
+  // outbound request (method/path/query/signed-headers/body-hash) with the
+  // brokered credentials, so `authorization`/`x-amz-date`/`x-amz-content-sha256`
+  // are internally consistent.
+  const signSecrets: ResolvedSecret[] = [];
+  let wireQuery = url.search.slice(1);
+  if (opts.sign?.type === "awsSigv4" && signCreds) {
+    const scope = inferSigV4Scope({
+      authorization: req.headers.get("authorization"),
+      credentialParam: url.searchParams.get("X-Amz-Credential"),
+      host,
+    });
+    if (!scope) {
+      return new Response(
+        `${label}: could not infer the AWS region/service to sign ` +
+          `${reqLike.method} ${reqLike.path} (no SigV4 Credential on the request, and host ` +
+          `${JSON.stringify(host)} is not a <service>.<region>.amazonaws.com name).\n`,
+        { status: 599, headers: { "content-type": "text/plain" } },
+      );
+    }
+    // Drop the app's own (dummy-signed) signing headers before re-signing.
+    for (const h of ["authorization", "x-amz-date", "x-amz-content-sha256", "x-amz-security-token"]) {
+      delete fwdHeaders[h];
+    }
+    // Fold the app's remaining `x-amz-*`/content-type headers (e.g.
+    // `x-amz-target`) into the signature; they stay on the outbound wire.
+    const additionalHeaders: Record<string, string> = {};
+    for (const [k, v] of Object.entries(fwdHeaders)) {
+      if (
+        k === "content-type" ||
+        (k.startsWith("x-amz-") &&
+          k !== "x-amz-date" &&
+          k !== "x-amz-content-sha256" &&
+          k !== "x-amz-security-token")
+      ) {
+        additionalHeaders[k] = v;
+      }
+    }
+    const forwardParams = stripAmzParams(url.searchParams);
+    const signed = signAwsV4Request({
+      method: reqLike.method,
+      path: reqLike.path,
+      query: forwardParams,
+      host,
+      body: bodyBytes,
+      credentials: signCreds,
+      scope,
+      additionalHeaders,
+    });
+    for (const [k, v] of Object.entries(signed.headers)) fwdHeaders[k] = v;
+    // Send the canonical query we signed (URLSearchParams' `+`-for-space
+    // encoding would diverge from the signed `%20`).
+    wireQuery = signed.canonicalQuery;
+    signSecrets.push(
+      { ref: opts.sign.accessKeyId, value: signCreds.accessKeyId },
+      { ref: opts.sign.secretAccessKey, value: signCreds.secretAccessKey },
+    );
+    if (signCreds.sessionToken && opts.sign.sessionToken) {
+      signSecrets.push({ ref: opts.sign.sessionToken, value: signCreds.sessionToken });
+    }
+  }
+
+  const pathWithQuery = reqLike.path + (wireQuery ? `?${wireQuery}` : "");
+
+  let upstream: UpstreamResponse;
+  try {
+    upstream = await forwardToRealUpstream({
+      ip: ips[0],
+      host,
+      method: reqLike.method,
+      pathWithQuery,
+      headers: fwdHeaders,
+      body: bodyBytes,
+      signal: req.signal,
+    });
+  } catch (err) {
+    return new Response(
+      `${label}: forward to real ${host} (${ips[0]}) failed: ${(err as Error).message}\n`,
+      { status: 599, headers: { "content-type": "text/plain" } },
+    );
+  }
+
+  return { upstream, secrets: [...brokered.secrets, ...signSecrets] };
+}

package/src/harness/activity.test.ts

@@ -0,0 +1,20 @@
+import { expect, test } from "bun:test";
+import { harnessMethods } from "../daemon";
+
+test("health reports pending work without acquiring its execution slot", async () => {
+  const state = {
+    inFlightTest: null as Promise<unknown> | null,
+    inFlightBootstrap: null,
+    inFlightProjectSetup: null,
+  };
+  const methods = harnessMethods(state);
+  expect((await methods.health({}) as any).inFlightTest).toBe(false);
+  let finish!: () => void;
+  state.inFlightTest = new Promise<void>((resolve) => { finish = resolve; });
+  expect((await methods.health({}) as any).inFlightTest).toBe(true);
+  finish();
+  await state.inFlightTest;
+  // Completion is acknowledged by the execution-slot owner, not just a request timeout.
+  state.inFlightTest = null;
+  expect((await methods.health({}) as any).inFlightTest).toBe(false);
+});

package/src/harness/fake-websocket.ts

@@ -1,3 +1,5 @@
+import { isInteractiveEnvironment } from "../replay-mode.js";
+import { liveFakeHandlers } from "./live-fake.js";
 import type { FakeWebSocketHandlers } from "../fake-websocket.js";
 import type { FakeContext, FakeDefinition } from "../index.js";
 import type { IngressServer } from "./ingress-server.js";
@@ -14,13 +16,14 @@ export async function dispatchFake<S>(
   state: S,
   ctx: FakeContext,
 ): Promise<Response> {
-  if (req.headers.get("upgrade")?.toLowerCase() !== "websocket" || !def.websocket) {
-    try { return await runUninstrumented(() => def.handler(req, state, ctx)); }
+  const implementation = def.live && isInteractiveEnvironment() ? liveFakeHandlers(def) : def;
+  if (req.headers.get("upgrade")?.toLowerCase() !== "websocket" || !implementation.websocket) {
+    try { return await runUninstrumented(() => implementation.handler(req, state, ctx)); }
     catch (err) { return fakeError(def.name, err); }
   }
   const lifetime = getWebSocketLifetime(req);
   try {
-    const accepted = await runUninstrumented(() => def.websocket!(req, state, ctx));
+    const accepted = await runUninstrumented(() => implementation.websocket!(req, state, ctx));
     if (accepted instanceof Response) { lifetime.end({ kind: "aborted" }); return accepted; }
     if (!accepted || typeof accepted !== "object") throw new Error("websocket must return handlers or a Response");
     if (accepted.protocol !== undefined && !parseSubprotocols(req.headers.get("sec-websocket-protocol")).includes(accepted.protocol)) {