@specific.dev/spectest 0.111.0 → 0.112.0

package/src/components/replayFake-handler.test.ts

@@ -163,6 +163,11 @@ describe("scripted behavior on replay fakes", () => {
         extends: fileURLToPath(new URL("../../tsconfig.json", import.meta.url)),
         compilerOptions: {
           noEmit: true,
+          // The WebSocket regression suite also typechecks the example key.
+          // Resolve its public import without relying on a local file: install.
+          paths: {
+            "@specific.dev/spectest/components": [fileURLToPath(new URL("./index.ts", import.meta.url))],
+          },
           rootDir: fileURLToPath(new URL("../../../", import.meta.url)),
           typeRoots: [fileURLToPath(new URL("../../node_modules/@types", import.meta.url))],
         },

package/src/components/replayFake-websocket.test.ts

@@ -262,3 +262,169 @@ test("an abandoned handshake cancels the connection before open", async () => {
   expect(h.state.websocketSessions.size).toBe(0);
   expect(h.state.websocketErrors).toEqual([]);
 });
+
+test("output hooks select previous candidates; record and replay preserve timing without quiet", async () => {
+  const server = new WebSocketServer({ port: 0, host: "127.0.0.1" });
+  await once(server, "listening");
+  cleanup.push(() => { for (const c of server.clients) c.terminate(); server.close(); });
+  server.on("connection", ws => {
+    let count = 0;
+    ws.send("opening");
+    ws.on("message", () => { if (++count === 3) setTimeout(() => { ws.send("answer"); ws.close(4001, "done"); }, 25); });
+  });
+  const key: WebSocketKey<{ outputs: number }> = {
+    initial: { outputs: 0 },
+    input(state, message) { return { state, key: message === "noise" ? null : message }; },
+    output(state, message) { return { state: { outputs: state.outputs + 1 }, anchorPreviousInput: message === "answer" }; },
+  };
+  let record!: CassetteWebSocket;
+  let complete!: () => void;
+  const done = new Promise<void>(resolve => complete = resolve);
+  const errors: Error[] = [];
+  const r = await recordWebSocket({
+    url: `ws://voice.test:${(server.address() as { port: number }).port}/live`, ip: "127.0.0.1", host: "voice.test", headers: {}, protocols: [], request,
+    match: websocketMatcher(key), redact: s => s, secrets: [], label: "anchored",
+    captured: value => record = value,
+    finished(_session, error) { if (error) errors.push(error); complete(); },
+  });
+  cleanup.push(() => r.session.cancel());
+  const socket: FakeWebSocket = { readyState: 1, send() {}, close() {} };
+  await r.handlers.open!(socket);
+  await r.handlers.message!(socket, "previous");
+  await r.handlers.message!(socket, "candidate");
+  await r.handlers.message!(socket, "candidate");
+  await done;
+  expect(errors).toEqual([]);
+  expect(record.inputMode).toBe("anchors");
+  expect(record.inputs).toEqual([websocketMatcher(() => "candidate")("x")!]);
+  expect(record.inputOccurrences).toEqual([2]);
+  expect(record.messages[0].afterInput).toBe(0);
+  expect(record.messages[1]).toMatchObject({ afterInput: 1, data: "answer" });
+  expect(record.messages[1].delayMs).toBeGreaterThanOrEqual(20);
+  expect(record.messages[1].delayMs).toBeLessThan(500);
+  const h = await harness([record], key), c = await h.connect();
+  await tick(); expect(c.sent).toEqual(["opening"]);
+  await c.message("noise"); await c.message("previous"); await c.message("candidate");
+  await tick(40); expect(c.sent).toEqual(["opening"]);
+  await c.message("candidate");
+  await tick(5); expect(c.sent).toEqual(["opening"]);
+  await tick(45); expect(c.sent).toEqual(["opening", "answer"]);
+  await c.close(4001, "done"); expect(h.state.websocketErrors).toEqual([]);
+  expect(key.initial.outputs).toBe(0);
+});
+
+test("candidate batches normalize chunking and output state feeds subsequent inputs", async () => {
+  const key: WebSocketKey<{ turn: number; position: number }> = {
+    initial: { turn: 0, position: 0 },
+    input(state, message) {
+      const keys = Array.from(String(message), () => ({ turn: state.turn, position: ++state.position }));
+      return { state, keys };
+    },
+    output(state, message) { return { state: message === "next" ? { turn: state.turn + 1, position: 0 } : state, anchorPreviousInput: true }; },
+  };
+  const hash = (turn: number, position: number) => websocketMatcher(() => ({ turn, position }))("x")!;
+  const h = await harness([{ request, inputMode: "anchors", inputs: [hash(0, 3), hash(1, 2)], inputOccurrences: [1, 1], messages: [
+    { afterInput: 1, delayMs: 0, type: "text", data: "next" },
+    { afterInput: 2, delayMs: 0, type: "text", data: "done" },
+  ] }], key);
+  const c = await h.connect();
+  await c.message("a"); await c.message("bc"); await tick(); expect(c.sent).toEqual(["next"]);
+  await c.message("de"); await tick(); expect(c.sent).toEqual(["next", "done"]);
+  await c.close(); expect(h.state.websocketErrors).toEqual([]);
+});
+
+test("unmatched candidates cannot extend the required anchor deadline", async () => {
+  const key: WebSocketKey<null> = { initial: null,
+    input(state, message) { return { state, key: message }; },
+    output(state) { return { state, anchorPreviousInput: true }; },
+  };
+  const h = await harness([{ request, inputMode: "anchors", inputs: [websocketMatcher(() => "wanted")("x")!], inputOccurrences: [1], messages: [] }], key, 30);
+  const c = await h.connect();
+  for (let i = 0; i < 6; i++) { await c.message("unrelated"); await tick(10); }
+  expect(h.state.websocketErrors.join()).toContain("required input never arrived");
+});
+
+test("anchor validation and hook failures are explicit", async () => {
+  const key: WebSocketKey<null> = { initial: null,
+    input(state, message) { return { state, key: message }; },
+    output(state) { return { state, anchorPreviousInput: true }; },
+  };
+  const h = await harness([], key);
+  const input = websocketMatcher(() => "x")("x")!;
+  for (const counts of [[], [0], [1.2], [2, 1]]) {
+    expect(() => h.helpers.replay(cassette([{ request, inputMode: "anchors", inputs: counts.length === 2 ? [input, input] : [input], inputOccurrences: counts, messages: [] }]))).toThrow("occurrence");
+  }
+  h.helpers.replay(cassette([{ request, inputs: [], messages: [] }]));
+  await expect(h.connect()).rejects.toThrow("re-record");
+  const asyncInput = websocketMatcher({ ...key, input: async () => ({ state: null, key: null }) } as any);
+  expect(() => asyncInput.candidates!("x")).toThrow("synchronous");
+  const asyncOutput = websocketMatcher({ ...key, output: async () => ({ state: null, anchorPreviousInput: true }) } as any);
+  expect(() => asyncOutput.output!("x")).toThrow("synchronous");
+  expect(() => websocketMatcher({ ...key, output: () => ({ state: null }) } as any).output!("x")).toThrow("anchorPreviousInput");
+});
+
+test("output hook cannot mutate binary playback and can select an opening with no input", async () => {
+  const key: WebSocketKey<null> = { initial: null,
+    input(state) { return { state, key: null }; },
+    output(state, message) { if (message instanceof Uint8Array) message.fill(0); return { state, anchorPreviousInput: true }; },
+  };
+  const h = await harness([{ request, inputMode: "anchors", inputs: [], inputOccurrences: [], messages: [
+    { afterInput: 0, delayMs: 0, type: "binary", data: "Af8=" },
+  ] }], key);
+  const c = await h.connect(); await tick();
+  expect(c.sent).toEqual([new Uint8Array([1, 255])]);
+  await c.close(); expect(h.state.websocketErrors).toEqual([]);
+});
+
+export function inferredHooks() {
+  return replayFake({ name: "inferred", host: "voice.test", websocketKey: {
+    initial: { position: 0 },
+    input(state, message) {
+      const position: number = state.position;
+      // @ts-expect-error state is inferred from initial
+      const wrong: string = state.position;
+      return { state: { position: position + 1 }, key: message };
+    },
+    output(state) { return { state, anchorPreviousInput: state.position > 0 }; },
+  } });
+}
+
+test("GPT-Live candidates normalize chunking and select fast speech without caller silence", async () => {
+  const { liveKey } = await import("../../../examples/phone-receptionist/spectest/live-key.js");
+  const audio = (pcm: Buffer) => JSON.stringify({ type: "session.input_audio.append", audio: pcm.toString("base64") });
+  const response = JSON.stringify({ type: "session.output_audio.delta", delta: Buffer.alloc(960, 16).toString("base64") });
+  const a = websocketMatcher(liveKey), b = websocketMatcher(liveKey);
+  expect(a.candidates!(audio(Buffer.alloc(1920)))).toEqual([]);
+  const speech = Buffer.alloc(960 * 3, 16);
+  const whole = a.candidates!(audio(speech));
+  const split = [speech.subarray(0, 300), speech.subarray(300, 1140), speech.subarray(1140)]
+    .flatMap(pcm => b.candidates!(audio(pcm)));
+  expect(whole).toEqual(split); expect(whole).toHaveLength(3);
+  expect(a.output!(response)).toBe(true); // No quiet samples at all.
+  expect(a.candidates!(audio(Buffer.alloc(960, 16)))).toEqual([]); // Same ongoing utterance.
+  expect(a.candidates!(audio(Buffer.alloc(960)))).toEqual([]);
+  expect(a.candidates!(audio(Buffer.alloc(960, 16)))).toHaveLength(1); // Next turn.
+  expect(a.output!(response)).toBe(true);
+});
+
+
+test("a reached response anchor catches up older output without adding its stale clock delay", async () => {
+  const key: WebSocketKey<null> = { initial: null,
+    input(state, message) { return { state, key: message }; },
+    output(state) { return { state, anchorPreviousInput: true }; },
+  };
+  const messages = Array.from({length: 200}, (_, index) => ({
+    afterInput: 0, delayMs: 3000 + index, type: "text" as const, data: `quiet-${index}`,
+  }));
+  const h = await harness([{ request, inputMode: "anchors",
+    inputs: [websocketMatcher(() => "go")("x")!], inputOccurrences: [1], messages: [
+      ...messages, {afterInput: 1, delayMs: 20, type: "text", data: "response"},
+    ],
+  }], key);
+  const c = await h.connect();
+  await tick(); expect(c.sent).toEqual([]);
+  await c.message("go");
+  await tick(60);
+  expect(c.sent).toEqual([...messages.map(m => m.data), "response"]);
+  await c.close(); expect(h.state.websocketErrors).toEqual([]);
+});

package/src/components/replayFake.ts

@@ -177,7 +177,10 @@ export interface ReplayFakeOptions<
    *  matching dimensions or fallback apply when a key is supplied. */
   key?: ReplayKey;
   /** Complete messages (string or Uint8Array), synchronously reduced per connection.
-   * Return null to emit no input. Raw inputs are never stored; changing this
+   * Functions and { initial, reduce } emit required keys (null skips an input).
+   * { initial, input, output } emits candidates; output can select the previous
+   * candidate as a recorded timing anchor. Replay waits for those selected keys.
+   * Raw inputs are never stored; changing this
    * matcher can require re-recording. Defaults to SHA-256(type + bytes). */
   websocketKey?: WebSocketKey<W>;
   /** Maximum wait for each required input key; null-producing messages do not

package/src/daemon.ts

@@ -1,3 +1,4 @@
+import { phoneScope, phoneRecordingChunk, releasePhoneRecording, type PhoneRecording } from "./phone.js";
 import { runWithBackgroundFailures } from "./harness/background-failures.js";
 import { generateHostCert as mintHostCert } from "./harness/certificate.js";
 import { componentBuildTasks, COMPONENT_RUNTIME } from "./harness/component-images.js";
@@ -3531,6 +3532,7 @@ async function runProjectSetupInner(hook: "setup" | "setupInteractive"): Promise
 // ────────────────────────────────────────────────────────────────────────
 
 interface RunResult {
+  phoneRecordings: PhoneRecording[];
   androidRecordings: AndroidRecording[];
   status: "passed" | "failed";
   durationMs: number;
@@ -5061,6 +5063,7 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
   const start = Date.now();
   const nativeAndroid = androidScope(testCase.id, start);
   let androidRecordings: AndroidRecording[] = [];
+  let phoneRecordings: PhoneRecording[] = [];
   const chunks: string[] = [];
   const origStdout = process.stdout.write.bind(process.stdout);
   const origStderr = process.stderr.write.bind(process.stderr);
@@ -5359,6 +5362,7 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
   // a real error rather than getting attributed to the test.
   const base = await spectestContext();
 
+  const phones = phoneScope(start);
   const ctx: TestContext<unknown> = {
     ...base,
     // exec/terminal/poll wrap their results at runtime (the recorder is
@@ -5372,6 +5376,7 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
     android: nativeAndroid.open,
     desktop: desktopOpener(start, testCase.id, sessions, browserDetaches),
     mcp: openMcp,
+    phone: phones.open,
     testName: testCase.name,
     parent,
   };
@@ -5415,6 +5420,9 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
     };
   } finally {
     if (timer) clearTimeout(timer);
+    phoneRecordings = await phones.finish();
+    const phoneErrors = phoneRecordings.flatMap(r => r.error ? [r.error] : []);
+    if (phoneErrors.length && outcome.status === "passed") outcome = { status: "failed", error: { message: phoneErrors.join("\n") } };
     nativeAndroid.deactivate();
     RECORDING_EXEC = undefined;
     INTERCEPTORS.endScope();
@@ -5497,6 +5505,7 @@ async function runOne(testCase: TestCase<unknown>): Promise<RunResult> {
   return {
     status: outcome.status,
     androidRecordings,
+    phoneRecordings,
     durationMs,
     log: chunks.join(""),
     events,
@@ -5915,6 +5924,7 @@ async function evalCode(code: string): Promise<EvalResult> {
   // snippets don't require a /load round-trip first.
   const base = await spectestContext();
 
+  const phones = phoneScope(start, { record: false });
   const ctx: TestContext<undefined> = {
     ...base,
     // Bun does not propagate the import() caller's AsyncLocalStorage into
@@ -5932,6 +5942,7 @@ async function evalCode(code: string): Promise<EvalResult> {
     android: nativeAndroid.open,
     desktop: desktopOpener(start, "eval", sessions, browserDetaches, artifactCollector),
     mcp: openMcp,
+    phone: phones.open,
     testName: "eval",
     parent: undefined,
   };
@@ -5978,6 +5989,7 @@ async function evalCode(code: string): Promise<EvalResult> {
       error: { message: explainEvalExportError(code, message), stack: e.stack },
     };
   } finally {
+    await phones.finish();
     fetchScope.active = false;
     restoreFetch();
     restoreConsole();
@@ -6408,6 +6420,8 @@ export function harnessMethods(state: RouteState): MethodTable {
 
     // One chunk of a parked replay bundle. Case ids are arbitrary user
     // strings, so they travel in params rather than a URL path.
+    phoneRecordingChunk: async (params) => phoneRecordingChunk(String(params.caseId ?? params.id), params.offset, params.limit),
+    releasePhoneRecording: async (params) => releasePhoneRecording(String(params.id)),
     androidRecordingChunk: async (params) => androidRecordingChunk(String(params.caseId ?? params.id), params.offset, params.limit),
     releaseAndroidRecording: async (params) => releaseAndroidRecording(String(params.id)),
     replayChunk: async (params) => {