@specific.dev/spectest 0.111.0 → 0.112.0

package/src/phone.test.ts

@@ -0,0 +1,474 @@
+import { afterAll, afterEach, beforeAll, expect, spyOn, test } from "bun:test";
+import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { phoneScope, phoneRecordingChunk, releasePhoneRecording } from "./phone.js";
+import { recordStep, startRecording, stopRecording } from "./recorder.js";
+import { trackHelpers } from "./helper-tracking.js";
+
+let dir: string;
+const original = process.env.SPECTEST_PHONE_PYTHON;
+let scope = phoneScope();
+beforeAll(async () => {
+  dir = await mkdtemp(join(tmpdir(), "spectest-phone-test-"));
+  const runtime = join(dir, "runtime");
+  await writeFile(
+    runtime,
+    `#!/usr/bin/env bun
+import { createInterface } from "node:readline";
+let connected = false, directory, sessionId, started, clockRate = 1;
+const offset = () => (Date.now() - started) / 1000 * clockRate;
+function finish() {
+  if (!sessionId) return;
+  const duration = offset();
+  const bytes = Buffer.alloc(128);
+  bytes.set([0x1a,0x45,0xdf,0xa3]);
+  require("node:fs").writeFileSync(directory + "/" + sessionId + ".webm", bytes);
+  console.log(JSON.stringify({event:"recording", sessionId, startedAtMs:started, duration, contentType:"audio/webm", encodingFinalizeMs:1, bytes:bytes.length, peaks:[.2,.8], callerPeaks:[.2,0], recipientPeaks:[0,.8]}));
+  sessionId = undefined;
+}
+createInterface({ input: process.stdin }).on("line", async line => {
+  const data = JSON.parse(line);
+  let result = {};
+  if (sessionId && data.action !== "call") console.log(JSON.stringify({event:"operation_started", id:data.id, offset:offset()}));
+  if (data.action === "open") directory = data.directory;
+  if (data.action === "call") {
+    sessionId = data.sessionId; started = Date.now();
+    clockRate = data.destination === "clock-drift" ? 0.25 : 1;
+    console.log(JSON.stringify({event:"operation_started", id:data.id, offset:0}));
+    if (data.destination === "timeout") return;
+    await Bun.sleep(20);
+    connected = true;
+  }
+  if (data.action === "hangup") { connected = false; finish(); }
+  if (data.action === "say") {
+    result = { error: "No connected call" };
+    if (connected) {
+      // Generation completes before any speech is sent. The start event must
+      // not resolve say(), and must survive a later playback failure.
+      await Bun.sleep(80);
+      console.log(JSON.stringify({event:"speech_started", id:data.id, sessionId, offset:offset()}));
+      await Bun.sleep(30);
+      result = data.text === "Interrupted speech"
+        ? { error: "Call disconnected during playback" }
+        : { duration: .03, offset:offset() };
+    }
+  }
+  if (data.action === "wait_silence") {
+    await Bun.sleep(Math.min(data.silenceMs, data.timeoutMs));
+    result = data.silenceMs > data.timeoutMs
+      ? {error:"SIP operation timed out"}
+      : {offset:offset()};
+  }
+  if (data.action === "position") result = {offset:offset()};
+  console.log(JSON.stringify({ id: data.id, ...result }));
+}).on("close", () => { finish(); process.exit(0); });
+`,
+  );
+  await chmod(runtime, 0o755);
+  process.env.SPECTEST_PHONE_PYTHON = runtime;
+});
+afterEach(async () => {
+  await scope.finish();
+  scope = phoneScope();
+  stopRecording();
+});
+afterAll(async () => {
+  if (original === undefined) delete process.env.SPECTEST_PHONE_PYTHON;
+  else process.env.SPECTEST_PHONE_PYTHON = original;
+  await rm(dir, { recursive: true, force: true });
+});
+const options = {
+  endpoint: "sip:5060",
+  from: "sip:caller@spectest.test",
+  password: "never-record-this",
+  language: "sv",
+};
+
+test("serializes dialing and speech, preserves the full utterance and excludes credentials", async () => {
+  startRecording();
+  const phone = await scope.open(options);
+  await Promise.all([
+    phone.call("123"),
+    phone.say("Hej! Jag vill boka en tid."),
+  ]);
+  await phone.hangup();
+  const events = stopRecording();
+  expect(events.map((e) => e.kind)).toEqual(["phone", "phone", "phone"]);
+  expect(events.map((e) => e.seq)).toEqual([0, 1, 2]);
+  const say = events[1] as any;
+  expect(say.status).toBe("passed");
+  expect(say.blocks[0].text).toBe("Hej! Jag vill boka en tid.");
+  expect((events[0] as any).blocks[0].waitedForSpeech).toBeUndefined();
+  expect(say.blocks[0].waitedForSpeech).toBeUndefined();
+  expect(say.blocks[0]).not.toHaveProperty("segments");
+  expect(JSON.stringify(events)).not.toContain(options.password);
+});
+
+test("failed speech retains text and scope cleanup fences later operations", async () => {
+  startRecording();
+  const phone = await scope.open(options);
+  await expect(phone.say("Hello.")).rejects.toThrow("No connected call");
+  const events = stopRecording() as any[];
+  expect(events[0].status).toBe("failed");
+  expect(events[0].blocks).toEqual([{ type: "text", text: "Hello." }]);
+  await scope.close();
+  startRecording();
+  await expect(phone.say("Late work")).rejects.toThrow("closed");
+  await expect(scope.open(options)).rejects.toThrow("ended");
+  expect(stopRecording()).toEqual([]);
+});
+
+test("times out an unanswered call and closes without leaking a subprocess", async () => {
+  const phone = await scope.open({ ...options, timeoutMs: 40 });
+  await expect(phone.call("timeout")).rejects.toThrow("timed out");
+  await phone.close();
+  await phone.close();
+});
+
+test("cancels an in-flight operation when a case ends", async () => {
+  const phone = await scope.open(options);
+  const call = phone.call("timeout");
+  const rejected = call.catch((error) => error);
+  await Bun.sleep(5);
+  await scope.close();
+  expect(String(await rejected)).toContain("closed");
+});
+
+test("rejects invalid inputs before contacting SIP", async () => {
+  await expect(
+    scope.open({ ...options, endpoint: "sip:5060\r\nInjected: header" }),
+  ).rejects.toThrow("endpoint");
+  const phone = await scope.open(options);
+  await expect(phone.say(" ")).rejects.toThrow("characters");
+  await expect(phone.say("x".repeat(2001))).rejects.toThrow("characters");
+  await expect(phone.say("hello", { speed: NaN })).rejects.toThrow("speed");
+  await expect(phone.call("a\r\nb")).rejects.toThrow("destination");
+  await expect(phone.call("123", { waitFor: true as any })).rejects.toThrow("predicate");
+});
+
+test("reports a missing golden runtime and still completes cleanup", async () => {
+  const runtime = process.env.SPECTEST_PHONE_PYTHON;
+  process.env.SPECTEST_PHONE_PYTHON = join(dir, "missing-runtime");
+  try {
+    await expect(scope.open(options)).rejects.toThrow("runtime unavailable");
+    await scope.close();
+  } finally {
+    process.env.SPECTEST_PHONE_PYTHON = runtime;
+  }
+});
+
+
+test("keeps one whole-call file across speech and idle gaps, with bounded chunk transfer", async () => {
+  startRecording();
+  const phone = await scope.open(options);
+  await phone.call("123");
+  await phone.say("One. Two.");
+  await Bun.sleep(60);
+  await phone.say("Three.");
+  await Bun.sleep(40);
+  // No hangup: finishing the case must seal the idle tail.
+  const records = await scope.finish();
+  const events = stopRecording() as any[];
+  expect(records).toHaveLength(1);
+  expect(records[0]!.error).toBeUndefined();
+  expect(records[0]!.duration).toBeGreaterThan(.1);
+  const first = events[1].blocks[0], second = events[2].blocks[0];
+  expect(first.sessionId).toBe(second.sessionId);
+  expect(first.end - first.start).toBeGreaterThanOrEqual(.02);
+  expect(records[0]!.duration - second.end).toBeGreaterThanOrEqual(.03);
+  expect(second.end - second.start).toBeGreaterThanOrEqual(.02);
+  expect(JSON.stringify(events)).not.toContain('"audio":');
+  const chunk = await phoneRecordingChunk(records[0]!.id, 0, 44);
+  expect([...Buffer.from(chunk.b64, "base64").subarray(0,4)]).toEqual([0x1a,0x45,0xdf,0xa3]);
+  expect(records[0]!.contentType).toBe("audio/webm");
+  expect(records[0]!.encodingFinalizeMs).toBeGreaterThanOrEqual(0);
+  expect(chunk.total).toBe(records[0]!.bytes);
+  await expect(phoneRecordingChunk(records[0]!.id, -1, 44)).rejects.toThrow("range");
+  await releasePhoneRecording(records[0]!.id);
+  await expect(phoneRecordingChunk(records[0]!.id, 0, 44)).rejects.toThrow("Unknown");
+});
+
+test("separates consecutive calls and discards unrecorded evaluation files", async () => {
+  const phone = await scope.open(options);
+  await phone.call("123");
+  await phone.hangup();
+  await phone.call("456");
+  const records = await scope.finish();
+  expect(records).toHaveLength(2);
+  expect(records[0]!.id).not.toBe(records[1]!.id);
+  scope = phoneScope(Date.now(), {record:false});
+  await expect(phoneRecordingChunk(records[0]!.id, 0, 44)).rejects.toThrow("Unknown");
+  const evaluated = await scope.open(options);
+  await evaluated.call("789");
+  expect(await scope.finish()).toEqual([]);
+});
+
+
+test("isolates say clips from unrelated waits and leaves synthesis in the full call", async () => {
+  startRecording();
+  const phone = await scope.open(options);
+  await phone.call("123");
+  await phone.say("First utterance");
+  await Bun.sleep(40); // Unrelated work, outside the step.
+  await phone.say("Second utterance");
+  await Bun.sleep(40);
+  await phone.hangup();
+  const [recording] = await scope.finish();
+  const events = stopRecording() as any[];
+  const [call, first, second] = events.map(e => e.blocks[0]);
+  // Each clip skips the fixture's synthesis delay and ends at its own completion.
+  expect(first.start - call.end).toBeGreaterThanOrEqual(.06);
+  expect(second.start - first.end).toBeGreaterThanOrEqual(.06);
+  // The speech_started event must not resolve the request before playback ends.
+  expect(first.end - first.start).toBeGreaterThanOrEqual(.02);
+  expect(second.end - second.start).toBeGreaterThanOrEqual(.02);
+  expect(recording!.duration - second.end).toBeGreaterThanOrEqual(.03);
+  expect(events[3].blocks).toEqual([]);
+  expect(events[1].durationMs).toBeGreaterThanOrEqual(100);
+  expect(recording!.duration).toBeGreaterThan(.3);
+});
+
+test("retains the transmission boundary when playback fails after speech starts", async () => {
+  startRecording();
+  const phone = await scope.open(options);
+  await phone.call("123");
+  await expect(phone.say("Interrupted speech")).rejects.toThrow("disconnected");
+  const [recording] = await scope.finish();
+  const events = stopRecording() as any[];
+  const call = events[0].blocks[0], say = events[1].blocks[0];
+  expect(events[1].status).toBe("failed");
+  expect(say.start - call.end).toBeGreaterThanOrEqual(.06);
+  expect(say.end - say.start).toBeGreaterThanOrEqual(.02);
+  expect(say.end).toBeLessThanOrEqual(recording!.duration);
+});
+
+test("maps every clip to PCM positions when the media clock falls behind wall time", async () => {
+  startRecording();
+  const phone = await scope.open(options);
+  await phone.call("clock-drift");
+  await phone.say("First");
+  await Bun.sleep(80);
+  await phone.say("Second");
+  await Bun.sleep(80);
+  await phone.hangup();
+  const [recording] = await scope.finish();
+  const events = stopRecording() as any[];
+  const [call, first, second] = events.map(e => e.blocks[0]);
+  expect(call.start).toBe(0);
+  expect(call.end).toBeLessThan(first.start);
+  expect(first.end).toBeLessThan(second.start);
+  expect(second.end).toBeLessThan(recording!.duration);
+  expect(events[3].blocks).toEqual([]);
+  // Each 30 ms utterance occupies only ~8 ms on this deliberately slow clock.
+  expect(first.end - first.start).toBeLessThan(.025);
+  expect(second.end - second.start).toBeLessThan(.025);
+  expect(recording!.duration).toBeLessThan(.2);
+});
+
+
+test("call waits for its greeting predicate then silence before the next say, and isolates its clip", async () => {
+  startRecording();
+  const phone = await scope.open(options);
+  let probes = 0, satisfiedAt = 0, finishedAt = 0;
+  const call = phone.call("123", {
+    waitFor: async () => {
+      recordStep({ kind: "probe", title: "Greeting observation" });
+      if (++probes < 3) return false;
+      satisfiedAt = Date.now();
+      return true;
+    },
+  }).then(() => { finishedAt = Date.now(); });
+  const say = phone.say("Hello");
+  await Promise.all([call, say]);
+  await scope.finish();
+  const events = stopRecording() as any[];
+  expect(probes).toBe(3);
+  expect(finishedAt - satisfiedAt).toBeGreaterThanOrEqual(650);
+  expect(events.map(e => e.kind)).toEqual(["phone", "phone"]);
+  const greeting = events[0].blocks[0], reply = events[1].blocks[0];
+  expect(greeting.waitedForSpeech).toBe(true);
+  expect(greeting.start).toBe(0);
+  expect(greeting.end).toBeGreaterThanOrEqual(.85);
+  expect(reply.start - greeting.end).toBeGreaterThanOrEqual(.06);
+  expect(events[0].durationMs).toBeGreaterThanOrEqual(850);
+});
+
+test("waitFor stores per-step transcript strings after silence and keeps boolean waits compatible", async () => {
+  startRecording();
+  const phone = await scope.open(options);
+  await phone.call("123", { waitFor: () => "Welcome. How can I help?" });
+  let probes = 0, returnedAt = 0;
+  const response = "Thursday at ten is available.\nWould that work for you?";
+  const fake = trackHelpers("fake", "voice", { transcript: async () => {
+    recordStep({ kind: "probe", title: "Read delivered transcript" });
+    if (++probes === 1) return false;
+    if (probes === 2) return "";
+    returnedAt = Date.now();
+    return response;
+  }});
+  await phone.say("A checkup, please", { waitFor: () => fake.transcript() });
+  expect(Date.now() - returnedAt).toBeGreaterThanOrEqual(650);
+  await phone.say("Yes", { waitFor: () => true });
+  await phone.say("Thanks");
+  await scope.finish();
+  const events = stopRecording() as any[];
+  expect(events.map(e => e.kind)).toEqual(["phone", "phone", "phone", "phone"]);
+  expect(probes).toBe(3);
+  expect(events[0].blocks[0].response).toBe("Welcome. How can I help?");
+  expect(events[1].blocks[0].response).toBe(response);
+  expect(events[2].blocks[0].response).toBeUndefined();
+  expect(events[3].blocks[0].response).toBeUndefined();
+});
+
+test("a greeting predicate failure fails the call step and preserves its recorded window", async () => {
+  startRecording();
+  const phone = await scope.open(options);
+  await expect(phone.call("123", { waitFor: () => { throw new Error("Greeting failed"); } })).rejects.toThrow("Greeting failed");
+  await Bun.sleep(60);
+  await phone.hangup();
+  const [recording] = await scope.finish();
+  const events = stopRecording() as any[];
+  expect(events[0].status).toBe("failed");
+  expect(events[0].blocks[0].start).toBe(0);
+  expect(events[0].blocks[0].end).toBeGreaterThan(0);
+  expect(recording!.duration - events[0].blocks[0].end).toBeGreaterThanOrEqual(.05);
+  expect(events[1].blocks).toEqual([]);
+});
+
+test("waits for the predicate then a fresh silence window inside one serialized say step", async () => {
+  startRecording();
+  const phone = await scope.open(options);
+  await phone.call("123");
+  let probes = 0, satisfiedAt = 0, finishedAt = 0;
+  const first = phone.say("Question", {
+    waitFor: async () => {
+      recordStep({ kind: "probe", title: "Internal observation" });
+      if (++probes < 3) return false;
+      satisfiedAt = Date.now();
+      return true;
+    },
+  }).then(() => { finishedAt = Date.now(); });
+  const second = phone.say("Next turn");
+  await Promise.all([first, second]);
+  await Bun.sleep(50);
+  const [recording] = await scope.finish();
+  const events = stopRecording() as any[];
+  expect(probes).toBe(3);
+  expect(finishedAt - satisfiedAt).toBeGreaterThanOrEqual(50);
+  expect(events.map(e => e.kind)).toEqual(["phone", "phone", "phone"]);
+  const a = events[1].blocks[0], b = events[2].blocks[0];
+  expect(a.end - a.start).toBeGreaterThanOrEqual(.12);
+  expect(b.start - a.end).toBeGreaterThanOrEqual(.06);
+  expect(recording!.duration - b.end).toBeGreaterThanOrEqual(.04);
+});
+
+test("predicate errors and hung predicates fail the say step without waiting for silence", async () => {
+  startRecording();
+  const phone = await scope.open(options);
+  await phone.call("123");
+  await expect(phone.say("Throw", { waitFor: () => { throw new Error("Application failed"); } })).rejects.toThrow("Application failed");
+  const nativeTimeout = globalThis.setTimeout;
+  const timer = spyOn(globalThis, "setTimeout").mockImplementation(((fn: any, ms: number, ...args: any[]) =>
+    nativeTimeout(fn, ms === 30_000 ? 40 : ms, ...args)) as typeof setTimeout);
+  try {
+    await expect(phone.say("Hang", { waitFor: () => new Promise(() => {}) })).rejects.toThrow("response wait timed out");
+  } finally { timer.mockRestore(); }
+  await scope.finish();
+  const events = stopRecording() as any[];
+  expect(events.slice(1).map(e => e.status)).toEqual(["failed", "failed"]);
+  expect(events[2].blocks[0].end - events[2].blocks[0].start).toBeGreaterThanOrEqual(.05);
+});
+
+test("case cleanup cancels a hung predicate and suppresses its late events", async () => {
+  startRecording();
+  const phone = await scope.open(options);
+  await phone.call("123");
+  let began!: () => void, release!: () => void;
+  const entered = new Promise<void>(resolve => { began = resolve; });
+  const gate = new Promise<void>(resolve => { release = resolve; });
+  const pending = phone.say("Question", { waitFor: async () => {
+    began();
+    await gate;
+    recordStep({ kind: "late", title: "Must not escape" });
+    return "Late transcript must not escape";
+  }}).catch(error => error);
+  await entered;
+  await scope.close();
+  expect(String(await pending)).toContain("closed");
+  release();
+  await Bun.sleep(10);
+  const events = stopRecording() as any[];
+  expect(events.map(e => e.kind)).toEqual(["phone", "phone"]);
+  expect(events[1].blocks[0].response).toBeUndefined();
+});
+
+test("real harness exposes ctx.phone, seals recordings on every outcome and serves audio chunks", async () => {
+  // Use the actual project loader, test runner and method table. Isolate daemon
+  // globals/instrumentation from the unit tests; only the native phone subprocess is a fixture.
+  const root = join(dir, "project");
+  await mkdir(join(root, "spectest"), { recursive: true });
+  const sdk = new URL("./index.ts", import.meta.url).href;
+  const daemon = new URL("./daemon.ts", import.meta.url).href;
+  await writeFile(join(root, "spectest/index.ts"), `
+    import { defineEnvironment } from ${JSON.stringify(sdk)};
+    const env = defineEnvironment({ name: "phone-integration", services: {} });
+    const cases = ["success", "failure", "timeout"].map(outcome => env.test(outcome,
+      { timeoutMs: outcome === "timeout" ? 350 : 2000 }, async ctx => {
+        const phone = await ctx.phone(${JSON.stringify(options)});
+        await phone.call("123");
+        await phone.say("One. Two.");
+        await new Promise(r => setTimeout(r, 30));
+        if (outcome === "failure") throw new Error("application assertion failed");
+        if (outcome === "timeout") await new Promise(() => {});
+        // The runner must close the live call before returning /run.
+      }));
+    export default env.project(cases);
+  `);
+  const script = `
+    import { strict as assert } from "node:assert";
+    import { METHODS } from ${JSON.stringify(daemon)};
+    const loaded = await METHODS.load({});
+    assert.equal(loaded.cases.length, 3);
+    for (const [index, test] of loaded.cases.entries()) {
+      const result = await METHODS.run({caseId:test.id});
+      assert.equal(result.status, index === 0 ? "passed" : "failed", JSON.stringify(result));
+      if (index === 1) assert.match(result.error.message, /application assertion failed/);
+      if (index === 2) assert.match(result.error.message, /timed out/);
+      assert.equal(result.phoneRecordings.length, 1);
+      const recording = result.phoneRecordings[0];
+      assert.equal(recording.error, undefined);
+      assert.ok(recording.duration > 0);
+      assert.deepEqual(recording.callerPeaks, [.2,0]);
+      assert.deepEqual(recording.recipientPeaks, [0,.8]);
+      const steps = result.events.filter(e => e.kind === "phone");
+      assert.equal(steps.length, 2);
+      assert.equal(steps[1].blocks[0].text, "One. Two.");
+      assert.equal(steps[1].blocks[0].sessionId, recording.id);
+      assert.ok(steps[1].blocks[0].end < recording.duration);
+      assert.ok(steps[1].blocks[0].end > steps[1].blocks[0].start);
+      assert.ok(!JSON.stringify(result).includes("never-record-this"));
+      assert.ok(!JSON.stringify(result).includes('"audio":'));
+      const chunk = await METHODS.phoneRecordingChunk({caseId:recording.id, offset:0, limit:44});
+      assert.equal(chunk.total, recording.bytes);
+      assert.deepEqual([...Buffer.from(chunk.b64, "base64").subarray(0,4)], [0x1a,0x45,0xdf,0xa3]);
+      assert.equal(recording.contentType, "audio/webm");
+      assert.ok(recording.encodingFinalizeMs >= 0);
+      await METHODS.releasePhoneRecording({id:recording.id});
+      await assert.rejects(METHODS.phoneRecordingChunk({caseId:recording.id}), /Unknown/);
+    }
+    const evaluated = await METHODS.eval({code:
+      'const p = await ctx.phone(' + ${JSON.stringify(JSON.stringify(options))} + '); await p.call("123"); await p.say("Eval speech"); export default "done";'
+    });
+    assert.equal(evaluated.ok, true, JSON.stringify(evaluated));
+    assert.equal(evaluated.result, "done");
+    process.exit(0);
+  `;
+  const child = Bun.spawn([process.execPath, "--eval", script], {
+    env: { ...process.env, SPECTEST_APP_DIR: root }, stdout: "pipe", stderr: "pipe",
+  });
+  const [code, stdout, stderr] = await Promise.all([
+    child.exited, new Response(child.stdout).text(), new Response(child.stderr).text(),
+  ]);
+  expect(code, stdout + stderr).toBe(0);
+}, 20_000);