@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

package/src/live-voice/__tests__/microphone-idle-gate.test.ts

@@ -0,0 +1,87 @@
+import { Buffer } from "node:buffer";
+import { describe, expect, test } from "bun:test";
+
+import { MicrophoneIdleGate } from "../microphone-idle-gate.js";
+
+function pcm(ms: number, amplitude: number, sampleRate = 24_000): Buffer {
+  const audio = Buffer.alloc(Math.round((sampleRate * ms) / 1_000) * 2);
+  for (let offset = 0; offset < audio.byteLength; offset += 2) {
+    audio.writeInt16LE(amplitude, offset);
+  }
+  return audio;
+}
+
+describe("MicrophoneIdleGate", () => {
+  test.each([16_000, 24_000, 48_000])(
+    "gates after one second of audio at %i Hz, including across a frame boundary",
+    (sampleRate) => {
+      const gate = new MicrophoneIdleGate(sampleRate);
+      const first = pcm(950, 100, sampleRate);
+      expect(gate.process(first, false)).toEqual(first);
+      expect(gate.closed).toBe(false);
+
+      const output = gate.process(pcm(350, 100, sampleRate), false);
+      expect(gate.closed).toBe(true);
+      expect(output).toEqual(
+        Buffer.concat([pcm(50, 100, sampleRate), pcm(100, 0, sampleRate)]),
+      );
+    },
+  );
+
+  test("restores the quiet onset once, without duplicating the audio timeline", () => {
+    const gate = new MicrophoneIdleGate(24_000);
+    const quiet = pcm(1_000, 100);
+    const oldNoise = pcm(200, 90);
+    const onset = pcm(200, 300);
+    const speech = pcm(100, 8_000);
+    const output = [
+      gate.process(quiet, false),
+      gate.process(oldNoise, false),
+      gate.process(onset, false),
+      gate.process(speech, true),
+    ];
+    expect(Buffer.concat(output)).toEqual(
+      Buffer.concat([quiet, pcm(200, 0), onset, speech]),
+    );
+    expect(gate.closed).toBe(false);
+    expect(gate.process(speech, true)).toEqual(speech);
+  });
+
+  test("each speech onset gets a fresh one-second tail", () => {
+    const gate = new MicrophoneIdleGate(24_000);
+    for (let index = 0; index < 3; index += 1) {
+      const speech = pcm(100, 8_000);
+      expect(gate.process(speech, true)).toEqual(speech);
+      const quiet = pcm(990, 100);
+      expect(gate.process(quiet, false)).toEqual(quiet);
+      expect(gate.closed).toBe(false);
+      const boundary = pcm(10, 100);
+      expect(gate.process(boundary, false)).toEqual(boundary);
+      expect(gate.closed).toBe(true);
+    }
+  });
+
+  test("long idle input keeps only 200ms pending and sends silence at input cadence", () => {
+    const gate = new MicrophoneIdleGate(24_000);
+    gate.process(pcm(1_000, 100), false);
+    for (let index = 0; index < 20; index += 1) {
+      expect(gate.process(pcm(10, 100), false).byteLength).toBe(0);
+    }
+    for (let index = 0; index < 1_000; index += 1) {
+      expect(gate.process(pcm(10, 100), false)).toEqual(pcm(10, 0));
+    }
+    const speech = pcm(10, 8_000);
+    expect(gate.process(speech, true)).toEqual(
+      Buffer.concat([pcm(200, 100), speech]),
+    );
+  });
+
+  test("reset discards held audio at a stream or client-interrupt boundary", () => {
+    const gate = new MicrophoneIdleGate(24_000);
+    gate.process(pcm(1_200, 100), false);
+    gate.reset();
+    expect(gate.closed).toBe(false);
+    const speech = pcm(10, 8_000);
+    expect(gate.process(speech, true)).toEqual(speech);
+  });
+});

package/src/live-voice/__tests__/session-controls.test.ts

@@ -3,6 +3,7 @@ import { describe, expect, test } from "bun:test";
 import type { VoiceProgressConfig } from "../../config/schemas/voice.js";
 import {
   FEWER_UPDATES_INTERVAL_MS,
+  lookFollowUpNote,
   progressConfigForCadence,
   requestedSessionControl,
   sessionControlTeaching,
@@ -18,6 +19,28 @@ const CONFIG: VoiceProgressConfig = {
   generationTimeoutMs: 1_500,
 };
 
+describe("lookFollowUpNote", () => {
+  test("quotes the caller request and preserves annotation intent", () => {
+    const callerUtterance = 'Show me the "New page" button.\nWhere is it?';
+    const note = lookFollowUpNote({ action: "look_screen", callerUtterance });
+    expect(note).toContain(JSON.stringify(callerUtterance));
+    expect(note).toContain("screen-annotation tools");
+    expect(note).toContain(
+      "If the request needs tools and this leg has none, escalate",
+    );
+  });
+
+  test("camera follow-ups resume their request without screen annotation guidance", () => {
+    const note = lookFollowUpNote({
+      action: "look_camera",
+      callerUtterance: "What is this plant?",
+    });
+    expect(note).toContain("fresh look at their camera");
+    expect(note).toContain(JSON.stringify("What is this plant?"));
+    expect(note).not.toContain("screen-annotation");
+  });
+});
+
 describe("progressConfigForCadence", () => {
   test("normal is the configured cadence, untouched", () => {
     expect(progressConfigForCadence(CONFIG, "normal")).toBe(CONFIG);

package/src/live-voice/__tests__/task-outcomes.test.ts

@@ -0,0 +1,117 @@
+import { describe, expect, test } from "bun:test";
+
+import { type VoiceTaskOutcome, VoiceTaskOutcomes } from "../task-outcomes.js";
+
+function notification(subagentId: string, message: string): VoiceTaskOutcome {
+  return { taskId: subagentId, message, metadata: {}, source: "subagent" };
+}
+
+describe("voice task delivery", () => {
+  test("finishing one task's playback preserves other tasks and newer updates", () => {
+    const queue = new VoiceTaskOutcomes();
+    const blocked = notification("task-1", "Needs a decision");
+    const other = notification("task-2", "Finished the comparison");
+    const completed = notification("task-1", "Finished the export");
+    queue.enqueue(blocked);
+    queue.enqueue(other);
+    queue.finish(blocked, 200);
+    queue.enqueue(completed);
+
+    queue.acknowledgePlayback(200);
+    expect(queue.drain(200)).toEqual([completed, other]);
+  });
+
+  test("an interruption during playback retains the outcome for delivery", () => {
+    const queue = new VoiceTaskOutcomes();
+    const result = notification("task-1", "Finished");
+    queue.enqueue(result);
+    queue.finish(result, 200);
+    queue.interruptPlayback(100);
+    queue.acknowledgePlayback(300);
+    expect(queue.next()).toBe(result);
+  });
+
+  test("hang-up returns unheard results once and excludes completed playback", () => {
+    const queue = new VoiceTaskOutcomes();
+    const heard = notification("task-1", "Finished");
+    const unheard = notification("task-2", "Failed");
+    queue.enqueue(heard);
+    queue.enqueue(unheard);
+    queue.finish(heard, 200);
+    expect(queue.drain(201)).toEqual([unheard]);
+    expect(queue.drain(201)).toEqual([]);
+  });
+  test("reading a continuation as context never consumes it", () => {
+    const queue = new VoiceTaskOutcomes();
+    const result: VoiceTaskOutcome = {
+      ...notification("continuation-1", "Report"),
+      source: "continuation",
+    };
+    queue.enqueue(result);
+    expect(queue.pendingOutcomes()).toEqual([result]);
+    queue.finishContextReply([result], 200, "Birds have feathers.");
+    queue.acknowledgePlayback(200);
+    expect(queue.next()).toBe(result);
+    expect(queue.deliveryContext(result)).toContain("Birds have feathers.");
+  });
+
+  test("a superseding completion retains the interrupted announcement state", () => {
+    const queue = new VoiceTaskOutcomes();
+    const progress = notification("task-1", "Initial findings");
+    const completed = notification("task-1", "Full report");
+    queue.enqueue(progress);
+    queue.finish(progress, 200, "Generated explanation");
+    queue.enqueue(completed);
+    queue.interruptPlayback(100);
+    expect(queue.next()).toBe(completed);
+    expect(queue.deliveryContext(completed)).toContain(
+      "interrupted before playback completed",
+    );
+    expect(queue.deliveryContext(completed)).not.toContain(
+      "Generated explanation",
+    );
+  });
+
+  test("an interruption before generation completes is retained across updates", () => {
+    const queue = new VoiceTaskOutcomes();
+    const progress = notification("task-1", "Initial findings");
+    queue.enqueue(progress);
+    queue.interrupt(progress);
+    const completed = notification("task-1", "Full report");
+    queue.enqueue(completed);
+    expect(queue.deliveryContext(completed)).toContain(
+      "interrupted before playback completed",
+    );
+  });
+
+  test("a later update can distinguish completed playback from generated history", () => {
+    const queue = new VoiceTaskOutcomes();
+    const progress = notification("task-1", "Initial findings");
+    queue.enqueue(progress);
+    queue.finish(progress, 200, "The export requires a connection.");
+    queue.acknowledgePlayback(200);
+    expect(queue.hasPending).toBe(false);
+    const completed = notification("task-1", "Full report");
+    queue.enqueue(completed);
+    expect(queue.deliveryContext(completed)).toContain(
+      "The export requires a connection.",
+    );
+    expect(queue.deliveryContext(completed)).not.toContain("was interrupted");
+  });
+
+  test("a cut-off context reply provides no completed playback receipt", () => {
+    const queue = new VoiceTaskOutcomes();
+    const result = notification("task-1", "Report");
+    queue.enqueue(result);
+    queue.finishContextReply([result], 200, "Generated explanation");
+    queue.interruptPlayback(100);
+    queue.acknowledgePlayback(300);
+    expect(queue.next()).toBe(result);
+    expect(queue.deliveryContext(result)).not.toContain(
+      "Generated explanation",
+    );
+    expect(queue.deliveryContext(result)).toContain(
+      "no completed audio playback receipt",
+    );
+  });
+});

package/src/live-voice/__tests__/voice-input-diagnostics.test.ts

@@ -0,0 +1,117 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  VoiceInputDiagnostics,
+  type VoiceInputSample,
+} from "../voice-input-diagnostics.js";
+
+function sample(
+  receivedAtMs: number,
+  overrides: Partial<VoiceInputSample> = {},
+): VoiceInputSample {
+  return {
+    receivedAtMs,
+    chunkMs: 50,
+    meanAmplitude: 100,
+    baseThreshold: 800,
+    roomNoiseFloor: null,
+    echoEnergy: 0,
+    echoOnsetLapsed: false,
+    echoWindowMs: 0,
+    echoGuardCarryover: false,
+    echoReferenceMs: 0,
+    echoProbeMs: 0,
+    echoCorrelation: null,
+    playbackRemainingMs: 0,
+    playbackEchoPossible: false,
+    speechMs: 0,
+    silenceMs: 50,
+    echoMs: 0,
+    ...overrides,
+  };
+}
+
+describe("voice input diagnostics", () => {
+  test("distinguishes arriving audio from gaps in submission to STT", () => {
+    const diagnostics = new VoiceInputDiagnostics();
+    diagnostics.observe(sample(0));
+    diagnostics.recordSttSubmission(0, 50);
+    diagnostics.observe(sample(50));
+    diagnostics.observe(sample(100));
+    diagnostics.recordSttSubmission(100, 50);
+    expect(diagnostics.flush()).toMatchObject({
+      chunks: 3,
+      audioMs: 150,
+      maxArrivalGapMs: 50,
+      sttSubmittedChunks: 2,
+      sttSubmittedAudioMs: 100,
+      sttMaxSubmissionGapMs: 100,
+    });
+    diagnostics.observe(sample(150));
+    diagnostics.recordSttSubmission(150, 50);
+    expect(diagnostics.flush()).toMatchObject({
+      sttSubmittedChunks: 1,
+      sttSubmittedAudioMs: 50,
+      sttMaxSubmissionGapMs: 50,
+    });
+  });
+  test("bounds traces by count and age, including burst arrivals", () => {
+    const diagnostics = new VoiceInputDiagnostics();
+    for (let index = 0; index < 100; index++) {
+      diagnostics.observe(sample(index));
+    }
+    const trace = diagnostics.snapshot(100);
+    expect(trace).toHaveLength(40);
+    expect(trace[0]?.receivedAtMs).toBe(60);
+    trace[0]!.meanAmplitude = 999;
+    expect(diagnostics.snapshot(100)[0]?.meanAmplitude).toBe(100);
+    expect(diagnostics.snapshot(2_100)).toEqual([]);
+  });
+
+  test("weights levels by audio duration and distinguishes silence from missing audio", () => {
+    const diagnostics = new VoiceInputDiagnostics();
+    expect(diagnostics.observe(sample(0, { meanAmplitude: 0 }))).toBeNull();
+    const summary = diagnostics.observe(
+      sample(5_000, {
+        chunkMs: 150,
+        meanAmplitude: 2_000,
+        speechMs: 150,
+        silenceMs: 0,
+      }),
+    );
+    expect(summary).toMatchObject({
+      firstAtMs: 0,
+      lastAtMs: 5_000,
+      audioMs: 200,
+      chunks: 2,
+      meanAmplitude: 1_500,
+      minAmplitude: 0,
+      maxAmplitude: 2_000,
+      zeroAudioMs: 50,
+      maxArrivalGapMs: 5_000,
+      speechMs: 150,
+      silenceMs: 50,
+    });
+    expect(diagnostics.flush()).toBeNull();
+    diagnostics.observe(sample(5_050));
+    expect(diagnostics.flush()).toMatchObject({
+      chunks: 1,
+      audioMs: 50,
+      maxArrivalGapMs: 50,
+    });
+  });
+
+  test("keeps buffered echo-probe decisions distinct from newly received audio", () => {
+    const diagnostics = new VoiceInputDiagnostics();
+    diagnostics.observe(sample(0, { silenceMs: 0, echoProbeMs: 50 }));
+    diagnostics.observe(
+      sample(50, { silenceMs: 0, echoMs: 100, echoCorrelation: 0.9 }),
+    );
+    expect(diagnostics.flush()).toMatchObject({
+      audioMs: 100,
+      echoMs: 100,
+      silenceMs: 0,
+    });
+    expect(diagnostics.snapshot(50)[1]?.echoCorrelation).toBe(0.9);
+  });
+});

package/src/live-voice/camera-mode-session.test.ts

@@ -0,0 +1,265 @@
+import { describe, expect, test } from "bun:test";
+
+import type { ModeSessionMode } from "../api/mode-session.js";
+import type {
+  ModeSessionSourceHandle,
+  ModeSessionTerminalDisposition,
+} from "../daemon/conversation-mode-session.js";
+import { CameraModeSessionProducer } from "./camera-mode-session.js";
+
+function createCoordinator() {
+  let activation = 0;
+  const owners = new Map<string, ModeSessionSourceHandle>();
+  const retired: Array<{
+    handle: ModeSessionSourceHandle;
+    disposition?: ModeSessionTerminalDisposition;
+  }> = [];
+  const rows: Array<{ turnId: string; messageId: string; at: number }> = [];
+  const releases: string[] = [];
+
+  const coordinator = {
+    activateSource(input: {
+      sourceId: string;
+      generation: number;
+      mode: ModeSessionMode;
+    }): ModeSessionSourceHandle {
+      activation += 1;
+      return {
+        sourceId: input.sourceId,
+        generation: input.generation,
+        activation,
+        id: `camera-${activation}`,
+        mode: input.mode,
+      };
+    },
+    claimTurn(
+      turnId: string,
+      handle: ModeSessionSourceHandle,
+    ): ModeSessionSourceHandle {
+      owners.set(turnId, handle);
+      return handle;
+    },
+    recordActivity: () => true,
+    retireSource(
+      handle: ModeSessionSourceHandle,
+      disposition?: ModeSessionTerminalDisposition,
+    ): boolean {
+      retired.push({ handle, disposition });
+      return true;
+    },
+    trackPersistedRow(turnId: string, messageId: string, at: number): void {
+      rows.push({ turnId, messageId, at });
+    },
+    releaseTurn(turnId: string): void {
+      releases.push(turnId);
+      owners.delete(turnId);
+    },
+  };
+  return { coordinator, owners, retired, rows, releases };
+}
+
+describe("CameraModeSessionProducer", () => {
+  test("starts before a frame and exposes the same owner to voice turns", () => {
+    const state = createCoordinator();
+    const producer = new CameraModeSessionProducer(
+      state.coordinator,
+      "voice-1",
+    );
+
+    const handle = producer.start(1, "live", 100);
+    expect(handle).toMatchObject({
+      generation: 1,
+      mode: "live_vision",
+      sourceId: "live-voice-camera:voice-1",
+    });
+    expect(producer.captureTurn()).toEqual(handle);
+    expect(producer.start(1, "live", 110)).toEqual(handle);
+  });
+
+  test("rolls back a partially claimed start and allows the epoch to retry", () => {
+    const state = createCoordinator();
+    const producer = new CameraModeSessionProducer(
+      state.coordinator,
+      "voice-1",
+    );
+    const claimTurn = state.coordinator.claimTurn.bind(state.coordinator);
+    let shouldThrow = true;
+    state.coordinator.claimTurn = (turnId, handle) => {
+      const owner = claimTurn(turnId, handle);
+      if (shouldThrow) {
+        shouldThrow = false;
+        throw new Error("session tracking unavailable");
+      }
+      return owner;
+    };
+
+    expect(() => producer.start(1, "ambient", 100)).toThrow(
+      "session tracking unavailable",
+    );
+    expect(producer.captureTurn()).toBeUndefined();
+    expect(state.owners.size).toBe(0);
+    expect(state.retired).toEqual([
+      expect.objectContaining({
+        disposition: {
+          status: "interrupted",
+          endReason: "camera_start_refused",
+        },
+      }),
+    ]);
+    expect(state.releases).toHaveLength(1);
+
+    expect(producer.start(1, "ambient", 110)).toMatchObject({
+      generation: 1,
+      activation: 2,
+      mode: "ambient",
+    });
+  });
+
+  test("releases a partial claim even when source retirement throws", () => {
+    const state = createCoordinator();
+    const producer = new CameraModeSessionProducer(
+      state.coordinator,
+      "voice-1",
+    );
+    const claimTurn = state.coordinator.claimTurn.bind(state.coordinator);
+    state.coordinator.claimTurn = (turnId, handle) => {
+      claimTurn(turnId, handle);
+      throw new Error("session tracking unavailable");
+    };
+    state.coordinator.retireSource = () => {
+      throw new Error("source retirement unavailable");
+    };
+
+    expect(() => producer.start(1, "live", 100)).toThrow(AggregateError);
+    expect(producer.captureTurn()).toBeUndefined();
+    expect(state.owners.size).toBe(0);
+    expect(state.releases).toHaveLength(1);
+  });
+
+  test("rolls back a refused synthetic owner", () => {
+    const state = createCoordinator();
+    const producer = new CameraModeSessionProducer(
+      state.coordinator,
+      "voice-1",
+    );
+    const claimTurn = state.coordinator.claimTurn.bind(state.coordinator);
+    state.coordinator.claimTurn = (turnId, handle) => {
+      claimTurn(turnId, handle);
+      return { ...handle, id: "another-session" };
+    };
+
+    expect(producer.start(1, "live", 100)).toBeUndefined();
+    expect(producer.captureTurn()).toBeUndefined();
+    expect(state.owners.size).toBe(0);
+    expect(state.retired).toHaveLength(1);
+    expect(state.releases).toHaveLength(1);
+  });
+
+  test("holds terminal publication until accepted keeps settle", () => {
+    const state = createCoordinator();
+    const producer = new CameraModeSessionProducer(
+      state.coordinator,
+      "voice-1",
+    );
+    producer.start(1, "ambient", 100);
+    const frame = producer.beginFrame(1, "ambient");
+
+    expect(producer.end(1)).toBe(true);
+    expect(state.releases).toEqual([]);
+    producer.finishFrame(frame, { messageId: "frame-1", at: 140 });
+
+    expect(state.rows).toEqual([
+      expect.objectContaining({ messageId: "frame-1", at: 140 }),
+    ]);
+    expect(state.releases).toHaveLength(1);
+    expect(state.retired[0]?.disposition).toEqual({
+      status: "completed",
+      endReason: "camera_stopped",
+    });
+  });
+
+  test("settles each accepted keep exactly once", () => {
+    const state = createCoordinator();
+    const producer = new CameraModeSessionProducer(
+      state.coordinator,
+      "voice-1",
+    );
+    producer.start(1, "ambient", 100);
+    const frame = producer.beginFrame(1, "ambient");
+    producer.end(1);
+
+    producer.finishFrame(frame, { messageId: "frame-1", at: 140 });
+    producer.finishFrame(frame, { messageId: "frame-1", at: 140 });
+
+    expect(state.rows).toHaveLength(1);
+    expect(state.releases).toHaveLength(1);
+  });
+
+  test("rejects stale frames and ends without requiring another message", () => {
+    const state = createCoordinator();
+    const producer = new CameraModeSessionProducer(
+      state.coordinator,
+      "voice-1",
+    );
+    producer.start(1, "live", 100);
+    producer.end(1);
+    producer.start(2, "live", 200);
+
+    expect(producer.start(1, "live", 210)).toBeUndefined();
+    expect(producer.beginFrame(1, "live")).toBeUndefined();
+    expect(producer.beginFrame(2, "ambient")).toBeUndefined();
+    expect(producer.end(1)).toBe(false);
+    expect(producer.captureTurn()).toMatchObject({ generation: 2 });
+    expect(state.retired).toHaveLength(1);
+    expect(state.releases).toHaveLength(1);
+  });
+
+  test("retires an unended run as interrupted when a replacement starts", () => {
+    const state = createCoordinator();
+    const producer = new CameraModeSessionProducer(
+      state.coordinator,
+      "voice-1",
+    );
+    producer.start(1, "live", 100);
+    producer.start(2, "ambient", 200);
+
+    expect(state.retired[0]?.disposition).toEqual({
+      status: "interrupted",
+      endReason: "camera_source_replaced",
+    });
+    expect(producer.captureTurn()).toMatchObject({ mode: "ambient" });
+  });
+
+  test("releases the local run when source retirement throws after mutation", () => {
+    const state = createCoordinator();
+    const producer = new CameraModeSessionProducer(
+      state.coordinator,
+      "voice-1",
+    );
+    producer.start(1, "live", 100);
+    const retireSource = state.coordinator.retireSource.bind(state.coordinator);
+    state.coordinator.retireSource = (handle, disposition) => {
+      retireSource(handle, disposition);
+      throw new Error("source retirement unavailable");
+    };
+
+    expect(() => producer.end(1)).toThrow("source retirement unavailable");
+    expect(producer.captureTurn()).toBeUndefined();
+    expect(state.releases).toHaveLength(1);
+    expect(producer.end(1)).toBe(false);
+  });
+
+  test("releases the synthetic turn when its source is already stale", () => {
+    const state = createCoordinator();
+    const producer = new CameraModeSessionProducer(
+      state.coordinator,
+      "voice-1",
+    );
+    producer.start(1, "live", 100);
+    state.coordinator.retireSource = () => false;
+
+    expect(producer.end(1)).toBe(false);
+    expect(producer.captureTurn()).toBeUndefined();
+    expect(state.releases).toHaveLength(1);
+  });
+});