@frockbot/plugin-voice 0.0.0 → 0.3.21

package/src/client/pending-notifications.test.ts

@@ -0,0 +1,63 @@
+import { describe, expect, mock, test } from "bun:test";
+import { deliverPendingVoiceNotificationsV1 } from "./pending-notifications.js";
+
+function storage() {
+  const values = new Map<string, string>();
+  return {
+    getItem: (key: string) => values.get(key) ?? null,
+    setItem: (key: string, value: string) => void values.set(key, value),
+  };
+}
+
+const answer = {
+  schemaVersion: 1 as const,
+  answerId: "ask-1",
+  botId: "research",
+  botName: "Research",
+  question: "What changed?",
+  answer: "The release landed.",
+  answeredAt: "2026-09-04T01:02:03.000Z",
+};
+
+describe("pending Voice notifications", () => {
+  test("fires the client notification seam once per pending answer", async () => {
+    const shared = storage();
+    const notify = mock((_intent: { title: string; body: string }) =>
+      Promise.resolve("web" as const),
+    );
+
+    expect(
+      await deliverPendingVoiceNotificationsV1([answer], {
+        storage: shared,
+        notify,
+      }),
+    ).toBe(1);
+    expect(
+      await deliverPendingVoiceNotificationsV1([answer], {
+        storage: shared,
+        notify,
+      }),
+    ).toBe(0);
+    expect(notify).toHaveBeenCalledWith({
+      title: "Research answered",
+      body: "The release landed.",
+    });
+    expect(notify).toHaveBeenCalledTimes(1);
+  });
+
+  test("returns an unavailable notification claim for a later retry", async () => {
+    const shared = storage();
+    const notify = mock((_intent: { title: string; body: string }) =>
+      Promise.resolve("unavailable" as const),
+    );
+    await deliverPendingVoiceNotificationsV1([answer], {
+      storage: shared,
+      notify,
+    });
+    await deliverPendingVoiceNotificationsV1([answer], {
+      storage: shared,
+      notify,
+    });
+    expect(notify).toHaveBeenCalledTimes(2);
+  });
+});

package/src/client/pending-notifications.ts

@@ -0,0 +1,80 @@
+import { showClientNotificationV1 } from "@frockbot/plugin-shell/client/notify";
+import type { VoicePendingAnswerV1 } from "../shared.js";
+
+const DELIVERED_KEY_V1 = "frockbot.voice.delivered-answers.v1";
+const DELIVERED_LIMIT_V1 = 200;
+type WritableStorage = Pick<Storage, "getItem" | "setItem">;
+
+function browserStorageV1(): WritableStorage | undefined {
+  try {
+    return typeof localStorage === "undefined" ? undefined : localStorage;
+  } catch {
+    return undefined;
+  }
+}
+
+function readIdsV1(storage: WritableStorage): string[] {
+  try {
+    const value: unknown = JSON.parse(
+      storage.getItem(DELIVERED_KEY_V1) ?? "[]",
+    );
+    return Array.isArray(value)
+      ? value.filter((entry): entry is string => typeof entry === "string")
+      : [];
+  } catch {
+    return [];
+  }
+}
+
+function claimV1(answerId: string, storage: WritableStorage | undefined) {
+  if (!storage) return true;
+  try {
+    const ids = readIdsV1(storage);
+    if (ids.includes(answerId)) return false;
+    storage.setItem(
+      DELIVERED_KEY_V1,
+      JSON.stringify([...ids, answerId].slice(-DELIVERED_LIMIT_V1)),
+    );
+    return true;
+  } catch {
+    return true;
+  }
+}
+
+function releaseV1(answerId: string, storage: WritableStorage | undefined) {
+  if (!storage) return;
+  try {
+    storage.setItem(
+      DELIVERED_KEY_V1,
+      JSON.stringify(readIdsV1(storage).filter((id) => id !== answerId)),
+    );
+  } catch {
+    // A denied store cannot silence a later notification attempt.
+  }
+}
+
+/** Shows each pending answer once per browser while durable Voice is off. */
+export async function deliverPendingVoiceNotificationsV1(
+  answers: readonly VoicePendingAnswerV1[],
+  options: {
+    storage?: WritableStorage;
+    notify?: typeof showClientNotificationV1;
+  } = {},
+): Promise<number> {
+  const storage = options.storage ?? browserStorageV1();
+  const notify = options.notify ?? showClientNotificationV1;
+  let delivered = 0;
+  for (const answer of answers) {
+    if (!claimV1(answer.answerId, storage)) continue;
+    const result = await notify({
+      title: `${answer.botName} answered`,
+      body: answer.answer,
+    });
+    if (result === "unavailable") {
+      releaseV1(answer.answerId, storage);
+      continue;
+    }
+    delivered += 1;
+  }
+  return delivered;
+}

package/src/client/playback.test.ts

@@ -0,0 +1,13 @@
+import { describe, expect, test } from "bun:test";
+import { pcm16LeToFloat32V1 } from "./playback.js";
+
+describe("Voice PCM playback", () => {
+  test("decodes little-endian signed PCM16 into browser samples", () => {
+    const pcm = new Uint8Array([0x00, 0x80, 0x00, 0x00, 0xff, 0x7f]);
+    expect([...pcm16LeToFloat32V1(pcm.buffer)]).toEqual([
+      -1,
+      0,
+      32_767 / 32_768,
+    ]);
+  });
+});

package/src/client/playback.ts

@@ -0,0 +1,79 @@
+/** PCM16LE playback for Gemini Live's 24 kHz output. */
+export interface VoicePlaybackV1 {
+  play(pcm16: ArrayBuffer): void;
+  interrupt(): void;
+  close(): Promise<void>;
+}
+
+interface AudioContextLikeV1 {
+  currentTime: number;
+  destination: AudioDestinationNode;
+  state: AudioContextState;
+  createBuffer(
+    channels: number,
+    length: number,
+    sampleRate: number,
+  ): AudioBuffer;
+  createBufferSource(): AudioBufferSourceNode;
+  resume(): Promise<void>;
+  close(): Promise<void>;
+}
+
+export function pcm16LeToFloat32V1(input: ArrayBuffer): Float32Array {
+  const view = new DataView(input);
+  const result = new Float32Array(Math.floor(input.byteLength / 2));
+  for (let index = 0; index < result.length; index += 1) {
+    result[index] = view.getInt16(index * 2, true) / 32_768;
+  }
+  return result;
+}
+
+export function createVoicePlaybackV1(
+  context: AudioContextLikeV1 = new AudioContext(),
+  sampleRate = 24_000,
+): VoicePlaybackV1 {
+  const playing = new Set<AudioBufferSourceNode>();
+  let nextAt = 0;
+  let closed = false;
+  if (context.state === "suspended") void context.resume();
+
+  return {
+    play(pcm16) {
+      if (closed || pcm16.byteLength < 2) return;
+      const samples = pcm16LeToFloat32V1(pcm16);
+      const buffer = context.createBuffer(1, samples.length, sampleRate);
+      const channel = buffer.getChannelData(0);
+      for (let index = 0; index < samples.length; index += 1) {
+        channel[index] = samples[index] ?? 0;
+      }
+      const source = context.createBufferSource();
+      source.buffer = buffer;
+      source.connect(context.destination);
+      const startAt = Math.max(context.currentTime, nextAt);
+      nextAt = startAt + buffer.duration;
+      playing.add(source);
+      source.onended = () => {
+        playing.delete(source);
+        source.disconnect();
+      };
+      source.start(startAt);
+    },
+    interrupt() {
+      nextAt = context.currentTime;
+      for (const source of playing) {
+        try {
+          source.stop();
+        } catch {
+          // It may have ended between enumeration and stop.
+        }
+      }
+      playing.clear();
+    },
+    async close() {
+      if (closed) return;
+      closed = true;
+      this.interrupt();
+      await context.close().catch(() => undefined);
+    },
+  };
+}

package/src/client/state.ts

@@ -0,0 +1,25 @@
+import type { VoiceAssistantLiveStateV1 } from "@frockbot/protocol";
+import type {
+  VoiceSessionRecordV1,
+  VoiceToolCallEntryV1,
+  VoiceTranscriptEntryV1,
+} from "../shared.js";
+import type { InjectionKey, Ref } from "vue";
+
+export interface VoiceClientStateV1 {
+  enabled: boolean;
+  status: VoiceAssistantLiveStateV1;
+  level: number;
+  quotaRemainingSeconds: number;
+  quotaLimitSeconds: number;
+  session?: Pick<VoiceSessionRecordV1, "sessionId" | "startedAt">;
+  transcript: VoiceTranscriptEntryV1[];
+  tools: VoiceToolCallEntryV1[];
+  message?: string;
+  refresh(): Promise<void>;
+  toggle(): Promise<void>;
+  open(): void;
+}
+
+export const voiceClientStateKey: InjectionKey<Ref<VoiceClientStateV1>> =
+  Symbol("frockbot.voice.client-state");

package/src/client/styles.css

@@ -0,0 +1,137 @@
+.voice-toggle {
+  position: relative;
+  display: grid;
+  width: var(--frock-avatar-sm);
+  height: var(--frock-avatar-sm);
+  place-items: center;
+  border: 0;
+  border-radius: var(--frock-radius-control);
+  color: var(--frock-text-muted);
+  background: transparent;
+  cursor: pointer;
+}
+
+.voice-toggle:hover,
+.voice-toggle--listening,
+.voice-toggle--speaking {
+  color: var(--frock-text);
+  background: var(--frock-fill-hover);
+}
+
+.voice-toggle:focus-visible {
+  outline: var(--frock-focus-ring);
+}
+
+.voice-toggle__state {
+  position: absolute;
+  right: 2px;
+  bottom: 2px;
+  width: 8px;
+  height: 8px;
+  border: 1px solid var(--frock-surface);
+  border-radius: 50%;
+  background: var(--frock-text-subtle);
+}
+
+.voice-toggle--connecting .voice-toggle__state {
+  background: var(--frock-warning);
+}
+
+.voice-toggle--listening .voice-toggle__state {
+  background: var(--frock-success);
+}
+
+.voice-toggle--speaking .voice-toggle__state {
+  background: var(--frock-action-primary);
+}
+
+.voice-surface {
+  display: flex;
+  min-height: 0;
+  flex-direction: column;
+  gap: 16px;
+  padding: 16px 24px 24px;
+  color: var(--frock-text);
+}
+
+.voice-surface__status {
+  display: grid;
+  grid-template-columns: auto minmax(0, 1fr) auto;
+  gap: 12px;
+  align-items: center;
+}
+
+.voice-surface__status h3,
+.voice-surface__status p,
+.voice-surface__message,
+.voice-surface__empty,
+.voice-surface__entry p {
+  margin: 0;
+}
+
+.voice-surface__status h3 {
+  font-size: var(--frock-text-base);
+  text-transform: capitalize;
+}
+
+.voice-surface__status p,
+.voice-surface__empty {
+  color: var(--frock-text-muted);
+  font-size: var(--frock-text-sm);
+}
+
+.voice-surface__pulse {
+  width: 12px;
+  height: 12px;
+  border-radius: 50%;
+  background: var(--frock-text-subtle);
+}
+
+.voice-surface__pulse--connecting {
+  background: var(--frock-warning);
+}
+
+.voice-surface__pulse--listening {
+  background: var(--frock-success);
+}
+
+.voice-surface__pulse--speaking {
+  background: var(--frock-action-primary);
+}
+
+.voice-surface__message {
+  padding: 10px 12px;
+  border-radius: var(--frock-radius-control);
+  color: var(--frock-text-muted);
+  background: var(--frock-surface-subtle);
+}
+
+.voice-surface__activity {
+  display: flex;
+  min-height: 0;
+  flex-direction: column;
+  gap: 10px;
+  overflow-y: auto;
+  margin: 0;
+  padding: 0;
+  list-style: none;
+}
+
+.voice-surface__entry {
+  padding: 10px 12px;
+  border-radius: var(--frock-radius-control);
+  background: var(--frock-surface-subtle);
+}
+
+.voice-surface__entry > span {
+  color: var(--frock-text-subtle);
+  font-size: var(--frock-text-xs);
+}
+
+.voice-surface__entry p {
+  margin-top: 4px;
+}
+
+.voice-surface__entry--tool {
+  border-left: 2px solid var(--frock-border-strong);
+}

package/src/index.ts

@@ -0,0 +1,6 @@
+export * from "./ask.js";
+export * from "./bot.js";
+export * from "./ledger.js";
+export * from "./prompt.js";
+export * from "./shared.js";
+export * from "./tools.js";

package/src/ledger.test.ts

@@ -0,0 +1,246 @@
+import { describe, expect, test } from "bun:test";
+import { VoiceLedgerV1, type VoiceLedgerStorageV1 } from "./ledger.js";
+
+function memoryStorage(): VoiceLedgerStorageV1 {
+  const values = new Map<string, unknown>();
+  const surface = {
+    async get<T>(key: string) {
+      return values.get(key) as T | undefined;
+    },
+    async put(key: string, value: unknown) {
+      values.set(key, value);
+    },
+    async delete(key: string) {
+      return values.delete(key);
+    },
+    async list<T>({ prefix, limit }: { prefix: string; limit?: number }) {
+      return new Map(
+        [...values]
+          .filter(([key]) => key.startsWith(prefix))
+          .slice(0, limit)
+          .map(([key, value]) => [key, value as T]),
+      );
+    },
+  };
+  return { ...surface, transaction: (callback) => callback(surface) };
+}
+
+const AT = "2026-09-04T01:02:03.000Z";
+
+describe("VoiceLedgerV1", () => {
+  test("records voice/ask, voice/answered, and voice/briefed in order", async () => {
+    const ledger = new VoiceLedgerV1(memoryStorage());
+    await ledger.recordAsk({
+      schemaVersion: 1,
+      type: "voice/ask",
+      askId: "ask-1",
+      sessionId: "session-1",
+      botId: "research",
+      botName: "Research",
+      question: "What changed?",
+      runId: "agent-1",
+      askedAt: AT,
+    });
+    await ledger.recordAnswered({
+      schemaVersion: 1,
+      type: "voice/answered",
+      askId: "ask-1",
+      botId: "research",
+      runId: "agent-1",
+      answer: "The release landed.",
+      answeredAt: "2026-09-04T01:03:03.000Z",
+    });
+
+    expect((await ledger.view()).pendingAnswers).toEqual([
+      expect.objectContaining({
+        answerId: "ask-1",
+        question: "What changed?",
+        answer: "The release landed.",
+      }),
+    ]);
+    expect(
+      await ledger.markBriefed({
+        askIds: ["ask-1", "ask-1"],
+        sessionId: "session-2",
+        at: "2026-09-04T01:04:03.000Z",
+      }),
+    ).toBe(1);
+    expect((await ledger.view()).pendingAnswers).toEqual([]);
+    expect(await ledger.readAsk("ask-1")).toEqual(
+      expect.objectContaining({
+        ask: expect.objectContaining({ type: "voice/ask" }),
+        answered: expect.objectContaining({ type: "voice/answered" }),
+        briefed: expect.objectContaining({
+          type: "voice/briefed",
+          sessionId: "session-2",
+        }),
+      }),
+    );
+  });
+
+  test("keeps an answer pending when Voice is off", async () => {
+    const ledger = new VoiceLedgerV1(memoryStorage());
+    await ledger.start({ sessionId: "session-1", deviceId: "phone", at: AT });
+    await ledger.end({
+      sessionId: "session-1",
+      at: "2026-09-04T01:02:04.000Z",
+      reason: "stopped",
+      seconds: 1,
+    });
+    await ledger.recordAsk({
+      schemaVersion: 1,
+      type: "voice/ask",
+      askId: "ask-offline",
+      sessionId: "session-1",
+      botId: "research",
+      botName: "Research",
+      question: "Are you done?",
+      runId: "agent-offline",
+      askedAt: AT,
+    });
+    await ledger.recordAnswered({
+      schemaVersion: 1,
+      type: "voice/answered",
+      askId: "ask-offline",
+      botId: "research",
+      runId: "agent-offline",
+      answer: "Yes.",
+      answeredAt: "2026-09-04T01:03:03.000Z",
+    });
+
+    const view = await ledger.view();
+    expect(view.state.enabled).toBe(false);
+    expect(view.pendingAnswers).toEqual([
+      expect.objectContaining({ answerId: "ask-offline", answer: "Yes." }),
+    ]);
+    expect((await ledger.readAsk("ask-offline"))?.briefed).toBeUndefined();
+  });
+
+  test("counts unanswered asks against the pending-answer bound", async () => {
+    const ledger = new VoiceLedgerV1(memoryStorage());
+    for (let index = 0; index < 32; index += 1) {
+      await expect(
+        ledger.recordAsk({
+          schemaVersion: 1,
+          type: "voice/ask",
+          askId: `ask-${index}`,
+          sessionId: `session-${index}`,
+          botId: "research",
+          botName: "Research",
+          question: "What changed?",
+          runId: `agent-${index}`,
+          askedAt: AT,
+        }),
+      ).resolves.toMatchObject({ status: "recorded" });
+    }
+
+    await expect(
+      ledger.recordAsk({
+        schemaVersion: 1,
+        type: "voice/ask",
+        askId: "ask-over-cap",
+        sessionId: "session-over-cap",
+        botId: "research",
+        botName: "Research",
+        question: "One more?",
+        runId: "agent-over-cap",
+        askedAt: AT,
+      }),
+    ).resolves.toEqual({
+      status: "refused",
+      reason: "Voice already has 32 Bot answers waiting or on the way.",
+    });
+  });
+
+  test("the newest device wins and the previous session ends durably", async () => {
+    const ledger = new VoiceLedgerV1(memoryStorage());
+    await ledger.start({ sessionId: "first", deviceId: "phone", at: AT });
+    const next = await ledger.start({
+      sessionId: "second",
+      deviceId: "laptop",
+      at: "2026-09-04T01:03:03.000Z",
+    });
+    expect(next.replacedSessionId).toBe("first");
+    const view = await ledger.view();
+    expect(view.state).toMatchObject({
+      enabled: true,
+      activeSessionId: "second",
+      activeDeviceId: "laptop",
+    });
+    expect(
+      view.sessions.find((session) => session.sessionId === "first"),
+    ).toMatchObject({
+      endedReason: "replaced",
+    });
+  });
+
+  test("transcript and tool writes are idempotent and never contain audio", async () => {
+    const ledger = new VoiceLedgerV1(memoryStorage());
+    await ledger.start({ sessionId: "one", deviceId: "phone", at: AT });
+    const transcript = {
+      schemaVersion: 1 as const,
+      id: "utterance-1",
+      speaker: "user" as const,
+      text: "What is happening?",
+      at: AT,
+    };
+    await ledger.appendTranscript("one", transcript);
+    await ledger.appendTranscript("one", transcript);
+    await ledger.appendToolCall("one", {
+      schemaVersion: 1,
+      id: "call-1",
+      name: "list_bots",
+      label: "Checked your Bots",
+      at: AT,
+    });
+    const session = (await ledger.view()).sessions[0]!;
+    expect(session.transcript).toEqual([transcript]);
+    expect(session.toolCalls).toHaveLength(1);
+    expect(JSON.stringify(session)).not.toContain("audio");
+  });
+
+  test("an old device cannot turn the replacement session off", async () => {
+    const ledger = new VoiceLedgerV1(memoryStorage());
+    await ledger.start({ sessionId: "first", deviceId: "phone", at: AT });
+    await ledger.start({
+      sessionId: "second",
+      deviceId: "laptop",
+      at: "2026-09-04T01:03:03.000Z",
+    });
+    const state = await ledger.end({
+      sessionId: "first",
+      at: "2026-09-04T01:04:03.000Z",
+      reason: "stopped",
+      seconds: 60,
+    });
+    expect(state.activeSessionId).toBe("second");
+    expect(state.enabled).toBe(true);
+  });
+
+  test("bounds retained sessions and pending answers", async () => {
+    const ledger = new VoiceLedgerV1(memoryStorage());
+    for (let index = 0; index < 40; index += 1) {
+      const suffix = String(index).padStart(2, "0");
+      await ledger.start({
+        sessionId: `session-${suffix}`,
+        deviceId: "phone",
+        at: `2026-09-04T01:${suffix}:00.000Z`,
+      });
+      await ledger.recordPendingAnswer({
+        schemaVersion: 1,
+        answerId: `answer-${suffix}`,
+        botId: "research",
+        botName: "Research",
+        question: "What changed?",
+        answer: `Answer ${suffix}`,
+        answeredAt: `2026-09-04T01:${suffix}:30.000Z`,
+      });
+    }
+
+    const view = await ledger.view();
+    expect(view.sessions).toHaveLength(24);
+    expect(view.sessions.at(-1)?.sessionId).toBe("session-16");
+    expect(view.pendingAnswers).toHaveLength(32);
+    expect(view.pendingAnswers[0]?.answerId).toBe("answer-08");
+  });
+});