@frockbot/plugin-voice 0.0.0 → 0.3.21

package/src/bot.ts

@@ -0,0 +1,158 @@
+// The Bot half of Voice answer delivery.
+//
+// A Voice ask is an ordinary agent-lane Turn owned by the target Bot. Once its
+// `turn/end` is durable, this projection takes the first text `send_to_user`
+// from that Turn and puts it in a bounded Bot-local outbox before crossing to
+// the User Durable Object. The User ledger is therefore never dependent on a
+// caller or socket remaining resident while the Bot works.
+import type { SessionEvent } from "@frockbot/kernel-contracts";
+import {
+  decodeVoiceAnswerDeliveryV1,
+  VOICE_ANSWER_OUTBOX_MAX_V1,
+  type VoiceAnswerDeliveryV1,
+} from "./shared.js";
+
+export const VOICE_ANSWER_OUTBOX_KEY_V1 = "voice:answer-outbox:v1";
+
+export interface VoiceAnswerSinkV1 {
+  recordVoiceAnswer(delivery: VoiceAnswerDeliveryV1): Promise<void>;
+}
+
+export interface VoiceAnswerOutboxStorageV1 {
+  get<T>(key: string): Promise<T | undefined>;
+  put<T>(key: string, value: T): Promise<void>;
+  delete(key: string): Promise<boolean>;
+}
+
+interface StoredVoiceAnswerOutboxV1 {
+  schemaVersion: 1;
+  deliveries: VoiceAnswerDeliveryV1[];
+  truncated: boolean;
+}
+
+function decodeOutboxV1(value: unknown): StoredVoiceAnswerOutboxV1 {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return { schemaVersion: 1, deliveries: [], truncated: false };
+  }
+  const candidate = value as Partial<StoredVoiceAnswerOutboxV1>;
+  if (candidate.schemaVersion !== 1 || !Array.isArray(candidate.deliveries)) {
+    return { schemaVersion: 1, deliveries: [], truncated: true };
+  }
+  const deliveries: VoiceAnswerDeliveryV1[] = [];
+  for (const delivery of candidate.deliveries.slice(
+    -VOICE_ANSWER_OUTBOX_MAX_V1,
+  )) {
+    try {
+      deliveries.push(decodeVoiceAnswerDeliveryV1(delivery));
+    } catch {
+      // The authoritative run remains reconstructable. `truncated` makes an
+      // invalid derived row visible instead of silently accepting it.
+    }
+  }
+  return {
+    schemaVersion: 1,
+    deliveries,
+    truncated:
+      candidate.truncated === true ||
+      candidate.deliveries.length !== deliveries.length,
+  };
+}
+
+export function voiceAnswerFromSettledTurnV1(input: {
+  userId: string;
+  botId: string;
+  runId: string;
+  turn: number;
+  origin?: { kind: string; messageId?: string };
+  events: readonly SessionEvent[];
+}): VoiceAnswerDeliveryV1 | undefined {
+  if (input.origin?.kind !== "voice" || !input.origin.messageId) return;
+  const ended = input.events.find(
+    (event) => event.type === "turn/end" && event.turn === input.turn,
+  );
+  if (!ended || ended.type !== "turn/end") return;
+  const sent = input.events.find(
+    (event) =>
+      event.type === "send/to-user" &&
+      event.turn === input.turn &&
+      event.payload.type === "text",
+  );
+  const base = {
+    schemaVersion: 1 as const,
+    userId: input.userId,
+    askId: input.origin.messageId,
+    botId: input.botId,
+    runId: input.runId,
+    at: sent?.timestamp ?? ended.timestamp,
+  };
+  return sent?.type === "send/to-user" && sent.payload.type === "text"
+    ? { ...base, outcome: "answered", answer: sent.payload.text }
+    : {
+        ...base,
+        outcome: "failed",
+        reason:
+          ended.outcome === "completed"
+            ? "The Bot finished without a text answer."
+            : "The Bot could not answer that Voice question.",
+      };
+}
+
+export class VoiceAnswerOutboxV1 {
+  constructor(private readonly storage: VoiceAnswerOutboxStorageV1) {}
+
+  private async read(): Promise<StoredVoiceAnswerOutboxV1> {
+    return decodeOutboxV1(
+      await this.storage.get<unknown>(VOICE_ANSWER_OUTBOX_KEY_V1),
+    );
+  }
+
+  private async write(outbox: StoredVoiceAnswerOutboxV1): Promise<void> {
+    if (outbox.deliveries.length === 0 && !outbox.truncated) {
+      await this.storage.delete(VOICE_ANSWER_OUTBOX_KEY_V1);
+      return;
+    }
+    await this.storage.put(VOICE_ANSWER_OUTBOX_KEY_V1, outbox);
+  }
+
+  async append(delivery: VoiceAnswerDeliveryV1): Promise<void> {
+    const decoded = decodeVoiceAnswerDeliveryV1(delivery);
+    const stored = await this.read();
+    if (
+      stored.deliveries.some(
+        (held) => held.askId === decoded.askId && held.runId === decoded.runId,
+      )
+    ) {
+      return;
+    }
+    stored.deliveries.push(decoded);
+    if (stored.deliveries.length > VOICE_ANSWER_OUTBOX_MAX_V1) {
+      stored.deliveries = stored.deliveries.slice(-VOICE_ANSWER_OUTBOX_MAX_V1);
+      stored.truncated = true;
+    }
+    await this.write(stored);
+  }
+
+  async drain(sink: VoiceAnswerSinkV1): Promise<void> {
+    const stored = await this.read();
+    for (const delivery of stored.deliveries) {
+      await sink.recordVoiceAnswer(delivery);
+      const current = await this.read();
+      await this.write({
+        ...current,
+        deliveries: current.deliveries.filter(
+          (candidate) =>
+            candidate.askId !== delivery.askId ||
+            candidate.runId !== delivery.runId,
+        ),
+      });
+    }
+  }
+
+  async state(): Promise<{ pending: number; truncated: boolean }> {
+    const stored = await this.read();
+    return {
+      pending: stored.deliveries.length,
+      truncated: stored.truncated,
+    };
+  }
+}

package/src/client/VoiceSurface.vue

@@ -0,0 +1,70 @@
+<script setup lang="ts">
+import { UiButton } from "@frockbot/client-ui";
+import { computed, inject } from "vue";
+import { voiceClientStateKey } from "./state.js";
+
+const provided = inject(voiceClientStateKey);
+if (!provided) throw new Error("Voice client state was not provided");
+const voice = provided;
+const quota = computed(() => {
+  const minutes = Math.ceil(voice.value.quotaRemainingSeconds / 60);
+  return `${minutes} minute${minutes === 1 ? "" : "s"} left this month`;
+});
+const activity = computed(() =>
+  [
+    ...voice.value.transcript.map((entry) => ({
+      id: entry.id,
+      at: entry.at,
+      kind: entry.speaker,
+      label: entry.speaker === "user" ? "You" : "Voice",
+      text: entry.text,
+    })),
+    ...voice.value.tools.map((entry) => ({
+      id: entry.id,
+      at: entry.at,
+      kind: "tool" as const,
+      label: "Looked up",
+      text: entry.label,
+    })),
+  ].sort((left, right) => left.at.localeCompare(right.at)),
+);
+</script>
+
+<template>
+  <section class="voice-surface">
+    <header class="voice-surface__status">
+      <span
+        class="voice-surface__pulse"
+        :class="`voice-surface__pulse--${voice.status}`"
+        aria-hidden="true"
+      ></span>
+      <div>
+        <h3>{{ voice.status }}</h3>
+        <p>{{ quota }}</p>
+      </div>
+      <UiButton
+        :variant="voice.status === 'offline' ? 'primary' : 'ghost'"
+        @click="voice.toggle()"
+        >{{ voice.status === "offline" ? "Turn on" : "Turn off" }}</UiButton
+      >
+    </header>
+
+    <p v-if="voice.message" class="voice-surface__message" role="status">
+      {{ voice.message }}
+    </p>
+    <p v-if="activity.length === 0" class="voice-surface__empty">
+      What you say and what Voice answers will appear here.
+    </p>
+    <ol v-else class="voice-surface__activity" aria-label="This Voice session">
+      <li
+        v-for="entry in activity"
+        :key="entry.id"
+        :class="`voice-surface__entry--${entry.kind}`"
+        class="voice-surface__entry"
+      >
+        <span>{{ entry.label }}</span>
+        <p>{{ entry.text }}</p>
+      </li>
+    </ol>
+  </section>
+</template>

package/src/client/VoiceToggle.vue

@@ -0,0 +1,32 @@
+<script setup lang="ts">
+import { UiIcon } from "@frockbot/client-ui";
+import { computed, inject, onMounted } from "vue";
+import { voiceClientStateKey } from "./state.js";
+
+const provided = inject(voiceClientStateKey);
+if (!provided) throw new Error("Voice client state was not provided");
+const voice = provided;
+const active = computed(() => voice.value.status !== "offline");
+const label = computed(() =>
+  active.value ? "Turn Voice off" : "Turn Voice on",
+);
+onMounted(() => {
+  void voice.value.refresh();
+});
+</script>
+
+<template>
+  <button
+    type="button"
+    class="voice-toggle"
+    :class="`voice-toggle--${voice.status}`"
+    :aria-label="label"
+    :aria-pressed="active"
+    :title="`${label} — ${voice.status}`"
+    @click="voice.toggle()"
+    @contextmenu.prevent="voice.open()"
+  >
+    <UiIcon name="mic" />
+    <span class="voice-toggle__state" aria-hidden="true"></span>
+  </button>
+</template>

package/src/client/index.test.ts

@@ -0,0 +1,95 @@
+import { describe, expect, test } from "bun:test";
+import {
+  clientSurfaceRegistryKey,
+  type ClientPluginContext,
+  type ClientSurfaceRegistry,
+} from "@frockbot/client-core";
+import { computed, ref } from "vue";
+import { voiceClientPlugin } from "./index.js";
+import { voiceClientStateKey, type VoiceClientStateV1 } from "./state.js";
+
+function surfaceRegistry(): ClientSurfaceRegistry {
+  const activeId = ref<string>();
+  return {
+    activeId,
+    active: computed(() => undefined),
+    register: () => () => {},
+    has: () => false,
+    open: (id) => {
+      activeId.value = id;
+    },
+    close: () => {
+      activeId.value = undefined;
+    },
+  };
+}
+
+function mount(
+  hostedRequest: NonNullable<ClientPluginContext["transport"]["hostedRequest"]>,
+): { calls: string[]; state: { value: VoiceClientStateV1 } } {
+  const calls: string[] = [];
+  const surfaces = surfaceRegistry();
+  let state: unknown;
+  const context: ClientPluginContext = {
+    transport: {
+      turn: () => Promise.resolve({ runId: "run", text: "", events: [] }),
+      hostedRequest: (path, method, body) => {
+        calls.push(path);
+        return hostedRequest(path, method, body);
+      },
+    },
+    inject: (key) => {
+      if (key === clientSurfaceRegistryKey) return surfaces as never;
+      throw new Error("unexpected client provider");
+    },
+    provide: (key, value) => {
+      if (key === voiceClientStateKey) state = value;
+      return () => {};
+    },
+    slot: () => () => {},
+  };
+  voiceClientPlugin(context);
+  return {
+    calls,
+    state: state as { value: VoiceClientStateV1 },
+  };
+}
+
+describe("Voice client contribution", () => {
+  test("does not read User voice state before authenticated chrome mounts", () => {
+    const mounted = mount(() => Promise.reject(new Error("unauthenticated")));
+
+    expect(mounted.calls).toEqual([]);
+  });
+
+  test("reads User voice state when authenticated chrome asks for it", async () => {
+    const mounted = mount(() =>
+      Promise.resolve({
+        schemaVersion: 1,
+        ledger: {
+          schemaVersion: 1,
+          state: {
+            schemaVersion: 1,
+            enabled: false,
+            updatedAt: "2026-09-04T00:00:00.000Z",
+          },
+          sessions: [],
+          pendingAnswers: [],
+        },
+        quota: {
+          schemaVersion: 1,
+          month: "2026-09",
+          usedSeconds: 120,
+          limitSeconds: 3_600,
+          remainingSeconds: 3_480,
+        },
+      }),
+    );
+
+    await mounted.state.value.refresh();
+
+    expect(mounted.calls).toEqual(["/api/voice"]);
+    expect(mounted.state.value.quotaRemainingSeconds).toBe(3_480);
+    expect(mounted.state.value.quotaLimitSeconds).toBe(3_600);
+  });
+});

package/src/client/index.ts

@@ -0,0 +1,330 @@
+import {
+  clientSurfaceRegistryKey,
+  type ClientPlugin,
+  type VoiceAssistantSessionV1,
+} from "@frockbot/client-core";
+import { defineClientContribution } from "@frockbot/kernel-contracts/contributions";
+import {
+  startVoiceMicrophoneV1,
+  voiceCaptureSupportedV1,
+  voiceMicrophoneRefusalV1,
+  type VoiceMicrophoneV1,
+} from "@frockbot/plugin-shell/client/voice-microphone";
+import { showClientNotificationV1 } from "@frockbot/plugin-shell/client/notify";
+import { VOICE_ASSISTANT_INPUT_SAMPLE_RATE_V1 } from "@frockbot/protocol";
+import { ref } from "vue";
+import {
+  decodeVoiceAssistantViewV1,
+  VOICE_MAX_TOOL_CALLS_V1,
+  VOICE_MAX_TRANSCRIPT_ENTRIES_V1,
+  type VoiceToolNameV1,
+} from "../shared.js";
+import { createVoicePlaybackV1, type VoicePlaybackV1 } from "./playback.js";
+import { deliverPendingVoiceNotificationsV1 } from "./pending-notifications.js";
+import VoiceSurface from "./VoiceSurface.vue";
+import VoiceToggle from "./VoiceToggle.vue";
+import { voiceClientStateKey, type VoiceClientStateV1 } from "./state.js";
+import "./styles.css";
+
+export const VOICE_SURFACE_ID_V1 = "voice";
+const DEVICE_KEY_V1 = "frockbot.voice.device.v1";
+const TOOL_NAMES: readonly VoiceToolNameV1[] = [
+  "list_bots",
+  "bot_activity",
+  "memory_search",
+  "pending_answers",
+  "ask_bot",
+];
+const VOICE_PENDING_POLL_INTERVAL_MS_V1 = 15_000;
+
+function deviceIdV1(): string {
+  if (typeof window === "undefined") return crypto.randomUUID();
+  try {
+    const stored = window.localStorage.getItem(DEVICE_KEY_V1);
+    if (stored && /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(stored)) {
+      return stored;
+    }
+    const created = crypto.randomUUID();
+    window.localStorage.setItem(DEVICE_KEY_V1, created);
+    return created;
+  } catch {
+    return crypto.randomUUID();
+  }
+}
+
+export const voiceClientPlugin: ClientPlugin = (ctx) => {
+  const request = ctx.transport.hostedRequest?.bind(ctx.transport);
+  const openAssistant = ctx.transport.openVoiceAssistant?.bind(ctx.transport);
+  const surfaces = ctx.inject(clientSurfaceRegistryKey);
+  let microphone: VoiceMicrophoneV1 | undefined;
+  let playback: VoicePlaybackV1 | undefined;
+  let session: VoiceAssistantSessionV1 | undefined;
+  let serverEnded = false;
+  /** Invalidates a permission prompt or socket callback from an older toggle. */
+  let attempt = 0;
+  let quotaTicker: ReturnType<typeof setInterval> | undefined;
+  let refreshInFlight: Promise<void> | undefined;
+
+  const releaseMedia = async (): Promise<void> => {
+    if (quotaTicker !== undefined) clearInterval(quotaTicker);
+    quotaTicker = undefined;
+    const heldMicrophone = microphone;
+    const heldPlayback = playback;
+    microphone = undefined;
+    playback = undefined;
+    await Promise.all([heldMicrophone?.stop(), heldPlayback?.close()]);
+  };
+
+  const notifyOffline = async (message: string): Promise<void> => {
+    await showClientNotificationV1({
+      title: "Voice is offline",
+      body: message,
+    });
+  };
+
+  const refresh = (): Promise<void> => {
+    if (!request) return Promise.resolve();
+    if (refreshInFlight) return refreshInFlight;
+    const currentAttempt = attempt;
+    refreshInFlight = request("/api/voice")
+      .then(decodeVoiceAssistantViewV1)
+      .then(async (view) => {
+        // The monthly ceiling is stable even if the User turns Voice on while
+        // this read is in flight. Live remaining time is owned by the socket.
+        state.value.quotaLimitSeconds = view.quota.limitSeconds;
+        if (currentAttempt !== attempt || state.value.status !== "offline") {
+          return;
+        }
+        state.value.enabled = view.ledger.state.enabled;
+        state.value.quotaRemainingSeconds = view.quota.remainingSeconds;
+        const latest = view.ledger.state.activeSessionId
+          ? view.ledger.sessions.find(
+              (entry) => entry.sessionId === view.ledger.state.activeSessionId,
+            )
+          : view.ledger.sessions[0];
+        if (latest) {
+          state.value.session = latest;
+          state.value.transcript = latest.transcript;
+          state.value.tools = latest.toolCalls;
+        }
+        if (view.ledger.state.enabled) {
+          state.value.message =
+            "Voice is ready to resume. Turn it on to reconnect this device.";
+        } else {
+          await deliverPendingVoiceNotificationsV1(view.ledger.pendingAnswers);
+        }
+      })
+      .catch(() => {
+        if (currentAttempt === attempt && state.value.status === "offline") {
+          state.value.message = "Voice status couldn't be loaded.";
+        }
+      })
+      .finally(() => {
+        refreshInFlight = undefined;
+      });
+    return refreshInFlight;
+  };
+
+  const state = ref<VoiceClientStateV1>({
+    enabled: false,
+    status: "offline",
+    level: 0,
+    quotaRemainingSeconds: 0,
+    quotaLimitSeconds: 0,
+    transcript: [],
+    tools: [],
+    refresh,
+    open() {
+      surfaces.open(VOICE_SURFACE_ID_V1);
+    },
+    async toggle() {
+      state.value.open();
+      if (state.value.status !== "offline") {
+        attempt += 1;
+        state.value.enabled = false;
+        state.value.status = "offline";
+        state.value.message = "Voice is off.";
+        const stopping = session;
+        session = undefined;
+        stopping?.stop();
+        await releaseMedia();
+        return;
+      }
+      if (!openAssistant || !voiceCaptureSupportedV1()) {
+        state.value.message = "Voice isn't available on this device.";
+        return;
+      }
+      state.value.status = "connecting";
+      state.value.message = undefined;
+      state.value.enabled = true;
+      serverEnded = false;
+      const currentAttempt = ++attempt;
+      let openingSession: VoiceAssistantSessionV1 | undefined;
+      let openedMicrophone: VoiceMicrophoneV1 | undefined;
+      let openedPlayback: VoicePlaybackV1 | undefined;
+      try {
+        openedMicrophone = await startVoiceMicrophoneV1({
+          sampleRate: VOICE_ASSISTANT_INPUT_SAMPLE_RATE_V1,
+          audio(audio) {
+            openingSession?.sendAudio(audio);
+          },
+          level(level) {
+            if (currentAttempt !== attempt) return;
+            state.value.level = level;
+          },
+        });
+        if (currentAttempt !== attempt) {
+          await openedMicrophone.stop();
+          return;
+        }
+        openedPlayback = createVoicePlaybackV1();
+        openingSession = openAssistant(deviceIdV1(), {
+          ready(sessionId, quotaRemainingSeconds) {
+            if (currentAttempt !== attempt) return;
+            if (quotaTicker !== undefined) clearInterval(quotaTicker);
+            const quotaStartedAt = Date.now();
+            state.value.session = {
+              sessionId,
+              startedAt: new Date().toISOString(),
+            };
+            state.value.transcript = [];
+            state.value.tools = [];
+            state.value.quotaRemainingSeconds = quotaRemainingSeconds;
+            state.value.status = "listening";
+            quotaTicker = setInterval(() => {
+              if (currentAttempt !== attempt) return;
+              const elapsed = Math.floor((Date.now() - quotaStartedAt) / 1_000);
+              state.value.quotaRemainingSeconds = Math.max(
+                0,
+                quotaRemainingSeconds - elapsed,
+              );
+            }, 1_000);
+          },
+          state(liveState) {
+            if (currentAttempt !== attempt) return;
+            state.value.status = liveState;
+          },
+          transcript(entry) {
+            if (currentAttempt !== attempt) return;
+            state.value.transcript = [
+              ...state.value.transcript,
+              { schemaVersion: 1 as const, ...entry },
+            ].slice(-VOICE_MAX_TRANSCRIPT_ENTRIES_V1);
+          },
+          tool(entry) {
+            if (currentAttempt !== attempt) return;
+            if (!TOOL_NAMES.includes(entry.name as VoiceToolNameV1)) return;
+            state.value.tools = [
+              ...state.value.tools,
+              {
+                schemaVersion: 1 as const,
+                ...entry,
+                name: entry.name as VoiceToolNameV1,
+              },
+            ].slice(-VOICE_MAX_TOOL_CALLS_V1);
+          },
+          audio(audio) {
+            if (currentAttempt !== attempt) return;
+            openedPlayback?.play(audio);
+          },
+          interrupted() {
+            if (currentAttempt !== attempt) return;
+            openedPlayback?.interrupt();
+            state.value.status = "listening";
+          },
+          offline(reason, message) {
+            if (currentAttempt !== attempt) return;
+            attempt += 1;
+            serverEnded = true;
+            session = undefined;
+            state.value.enabled = false;
+            state.value.status = "offline";
+            state.value.message = message;
+            void releaseMedia();
+            if (reason === "quota" || reason === "error") {
+              void notifyOffline(message);
+            }
+          },
+          failed(message) {
+            if (currentAttempt !== attempt || serverEnded) return;
+            attempt += 1;
+            serverEnded = true;
+            session = undefined;
+            state.value.enabled = false;
+            state.value.status = "offline";
+            state.value.message = message;
+            void releaseMedia();
+            void notifyOffline(message);
+          },
+          closed() {
+            if (currentAttempt !== attempt) return;
+            attempt += 1;
+            session = undefined;
+            if (serverEnded || state.value.status === "offline") return;
+            state.value.status = "offline";
+            state.value.message =
+              "Voice is still ready. Turn it on to reconnect this device.";
+            void releaseMedia();
+          },
+        });
+        if (currentAttempt !== attempt) {
+          openingSession.close();
+          await Promise.all([openedMicrophone.stop(), openedPlayback.close()]);
+          return;
+        }
+        microphone = openedMicrophone;
+        playback = openedPlayback;
+        session = openingSession;
+      } catch (error) {
+        if (currentAttempt !== attempt) {
+          openingSession?.close();
+          await Promise.all([
+            openedMicrophone?.stop(),
+            openedPlayback?.close(),
+          ]);
+          return;
+        }
+        attempt += 1;
+        state.value.enabled = false;
+        state.value.status = "offline";
+        state.value.message = voiceMicrophoneRefusalV1(error);
+        openingSession?.stop();
+        session = undefined;
+        await Promise.all([openedMicrophone?.stop(), openedPlayback?.close()]);
+      }
+    },
+  });
+
+  const pendingPoll = setInterval(() => {
+    if (state.value.status === "offline") void refresh();
+  }, VOICE_PENDING_POLL_INTERVAL_MS_V1);
+
+  return [
+    ctx.provide(voiceClientStateKey, state),
+    surfaces.register({
+      id: VOICE_SURFACE_ID_V1,
+      title: "Voice",
+      component: VoiceSurface,
+      placement: "panel",
+    }),
+    ctx.slot({
+      slot: "frockbot.header-actions",
+      order: 20,
+      component: VoiceToggle,
+    }),
+    () => {
+      clearInterval(pendingPoll);
+      attempt += 1;
+      session?.close();
+      session = undefined;
+      void releaseMedia();
+    },
+  ];
+};
+
+export const clientContribution = defineClientContribution<ClientPlugin>({
+  specifier: "@frockbot/plugin-voice/client",
+  plugin: voiceClientPlugin,
+});
+
+export default voiceClientPlugin;