@frockbot/plugin-shell 0.3.12 → 0.3.14

package/src/client/FrockBotApp.vue

@@ -28,6 +28,19 @@ import {
   type WebToolActivity,
 } from "../shared.js";
 import { ComposerDraftStore } from "./composer-draft.js";
+import {
+  applyDictationTailV1,
+  voiceButtonLabelV1,
+  voiceWaveBarsV1,
+  VoiceDictationTranscriptV1,
+  type VoiceDictationStateV1,
+} from "./voice-dictation.js";
+import {
+  startVoiceMicrophoneV1,
+  voiceMicrophoneRefusalV1,
+  type VoiceMicrophoneV1,
+} from "./voice-microphone.js";
+import type { VoiceDictationSessionV1 } from "@frockbot/client-core";
 import {
   activityTrailBeginV1,
   activityTrailSampleV1,
@@ -372,6 +385,149 @@ const canSend = computed(
  */
 const showStop = computed(() => isRunning.value && !canSend.value);
 
+/*
+ * Dictation (voice plan D4).
+ *
+ * The send button's slot already switches between Send and Stop, and this is
+ * the third thing it holds: with nothing to send and nothing to stop, the
+ * button offers to listen instead. While it is listening the slot carries a
+ * bin and a send, because those are the only two things a person wants next.
+ *
+ * Nothing here is a second composer. Speech lands in the same draft the
+ * keyboard writes to, through the same `ComposerDraftStore`, and Send is the
+ * ordinary `sendMessage` — so a message that is half spoken and half typed
+ * behaves exactly like one that is entirely typed, including a refusal that
+ * gives the draft back.
+ */
+const voiceState = ref<VoiceDictationStateV1>("idle");
+const voiceError = ref<string | undefined>(undefined);
+const voiceBars = ref<number[]>(voiceWaveBarsV1(0, []));
+const voiceTranscript = new VoiceDictationTranscriptV1();
+let voiceSession: VoiceDictationSessionV1 | undefined;
+let voiceMicrophone: VoiceMicrophoneV1 | undefined;
+/** The exact text dictation last wrote, so a typed edit around it survives. */
+let voiceTail = "";
+/** Send was pressed; the last transcript is what we are waiting for. */
+let voiceSendOnFinal = false;
+
+const dictating = computed(() => voiceState.value !== "idle");
+const voiceButtonLabel = computed(() => voiceButtonLabelV1(voiceState.value));
+/**
+ * The wave button takes the slot only when the slot is otherwise idle: an
+ * empty draft, no Turn to stop, and a platform that can actually listen.
+ */
+const showVoiceButton = computed(
+  () =>
+    state.value.voiceAvailable &&
+    !dictating.value &&
+    !showStop.value &&
+    draftText.value.length === 0,
+);
+
+function writeDictationIntoDraft(): void {
+  const applied = applyDictationTailV1(
+    draft.value,
+    voiceTail,
+    voiceTranscript.text(),
+  );
+  voiceTail = applied.tail;
+  draft.value = applied.draft;
+  void nextTick(syncComposerHeight);
+}
+
+async function startDictation(): Promise<void> {
+  if (dictating.value || !state.value.voiceAvailable) return;
+  voiceError.value = undefined;
+  voiceTranscript.reset();
+  voiceTail = "";
+  voiceSendOnFinal = false;
+  voiceBars.value = voiceWaveBarsV1(0, []);
+  voiceState.value = "starting";
+  const session = web.value.openVoiceDictation({
+    ready: () => {
+      if (voiceState.value === "starting") voiceState.value = "listening";
+    },
+    delta: (text) => {
+      voiceTranscript.delta(text);
+      writeDictationIntoDraft();
+    },
+    transcript: (text) => {
+      voiceTranscript.settle(text);
+      writeDictationIntoDraft();
+    },
+    final: () => {
+      void completeDictation();
+    },
+    failed: (message) => {
+      voiceError.value = message;
+      void stopDictation();
+    },
+    closed: () => {
+      // A socket that goes away mid-capture leaves the draft exactly where it
+      // is; the person can still type the rest and send it.
+      if (dictating.value) void stopDictation();
+    },
+  });
+  if (!session) {
+    voiceState.value = "idle";
+    voiceError.value = "Dictation isn't available on this device.";
+    return;
+  }
+  voiceSession = session;
+  try {
+    voiceMicrophone = await startVoiceMicrophoneV1({
+      audio: (pcm16) => session.sendAudio(pcm16),
+      level: (value) => {
+        voiceBars.value = voiceWaveBarsV1(value, voiceBars.value);
+      },
+    });
+  } catch (error) {
+    voiceError.value = voiceMicrophoneRefusalV1(error);
+    await stopDictation();
+  }
+}
+
+/** Everything captured has been transcribed; send it if that is why we stopped. */
+async function completeDictation(): Promise<void> {
+  const send = voiceSendOnFinal;
+  await stopDictation();
+  if (send) await sendMessage();
+}
+
+async function stopDictation(): Promise<void> {
+  voiceState.value = "idle";
+  voiceSendOnFinal = false;
+  voiceBars.value = voiceWaveBarsV1(0, []);
+  const microphone = voiceMicrophone;
+  const session = voiceSession;
+  voiceMicrophone = undefined;
+  voiceSession = undefined;
+  await microphone?.stop();
+  session?.close();
+}
+
+/** The bin. Discards what was dictated, exactly as D4 says it does. */
+function discardDictation(): void {
+  voiceSession?.cancel();
+  voiceTranscript.reset();
+  voiceTail = "";
+  draft.value = "";
+  voiceError.value = undefined;
+  void stopDictation();
+  void nextTick(syncComposerHeight);
+}
+
+/**
+ * Send, mid-dictation. The audio is committed and the message waits for the
+ * last transcript rather than sending half a sentence.
+ */
+function sendDictation(): void {
+  if (voiceState.value !== "listening") return;
+  voiceSendOnFinal = true;
+  voiceState.value = "finishing";
+  voiceSession?.commit();
+}
+
 /*
  * Tool activity is internal to the Turn. A Turn that produced only tool calls
  * shows the Bot avatar while it runs and nothing once it finishes with no
@@ -542,16 +698,18 @@ let trailRunId: string | undefined;
 let trailSeq = 0;
 let trailTick = 0;
 
-/** The one Turn still going, if any. Only one runs at a time. */
-const workingMessage = computed(() =>
-  messages.value.find(
+/**
+ * The Turn still going, if any. Only one executes at a time, but a Turn queued
+ * behind it is streaming-shaped too and has produced nothing yet, so the
+ * executing one wins: the trail keeps reading the words that are arriving
+ * rather than restarting on a Turn that has not begun.
+ */
+const workingMessage = computed(() => {
+  const streaming = messages.value.filter(
     (message) => message.role === "assistant" && message.status === "streaming",
-  ),
-);
-
-function isWorking(message: WebChatMessage): boolean {
-  return message.id === workingMessage.value?.id;
-}
+  );
+  return streaming.find((message) => !message.pending) ?? streaming.at(-1);
+});
 
 const workingSample = computed(() => {
   const message = workingMessage.value;
@@ -559,7 +717,16 @@ const workingSample = computed(() => {
   return activityTrailSampleV1({
     text: message.text,
     toolStatuses: message.tools.map((tool) => tool.status),
-    sends: message.sends.length,
+    // Counted across the whole Turn, because every send the Bot delivers is a
+    // message of its own and the working line carries none of them. A send is
+    // still a beat the trail has to feel.
+    sends: messages.value.reduce(
+      (total, candidate) =>
+        candidate.runId === message.runId
+          ? total + candidate.sends.length
+          : total,
+      0,
+    ),
     status: message.status,
   });
 });
@@ -807,6 +974,9 @@ onBeforeUnmount(() => {
   window.removeEventListener("hashchange", applySettingsDeepLink);
   phoneLayoutMedia?.removeEventListener("change", onPhoneLayoutChange);
   window.removeEventListener("keydown", onRootKeydown);
+  // A microphone outlives a component that stops drawing it unless it is told
+  // not to, and a browser shows the recording indicator for as long as it does.
+  void stopDictation();
 });
 
 watch(
@@ -1047,8 +1217,20 @@ function handleComposerKeydown(event: KeyboardEvent): void {
       }
     }
   }
+  if (event.key === "Escape" && dictating.value) {
+    // Escape is the keyboard's bin, the same as it is for the Skill popover.
+    event.preventDefault();
+    discardDictation();
+    return;
+  }
   if (event.key !== "Enter" || event.shiftKey) return;
   event.preventDefault();
+  // Enter means send either way; mid-dictation it commits the audio first so
+  // the last word spoken is in the message.
+  if (dictating.value) {
+    sendDictation();
+    return;
+  }
   void sendMessage();
 }
 </script>
@@ -1183,23 +1365,6 @@ function handleComposerKeydown(event: KeyboardEvent): void {
                 <div v-if="message.text" class="message-bubble">
                   <UiMarkdown :text="message.text" />
                 </div>
-                <!--
-                  The working state. Until the model has produced a word there
-                  was nothing beside the avatar at all, for twenty seconds and
-                  occasionally for two minutes, and the only other signal — the
-                  composer's stop button — is at the far end of the window from
-                  where the reply will appear.
-                -->
-                <div
-                  v-else-if="message.status === 'streaming'"
-                  class="message-bubble message-working"
-                  role="status"
-                  aria-label="Working on a reply"
-                >
-                  <span class="working-dots" aria-hidden="true">
-                    <i></i><i></i><i></i>
-                  </span>
-                </div>
                 <!--
                   Why the Turn ends where it does, under whatever it had
                   already said rather than in place of it.
@@ -1324,49 +1489,55 @@ function handleComposerKeydown(event: KeyboardEvent): void {
                   </button>
                 </div>
               </div>
-              <!--
-                The working row: the Bot's own avatar on its own line at the
-                end of the thread, with the comet trail streaming off to its
-                right. It appears only while the Turn is running. Every line in
-                this transcript is from the same Bot — there are no group
-                conversations yet (issue 152) — so a sheep beside a settled
-                reply named nobody the reader did not already know; the one
-                under a running Turn is the whole account of what the Bot is
-                doing.
-
-                The art comes from whichever Package owns Bot identity; when no
-                Package fills the slot the sparkle tile is the only child and
-                shows through. The row carries the status role, and the canvas
-                beside it is hidden from assistive technology: the trail is a
-                picture of a fact the label already states.
-              -->
-              <Transition name="bot-working">
-                <div
-                  v-if="isWorking(message)"
-                  class="bot-working"
-                  role="status"
-                  aria-label="Working"
-                >
-                  <div
-                    class="bot-avatar bot-avatar-live"
-                    :class="{ 'bot-avatar-waiting': !message.text }"
-                  >
-                    <span class="bot-avatar-fallback" aria-hidden="true"
-                      ><UiIcon name="sparkle" size="sm"
-                    /></span>
-                    <k-slot name="frockbot.bot-avatar" />
-                  </div>
-                  <UiActivityTrail
-                    class="bot-working-indicator"
-                    :rate="trailRate"
-                    :bursts="trailBursts"
-                    :state="trailState"
-                  />
-                </div>
-              </Transition>
             </template>
             <div v-else class="message-bubble">{{ message.text }}</div>
           </article>
+          <!--
+            The working row: the Bot's own avatar on its own line at the end of
+            the thread, with the comet trail streaming off to its right. It
+            appears only while a Turn is running. Every line in this transcript
+            is from the same Bot — there are no group conversations yet (issue
+            152) — so a sheep beside a settled reply named nobody the reader did
+            not already know; the one under a running Turn is the whole account
+            of what the Bot is doing.
+
+            It is a child of the thread rather than of the running Turn's
+            article, so it is always the last thing in the transcript. Send a
+            message while the Bot is still winding down and the new message
+            lands above the sheep, where a reader looking at the bottom of the
+            thread expects the newest thing to be — not underneath a running
+            Turn that is already over.
+
+            The art comes from whichever Package owns Bot identity; when no
+            Package fills the slot the sparkle tile is the only child and shows
+            through. The row carries the status role, and the canvas beside it
+            is hidden from assistive technology: the trail is a picture of a
+            fact the label already states.
+          -->
+          <Transition name="bot-working">
+            <div
+              v-if="workingMessage"
+              class="bot-working"
+              role="status"
+              aria-label="Working"
+            >
+              <div
+                class="bot-avatar bot-avatar-live"
+                :class="{ 'bot-avatar-waiting': !workingMessage.text }"
+              >
+                <span class="bot-avatar-fallback" aria-hidden="true"
+                  ><UiIcon name="sparkle" size="sm"
+                /></span>
+                <k-slot name="frockbot.bot-avatar" />
+              </div>
+              <UiActivityTrail
+                class="bot-working-indicator"
+                :rate="trailRate"
+                :bursts="trailBursts"
+                :state="trailState"
+              />
+            </div>
+          </Transition>
         </section>
 
         <Transition name="banner">
@@ -1492,6 +1663,36 @@ function handleComposerKeydown(event: KeyboardEvent): void {
             >
               {{ draftCounterLabel }}
             </p>
+            <!--
+              The capture animation, and the only place the state is written
+              out. It is level-driven from the microphone: bars that move while
+              the room is silent would say the microphone works when it does
+              not.
+            -->
+            <p
+              v-if="dictating"
+              class="voice-capture"
+              role="status"
+              aria-live="polite"
+            >
+              <span class="voice-wave" aria-hidden="true">
+                <span
+                  v-for="(bar, index) in voiceBars"
+                  :key="index"
+                  class="voice-wave-bar"
+                  :style="{ transform: `scaleY(${bar})` }"
+                />
+              </span>
+              <span>{{ voiceButtonLabel }}</span>
+            </p>
+            <!--
+              Why dictation stopped, in the words the server or the browser
+              used. It sits under the draft it could not add to, and the draft
+              itself is untouched.
+            -->
+            <p v-if="voiceError" class="voice-error" role="alert">
+              {{ voiceError }}
+            </p>
           </div>
           <!--
             Start a new conversation. Sits beside the composer because that is
@@ -1506,14 +1707,44 @@ function handleComposerKeydown(event: KeyboardEvent): void {
             :disabled="isRunning"
             @click="web.startConversation()"
           />
+          <!--
+            The send slot, and its four states. Dictation replaces the one
+            button with two, because while it is listening the only two things
+            worth offering are "throw this away" and "that's the message".
+          -->
+          <template v-if="dictating">
+            <UiIconButton
+              class="voice-discard-button"
+              icon="trash"
+              label="Discard dictation"
+              variant="ghost"
+              @click="discardDictation"
+            />
+            <UiIconButton
+              class="voice-send-button"
+              icon="arrow-up"
+              label="Send dictated message"
+              variant="primary"
+              :disabled="voiceState !== 'listening'"
+              @click="sendDictation"
+            />
+          </template>
           <UiIconButton
-            v-if="showStop"
+            v-else-if="showStop"
             class="stop-button"
             icon="stop"
             label="Stop generating"
             variant="primary"
             @click="web.stopRun()"
           />
+          <UiIconButton
+            v-else-if="showVoiceButton"
+            class="voice-button"
+            icon="waveform"
+            :label="voiceButtonLabel"
+            variant="primary"
+            @click="startDictation"
+          />
           <UiIconButton
             v-else
             type="submit"

package/src/client/applets-client.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, test } from "bun:test";
 import {
+  appletSourceFingerprintV1,
   mostRecentlyChangedFileV1,
   readAppletList,
   readAppletSource,
@@ -119,6 +120,67 @@ describe("Applet routes", () => {
       }),
     ).toBe("ui.tsx");
   });
+
+  test("the source fingerprint is the same for a re-read of the same files", () => {
+    // The canvas moves the User to the code when a Turn *writes* source. It
+    // re-reads the source on every poll and gets a fresh view object each
+    // time, so the identity that decides has to be the files themselves —
+    // otherwise a Turn that only called an Applet's own tool throws the User
+    // off the live Applet.
+    const file = {
+      path: "server.ts",
+      text: "export default class {}",
+      generationId: "w-1",
+      changedAt: "2026-09-03T00:01:00.000Z",
+    };
+    const view = {
+      appletId: "u1abc.todo",
+      truncated: false,
+      files: [file, { ...file, path: "ui.tsx", generationId: "w-2" }],
+    };
+    expect(appletSourceFingerprintV1(view)).toBe(
+      appletSourceFingerprintV1(structuredClone(view)),
+    );
+    // Order is not a change either: two reads may sort the store differently.
+    expect(
+      appletSourceFingerprintV1({ ...view, files: view.files.toReversed() }),
+    ).toBe(appletSourceFingerprintV1(view));
+    expect(appletSourceFingerprintV1(undefined)).toBe("");
+  });
+
+  test("the source fingerprint changes when a Turn writes a file", () => {
+    const view = {
+      appletId: "u1abc.todo",
+      truncated: false,
+      files: [
+        {
+          path: "server.ts",
+          text: "",
+          generationId: "w-1",
+          changedAt: "2026-09-03T00:01:00.000Z",
+        },
+      ],
+    };
+    expect(
+      appletSourceFingerprintV1({
+        ...view,
+        files: [
+          {
+            ...view.files[0]!,
+            generationId: "w-2",
+            changedAt: "2026-09-03T00:02:00.000Z",
+          },
+        ],
+      }),
+    ).not.toBe(appletSourceFingerprintV1(view));
+    // A new file is a write too.
+    expect(
+      appletSourceFingerprintV1({
+        ...view,
+        files: [...view.files, { ...view.files[0]!, path: "ui.tsx" }],
+      }),
+    ).not.toBe(appletSourceFingerprintV1(view));
+  });
 });
 
 describe("the applets feed a page receives", () => {

package/src/client/applets-client.ts

@@ -137,3 +137,22 @@ export function mostRecentlyChangedFileV1(
   });
   return ordered[0]?.path;
 }
+
+/**
+ * A stable identity for the source the canvas is showing.
+ *
+ * The canvas follows the Turn: a Turn that writes source lands the User on the
+ * code. "Wrote source" has to be a fact about the files, though, not about the
+ * store having been re-read — `refreshAppletCanvas` assigns a fresh view object
+ * on every poll, so a watcher on the array itself fired on Turns that touched
+ * no file at all and yanked the User off the live Applet.
+ */
+export function appletSourceFingerprintV1(
+  source: AppletSourceViewV1 | undefined,
+): string {
+  if (!source) return "";
+  return source.files
+    .map((file) => `${file.path}@${file.generationId}@${file.changedAt ?? ""}`)
+    .toSorted()
+    .join("\n");
+}