@frockbot/plugin-shell 0.3.12 → 0.3.14

package/src/client/styles.css

@@ -360,11 +360,13 @@
  * notices and sends stack inside it, so a one-word reply is a bubble the width
  * of its word rather than a sliver of a shared row.
  *
- * The avatar is not in that column. While the Bot is working it sits on its
- * own row underneath, so the trail streaming off its right has the width of
- * the transcript to run through rather than the gutter beside a bubble. Every
- * reply here is from the same Bot — there are no group conversations — so a
- * sheep on every settled line said nothing and cost the column its left edge.
+ * The avatar is not in that column, and not in the article at all: while the
+ * Bot is working it sits on its own row at the very end of the thread, so the
+ * trail streaming off its right has the width of the transcript to run through
+ * rather than the gutter beside a bubble, and a message sent mid-Turn lands
+ * above it. Every reply here is from the same Bot — there are no group
+ * conversations — so a sheep on every settled line said nothing and cost the
+ * column its left edge.
  */
 .message-assistant {
   flex-direction: column;
@@ -383,11 +385,16 @@
 /*
  * The working row. Its height is the trail's canvas: tall enough for the
  * wobble to be visible, short enough that it reads as one line of the thread.
+ * It is the thread's last child, so it takes the same top margin every message
+ * has and adds no width of its own — the bubbles above never move sideways
+ * when it appears or goes.
  */
 .bot-working {
   display: flex;
+  width: 100%;
   height: 44px;
   align-items: center;
+  margin-top: 6px;
   gap: 0;
 }
 
@@ -461,62 +468,6 @@
   }
 }
 
-@keyframes frock-halo {
-  0%,
-  100% {
-    opacity: 0;
-    transform: scale(0.94);
-  }
-
-  50% {
-    opacity: 1;
-    transform: scale(1.06);
-  }
-}
-
-/*
- * The bubble a Turn has before it has words: three dots that say the Bot is
- * working, where the reply itself will appear.
- */
-.message-working {
-  display: flex;
-  align-items: center;
-  min-height: 20px;
-}
-
-.working-dots {
-  display: inline-flex;
-  gap: 4px;
-}
-
-.working-dots i {
-  width: 6px;
-  height: 6px;
-  border-radius: 50%;
-  background: var(--frock-text-muted);
-  animation: frock-working-dot 1.2s ease-in-out infinite;
-}
-
-.working-dots i:nth-child(2) {
-  animation-delay: 0.15s;
-}
-
-.working-dots i:nth-child(3) {
-  animation-delay: 0.3s;
-}
-
-@keyframes frock-working-dot {
-  0%,
-  60%,
-  100% {
-    opacity: 0.3;
-  }
-
-  30% {
-    opacity: 1;
-  }
-}
-
 /*
  * A bubble runs nearly the width of the transcript, leaving one clear margin
  * on the side it is not anchored to: the Bot's at the end, the User's at the
@@ -807,6 +758,50 @@
   color: var(--frock-danger-text);
 }
 
+/*
+ * Dictation.
+ *
+ * The capture line sits under the draft it is writing, inside the composer,
+ * because the text arriving in the textarea is the thing it is describing. It
+ * is a status line rather than an overlay: nothing about the composer moves
+ * when it appears beyond the row it adds.
+ */
+.voice-capture {
+  display: flex;
+  margin: 0;
+  align-items: center;
+  gap: 8px;
+  color: var(--frock-text-muted);
+  font-size: var(--frock-text-xs);
+}
+
+.voice-wave {
+  display: flex;
+  height: 14px;
+  align-items: center;
+  gap: 2px;
+}
+
+/*
+ * Each bar is scaled from the microphone's own level, so the animation stops
+ * when the room does. `transform` rather than `height`: it is the one property
+ * that animates without a layout pass, and this runs at frame rate.
+ */
+.voice-wave-bar {
+  width: 2px;
+  height: 100%;
+  border-radius: 999px;
+  background: var(--frock-action-primary);
+  transform-origin: center;
+  transition: transform var(--frock-motion-fast);
+}
+
+.voice-error {
+  margin: 0;
+  color: var(--frock-danger-text);
+  font-size: var(--frock-text-xs);
+}
+
 .skill-chips {
   display: flex;
   margin: 0;

package/src/client/voice-dictation.test.ts

@@ -0,0 +1,105 @@
+import { describe, expect, test } from "bun:test";
+import {
+  applyDictationTailV1,
+  voiceButtonLabelV1,
+  voiceWaveBarsV1,
+  VoiceDictationTranscriptV1,
+} from "./voice-dictation.js";
+
+describe("writing dictation into a draft somebody may be editing", () => {
+  test("appends the first words to an empty draft", () => {
+    expect(applyDictationTailV1("", "", "hello")).toEqual({
+      draft: "hello",
+      tail: "hello",
+    });
+  });
+
+  test("grows in place rather than repeating itself", () => {
+    const first = applyDictationTailV1("", "", "hello");
+    const second = applyDictationTailV1(first.draft, first.tail, "hello there");
+    expect(second.draft).toBe("hello there");
+  });
+
+  test("keeps a typed prefix, and a typed suffix, around what is spoken", () => {
+    const first = applyDictationTailV1("Note: ", "", "buy milk");
+    expect(first.draft).toBe("Note: buy milk");
+    // The person types on the end while the next words arrive.
+    const edited = `${first.draft} today`;
+    const second = applyDictationTailV1(
+      edited,
+      first.tail,
+      "buy milk and bread",
+    );
+    expect(second.draft).toBe("Note: buy milk and bread today");
+  });
+
+  test("rewrites the last occurrence, so a repeated word moves the right one", () => {
+    // "go" was typed, then "go" was dictated. Growing the dictated one must
+    // not reach back and rewrite the typed one.
+    expect(applyDictationTailV1("go go", "go", "gone")).toEqual({
+      draft: "go gone",
+      tail: "gone",
+    });
+  });
+
+  test("appends rather than fighting when the person deleted what was dictated", () => {
+    const first = applyDictationTailV1("", "", "hello");
+    const cleared = applyDictationTailV1("", first.tail, "hello there");
+    expect(cleared).toEqual({ draft: "hello there", tail: "hello there" });
+  });
+
+  test("never glues two words together", () => {
+    expect(applyDictationTailV1("Note:", "", "one").draft).toBe("Note: one");
+    expect(applyDictationTailV1("Note: ", "", "one").draft).toBe("Note: one");
+  });
+});
+
+describe("what has been heard so far", () => {
+  test("streams deltas, then lets the finished segment replace them", () => {
+    const transcript = new VoiceDictationTranscriptV1();
+    expect(transcript.empty()).toBe(true);
+    transcript.delta("hello ");
+    transcript.delta("their");
+    expect(transcript.text()).toBe("hello their");
+    transcript.settle("Hello there.");
+    expect(transcript.text()).toBe("Hello there.");
+    expect(transcript.empty()).toBe(false);
+  });
+
+  test("joins segments with a single space and drops an empty one", () => {
+    const transcript = new VoiceDictationTranscriptV1();
+    transcript.settle("  One. ");
+    transcript.settle("");
+    transcript.settle("Two.");
+    transcript.delta("thr");
+    expect(transcript.text()).toBe("One. Two. thr");
+  });
+
+  test("resets to nothing, so a binned capture leaves no tail behind", () => {
+    const transcript = new VoiceDictationTranscriptV1();
+    transcript.settle("One.");
+    transcript.reset();
+    expect(transcript.empty()).toBe(true);
+    expect(transcript.text()).toBe("");
+  });
+});
+
+describe("the composer's dictation chrome", () => {
+  test("names each state for the aria label and the tooltip alike", () => {
+    expect(voiceButtonLabelV1("idle")).toBe("Dictate a message");
+    expect(voiceButtonLabelV1("listening")).toBe("Listening");
+    expect(voiceButtonLabelV1("finishing")).toBe("Finishing dictation");
+  });
+
+  test("the wave follows the microphone and never collapses to a line", () => {
+    const silent = voiceWaveBarsV1(0, []);
+    expect(silent).toHaveLength(4);
+    expect(silent.every((bar) => bar >= 0.2)).toBe(true);
+    const loud = voiceWaveBarsV1(1, silent);
+    expect(loud[0]).toBeCloseTo(1);
+    // The level travels along the bars rather than moving all of them at once.
+    expect(loud[1]).toBeCloseTo(silent[0]!);
+    expect(voiceWaveBarsV1(Number.NaN, silent)[0]).toBeCloseTo(0.2);
+    expect(voiceWaveBarsV1(9, silent)[0]).toBeCloseTo(1);
+  });
+});

package/src/client/voice-dictation.ts

@@ -0,0 +1,137 @@
+// Dictation, as text arriving in a draft somebody may be editing at the same
+// time (voice plan D4).
+//
+// The composer writes what it hears into the textarea rather than into a
+// separate box, so the message is editable while it is being spoken, a
+// rejected send restores it like any other draft, and Send is the ordinary
+// Send. That makes one thing hard and this module is that one thing: knowing
+// which part of the draft dictation put there, when the person is free to
+// type in the middle of it.
+//
+// The answer is a *tail*: the exact text last written on dictation's behalf.
+// Each update replaces the last occurrence of the previous tail with the next
+// one. Delete it, retype around it, paste over it — if the previous tail is
+// no longer in the draft, the new text is appended rather than forced back
+// into a position nobody asked for. Nothing here reads the caret, so it holds
+// on a phone keyboard and a desktop one alike.
+
+export type VoiceDictationStateV1 =
+  /** No microphone. The send button is the wave button when the draft is empty. */
+  | "idle"
+  /** Asked for the microphone, or waiting for the upstream to say `ready`. */
+  | "starting"
+  /** Capturing. Bin and Send have replaced the wave button. */
+  | "listening"
+  /** Send was pressed; the last of the audio is being transcribed. */
+  | "finishing";
+
+export interface DictationDraftV1 {
+  draft: string;
+  tail: string;
+}
+
+/** Joins dictation onto a draft without gluing two words together. */
+function joined(head: string, tail: string): string {
+  if (!head) return tail;
+  if (!tail) return head;
+  return /\s$/u.test(head) ? `${head}${tail}` : `${head} ${tail}`;
+}
+
+/**
+ * Puts `nextTail` where `previousTail` was, or on the end when it has gone.
+ *
+ * `lastIndexOf`, not `indexOf`: dictating the same short word twice must
+ * rewrite the second one.
+ */
+export function applyDictationTailV1(
+  draft: string,
+  previousTail: string,
+  nextTail: string,
+): DictationDraftV1 {
+  if (!previousTail) return { draft: joined(draft, nextTail), tail: nextTail };
+  const at = draft.lastIndexOf(previousTail);
+  if (at < 0) return { draft: joined(draft, nextTail), tail: nextTail };
+  const before = draft.slice(0, at);
+  const after = draft.slice(at + previousTail.length);
+  return { draft: `${before}${nextTail}${after}`, tail: nextTail };
+}
+
+/**
+ * What has been heard so far: the finished segments, and the deltas of the
+ * one still being spoken.
+ *
+ * A provider streams a segment as deltas and then re-sends it, punctuated and
+ * capitalised, as a `completed` transcript. Keeping the two apart is what lets
+ * the finished form replace the rough one in place instead of appearing twice.
+ */
+export class VoiceDictationTranscriptV1 {
+  #settled: string[] = [];
+  #pending = "";
+
+  delta(text: string): void {
+    this.#pending += text;
+  }
+
+  /** One finished segment; it replaces the deltas that built it. */
+  settle(text: string): void {
+    const trimmed = text.trim();
+    if (trimmed) this.#settled.push(trimmed);
+    this.#pending = "";
+  }
+
+  /** Everything dictated in this session, as one string. */
+  text(): string {
+    const settled = this.#settled.join(" ");
+    const pending = this.#pending.trim();
+    if (!settled) return pending;
+    if (!pending) return settled;
+    return `${settled} ${pending}`;
+  }
+
+  /** True until the first word arrives, so an empty capture sends nothing. */
+  empty(): boolean {
+    return this.text().length === 0;
+  }
+
+  reset(): void {
+    this.#settled = [];
+    this.#pending = "";
+  }
+}
+
+/** The composer's label for each state, so the aria text and the title agree. */
+export function voiceButtonLabelV1(state: VoiceDictationStateV1): string {
+  switch (state) {
+    case "idle":
+      return "Dictate a message";
+    case "starting":
+      return "Starting dictation";
+    case "listening":
+      return "Listening";
+    case "finishing":
+      return "Finishing dictation";
+  }
+}
+
+/**
+ * The scale factor of each bar of the capture animation, newest first.
+ *
+ * Level-driven rather than time-driven: a wave that moves while the room is
+ * silent says the microphone is working when it is not. Each call shifts the
+ * previous bars along, so a syllable travels across the control instead of
+ * every bar jumping at once.
+ *
+ * `previous` is the value this function last returned, so the whole animation
+ * is one `ref` and one assignment.
+ */
+export function voiceWaveBarsV1(
+  level: number,
+  previous: readonly number[],
+  bars = 4,
+): number[] {
+  const clamped = Number.isFinite(level) ? Math.min(1, Math.max(0, level)) : 0;
+  // A floor, so the control never collapses to a line and loses its shape.
+  const next = [0.2 + clamped * 0.8, ...previous].slice(0, bars);
+  while (next.length < bars) next.push(0.2);
+  return next;
+}

package/src/client/voice-microphone.ts

@@ -0,0 +1,125 @@
+// The browser's half of dictation: a microphone, resampled to the PCM16 the
+// upstream transcription session expects.
+//
+// The capture itself runs on the audio thread; its source and the reason it is
+// served as a first-party asset are in `voice-worklet.ts`. This module is the
+// graph around it: permission, the context, the silent sink that keeps the
+// node pulled, and the teardown that actually releases the microphone.
+import {
+  VOICE_CAPTURE_WORKLET_PATH_V1,
+  VOICE_CAPTURE_WORKLET_PROCESSOR_V1,
+} from "./voice-worklet.js";
+
+/** What the upstream is told to expect, and therefore what leaves here. */
+export const VOICE_CAPTURE_SAMPLE_RATE_V1 = 16_000;
+
+/** Samples per frame at 16 kHz: 32 ms, small enough to feel live. */
+const FRAME_SAMPLES = 512;
+
+export interface VoiceMicrophoneV1 {
+  stop(): Promise<void>;
+}
+
+export interface VoiceMicrophoneOptionsV1 {
+  /** One frame of PCM16, little-endian, mono, 16 kHz. */
+  audio(pcm16: ArrayBuffer): void;
+  /** Peak amplitude of the frame, 0…1, for the capture animation. */
+  level(value: number): void;
+}
+
+/** False on a platform with no microphone API at all; the button stays hidden. */
+export function voiceCaptureSupportedV1(): boolean {
+  return (
+    typeof navigator !== "undefined" &&
+    typeof navigator.mediaDevices?.getUserMedia === "function" &&
+    typeof (globalThis as { AudioContext?: unknown }).AudioContext ===
+      "function"
+  );
+}
+
+/**
+ * The refusal a person reads when the browser will not give up the
+ * microphone. Named cases only: anything else says what the browser said.
+ */
+export function voiceMicrophoneRefusalV1(error: unknown): string {
+  const name =
+    typeof error === "object" && error !== null && "name" in error
+      ? String((error as { name: unknown }).name)
+      : "";
+  if (name === "NotAllowedError" || name === "SecurityError") {
+    return "FrockBot needs permission to use your microphone. Allow it in your browser, then try again.";
+  }
+  if (name === "NotFoundError" || name === "OverconstrainedError") {
+    return "No microphone was found. Plug one in, then try again.";
+  }
+  return `The microphone couldn't start: ${
+    error instanceof Error && error.message ? error.message : "unknown error"
+  }`;
+}
+
+export async function startVoiceMicrophoneV1(
+  options: VoiceMicrophoneOptionsV1,
+): Promise<VoiceMicrophoneV1> {
+  const stream = await navigator.mediaDevices.getUserMedia({
+    audio: {
+      channelCount: 1,
+      echoCancellation: true,
+      noiseSuppression: true,
+      autoGainControl: true,
+    },
+  });
+  const context = new AudioContext();
+  let node: AudioWorkletNode | undefined;
+  let source: MediaStreamAudioSourceNode | undefined;
+  let silence: GainNode | undefined;
+  try {
+    // Same-origin, so `script-src 'self'` admits it. See `voice-worklet.ts`.
+    await context.audioWorklet.addModule(VOICE_CAPTURE_WORKLET_PATH_V1);
+    node = new AudioWorkletNode(context, VOICE_CAPTURE_WORKLET_PROCESSOR_V1, {
+      numberOfInputs: 1,
+      // One silent output, connected below. A graph is pulled from the
+      // destination, so a node with no path to it is never asked to process
+      // and the microphone produces nothing at all — silently.
+      numberOfOutputs: 1,
+      outputChannelCount: [1],
+      processorOptions: {
+        targetRate: VOICE_CAPTURE_SAMPLE_RATE_V1,
+        frameSamples: FRAME_SAMPLES,
+      },
+    });
+    node.port.onmessage = (event: MessageEvent) => {
+      const message = event.data as { pcm?: ArrayBuffer; level?: number };
+      if (message.pcm) options.audio(message.pcm);
+      if (typeof message.level === "number") options.level(message.level);
+    };
+    source = context.createMediaStreamSource(stream);
+    source.connect(node);
+    // Silenced at the sink rather than left unconnected: the person must not
+    // hear themselves, and the node must still be pulled.
+    silence = context.createGain();
+    silence.gain.value = 0;
+    node.connect(silence);
+    silence.connect(context.destination);
+    // A suspended context produces silence and no error at all; Safari hands
+    // one back whenever the gesture that opened it has already finished.
+    if (context.state === "suspended") await context.resume();
+  } catch (error) {
+    for (const track of stream.getTracks()) track.stop();
+    await context.close().catch(() => undefined);
+    throw error;
+  }
+
+  let stopped = false;
+  return {
+    async stop() {
+      if (stopped) return;
+      stopped = true;
+      if (node) node.port.onmessage = null;
+      source?.disconnect();
+      node?.disconnect();
+      silence?.disconnect();
+      for (const track of stream.getTracks()) track.stop();
+      await context.close().catch(() => undefined);
+    },
+  };
+}

package/src/client/voice-worklet.ts

@@ -0,0 +1,75 @@
+// The dictation capture worklet, and the path the app serves it from.
+//
+// It is a first-party asset on the app's own origin rather than a blob URL,
+// and that is a security decision, not a packaging one: the hosted client is
+// served under `script-src 'self'`, which a `blob:` module does not satisfy —
+// `context.audioWorklet.addModule(blobUrl)` fails with "Unable to load a
+// worklet's module". Widening the policy to admit blob scripts everywhere, to
+// load one 40-line file, is the wrong trade. So the source lives here as a
+// string, the application Worker answers `GET` on the path below with it
+// (`apps/cloudflare/src/user-application.ts`), and the policy is untouched.
+//
+// Its own module so the Worker can serve the source without pulling the
+// browser-only microphone plumbing in beside it.
+
+/** Where the application Worker serves {@link VOICE_CAPTURE_WORKLET_SOURCE_V1}. */
+export const VOICE_CAPTURE_WORKLET_PATH_V1 = "/voice-capture-worklet.js";
+
+/** The processor's registered name, shared by the source and the node. */
+export const VOICE_CAPTURE_WORKLET_PROCESSOR_V1 = "frock-voice-capture";
+
+/**
+ * Microphone audio, decimated to the target rate and framed as PCM16.
+ *
+ * On the audio thread rather than the main one, because the alternative
+ * (`ScriptProcessorNode`) shares the thread with Vue's renderer and drops
+ * audio exactly when the composer is busiest — while the draft it is writing
+ * is being re-laid out.
+ *
+ * The rate is reached by decimating whatever the context gives us rather than
+ * by asking for it: iOS Safari ignores `new AudioContext({ sampleRate })` and
+ * hands back 48 kHz regardless, and audio at the wrong rate transcribes as
+ * gibberish rather than failing.
+ */
+export const VOICE_CAPTURE_WORKLET_SOURCE_V1 = `
+class FrockVoiceCapture extends AudioWorkletProcessor {
+  constructor(options) {
+    super();
+    this.target = options.processorOptions.targetRate;
+    this.frame = options.processorOptions.frameSamples;
+    this.buffer = new Float32Array(this.frame);
+    this.filled = 0;
+    this.position = 0;
+  }
+  process(inputs) {
+    const channel = inputs[0] && inputs[0][0];
+    if (!channel) return true;
+    const step = sampleRate / this.target;
+    let peak = 0;
+    for (let index = 0; index < channel.length; index += 1) {
+      const value = channel[index];
+      const magnitude = value < 0 ? -value : value;
+      if (magnitude > peak) peak = magnitude;
+    }
+    // Decimation: walk the block at a fractional step, carrying the remainder
+    // across blocks so no drift accumulates. Enough for speech, and cheap.
+    while (this.position < channel.length) {
+      this.buffer[this.filled] = channel[Math.floor(this.position)];
+      this.position += step;
+      this.filled += 1;
+      if (this.filled === this.frame) {
+        const pcm = new Int16Array(this.frame);
+        for (let sample = 0; sample < this.frame; sample += 1) {
+          const clamped = Math.max(-1, Math.min(1, this.buffer[sample]));
+          pcm[sample] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
+        }
+        this.port.postMessage({ pcm: pcm.buffer, level: peak }, [pcm.buffer]);
+        this.filled = 0;
+      }
+    }
+    this.position -= channel.length;
+    return true;
+  }
+}
+registerProcessor("${VOICE_CAPTURE_WORKLET_PROCESSOR_V1}", FrockVoiceCapture);
+`;