@devaccel-labs/app-comments-toolbar 1.1.0 → 1.3.0

package/src/runtime/dictation-ui.tsx

@@ -0,0 +1,363 @@
+import {
+  useCallback,
+  useEffect,
+  useLayoutEffect,
+  useMemo,
+  useRef,
+  useState,
+  type KeyboardEvent as ReactKeyboardEvent,
+} from "react";
+import { AttachButton } from "./attachments.js";
+import { dictationPrompt, insertDictated, rebaseAnchor } from "./dictation.js";
+import {
+  Dictation,
+  dictationSupported,
+  idleDictation,
+  voiceRecordingSupported,
+  type DictationDeps,
+  type DictationState,
+} from "./dictation-engine.js";
+import * as Icon from "./icons.js";
+
+type Options = {
+  transcribe: DictationDeps["transcribe"];
+  /** The writing box's text, and how to change it. Dictated words go through the same path as typing. */
+  value: string;
+  onChange: (next: string) => void;
+  /** People who may be named: Whisper spells them right when it knows them. */
+  names: readonly string[];
+  onNote: (note: string) => void;
+  /** The take as a voice file, attached to the box's message. Without it nothing is recorded. */
+  onAudio?: (file: File) => void;
+  /** "Keep voice": whether the recording is attached when the take ends. */
+  keepVoice?: boolean;
+  onKeepVoice?: (keep: boolean) => void;
+  /** Each piece's final words with when they were spoken, for a clip recording meanwhile. */
+  onSaid?: (text: string, spokenAt: number) => void;
+};
+
+export type DictationControl = ReturnType<typeof useDictation>;
+
+/**
+ * Dictation into one writing box. Words land where the caret was when dictation started and keep landing
+ * after each other there, even while the person types elsewhere in the box (the point moves with their
+ * edits). The take is cancelled when the box's owner unmounts.
+ */
+export function useDictation(options: Options) {
+  const latest = useRef(options);
+  const anchor = useRef(0);
+  const seen = useRef("");
+  const field = useRef<HTMLTextAreaElement | null>(null);
+  const [state, setState] = useState<DictationState>(idleDictation);
+  const [engine] = useState(() => new Dictation());
+  useLayoutEffect(() => {
+    latest.current = options;
+    engine.connect({
+      transcribe: (wav, prompt, signal) => latest.current.transcribe(wav, prompt, signal),
+      prompt: () => {
+        const { value, names } = latest.current;
+        return dictationPrompt(
+          value.slice(0, rebaseAnchor(seen.current, value, anchor.current)),
+          names,
+        );
+      },
+      write: (text) => {
+        const { value, onChange } = latest.current;
+        const at = rebaseAnchor(seen.current, value, anchor.current);
+        const next = insertDictated(value, at, text);
+        const box = field.current;
+        const follow =
+          box?.isConnected &&
+          document.activeElement === box &&
+          box.selectionStart === box.selectionEnd &&
+          box.selectionEnd === at;
+        seen.current = next.value;
+        anchor.current = next.caret;
+        // A second piece can land before React re-renders with the first.
+        latest.current = { ...latest.current, value: next.value };
+        onChange(next.value);
+        if (box && follow)
+          requestAnimationFrame(() => {
+            if (box.value === next.value) box.setSelectionRange(next.caret, next.caret);
+          });
+      },
+      onState: setState,
+      note: (text) => latest.current.onNote(text),
+      said: (text, spokenAt) => latest.current.onSaid?.(text, spokenAt),
+      ...(options.onAudio
+        ? {
+            audio: (file: File) => latest.current.onAudio?.(file),
+            keepAudio: () => latest.current.keepVoice !== false,
+          }
+        : {}),
+    });
+  });
+  useEffect(() => () => engine.cancel(), [engine]);
+
+  /** `recordVoice: false` leaves this take's voice out (the screen recording has it). */
+  const start = useCallback(
+    (box: HTMLTextAreaElement | null, recordVoice = true) => {
+      const { value } = latest.current;
+      field.current = box;
+      // Words from the last take still on their way keep their place; a fresh take starts at the caret.
+      if (engine.current.queued === 0 && engine.current.stalled === null) {
+        seen.current = value;
+        anchor.current = Math.min(
+          box?.value === value ? box.selectionEnd : value.length,
+          value.length,
+        );
+      }
+      void engine.start({ recordVoice });
+    },
+    [engine],
+  );
+  const toggle = useCallback(
+    (box: HTMLTextAreaElement | null, recordVoice = true) => {
+      if (engine.current.phase === "idle") start(box, recordVoice);
+      else engine.stop();
+    },
+    [engine, start],
+  );
+  const stop = useCallback(() => engine.stop(), [engine]);
+  const cancel = useCallback(() => engine.cancel(), [engine]);
+  const setKeepVoice = useCallback((keep: boolean) => latest.current.onKeepVoice?.(keep), []);
+  const listening = state.phase !== "idle";
+  const recordsVoice = Boolean(options.onAudio) && voiceRecordingSupported();
+  const keepVoice = options.keepVoice !== false;
+  return useMemo(
+    () => ({
+      engine,
+      state,
+      start,
+      toggle,
+      stop,
+      cancel,
+      /** The mic is open or being opened. */
+      listening,
+      /** Anything still to write: posting now would lose words. */
+      busy: listening || state.queued > 0,
+      /** The pill replaces the attach buttons. */
+      shown: listening || state.queued > 0 || state.stalled !== null,
+      /** The take is recorded too; the pill offers "Keep voice". */
+      recordsVoice,
+      keepVoice,
+      setKeepVoice,
+    }),
+    [engine, state, start, toggle, stop, cancel, listening, recordsVoice, keepVoice, setKeepVoice],
+  );
+}
+
+/** ⌘⇧Space / Ctrl+Shift+Space in a writing box starts or stops dictation. */
+export function isDictationKey(event: ReactKeyboardEvent<HTMLTextAreaElement>) {
+  return (
+    (event.metaKey || event.ctrlKey) && event.shiftKey && !event.altKey && event.code === "Space"
+  );
+}
+
+function clock(ms: number) {
+  const seconds = Math.floor(ms / 1000);
+  return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;
+}
+
+const BARS = 7;
+
+/**
+ * The live part of the pill: level bars and elapsed time, painted straight to the DOM every frame so the
+ * rest of the toolbar never re-renders for them.
+ */
+function Meter({
+  engine,
+  onSilentMic,
+}: {
+  engine: Dictation;
+  onSilentMic: (silent: boolean) => void;
+}) {
+  const wave = useRef<HTMLSpanElement>(null);
+  const time = useRef<HTMLSpanElement>(null);
+  const report = useRef(onSilentMic);
+  useLayoutEffect(() => {
+    report.current = onSilentMic;
+  });
+  useEffect(() => {
+    const history = Array.from({ length: BARS }, () => 0);
+    let frame = 0;
+    let lastShift = 0;
+    let lastClock = "";
+    let silent: boolean | null = null;
+    const paint = (now: number) => {
+      const meter = engine.meter();
+      if (now - lastShift > 70) {
+        history.shift();
+        history.push(meter.level);
+        lastShift = now;
+        const bars = wave.current?.children;
+        if (bars)
+          for (let index = 0; index < bars.length; index++) {
+            const bar = bars[index];
+            if (bar instanceof HTMLElement)
+              bar.style.transform = `scaleY(${0.18 + (history[index] ?? 0) * 0.82})`;
+          }
+      }
+      const shown = clock(meter.elapsedMs);
+      if (shown !== lastClock && time.current) {
+        time.current.textContent = shown;
+        lastClock = shown;
+      }
+      if (meter.silentMic !== silent) {
+        silent = meter.silentMic;
+        report.current(silent);
+      }
+      frame = requestAnimationFrame(paint);
+    };
+    frame = requestAnimationFrame(paint);
+    return () => cancelAnimationFrame(frame);
+  }, [engine]);
+  return (
+    <>
+      <span className="cs-dictate-wave" ref={wave} aria-hidden="true">
+        {Array.from({ length: BARS }, (_, index) => (
+          <i key={index} />
+        ))}
+      </span>
+      <span className="cs-dictate-time" ref={time}>
+        0:00
+      </span>
+    </>
+  );
+}
+
+function fieldFor(button: HTMLElement, name: string) {
+  return (
+    button.closest("form")?.querySelector<HTMLTextAreaElement>(`[data-cs-ui="${name}"]`) ?? null
+  );
+}
+
+/**
+ * The mic button, and while dictating the pill that replaces it: stop, live level, time, and what is
+ * happening (listening, writing, or a piece that needs a retry). `field` is the `data-cs-ui` of the box
+ * in the same form that the words go into.
+ */
+export function DictateButton({
+  dictation,
+  field,
+  disabled,
+  compact = false,
+}: {
+  dictation: DictationControl;
+  field: string;
+  disabled: boolean;
+  compact?: boolean;
+}) {
+  const [silentMic, setSilentMic] = useState(false);
+  const { state, engine } = dictation;
+  const supported = dictationSupported();
+
+  if (!dictation.shown) {
+    return (
+      <span className="cs-attach">
+        <AttachButton
+          className="cs-icon-button"
+          data-cs-ui="dictate"
+          data-tip={supported ? "Dictate  ⌘⇧Space" : "Dictation needs https"}
+          aria-label="Dictate"
+          aria-keyshortcuts="Meta+Shift+Space Control+Shift+Space"
+          disabled={disabled}
+          // Keeps the caret in the box, so dictation starts where it is and typing can carry on.
+          onMouseDown={(event) => event.preventDefault()}
+          onClick={(event) => dictation.start(fieldFor(event.currentTarget, field))}
+        >
+          <Icon.Mic />
+        </AttachButton>
+      </span>
+    );
+  }
+
+  const status = state.stalled
+    ? null
+    : state.phase === "starting"
+      ? "Allow the microphone…"
+      : state.phase === "listening"
+        ? silentMic
+          ? "No sound. Is the mic muted?"
+          : state.queued > 0
+            ? "Writing…"
+            : "Listening…"
+        : "Writing…";
+
+  return (
+    <div
+      className={[
+        "cs-dictate",
+        dictation.listening ? "is-live" : "",
+        state.stalled ? "is-stalled" : "",
+        compact ? "is-compact" : "",
+      ].join(" ")}
+      role="group"
+      aria-label="Dictation"
+      data-cs-ui="dictation"
+      data-cs-phase={state.phase}
+    >
+      {dictation.listening ? (
+        <button
+          type="button"
+          className="cs-dictate-stop"
+          data-cs-ui="dictate-stop"
+          aria-label="Stop dictation"
+          title="Stop (Esc)"
+          onMouseDown={(event) => event.preventDefault()}
+          onClick={() => dictation.stop()}
+        >
+          <span />
+        </button>
+      ) : state.stalled ? null : (
+        <span className="cs-dictate-spin" aria-hidden="true" />
+      )}
+      {state.phase === "listening" ? <Meter engine={engine} onSilentMic={setSilentMic} /> : null}
+      {dictation.listening && dictation.recordsVoice && !compact ? (
+        <button
+          type="button"
+          className="cs-dictate-keep"
+          data-cs-ui="dictate-keep-voice"
+          aria-pressed={dictation.keepVoice}
+          title={
+            dictation.keepVoice
+              ? "The recording is attached when you stop. Click to keep only the text."
+              : "Only the text is kept. Click to attach the recording too."
+          }
+          onMouseDown={(event) => event.preventDefault()}
+          onClick={() => dictation.setKeepVoice(!dictation.keepVoice)}
+        >
+          Keep voice
+        </button>
+      ) : null}
+      {state.stalled ? (
+        <span className="cs-dictate-stalled" title={state.stalled} role="alert">
+          <span className="cs-dictate-status">Couldn’t transcribe</span>
+          <button
+            type="button"
+            className="cs-link"
+            data-cs-ui="dictate-retry"
+            onClick={() => engine.retry()}
+          >
+            Retry
+          </button>
+          <button
+            type="button"
+            className="cs-link"
+            data-cs-ui="dictate-skip"
+            onClick={() => engine.skip()}
+          >
+            Skip
+          </button>
+        </span>
+      ) : (
+        <span
+          className={`cs-dictate-status${silentMic && state.phase === "listening" ? " is-warn" : ""}`}
+          aria-live="polite"
+        >
+          {status}
+        </span>
+      )}
+    </div>
+  );
+}

package/src/runtime/dictation.ts

@@ -0,0 +1,271 @@
+/**
+ * Dictation's pure parts (ADR-APPC-010): resample the microphone to 16 kHz, cut the take into pieces at
+ * pauses, write each piece as WAV, and place the words that come back into the writing box. No browser
+ * audio APIs here, so all of it runs under unit tests. `dictation-engine.ts` drives it from a live mic.
+ */
+
+import { decibels } from "./dictation-model.js";
+
+export const SAMPLE_RATE = 16_000;
+const FRAME = 480;
+const FRAME_SECONDS = FRAME / SAMPLE_RATE;
+const frames = (seconds: number) => Math.round(seconds / FRAME_SECONDS);
+
+/** Sound kept from before the first loud frame, so a soft first syllable isn't clipped. */
+const PREROLL = frames(0.3);
+/** A pause this long ends a piece. Pauses between words are shorter; between sentences, longer. */
+const END_PAUSE = frames(0.7);
+/** Quiet kept after the last word: enough for Whisper to hear the end, not enough to invent more. */
+const TAIL = frames(0.3);
+/** Less speech than this is a click or a breath and is never sent. */
+const MIN_SPEECH = frames(0.25);
+/** Whisper reads 30 s windows; a piece is cut before that even when nobody pauses. */
+export const MAX_PIECE_SECONDS = 28;
+const MAX_FRAMES = frames(MAX_PIECE_SECONDS);
+/** A forced cut lands on the quietest frame of the last few seconds, which is usually between words. */
+const CUT_SEARCH = frames(3);
+
+export type Piece = { samples: Float32Array; seconds: number; speechSeconds: number };
+
+/**
+ * Streaming sample-rate conversion. Downsampling averages every input sample an output sample covers (a
+ * box filter, enough low-pass for speech); upsampling interpolates. State carries across chunks, so the
+ * output is the same however the input is split.
+ */
+export class Resampler {
+  private readonly step: number;
+  private rest = new Float32Array(0);
+  private offset = 0;
+
+  constructor(
+    readonly from: number,
+    readonly to = SAMPLE_RATE,
+  ) {
+    this.step = from / to;
+  }
+
+  push(input: Float32Array): Float32Array {
+    if (this.from === this.to) return input.slice();
+    const data = new Float32Array(this.rest.length + input.length);
+    data.set(this.rest);
+    data.set(input, this.rest.length);
+    const out: number[] = [];
+    let start = this.offset;
+    const step = this.step;
+    const ready = () =>
+      step >= 1 ? start + step <= data.length : Math.floor(start) + 1 < data.length;
+    while (ready()) {
+      if (step >= 1) {
+        const end = start + step;
+        let sum = 0;
+        let weight = 0;
+        for (let index = Math.floor(start); index < Math.ceil(end); index++) {
+          const covered = Math.min(index + 1, end) - Math.max(index, start);
+          sum += data[index]! * covered;
+          weight += covered;
+        }
+        out.push(weight ? sum / weight : 0);
+      } else {
+        const index = Math.floor(start);
+        const fraction = start - index;
+        out.push(data[index]! * (1 - fraction) + data[index + 1]! * fraction);
+      }
+      start += step;
+    }
+    const keep = Math.floor(start);
+    this.rest = data.slice(keep);
+    this.offset = start - keep;
+    return Float32Array.from(out);
+  }
+}
+
+/**
+ * Cuts 16 kHz audio into pieces at pauses. Speech is a frame louder than the room: a noise floor that falls
+ * fast and rises slowly, plus a margin, never quieter than −55 dBFS. A piece ends after `END_PAUSE` of quiet,
+ * or is cut at `MAX_PIECE_SECONDS`; pieces with almost no speech are dropped, so silence never reaches
+ * Whisper (which turns silence into "Thank you.").
+ */
+export class Segmenter {
+  /** Loudness of the newest frame. */
+  lastDb = -100;
+  /** Loudest frame so far: −100 means the mic delivered digital silence. */
+  peakDb = -100;
+  private floor = -65;
+  private partial = new Float32Array(FRAME);
+  private filled = 0;
+  private preroll: Float32Array[] = [];
+  private active: Array<{ samples: Float32Array; db: number; speech: boolean }> | null = null;
+
+  push(samples: Float32Array): Piece[] {
+    const out: Piece[] = [];
+    let at = 0;
+    while (at < samples.length) {
+      const take = Math.min(FRAME - this.filled, samples.length - at);
+      this.partial.set(samples.subarray(at, at + take), this.filled);
+      this.filled += take;
+      at += take;
+      if (this.filled < FRAME) break;
+      const piece = this.frame(this.partial);
+      if (piece) out.push(piece);
+      this.partial = new Float32Array(FRAME);
+      this.filled = 0;
+    }
+    return out;
+  }
+
+  /** The audio of the piece still being spoken (not yet cut), for a live preview; null between pieces. */
+  pending(): Float32Array | null {
+    const active = this.active;
+    if (!active?.some((frame) => frame.speech)) return null;
+    const samples = new Float32Array(active.length * FRAME);
+    active.forEach((frame, index) => samples.set(frame.samples, index * FRAME));
+    return samples;
+  }
+
+  /** The piece in progress, when the take ends. */
+  flush(): Piece | null {
+    const active = this.active;
+    this.active = null;
+    this.preroll = [];
+    if (!active) return null;
+    let quiet = 0;
+    while (quiet < active.length && !active[active.length - 1 - quiet]!.speech) quiet++;
+    return this.piece(active.slice(0, active.length - Math.max(0, quiet - TAIL)));
+  }
+
+  private frame(samples: Float32Array): Piece | null {
+    const db = decibels(samples);
+    this.lastDb = db;
+    this.peakDb = Math.max(this.peakDb, db);
+    const threshold = Math.min(-25, Math.max(-55, this.floor + 12));
+    const speech = db > threshold;
+    this.floor =
+      db < this.floor
+        ? this.floor * 0.7 + db * 0.3
+        : this.floor + (db - this.floor) * (speech ? 0.002 : 0.01);
+
+    if (!this.active) {
+      if (!speech) {
+        this.preroll.push(samples);
+        if (this.preroll.length > PREROLL) this.preroll.shift();
+        return null;
+      }
+      this.active = this.preroll.map((frame) => ({ samples: frame, db: -100, speech: false }));
+      this.preroll = [];
+    }
+    const active = this.active;
+    active.push({ samples, db, speech });
+
+    let quiet = 0;
+    while (quiet < active.length && !active[active.length - 1 - quiet]!.speech) quiet++;
+    if (quiet >= END_PAUSE) {
+      const kept = active.length - (quiet - TAIL);
+      this.active = null;
+      this.preroll = active.slice(-PREROLL).map((frame) => frame.samples);
+      return this.piece(active.slice(0, kept));
+    }
+    if (active.length >= MAX_FRAMES) {
+      let cut = active.length - 1;
+      for (let index = active.length - CUT_SEARCH; index < active.length; index++) {
+        if (active[index]!.db < active[cut]!.db) cut = index;
+      }
+      this.active = active.slice(cut);
+      return this.piece(active.slice(0, cut));
+    }
+    return null;
+  }
+
+  private piece(list: Array<{ samples: Float32Array; speech: boolean }>): Piece | null {
+    const speech = list.filter((frame) => frame.speech).length;
+    if (speech < MIN_SPEECH) return null;
+    const samples = new Float32Array(list.length * FRAME);
+    list.forEach((frame, index) => samples.set(frame.samples, index * FRAME));
+    return {
+      samples,
+      seconds: list.length * FRAME_SECONDS,
+      speechSeconds: speech * FRAME_SECONDS,
+    };
+  }
+}
+
+/**
+ * 16-bit mono PCM WAV. Quiet takes are raised (up to 8x, to a 0.9 peak) so a far-away mic still reads
+ * clearly; loud ones are left alone.
+ */
+export function encodeWav(samples: Float32Array, rate = SAMPLE_RATE): ArrayBuffer {
+  let peak = 0;
+  for (const sample of samples) peak = Math.max(peak, Math.abs(sample));
+  const gain = peak > 0 && peak < 0.3 ? Math.min(8, 0.9 / peak) : 1;
+  const buffer = new ArrayBuffer(44 + samples.length * 2);
+  const view = new DataView(buffer);
+  const text = (at: number, value: string) => {
+    for (let index = 0; index < value.length; index++)
+      view.setUint8(at + index, value.charCodeAt(index));
+  };
+  text(0, "RIFF");
+  view.setUint32(4, 36 + samples.length * 2, true);
+  text(8, "WAVE");
+  text(12, "fmt ");
+  view.setUint32(16, 16, true);
+  view.setUint16(20, 1, true);
+  view.setUint16(22, 1, true);
+  view.setUint32(24, rate, true);
+  view.setUint32(28, rate * 2, true);
+  view.setUint16(32, 2, true);
+  view.setUint16(34, 16, true);
+  text(36, "data");
+  view.setUint32(40, samples.length * 2, true);
+  samples.forEach((sample, index) => {
+    const value = Math.max(-1, Math.min(1, sample * gain));
+    view.setInt16(44 + index * 2, value < 0 ? value * 0x8000 : value * 0x7fff, true);
+  });
+  return buffer;
+}
+
+export {
+  decibels,
+  dictationPrompt,
+  insertDictated,
+  meterLevel,
+  rebaseAnchor,
+  voiceRecordingName,
+} from "./dictation-model.js";
+
+/**
+ * What to tell someone when the browser won't give us the microphone. A refusal can come from the site
+ * setting, from the operating system, or from a browser that never shows a prompt at all (an app's built-in
+ * browser, or Chrome after the prompt was dismissed a few times); `permission` (from the Permissions API, when
+ * the browser has it) and the error's text tell those apart, because each needs a different fix.
+ */
+export function microphoneError(
+  error: unknown,
+  permission?: PermissionState | null,
+  userAgent = typeof navigator === "undefined" ? "" : navigator.userAgent,
+  /** The page's Permissions-Policy verdict for `microphone`, when the browser exposes it. */
+  pageAllows: boolean | null = null,
+) {
+  const name = error instanceof Error || error instanceof DOMException ? error.name : "";
+  const message = error instanceof Error || error instanceof DOMException ? error.message : "";
+  if (name === "NotAllowedError" || name === "SecurityError" || name === "PermissionDeniedError") {
+    // An Electron app's built-in browser (T3 Code, Slack, …) answers "denied" without ever prompting, so the
+    // address-bar advice would send the person looking for a setting that doesn't exist.
+    if (pageAllows === false)
+      return "This page’s Permissions-Policy turns the microphone off, so the browser can’t ask for it.";
+    if (/\bElectron\//.test(userAgent))
+      return "This app’s built-in browser doesn’t allow the microphone. Open the page in Chrome, Safari or Edge to dictate.";
+    if (/system/i.test(message))
+      return "Your computer blocks this browser from the microphone. Allow it in System Settings → Privacy & Security → Microphone, then try again.";
+    if (permission === "denied")
+      return "Microphone access is blocked for this site. Click the icon left of the address, set Microphone to Allow, then reload.";
+    return "This browser didn’t let the page ask for the microphone. Allow it for this site from the address bar, or open the page in Chrome, Safari or Edge if you’re in an app’s built-in browser.";
+  }
+  if (
+    name === "NotFoundError" ||
+    name === "OverconstrainedError" ||
+    name === "DevicesNotFoundError"
+  )
+    return "No microphone found. Plug one in and try again.";
+  if (name === "NotReadableError" || name === "AbortError" || name === "TrackStartError")
+    return "The microphone is in use by another app, or your system blocked it.";
+  return "Couldn’t start the microphone.";
+}