@dxos/react-ui-transcription 0.0.0

package/src/components/Transcription/transcription-extension.ts

@@ -0,0 +1,203 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Extension, RangeSetBuilder } from '@codemirror/state';
+import { EditorView, GutterMarker, ViewPlugin, type ViewUpdate, gutter } from '@codemirror/view';
+import { format } from 'date-fns/format';
+import { intervalToDuration } from 'date-fns/intervalToDuration';
+
+import { type CleanupFn, addEventListener, combine } from '@dxos/async';
+import { type Message } from '@dxos/types';
+
+import { EditorChunkDocument, type TranscriptModel } from '../../model';
+
+/**
+ * Data structure that maps Chunks queue to lines with transcript state.
+ */
+export type TranscriptionOptions = {
+  model: TranscriptModel<Message.Message>;
+  started?: Date;
+};
+
+/**
+ * Scrolling transcript with timestamps.
+ */
+export const transcription = ({ model, started }: TranscriptionOptions): Extension => {
+  return [
+    // Show timestamps in the gutter.
+    gutter({
+      class: 'cm-timestamp-gutter',
+      lineMarkerChange: (update) => update.docChanged || update.viewportChanged,
+      markers: (view) => {
+        const start = getStartTime(started, model.chunks[0]);
+        const builder = new RangeSetBuilder<GutterMarker>();
+        for (const { from, to } of view.visibleRanges) {
+          let line = view.state.doc.lineAt(from);
+          while (line.from <= to) {
+            const block = model.getChunkAtLine(line.number)?.blocks[0];
+            const timestamp = block?._tag === 'transcript' && block.started;
+            if (timestamp) {
+              builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
+            }
+
+            if (line.to + 1 > view.state.doc.length) {
+              break;
+            }
+
+            line = view.state.doc.lineAt(line.to + 1);
+          }
+        }
+
+        return builder.finish();
+      },
+    }),
+
+    // Listen for model updates.
+    ViewPlugin.fromClass(
+      class {
+        private readonly _controls?: HTMLDivElement;
+        private readonly _cleanup: CleanupFn;
+        private readonly _adapter: EditorChunkDocument;
+        private _initialized = false;
+
+        constructor(view: EditorView) {
+          this._adapter = new EditorChunkDocument(view);
+
+          const scroller = view.scrollDOM;
+          let isAutoScrolling = false;
+          let hasScrolled = false;
+
+          let timeout: NodeJS.Timeout | undefined;
+          const scrollToBottom = (smooth = false) => {
+            scroller.style.scrollBehavior = smooth ? 'smooth' : '';
+
+            // Temporarily hide scrollbar to prevent flicker.
+            scroller.classList.add('cm-hide-scrollbar');
+            isAutoScrolling = true;
+            clearTimeout(timeout);
+            timeout = setTimeout(() => {
+              this._controls?.classList.add('opacity-0');
+              scroller.classList.remove('cm-hide-scrollbar');
+              isAutoScrolling = false;
+            }, 500);
+
+            // Scroll to bottom.
+            view.dispatch({
+              effects: EditorView.scrollIntoView(view.state.doc.length, { y: 'end' }),
+            });
+          };
+
+          // Event listeners.
+          this._cleanup = combine(
+            addEventListener(view.scrollDOM, 'scroll', () => {
+              if (!isAutoScrolling) {
+                hasScrolled = true;
+                this._controls?.classList.remove('opacity-0');
+              }
+            }),
+
+            model.update.on(() => {
+              // Check if clamped to bottom.
+              const autoScroll =
+                scroller.scrollHeight - scroller.scrollTop - scroller.clientHeight === 0 || !hasScrolled;
+
+              // Sync.
+              model.sync(this._adapter);
+
+              // Scroll.
+              if (autoScroll) {
+                scrollToBottom(true);
+              }
+            }),
+          );
+        }
+
+        update(_update: ViewUpdate) {
+          // Initial sync.
+          if (!this._initialized) {
+            this._initialized = true;
+            setTimeout(() => {
+              model.sync(this._adapter);
+            });
+          }
+        }
+
+        destroy() {
+          this._controls?.remove();
+          this._cleanup();
+        }
+      },
+    ),
+
+    EditorView.theme({
+      '.cm-gutters': {
+        backgroundColor: 'var(--color-base-surface)',
+      },
+      '.cm-timestamp-gutter': {
+        width: '6rem',
+        paddingLeft: '0.5rem',
+        paddingRight: '1rem',
+      },
+      '.cm-timestamp-gutter > .cm-gutterElement > div': {
+        display: 'inline-flex',
+        textAlign: 'right',
+        padding: '3px',
+      },
+    }),
+  ];
+};
+
+/**
+ * Gutter marker that displays a timestamp.
+ */
+class TimestampMarker extends GutterMarker {
+  constructor(
+    readonly _line: number,
+    readonly _timestamp: Date,
+    readonly _started?: Date,
+  ) {
+    super();
+  }
+
+  override eq(other: this) {
+    return this._timestamp === other._timestamp;
+  }
+
+  override toDOM(view: EditorView) {
+    const el = document.createElement('div');
+    el.className = 'text-sm text-subdued hover:bg-hover-surface cursor-pointer';
+    el.textContent = formatTimestamp(this._timestamp, this._started);
+    // TODO(burdon): Click to bookmark or copy hyperlink.
+    el.onclick = () => {
+      const pos = view.state.doc.line(this._line).from;
+      view.dispatch({
+        effects: EditorView.scrollIntoView(pos, { y: 'start' }),
+      });
+    };
+
+    return el;
+  }
+}
+
+const getStartTime = (started?: Date, message?: Message.Message): Date | undefined => {
+  if (started) {
+    return started;
+  }
+
+  if (message?.blocks[0]?._tag === 'transcript' && message.blocks[0].started) {
+    return new Date(message.blocks[0].started);
+  }
+
+  return undefined;
+};
+
+const formatTimestamp = (timestamp: Date, relative?: Date) => {
+  if (relative) {
+    const pad = (n = 0) => String(n).padStart(2, '0');
+    const { hours, minutes, seconds } = intervalToDuration({ start: relative, end: timestamp });
+    return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
+  } else {
+    return format(timestamp, 'HH:mm:ss');
+  }
+};

package/src/components/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+export * from './MicSettings';
+export * from './PipelineStatus';
+export * from './Transcription';

package/src/hooks/index.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+export * from './useAudioFile';
+export * from './useAudioTrack';
+export * from './useFeedModelAdapter';
+export * from './useIsSpeaking';
+export * from './useRecordingPipeline';
+export * from './useSpeechRecognition';
+export * from './useTranscriber';

package/src/hooks/useAudioFile.ts

@@ -0,0 +1,88 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useEffect, useState } from 'react';
+
+import { scheduleTask } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { log } from '@dxos/log';
+
+export type UseAudioState = {
+  audio?: HTMLAudioElement;
+  stream?: MediaStream;
+  track?: MediaStreamTrack;
+};
+
+export const useAudioFile = (audioUrl: string, constraints?: MediaTrackConstraints): UseAudioState => {
+  const [{ audio, stream, track }, setStream] = useState<UseAudioState>({});
+  useEffect(() => {
+    const ctx = new Context();
+
+    scheduleTask(ctx, async () => {
+      // TODO(wittjosiah): Fetch to external url fails in headless storybook test.
+      try {
+        const response = await fetch(audioUrl);
+        const blob = await response.blob();
+        const objectUrl = URL.createObjectURL(blob);
+        ctx.onDispose(() => {
+          URL.revokeObjectURL(objectUrl);
+        });
+
+        const audio = new Audio();
+        audio.src = objectUrl;
+
+        await new Promise<void>((resolve, reject) => {
+          audio.addEventListener(
+            'error',
+            (err) => {
+              log.error('error', { err });
+              reject(err);
+            },
+            { once: true },
+          );
+          audio.addEventListener(
+            'canplay',
+            async () => {
+              resolve();
+            },
+            { once: true },
+          );
+          audio.load();
+        });
+
+        const audioCtx = new AudioContext();
+
+        // Resume AudioContext if it's suspended.
+        if (audioCtx.state === 'suspended') {
+          await audioCtx.resume();
+        }
+
+        const destination = audioCtx.createMediaStreamDestination();
+        destination.channelCount = 1;
+
+        const source = audioCtx.createMediaElementSource(audio);
+        source.connect(destination);
+        const track = destination.stream.getAudioTracks()[0];
+        if (constraints) {
+          await track.applyConstraints(constraints).catch((err) => log.catch(err));
+        }
+
+        // Also connect to speakers so audio is audible.
+        setStream({
+          audio,
+          stream: destination.stream,
+          track: destination.stream.getAudioTracks()[0],
+        });
+      } catch (err) {
+        log.catch(err);
+      }
+    });
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [audioUrl]);
+
+  return { audio, stream, track };
+};

package/src/hooks/useAudioTrack.ts

@@ -0,0 +1,56 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useEffect, useState } from 'react';
+
+/**
+ * Acquires a microphone audio track while `active`. Re-acquires when the `constraints` change (e.g.
+ * a different input device is selected) so the new device takes effect without a remount.
+ */
+// TODO(burdon): Reconcile with react-ui-experimental and plugin-calls.
+export const useAudioTrack = (active?: boolean, constraints?: MediaTrackConstraints): MediaStreamTrack | undefined => {
+  const [track, setTrack] = useState<MediaStreamTrack>();
+  // Re-acquire on a value change, not on every re-created constraints object identity.
+  const constraintsKey = constraints ? JSON.stringify(constraints) : '';
+  useEffect(() => {
+    if (!active) {
+      track?.stop();
+      setTrack(undefined);
+      return;
+    }
+
+    let cancelled = false;
+    let acquired: MediaStreamTrack | undefined;
+    // Drop any previous (now-stopped) track while reacquiring, so callers never hold a dead track.
+    setTrack(undefined);
+    const initAudio = async () => {
+      try {
+        const stream = await navigator.mediaDevices.getUserMedia({ audio: constraints ?? true });
+        const [next] = stream.getAudioTracks();
+        if (cancelled || !next) {
+          next?.stop();
+          return;
+        }
+        acquired = next;
+        setTrack(next);
+      } catch {
+        // Permission denied / device unavailable: leave the track cleared rather than rejecting.
+        if (!cancelled) {
+          setTrack(undefined);
+        }
+      }
+    };
+
+    void initAudio();
+
+    return () => {
+      cancelled = true;
+      acquired?.stop();
+    };
+    // `constraints` is read via the serialized `constraintsKey`.
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [active, constraintsKey]);
+
+  return track;
+};

package/src/hooks/useFeedModelAdapter.ts

@@ -0,0 +1,46 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useEffect, useMemo } from 'react';
+
+import type { Obj } from '@dxos/echo';
+
+import { type ChunkRenderer, TranscriptModel } from '../model';
+
+/**
+ * Feeds an array of objects (typically from `useQuery(db, Query.from(feed))`) into a
+ * `TranscriptModel`, appending each new tail item as the array grows.
+ *
+ * @param renderer - chunk renderer.
+ * @param objects - current snapshot of feed items.
+ * @param initialChunks - chunks to seed the model with on first render.
+ */
+export const useFeedModelAdapter = <T extends Obj.Unknown>(
+  renderer: ChunkRenderer<T>,
+  objects: readonly T[],
+  initialChunks: T[] = [],
+): TranscriptModel<T> => {
+  const model = useMemo(() => new TranscriptModel<T>(renderer, initialChunks), [renderer]);
+
+  // Seed the model with the current snapshot on first render and whenever the
+  // identity of `objects` changes (e.g. feed swapped).
+  useEffect(() => {
+    for (const chunk of objects) {
+      model.appendChunk(chunk);
+    }
+  }, [model, objects]);
+
+  // Append new tail items as the array grows.
+  useEffect(() => {
+    if (model.chunks.length === objects.length) {
+      return;
+    }
+    const chunk = objects.at(-1);
+    if (chunk) {
+      model.appendChunk(chunk);
+    }
+  }, [model, objects.length]);
+
+  return model;
+};

package/src/hooks/useIsSpeaking.ts

@@ -0,0 +1,42 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { useEffect, useMemo, useState } from 'react';
+
+import { scheduleTask } from '@dxos/async';
+import { SpeakingMonitor } from '@dxos/av';
+import { Context } from '@dxos/context';
+
+/**
+ * Tracks voice activity on an audio track via {@link SpeakingMonitor}.
+ */
+export const useIsSpeaking = (track?: MediaStreamTrack) => {
+  const [isSpeaking, setIsSpeaking] = useState(false);
+  const speakingMonitor = useMemo(() => {
+    if (!track) {
+      return;
+    }
+
+    return new SpeakingMonitor(track);
+  }, [track]);
+
+  useEffect(() => {
+    if (!speakingMonitor) {
+      return;
+    }
+
+    const ctx = new Context();
+    scheduleTask(ctx, async () => {
+      speakingMonitor.speakingChanged.on(ctx, () => setIsSpeaking(speakingMonitor.isSpeaking));
+      await speakingMonitor.open();
+      ctx.onDispose(() => speakingMonitor.close());
+    });
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [speakingMonitor]);
+
+  return isSpeaking;
+};

package/src/hooks/useRecordingPipeline.ts

@@ -0,0 +1,166 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { useEffect, useMemo, useRef, useState } from 'react';
+
+import { log } from '@dxos/log';
+import {
+  type AsrPipeline,
+  type CommitFn,
+  type EntityLookup,
+  type Stage,
+  type TranscribeConfig,
+  runAsrPipeline,
+} from '@dxos/transcription-pipeline';
+import { type ContentBlock } from '@dxos/types';
+
+import { MediaStreamRecorder } from '../capture';
+import { type PipelinePhase } from '../types';
+import { useAudioTrack } from './useAudioTrack';
+
+// Recorder chunk interval; the transcriber's chunk threshold derives from this.
+const RECORDER_INTERVAL_MS = 200;
+
+// Silence reported on drain so on-silence stages (e.g. summarization) fire over the final window.
+const DRAIN_SILENCE_MS = 5_000;
+
+const DEFAULT_TRANSCRIBE_CONFIG: TranscribeConfig = {
+  transcribeAfterChunksAmount: 50,
+  prefixBufferChunksAmount: 10,
+};
+
+export type RecordingPipelineOptions = {
+  /** Override the transcriber chunk configuration. */
+  config?: Partial<TranscribeConfig>;
+  /** Re-segment ASR output into complete sentences before the pipeline (merges mid-sentence cuts). */
+  segmentSentences?: boolean;
+  /**
+   * Raw capture flag — recording (mic) or playing (file). When it flips false the hook drains
+   * (flushing the transcriber's buffered audio) before returning to idle.
+   */
+  active: boolean;
+  /** Use the microphone as the ASR source; the hook owns the track and keeps it alive through drain. */
+  microphone?: boolean;
+  /** Externally-supplied track (file source). Used only when `microphone` is false. */
+  track?: MediaStreamTrack;
+  /** Audio constraints for the microphone source. */
+  audioConstraints?: MediaTrackConstraints;
+  /** Ordered pipeline stages applied to captured blocks. */
+  stages: readonly Stage<any, any>[];
+  /** Commit sink for enriched stage output. */
+  commit: CommitFn;
+  /** Optional entity-reference resolver injected into stages. */
+  lookup?: EntityLookup;
+  /** Optional raw-segment hook (pre-pipeline), e.g. for live display before enrichment. */
+  onSegment?: (block: ContentBlock.Transcript) => void;
+};
+
+export type RecordingPipeline = {
+  /** Lifecycle phase (idle → recording → draining → idle). */
+  phase: PipelinePhase;
+};
+
+/**
+ * Single ASR-capture engine binding for React consumers: drives an {@link AsrPipeline}
+ * ({@link runAsrPipeline}) off a capture flag, sourcing audio from the microphone (owned here) or an
+ * external track. The editor driver, message-list recording, and stories all build on this — varying
+ * only by source, stages, and commit sink.
+ */
+export const useRecordingPipeline = ({
+  active,
+  microphone,
+  track: externalTrack,
+  audioConstraints,
+  stages,
+  commit,
+  lookup,
+  onSegment,
+  config,
+  segmentSentences,
+}: RecordingPipelineOptions): RecordingPipeline => {
+  // Phase machine: hold 'recording' for the commit when `active` flips off, then 'draining' until the
+  // transcriber flushes, then 'idle' — so the tail of speech and its enrichment are not lost.
+  const [phase, setPhase] = useState<PipelinePhase>('idle');
+  const inUse = phase !== 'idle';
+  useEffect(() => {
+    setPhase((current) => (active ? 'recording' : current === 'recording' ? 'draining' : current));
+  }, [active]);
+
+  // Keep the mic alive through the drain; external (file) tracks are owned by the caller.
+  const micTrack = useAudioTrack(!!microphone && inUse, audioConstraints);
+  const track = microphone ? micTrack : externalTrack;
+
+  // A fresh recorder per track so a stopped (encoder-flushed) instance is never reused.
+  const recorder = useMemo(
+    () =>
+      track
+        ? new MediaStreamRecorder({ mediaStreamTrack: track, config: { interval: RECORDER_INTERVAL_MS } })
+        : undefined,
+    [track],
+  );
+
+  const pipelineRef = useRef<AsrPipeline | null>(null);
+
+  // Start a live ASR pipeline session when recording. Stages/commit/lookup are captured for the
+  // lifetime of the session (a config change mid-recording would orphan the run).
+  useEffect(() => {
+    if (phase !== 'recording' || !recorder) {
+      return;
+    }
+    const asr = runAsrPipeline({
+      recorder,
+      config: { ...DEFAULT_TRANSCRIBE_CONFIG, ...config },
+      segmentSentences,
+      stages,
+      commit,
+      lookup,
+      onSegment,
+      drainSilenceMs: DRAIN_SILENCE_MS,
+    });
+    pipelineRef.current = asr;
+    let cancelled = false;
+    void asr.open().then(
+      () => {
+        if (!cancelled) {
+          asr.start();
+        }
+      },
+      (err) => {
+        if (!cancelled) {
+          log.catch(err);
+          setPhase('idle');
+        }
+      },
+    );
+    return () => {
+      cancelled = true;
+    };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [phase, recorder]);
+
+  // Drain on stop: flush buffered audio, signal silence, end the session.
+  useEffect(() => {
+    if (phase !== 'draining') {
+      return;
+    }
+    let cancelled = false;
+    void (async () => {
+      try {
+        await pipelineRef.current?.end();
+      } catch (err) {
+        log.catch(err);
+      } finally {
+        pipelineRef.current = null;
+        if (!cancelled) {
+          setPhase((current) => (current === 'draining' ? 'idle' : current));
+        }
+      }
+    })();
+    return () => {
+      cancelled = true;
+    };
+  }, [phase]);
+
+  return { phase };
+};

package/src/hooks/useSpeechRecognition.ts

@@ -0,0 +1,56 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { useEffect, useMemo } from 'react';
+
+import { log } from '@dxos/log';
+
+const cleanText = (text: string) =>
+  text
+    .toLowerCase()
+    .replace(/[.,!?]/g, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+
+export type UseSpeechRecognitionProps = {
+  active: boolean;
+  onTranscript: (transcript: string) => void;
+};
+
+/**
+ * Use browser speech recognition API to transcribe speech to text.
+ */
+export const useSpeechRecognition = ({ active, onTranscript }: UseSpeechRecognitionProps) => {
+  const recognition = useMemo(() => {
+    const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
+    const SpeechGrammarList = (window as any).SpeechGrammarList || (window as any).webkitSpeechGrammarList;
+    if (!SpeechRecognition || !SpeechGrammarList) {
+      log.error('Speech recognition not supported');
+      return;
+    }
+
+    return Object.assign(new SpeechRecognition(), {
+      grammars: new SpeechGrammarList(),
+      continuous: true,
+      interimResults: true,
+      lang: 'en-US',
+      maxAlternatives: 1,
+      onresult: (event: any) => {
+        const transcript = cleanText(event.results[event.resultIndex][0].transcript);
+        onTranscript(transcript);
+      },
+      onerror: (event: any) => {
+        log.error('Speech recognition error', { error: event.error });
+      },
+    });
+  }, [onTranscript]);
+
+  useEffect(() => {
+    if (active) {
+      recognition?.start();
+    } else {
+      recognition?.stop();
+    }
+  }, [active, recognition]);
+};