@dxos/plugin-transcription 0.9.0 → 0.9.1-staging.ee54ba693a

package/src/capabilities/transcriber.ts

@@ -9,28 +9,12 @@ import { ClientCapabilities } from '@dxos/plugin-client';
 
 import { TranscriptionCapabilities } from '#types';
 
-import { MediaStreamRecorder, Transcriber, TranscriptionManager } from '../transcriber';
-
-// TODO(burdon): Move to config?
-
-/**
- * Length of the chunk in ms.
- */
-const RECORD_INTERVAL = 200;
-
-/**
- * Number of chunks to save before the user starts speaking.
- */
-const PREFIXED_CHUNKS_AMOUNT = 10;
+import { TranscriptionManagerImpl } from '../transcription-manager';
 
 /**
- * Number of chunks to transcribe automatically after.
- * Combined should be mess than 25MB or whisper would fail.
- */
-const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
-
-/**
- * Records audio while user is speaking and transcribes it after user is done speaking.
+ * Provides the higher-level transcription manager to the app-framework so other plugins can obtain it
+ * via DI. The low-level construction lives in `@dxos/react-ui-transcription`; this module is the
+ * provision seam.
  */
 export default Capability.makeModule(
   Effect.fnUntraced(function* () {
@@ -38,37 +22,11 @@ export default Capability.makeModule(
     const capabilities = yield* Capability.Service;
     const registry = yield* Capability.get(Capabilities.AtomRegistry);
 
-    const transcriberProvider: TranscriptionCapabilities.TranscriberProvider = ({
-      audioStreamTrack,
-      onSegments,
-      transcriberConfig,
-      recorderConfig,
-      transcribe,
-    }) => {
-      // Initialize audio transcription.
-      return new Transcriber({
-        config: {
-          transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
-          prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
-          ...transcriberConfig,
-        },
-        recorder: new MediaStreamRecorder({
-          mediaStreamTrack: audioStreamTrack,
-          config: {
-            interval: RECORD_INTERVAL,
-            ...recorderConfig,
-          },
-        }),
-        onSegments,
-        transcribe,
-      });
-    };
-
     const transcriptionManagerProvider: TranscriptionCapabilities.TranscriptionManagerProvider = ({
       messageEnricher,
     }) => {
       const client = capabilities.get(ClientCapabilities.Client);
-      const transcriptionManager = new TranscriptionManager({
+      const transcriptionManager = new TranscriptionManagerImpl({
         edgeClient: client.edge.http,
         messageEnricher,
         registry,
@@ -83,7 +41,6 @@ export default Capability.makeModule(
     };
 
     return [
-      Capability.contributes(TranscriptionCapabilities.TranscriberProvider, transcriberProvider),
       Capability.contributes(TranscriptionCapabilities.TranscriptionManagerProvider, transcriptionManagerProvider),
     ];
   }),

package/src/capabilities/transcription-driver.tsx

@@ -0,0 +1,263 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import React, { Fragment, useCallback, useEffect, useMemo, useRef, useState } from 'react';
+
+import { Capabilities, Capability } from '@dxos/app-framework';
+import { useAtomCapability, useAtomCapabilityState, useCapabilities } from '@dxos/app-framework/ui';
+import { log } from '@dxos/log';
+import { MarkdownCapabilities } from '@dxos/plugin-markdown/types';
+import { useAudioTrack, useTranscriber } from '@dxos/react-ui-transcription';
+import { linkEntities } from '@dxos/transcription-pipeline';
+import { type ContentBlock } from '@dxos/types';
+import { PendingTextStreamer, cancelPendingText, editorPendingTextSink, pendingTextState } from '@dxos/ui-editor';
+
+import { meta } from '#meta';
+import { TranscriptionCapabilities } from '#types';
+
+// Recorder chunk interval; the transcriber's chunk threshold is derived from the buffering setting.
+const RECORDER_INTERVAL_MS = 200;
+
+/**
+ * Headless controller driving live transcription into a Markdown editor. Reads the active recording
+ * session, captures audio while recording, and dispatches the transcribed text into the target
+ * editor as pending-text effects. The user confirms or discards the pending text via the editor's
+ * inline affordances. Mounted once, app-wide, via `Capabilities.ReactContext`.
+ */
+const TranscriptionDriver = () => {
+  const [session, setSession] = useAtomCapabilityState(TranscriptionCapabilities.RecordingSession);
+  // `useCapabilities` (not `useCapability`) so the driver does not throw when plugin-markdown is absent.
+  const [editorViews] = useCapabilities(MarkdownCapabilities.EditorViews);
+  // Injected entity resolver (full-text/vector/…); the driver stays decoupled from the database.
+  const [lookup] = useCapabilities(TranscriptionCapabilities.EntityLookup);
+  const settings = useAtomCapability(TranscriptionCapabilities.Settings);
+
+  const [, setStatus] = useAtomCapabilityState(TranscriptionCapabilities.PipelineStatus);
+
+  const recording = session?.recording ?? false;
+  const liveSessionId = session?.id;
+
+  // Only this driver's editor-backed sessions are captured here: if no editor view is registered for
+  // the session there is nowhere to stream transcription, so the driver stays idle and leaves the
+  // mic to whoever owns that session (e.g. a non-editor consumer running its own transcriber).
+  const hasEditorForSession = !!(liveSessionId && editorViews?.get(liveSessionId));
+
+  // Lifecycle phase. Driven from `recording` via an effect (not derived synchronously) so that when
+  // the mic switches off the phase is still 'recording' for that commit — nothing tears down until
+  // the drain finishes and sets 'idle'. This lets the final transcription + enrichment complete.
+  const [phase, setPhase] = useState<'idle' | 'recording' | 'draining'>('idle');
+  const active = phase !== 'idle';
+
+  // The toolbar clears the RecordingSession on stop, but the driver must keep targeting the same
+  // editor through the drain — capture the session id for the lifetime of the active phase (the
+  // editor stays open). Cleared once the phase returns to idle.
+  const [sessionId, setSessionId] = useState<string>();
+  useEffect(() => {
+    if (recording && liveSessionId && hasEditorForSession) {
+      setSessionId(liveSessionId);
+    }
+  }, [recording, liveSessionId, hasEditorForSession]);
+
+  useEffect(() => {
+    setPhase((current) =>
+      recording && hasEditorForSession ? 'recording' : current === 'recording' ? 'draining' : current,
+    );
+  }, [recording, hasEditorForSession]);
+
+  useEffect(() => {
+    if (phase === 'idle') {
+      setSessionId(undefined);
+    }
+  }, [phase]);
+
+  // Publish the phase so the toolbar / testbench can reflect mic + pipeline state in real time.
+  useEffect(() => {
+    setStatus(() => ({ phase }));
+  }, [phase, setStatus]);
+
+  // `ideal` (not `exact`) so an unavailable device falls back to the default rather than failing.
+  const audioConstraints = useMemo<MediaTrackConstraints | undefined>(
+    () => (settings?.audioDeviceId ? { deviceId: { ideal: settings.audioDeviceId } } : undefined),
+    [settings?.audioDeviceId],
+  );
+
+  // Keep the track (and thus the transcriber) alive across the whole active session — including the
+  // drain — so the transcriber can flush its buffered audio before teardown. Released at idle. The
+  // mic indicator therefore lingers for the brief drain after stop.
+  const track = useAudioTrack(active, audioConstraints);
+
+  // Refs so `handleSegments` stays stable (changing it would recreate the transcriber).
+  const sessionIdRef = useRef(sessionId);
+  sessionIdRef.current = sessionId;
+  const editorViewsRef = useRef(editorViews);
+  editorViewsRef.current = editorViews;
+
+  // The user cancelled (✕) if the pending decoration was cleared mid-session; guards against
+  // resurrecting it with late transcription / post-process writes during the drain.
+  const isCancelled = useCallback(() => {
+    const id = sessionIdRef.current;
+    const view = id ? editorViewsRef.current?.get(id)?.view : undefined;
+    return view ? view.state.field(pendingTextState, false) == null : false;
+  }, []);
+
+  // Streams transcribed text into the editor's pending-text block (buffering + word-by-word reveal).
+  // Kept alive across the whole session (incl. drain); disposed only when the phase returns to idle.
+  const streamerRef = useRef<PendingTextStreamer | null>(null);
+
+  useEffect(() => {
+    if (!active || !editorViews || !sessionId) {
+      return;
+    }
+    const entry = editorViews.get(sessionId);
+    if (!entry) {
+      return;
+    }
+
+    // Clicking the toolbar blurs the editor and hides the caret, so anchor + placeholder give
+    // immediate feedback while the first transcription is buffered.
+    // When entity extraction is enabled and a lookup is available, rewrite settled text with inline
+    // `[noun](echo:/<id>)` links — decorated as dx-anchors by the markdown editor's preview extension.
+    const entityExtraction = settings?.entityExtraction !== false;
+    const streamer = new PendingTextStreamer(editorPendingTextSink(entry.view), {
+      mode: settings?.streamMode ?? 'word',
+      wordIntervalMs: settings?.wordIntervalMs ?? 80,
+      postProcess: entityExtraction && lookup ? (text) => linkEntities(text, lookup) : undefined,
+    });
+    streamer.start({ anchor: entry.view.state.selection.main.head, placeholder: 'Recording…' });
+    streamerRef.current = streamer;
+
+    return () => {
+      streamer.dispose();
+      streamerRef.current = null;
+      // On teardown, drop the placeholder if nothing was transcribed; otherwise leave it for review.
+      const current = editorViews.get(sessionId);
+      const pending = current?.view.state.field(pendingTextState, false);
+      if (current && pending && pending.final.length === 0) {
+        current.view.dispatch({ effects: cancelPendingText.of() });
+      }
+    };
+  }, [
+    active,
+    sessionId,
+    editorViews,
+    lookup,
+    settings?.streamMode,
+    settings?.wordIntervalMs,
+    settings?.entityExtraction,
+  ]);
+
+  const handleSegments = useCallback(
+    async (segments: ContentBlock.Transcript[]) => {
+      const text = segments
+        .map((segment) => segment.text)
+        .join(' ')
+        .trim();
+      if (text.length > 0 && !isCancelled()) {
+        streamerRef.current?.push(text);
+      }
+    },
+    [isCancelled],
+  );
+
+  // Derive the transcriber's chunk threshold from the configured initial buffering time.
+  const transcriberConfig = useMemo(
+    () => ({
+      transcribeAfterChunksAmount: Math.max(
+        1,
+        Math.round((settings?.transcribeAfterMs ?? 4000) / RECORDER_INTERVAL_MS),
+      ),
+    }),
+    [settings?.transcribeAfterMs],
+  );
+
+  // Stable identity: a fresh object would change useTranscriber's memo deps every render, recreating
+  // (and reopening) the transcriber and discarding its buffered audio.
+  const recorderConfig = useMemo(() => ({ interval: RECORDER_INTERVAL_MS }), []);
+  const transcriber = useTranscriber({
+    audioStreamTrack: track,
+    onSegments: handleSegments,
+    transcriberConfig,
+    recorderConfig,
+  });
+  const transcriberRef = useRef(transcriber);
+  transcriberRef.current = transcriber;
+
+  useEffect(() => {
+    if (!transcriber) {
+      return;
+    }
+    let cancelled = false;
+    // Clear the session if `open()` rejects (e.g. mic permission denied) so the toolbar does not
+    // report "stop" forever while nothing is captured.
+    void transcriber.open().then(
+      () => {
+        if (!cancelled && phase === 'recording') {
+          transcriber.startChunksRecording();
+        }
+      },
+      (err) => {
+        if (!cancelled) {
+          log.catch(err);
+          setSession(() => null);
+        }
+      },
+    );
+    return () => {
+      cancelled = true;
+    };
+  }, [transcriber, phase, setSession]);
+
+  // Drain on stop: flush buffered audio (final transcription) and the streamer's post-process
+  // (entity linking) before tearing down — so the tail of speech and its enrichment are not lost.
+  useEffect(() => {
+    if (phase !== 'draining') {
+      return;
+    }
+    let cancelled = false;
+    // Hard-abort the in-flight transcription request if the user cancels (✕) during the drain.
+    const abortTimer = setInterval(() => {
+      if (isCancelled()) {
+        transcriberRef.current?.abort();
+      }
+    }, 150);
+    void (async () => {
+      try {
+        // Flush buffered audio (final transcription) then settle the streamer's post-process, unless
+        // the user cancelled (✕) — in which case skip the writes so the decoration stays cleared.
+        await transcriberRef.current?.flush();
+        if (!isCancelled()) {
+          await streamerRef.current?.flush();
+        }
+      } catch (err) {
+        log.catch(err);
+      } finally {
+        clearInterval(abortTimer);
+        if (!cancelled) {
+          setPhase((current) => (current === 'draining' ? 'idle' : current));
+        }
+      }
+    })();
+    return () => {
+      cancelled = true;
+      clearInterval(abortTimer);
+    };
+  }, [phase, isCancelled]);
+
+  return null;
+};
+
+export default Capability.makeModule(
+  Effect.fnUntraced(function* () {
+    return Capability.contributes(Capabilities.ReactContext, {
+      id: meta.profile.key,
+      context: ({ children }) => (
+        <Fragment>
+          {children}
+          <TranscriptionDriver />
+        </Fragment>
+      ),
+    });
+  }),
+);

package/src/components/Mic/Mic.stories.tsx

@@ -0,0 +1,66 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import React, { useState } from 'react';
+
+import { MicButton, Toolbar } from '@dxos/react-ui';
+import { type AudioInputDevice, MicSettings, type RecordMode } from '@dxos/react-ui-transcription';
+import { withLayout, withTheme } from '@dxos/react-ui/testing';
+
+import { meta as pluginMeta } from '#meta';
+import { translations } from '#translations';
+
+const DEVICES: AudioInputDevice[] = [
+  { deviceId: 'default', label: 'Default — MacBook Pro Microphone' },
+  { deviceId: 'usb-1', label: 'USB Audio Interface' },
+  { deviceId: 'bt-1', label: 'AirPods Pro' },
+];
+
+// Drives the presentational pieces from local state so the dropdown, mode switch, and
+// press-and-hold can be exercised without capabilities.
+const DefaultStory = () => {
+  const [recording, setRecording] = useState(false);
+  const [recordMode, setRecordMode] = useState<RecordMode>('toggle');
+  const [entityExtraction, setEntityExtraction] = useState(true);
+  const [selectedDeviceId, setSelectedDeviceId] = useState('');
+
+  return (
+    <Toolbar.Root>
+      <MicButton
+        iconOnly
+        variant='ghost'
+        label={recording ? 'Stop recording' : recordMode === 'hold' ? 'Hold to record' : 'Start recording'}
+        recording={recording}
+        mode={recordMode}
+        onToggle={() => setRecording((value) => !value)}
+        onPressStart={() => setRecording(true)}
+        onPressEnd={() => setRecording(false)}
+      />
+      <MicSettings
+        translationNs={pluginMeta.profile.key}
+        recordMode={recordMode}
+        entityExtraction={entityExtraction}
+        devices={DEVICES}
+        selectedDeviceId={selectedDeviceId}
+        onRecordModeChange={setRecordMode}
+        onEntityExtractionChange={setEntityExtraction}
+        onSelectDevice={setSelectedDeviceId}
+      />
+    </Toolbar.Root>
+  );
+};
+
+const meta = {
+  title: 'plugins/plugin-transcription/components/Mic',
+  render: DefaultStory,
+  decorators: [withTheme(), withLayout({ layout: 'centered' })],
+  parameters: { translations },
+} satisfies Meta<typeof DefaultStory>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {};

package/src/components/Mic/Mic.tsx

@@ -0,0 +1,136 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import React, { useCallback, useEffect, useRef, useState } from 'react';
+
+import { useAtomCapabilityState } from '@dxos/app-framework/ui';
+import { MicButton, useTranslation } from '@dxos/react-ui';
+import { useSoundEffect } from '@dxos/react-ui-audio';
+import { type AudioInputDevice, MicSettings } from '@dxos/react-ui-transcription';
+
+import { meta } from '#meta';
+import { type Settings, TranscriptionCapabilities } from '#types';
+
+export type MicProps = {
+  /** Attendable id of the target editor; keys the recording session. */
+  docId: string;
+};
+
+/**
+ * Connects the {@link MicButton} and {@link MicSettings} to the recording session and settings
+ * capabilities, and enumerates available audio inputs. Mounted from the toolbar's custom action, so
+ * capability hooks resolve.
+ */
+export const Mic = ({ docId }: MicProps) => {
+  const { t } = useTranslation(meta.profile.key);
+  const [session, setSession] = useAtomCapabilityState(TranscriptionCapabilities.RecordingSession);
+  const [settings, setSettings] = useAtomCapabilityState(TranscriptionCapabilities.Settings);
+
+  const recording = !!session?.recording && session.id === docId;
+
+  // Play start/stop cues as this session's recording flag flips (skip the initial mount).
+  const soundStart = useSoundEffect('StartRecording');
+  const soundStop = useSoundEffect('StopRecording');
+  const wasRecording = useRef(recording);
+  useEffect(() => {
+    if (wasRecording.current === recording) {
+      return;
+    }
+    wasRecording.current = recording;
+    void (recording ? soundStart : soundStop).play();
+  }, [recording, soundStart, soundStop]);
+
+  const recordMode: Settings.RecordMode = settings?.recordMode ?? 'toggle';
+  const entityExtraction = settings?.entityExtraction !== false;
+  const selectedDeviceId = settings?.audioDeviceId ?? '';
+
+  const [devices, setDevices] = useState<AudioInputDevice[]>([]);
+  useEffect(() => {
+    if (!navigator.mediaDevices?.enumerateDevices) {
+      return;
+    }
+    let cancelled = false;
+    const refresh = async () => {
+      const all = await navigator.mediaDevices.enumerateDevices();
+      if (cancelled) {
+        return;
+      }
+      // Labels are blank until microphone permission is granted; fall back to an ordinal name.
+      setDevices(
+        all
+          .filter((device) => device.kind === 'audioinput' && device.deviceId)
+          .map((device, index) => ({
+            deviceId: device.deviceId,
+            label: device.label || `Microphone ${index + 1}`,
+          })),
+      );
+    };
+    void refresh();
+    navigator.mediaDevices.addEventListener('devicechange', refresh);
+    return () => {
+      cancelled = true;
+      navigator.mediaDevices.removeEventListener('devicechange', refresh);
+    };
+  }, []);
+
+  const handleToggle = useCallback(() => {
+    setSession((current) => (current?.recording && current.id === docId ? null : { id: docId, recording: true }));
+  }, [setSession, docId]);
+
+  const handlePressStart = useCallback(() => {
+    setSession(() => ({ id: docId, recording: true }));
+  }, [setSession, docId]);
+
+  const handlePressEnd = useCallback(() => {
+    // Only stop our own session; a stray release must not clear another editor's recording.
+    setSession((current) => (current?.id === docId ? null : current));
+  }, [setSession, docId]);
+
+  const handleRecordModeChange = useCallback(
+    (mode: Settings.RecordMode) => setSettings((current) => ({ ...current, recordMode: mode })),
+    [setSettings],
+  );
+
+  const handleEntityExtractionChange = useCallback(
+    (value: boolean) => setSettings((current) => ({ ...current, entityExtraction: value })),
+    [setSettings],
+  );
+
+  const handleSelectDevice = useCallback(
+    (deviceId: string) => setSettings((current) => ({ ...current, audioDeviceId: deviceId || undefined })),
+    [setSettings],
+  );
+
+  const recordLabel = recording
+    ? t('stop-recording.label')
+    : recordMode === 'hold'
+      ? t('hold-to-record.label')
+      : t('start-recording.label');
+
+  return (
+    <div className='flex items-center'>
+      <MicButton
+        iconOnly
+        variant='ghost'
+        label={recordLabel}
+        recording={recording}
+        mode={recordMode}
+        onToggle={handleToggle}
+        onPressStart={handlePressStart}
+        onPressEnd={handlePressEnd}
+        data-testid='transcription.record'
+      />
+      <MicSettings
+        translationNs={meta.profile.key}
+        recordMode={recordMode}
+        entityExtraction={entityExtraction}
+        devices={devices}
+        selectedDeviceId={selectedDeviceId}
+        onRecordModeChange={handleRecordModeChange}
+        onEntityExtractionChange={handleEntityExtractionChange}
+        onSelectDevice={handleSelectDevice}
+      />
+    </div>
+  );
+};

package/src/components/Mic/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+export * from './Mic';

package/src/components/index.ts

@@ -2,5 +2,4 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './Oracle';
-export * from './Transcription';
+export * from './Mic';

package/src/containers/TranscriptionArticle/TranscriptionArticle.stories.tsx

@@ -7,6 +7,7 @@ import * as Effect from 'effect/Effect';
 import React from 'react';
 
 import { withPluginManager } from '@dxos/app-framework/testing';
+import { AppActivationEvents } from '@dxos/app-toolkit';
 import { Feed, Query, Ref } from '@dxos/echo';
 import { ClientPlugin, initializeIdentity } from '@dxos/plugin-client/testing';
 import { corePlugins } from '@dxos/plugin-testing';
@@ -36,6 +37,9 @@ const meta = {
   decorators: [
     withLayout({ layout: 'column' }),
     withPluginManager({
+      // The app-wide TranscriptionDriver (ReactContext) reads the recording-session/settings
+      // capabilities, which activate on SetupSettings — fire it so the driver does not throw.
+      setupEvents: [AppActivationEvents.SetupSettings],
       plugins: [
         ...corePlugins(),
         ClientPlugin({

package/src/containers/TranscriptionArticle/TranscriptionArticle.tsx

@@ -9,14 +9,12 @@ import { Filter, Obj, Query } from '@dxos/echo';
 import { useMembers, useQuery } from '@dxos/react-client/echo';
 import { Panel } from '@dxos/react-ui';
 import { Menu, MenuBuilder, useMenuBuilder } from '@dxos/react-ui-menu';
+import { Transcription, renderByline, useFeedModelAdapter } from '@dxos/react-ui-transcription';
 import { Message, type Transcript } from '@dxos/types';
 
-import { Transcription } from '#components';
-import { useFeedModelAdapter, useTranscriptionRecording } from '#hooks';
+import { useTranscriptionRecording } from '#hooks';
 import { meta } from '#meta';
 
-import { renderByline } from '../../util';
-
 export type TranscriptionArticleProps = AppSurface.ObjectArticleProps<Transcript.Transcript>;
 
 export const TranscriptionArticle = ({ role, subject: transcript, attendableId }: TranscriptionArticleProps) => {
@@ -28,15 +26,16 @@ export const TranscriptionArticle = ({ role, subject: transcript, attendableId }
     feed ? Query.select(Filter.type(Message.Message)).from(feed) : Query.select(Filter.nothing()),
   );
   const model = useFeedModelAdapter(renderByline(members), messages);
-  const { recording, toggleRecording } = useTranscriptionRecording(transcript);
 
+  // TODO(burdon): Remove if not mutable. E.g., finalized transcript.
+  const { recording, toggleRecording } = useTranscriptionRecording(transcript);
   const menuActions = useMenuBuilder(
     () =>
       MenuBuilder.make()
         .action(
           'toggle-recording',
           {
-            label: [recording ? 'stop-recording.label' : 'start-recording.label', { ns: meta.id }],
+            label: [recording ? 'stop-recording.label' : 'start-recording.label', { ns: meta.profile.key }],
             icon: recording ? 'ph--stop-circle--regular' : 'ph--microphone--regular',
             disposition: 'toolbar',
             testId: 'transcription.toggle-recording',

package/src/hooks/index.ts

@@ -2,9 +2,4 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './useAudioFile';
-export * from './useAudioTrack';
-export * from './useFeedModelAdapter';
-export * from './useTranscriber';
 export * from './useTranscriptionRecording';
-export * from './useVoiceInput';