@dxos/react-ui-transcription 0.0.0

package/package.json

@@ -0,0 +1,69 @@
+{
+  "name": "@dxos/react-ui-transcription",
+  "version": "0.0.0",
+  "description": "Browser audio capture, transcription hooks, and UI for the transcription pipeline.",
+  "homepage": "https://dxos.org",
+  "bugs": "https://github.com/dxos/dxos/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
+  "license": "FSL-1.1-Apache-2.0",
+  "author": "DXOS.org",
+  "type": "module",
+  "exports": {
+    ".": {
+      "source": "./src/index.ts",
+      "types": "./dist/types/src/index.d.ts",
+      "browser": "./dist/lib/browser/index.mjs",
+      "node": "./dist/lib/node-esm/index.mjs"
+    }
+  },
+  "types": "dist/types/src/index.d.ts",
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@codemirror/state": "catalog:",
+    "@codemirror/view": "catalog:",
+    "@dxos/async": "workspace:*",
+    "@dxos/av": "workspace:*",
+    "@dxos/context": "workspace:*",
+    "@dxos/echo": "workspace:*",
+    "@dxos/invariant": "workspace:*",
+    "@dxos/log": "workspace:*",
+    "@dxos/react-client": "workspace:*",
+    "@dxos/react-ui": "workspace:*",
+    "@dxos/react-ui-editor": "workspace:*",
+    "@dxos/tracing": "workspace:*",
+    "@dxos/transcription-pipeline": "workspace:*",
+    "@dxos/types": "workspace:*",
+    "@dxos/ui-editor": "workspace:*",
+    "@dxos/ui-theme": "workspace:*",
+    "@dxos/ui-types": "workspace:*",
+    "@dxos/util": "workspace:*",
+    "date-fns": "catalog:",
+    "extendable-media-recorder": "catalog:",
+    "extendable-media-recorder-wav-encoder": "catalog:",
+    "wavefile": "catalog:"
+  },
+  "devDependencies": {
+    "@types/react": "catalog:",
+    "@types/react-dom": "catalog:",
+    "effect": "catalog:",
+    "react": "catalog:",
+    "react-dom": "catalog:",
+    "vite": "catalog:"
+  },
+  "peerDependencies": {
+    "@dxos/react-ui": "workspace:*",
+    "@dxos/ui-theme": "workspace:*",
+    "effect": "catalog:",
+    "react": "catalog:",
+    "react-dom": "catalog:"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/capture/create-transcriber.ts

@@ -0,0 +1,49 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type TranscribeConfig, type TranscribeFn, Transcriber } from '@dxos/transcription-pipeline';
+import { type ContentBlock } from '@dxos/types';
+
+import { MediaStreamRecorder } from './media-stream-recorder';
+
+// Recorder chunk interval (ms).
+const RECORD_INTERVAL = 200;
+// Chunks retained before the user starts speaking, to catch the onset of speech.
+const PREFIXED_CHUNKS_AMOUNT = 10;
+// Chunks transcribed automatically; combined should stay under ~25MB or Whisper fails.
+const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
+
+export type CreateTranscriberOptions = {
+  audioStreamTrack: MediaStreamTrack;
+  onSegments: (segments: ContentBlock.Transcript[]) => Promise<void>;
+  transcriberConfig?: Partial<TranscribeConfig>;
+  recorderConfig?: { interval?: number };
+  transcribe?: TranscribeFn;
+};
+
+/**
+ * Builds a {@link Transcriber} backed by a browser {@link MediaStreamRecorder} for the given track —
+ * the low-level audio→text construction, co-located with the recorder so it can be tested outside the
+ * plugin. Consumed directly via the {@link useTranscriber} hook.
+ */
+export const createTranscriber = ({
+  audioStreamTrack,
+  onSegments,
+  transcriberConfig,
+  recorderConfig,
+  transcribe,
+}: CreateTranscriberOptions): Transcriber =>
+  new Transcriber({
+    config: {
+      transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
+      prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
+      ...transcriberConfig,
+    },
+    recorder: new MediaStreamRecorder({
+      mediaStreamTrack: audioStreamTrack,
+      config: { interval: RECORD_INTERVAL, ...recorderConfig },
+    }),
+    onSegments,
+    transcribe,
+  });

package/src/capture/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+export * from './create-transcriber';
+export * from './media-stream-recorder';

package/src/capture/media-stream-recorder.ts

@@ -0,0 +1,137 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type IBlobEvent, type IMediaRecorder } from 'extendable-media-recorder';
+import { WaveFile } from 'wavefile';
+
+import { synchronized } from '@dxos/async';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+import { trace } from '@dxos/tracing';
+import { type AudioChunk, type AudioRecorder, type WavConfig } from '@dxos/transcription-pipeline';
+
+// `extendable-media-recorder` references browser-only globals (Worker/AudioWorklet) at module load,
+// so it is imported lazily — keeping this package importable in non-browser contexts (node tests,
+// Cloudflare Workers) where the recorder is never started.
+let MediaRecorderClass: typeof import('extendable-media-recorder').MediaRecorder | undefined;
+let initializingPromise: Promise<void> | undefined;
+
+const initializeExtendableMediaRecorder = async () => {
+  const { MediaRecorder, register } = await import('extendable-media-recorder');
+  const { connect } = await import('extendable-media-recorder-wav-encoder');
+  await register(await connect());
+  MediaRecorderClass = MediaRecorder;
+};
+
+export type MediaStreamRecorderProps = {
+  mediaStreamTrack: MediaStreamTrack;
+  config: {
+    interval: number;
+  };
+};
+
+/**
+ * Recorder that uses the MediaContext API and AudioNode API to record audio.
+ *
+ * It records MediaStream using https://www.npmjs.com/package/extendable-media-recorder.
+ */
+@trace.resource()
+export class MediaStreamRecorder implements AudioRecorder {
+  private readonly _mediaStreamTrack: MediaStreamTrack;
+  /**
+   * Default MediaRecorder implementation do not support wav encoding.
+   */
+  private readonly _config: MediaStreamRecorderProps['config'];
+  private _mediaRecorder?: IMediaRecorder = undefined;
+  private _header?: Uint8Array = undefined;
+  private _onChunk?: (chunk: AudioChunk) => void;
+
+  constructor({ mediaStreamTrack, config }: MediaStreamRecorderProps) {
+    this._mediaStreamTrack = mediaStreamTrack;
+    this._config = config;
+  }
+
+  get wavConfig(): WavConfig {
+    const settings = this._mediaStreamTrack.getSettings();
+    return {
+      channels: settings.channelCount ?? 1,
+      sampleRate: settings.sampleRate ?? 16000,
+      bitDepthCode: settings.sampleSize ? String(settings.sampleSize) : '16',
+    };
+  }
+
+  get mediaStreamTrack() {
+    return this._mediaStreamTrack;
+  }
+
+  setOnChunk(onChunk: (chunk: AudioChunk) => void): void {
+    this._onChunk = onChunk;
+  }
+
+  // `@synchronized` serializes start/stop (they share the instance lock) so a rapid stop→start cycle
+  // (e.g. React StrictMode double-mount) cannot initiate a new WAV encoding before the previous one
+  // has flushed — which throws "Another request was made to initiate an encoding" and kills capture.
+  @synchronized
+  async start(): Promise<void> {
+    await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) =>
+      log.info('initializeExtendableMediaRecorder', { err }),
+    );
+
+    if (this._mediaRecorder?.state === 'recording') {
+      return;
+    }
+    invariant(this._onChunk, 'MediaStreamRecorder: onChunk is not set');
+    invariant(MediaRecorderClass, 'MediaStreamRecorder: extendable-media-recorder not initialized');
+    // Fresh recorder per session so a previously-stopped (encoder-flushed) instance is never reused.
+    const stream = new MediaStream([this._mediaStreamTrack]);
+    const recorder = new MediaRecorderClass(stream, { mimeType: 'audio/wav' });
+    recorder.ondataavailable = (event) => this._ondataavailable(event);
+    this._mediaRecorder = recorder;
+    recorder.start(this._config.interval);
+  }
+
+  @synchronized
+  async stop(): Promise<void> {
+    const recorder = this._mediaRecorder;
+    if (!recorder || recorder.state !== 'recording') {
+      return;
+    }
+    // Await the actual stop (final dataavailable + encoder flush) before releasing the recorder, so a
+    // subsequent start() does not collide with an in-progress encoding.
+    await new Promise<void>((resolve) => {
+      recorder.addEventListener('stop', () => resolve(), { once: true });
+      recorder.stop();
+    });
+    this._mediaRecorder = undefined;
+  }
+
+  @synchronized
+  private async _ondataavailable(event: IBlobEvent): Promise<void> {
+    const blob = event.data;
+    const uint8Array = new Uint8Array(await blob.arrayBuffer());
+
+    // First chunk from the MediaRecorder has a header.
+    const isHeader =
+      uint8Array[0] === 0x52 && // R
+      uint8Array[1] === 0x49 && // I
+      uint8Array[2] === 0x46 && // F
+      uint8Array[3] === 0x46; // F
+    let wav: WaveFile;
+
+    if (isHeader) {
+      wav = new WaveFile(uint8Array);
+      this._header = uint8Array.slice(0, 44);
+    } else if (this._header) {
+      wav = new WaveFile(new Uint8Array([...this._header, ...uint8Array]));
+    } else {
+      log.warn('MediaStreamRecorder: no header');
+      return;
+    }
+
+    this._onChunk!({
+      timestamp: Date.now(),
+      data: wav.getSamples(),
+    });
+  }
+}

package/src/components/MicSettings/MicSettings.tsx

@@ -0,0 +1,126 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import React from 'react';
+
+import { DropdownMenu, Icon, IconButton, useTranslation } from '@dxos/react-ui';
+
+/** Recording trigger mode: toggle on/off, or hold-to-record (push-to-talk). */
+export type RecordMode = 'toggle' | 'hold';
+
+export type AudioInputDevice = {
+  deviceId: string;
+  label: string;
+};
+
+export type MicSettingsProps = {
+  /** Translation namespace for the option labels (owned by the consuming plugin). */
+  translationNs: string;
+  recordMode: RecordMode;
+  entityExtraction: boolean;
+  devices: AudioInputDevice[];
+  /** Empty string selects the system default device. */
+  selectedDeviceId: string;
+  onRecordModeChange: (mode: RecordMode) => void;
+  onEntityExtractionChange: (value: boolean) => void;
+  onSelectDevice: (deviceId: string) => void;
+};
+
+/**
+ * Dropdown of recording options for the mic: record mode (toggle/push-to-talk), the input device,
+ * and whether entity extraction is applied. Presentational — state lives in the consumer.
+ */
+export const MicSettings = ({
+  translationNs,
+  recordMode,
+  entityExtraction,
+  devices,
+  selectedDeviceId,
+  onRecordModeChange,
+  onEntityExtractionChange,
+  onSelectDevice,
+}: MicSettingsProps) => {
+  const { t } = useTranslation(translationNs);
+
+  return (
+    <DropdownMenu.Root>
+      <DropdownMenu.Trigger asChild>
+        <IconButton
+          icon='ph--caret-down--regular'
+          iconOnly
+          label={t('recording-options.label')}
+          variant='ghost'
+          // Half-width slim caret abutting the mic (split-control affordance).
+          classNames='w-4'
+          data-testid='transcription.record.options'
+        />
+      </DropdownMenu.Trigger>
+      <DropdownMenu.Portal>
+        <DropdownMenu.Content>
+          <DropdownMenu.Viewport>
+            <DropdownMenu.GroupLabel>{t('record-mode.label')}</DropdownMenu.GroupLabel>
+            <SelectableItem
+              label={t('record-mode.toggle.label')}
+              selected={recordMode === 'toggle'}
+              onSelect={() => onRecordModeChange('toggle')}
+            />
+            <SelectableItem
+              label={t('record-mode.hold.label')}
+              selected={recordMode === 'hold'}
+              onSelect={() => onRecordModeChange('hold')}
+            />
+
+            <DropdownMenu.Separator />
+
+            <DropdownMenu.GroupLabel>{t('audio-device.label')}</DropdownMenu.GroupLabel>
+            <SelectableItem
+              label={t('audio-device.default.label')}
+              selected={selectedDeviceId === ''}
+              onSelect={() => onSelectDevice('')}
+            />
+            {devices.map((device) => (
+              <SelectableItem
+                key={device.deviceId}
+                label={device.label}
+                selected={selectedDeviceId === device.deviceId}
+                onSelect={() => onSelectDevice(device.deviceId)}
+              />
+            ))}
+
+            <DropdownMenu.Separator />
+
+            <DropdownMenu.CheckboxItem
+              checked={entityExtraction}
+              onCheckedChange={onEntityExtractionChange}
+              classNames='gap-2'
+              data-testid='transcription.entity-extraction'
+            >
+              <span className='grow truncate'>{t('settings.entity-extraction.label')}</span>
+              <DropdownMenu.ItemIndicator asChild>
+                <Icon icon='ph--check--regular' size={4} />
+              </DropdownMenu.ItemIndicator>
+            </DropdownMenu.CheckboxItem>
+          </DropdownMenu.Viewport>
+          <DropdownMenu.Arrow />
+        </DropdownMenu.Content>
+      </DropdownMenu.Portal>
+    </DropdownMenu.Root>
+  );
+};
+
+type SelectableItemProps = {
+  label: string;
+  selected: boolean;
+  onSelect: () => void;
+};
+
+// The design system's `DropdownMenu.RadioItem` renders a plain item (no radio semantics), so
+// single-select is modelled with a plain item and an explicit trailing check. `onSelect` (not
+// `onClick`) so keyboard activation works.
+const SelectableItem = ({ label, selected, onSelect }: SelectableItemProps) => (
+  <DropdownMenu.Item classNames='gap-2' onSelect={onSelect}>
+    <span className='grow truncate'>{label}</span>
+    {selected && <Icon icon='ph--check--regular' size={4} />}
+  </DropdownMenu.Item>
+);

package/src/components/MicSettings/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+export * from './MicSettings';

package/src/components/PipelineStatus/PipelineStatus.tsx

@@ -0,0 +1,71 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import React, { Fragment } from 'react';
+
+import { type TelemetryEvent } from '@dxos/transcription-pipeline';
+
+import { type PipelinePhase } from '../../types';
+
+export type PipelineStatusProps = {
+  /** Live lifecycle phase published by the transcription driver. */
+  phase?: PipelinePhase;
+  /** Ordered stage ids being run (e.g. correct → extract → summarize). */
+  stages?: readonly string[];
+  /** Per-stage telemetry emitted by `PipelineRuntime`. */
+  telemetry?: readonly TelemetryEvent[];
+  /** Cumulative summary produced by the summarization stage. */
+  summary?: string;
+};
+
+/**
+ * Read-only status panel reflecting the transcription/pipeline state: mic + pipeline phase, the
+ * active stage list, per-stage telemetry, and the running summary. Driven by the `PipelineStatus`
+ * capability (phase) and `PipelineRuntime` telemetry.
+ */
+export const PipelineStatus = ({ phase = 'idle', stages, telemetry, summary }: PipelineStatusProps) => (
+  <div className='flex flex-col gap-3 overflow-y-auto p-2 text-sm'>
+    <Section title='Status'>
+      <div className='grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-description'>
+        <span>Mic</span>
+        <span>{phase === 'recording' ? '● on' : '○ off'}</span>
+        <span>Pipeline</span>
+        <span>{phase}</span>
+      </div>
+    </Section>
+
+    {stages && (
+      <Section title='Stages'>
+        <div className='text-description'>{stages.join(' → ') || '(none)'}</div>
+      </Section>
+    )}
+
+    {telemetry && telemetry.length > 0 && (
+      <Section title='Telemetry'>
+        <div className='grid w-fit grid-cols-[1fr_auto_auto] gap-x-4 gap-y-1'>
+          {telemetry.map((event, index) => (
+            <Fragment key={index}>
+              <span>{event.stageId}</span>
+              <span className='text-description'>{event.outcome}</span>
+              <span className='text-right tabular-nums text-description'>{event.durationMs}ms</span>
+            </Fragment>
+          ))}
+        </div>
+      </Section>
+    )}
+
+    {summary && (
+      <Section title='Summary'>
+        <p className='text-description'>{summary}</p>
+      </Section>
+    )}
+  </div>
+);
+
+const Section = ({ title, children }: { title: string; children: React.ReactNode }) => (
+  <div>
+    <h3 className='mb-1 font-medium'>{title}</h3>
+    {children}
+  </div>
+);

package/src/components/PipelineStatus/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+export * from './PipelineStatus';

package/src/components/Transcription/Transcription.tsx

@@ -0,0 +1,67 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import React from 'react';
+
+import { composable, composableProps, useThemeContext } from '@dxos/react-ui';
+import { useTextEditor } from '@dxos/react-ui-editor';
+import { type Message, type Transcript } from '@dxos/types';
+import {
+  AnchorWidget,
+  type XmlWidgetProps,
+  type XmlWidgetRegistry,
+  createBasicExtensions,
+  createMarkdownExtensions,
+  createThemeExtensions,
+  decorateMarkdown,
+  documentSlots,
+  scroller,
+  xmlTags,
+} from '@dxos/ui-editor';
+
+import { type TranscriptModel } from '../../model';
+import { transcription } from './transcription-extension';
+
+const inlinePreviewRegistry: XmlWidgetRegistry = {
+  'link-preview': {
+    block: false,
+    urlSchemes: ['dxn:', 'echo:'],
+    factory: ({ label, dxn }: XmlWidgetProps<{ label: string; dxn: string }>) =>
+      typeof label === 'string' && typeof dxn === 'string' ? new AnchorWidget(label, dxn) : null,
+  },
+};
+
+export type TranscriptionProps = {
+  transcript?: Transcript.Transcript;
+  model: TranscriptModel<Message.Message>;
+};
+
+// TODO(burdon): Rename Transcript.
+export const Transcription = composable<HTMLDivElement, TranscriptionProps>(
+  ({ transcript: object, model, children, ...props }, forwardedRef) => {
+    const { themeMode } = useThemeContext();
+    const { parentRef } = useTextEditor(() => {
+      return {
+        extensions: [
+          createBasicExtensions({ readOnly: true, lineWrapping: true, search: true }),
+          createThemeExtensions({ themeMode, slots: documentSlots }),
+          createMarkdownExtensions(),
+          // xmlTags() handles dxn:/echo: links via url-scheme widgets; skip here to avoid double-processing.
+          decorateMarkdown({ skip: ({ url }) => url.startsWith('dxn:') || url.startsWith('echo:') }),
+          xmlTags({ registry: inlinePreviewRegistry }),
+          transcription({ model, started: object?.started ? new Date(object.started) : undefined }),
+          scroller(),
+        ],
+      };
+    }, [model, themeMode]);
+
+    return (
+      <div
+        {...composableProps(props, { classNames: 'dx-container' })}
+        data-popover-collision-boundary={true}
+        ref={parentRef}
+      />
+    );
+  },
+);

package/src/components/Transcription/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './Transcription';
+export * from './transcription-extension';