@dxos/plugin-transcription 0.9.1-main.c7dcc2e112 → 0.10.0

package/src/enrichment/parser.ts

@@ -1,101 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-// Parser for the human-authored conversation fixture (see `conversation.txt`).
-// Captures both the seed entities (Persons, Organizations, …) and the speaker-tagged
-// dialogue used by `stream-simulator.ts` to feed the enrichment pipeline.
-
-export type ConversationEntity = {
-  /** Type name as written in the fixture, e.g. `Person`, `Organization`. */
-  kind: string;
-  /** Free-form field values; consumers map these to the relevant ECHO schema. */
-  fields: Record<string, string>;
-};
-
-export type ConversationUtterance = {
-  /** Speaker label as written in the fixture (e.g. `A`, `B`, `Rich`). */
-  speaker: string;
-  /** Raw utterance text. */
-  text: string;
-};
-
-export type ParsedConversation = {
-  entities: ConversationEntity[];
-  utterances: ConversationUtterance[];
-  /** Distinct speaker labels in first-seen order. */
-  speakers: string[];
-};
-
-/**
- * Parse the on-disk conversation format into structured data.
- *
- * Tolerates leading whitespace inside blocks and ignores blank / `#` comment lines.
- * Throws if the input doesn't contain a `Conversation:` block — `Entities:` is optional.
- */
-export const parseConversation = (input: string): ParsedConversation => {
-  const lines = input.split('\n').map((line) => line.replace(/\s+$/, ''));
-
-  const entities: ConversationEntity[] = [];
-  const utterances: ConversationUtterance[] = [];
-  const speakerOrder = new Set<string>();
-
-  type Section = 'none' | 'entities' | 'conversation';
-  let section: Section = 'none';
-  let currentEntity: ConversationEntity | undefined;
-
-  for (const raw of lines) {
-    const line = raw.trim();
-    if (line.length === 0 || line.startsWith('#')) {
-      continue;
-    }
-
-    // Section headers (must be left-aligned to switch sections).
-    if (raw.startsWith('Entities:') || raw.startsWith('Entitites:')) {
-      // tolerate the typo present in earlier fixtures
-      section = 'entities';
-      currentEntity = undefined;
-      continue;
-    }
-    if (raw.startsWith('Conversation:')) {
-      section = 'conversation';
-      currentEntity = undefined;
-      continue;
-    }
-
-    if (section === 'entities') {
-      // `<TypeName>:` starts a new entity; nested `<field>: <value>` lines fill it.
-      const typeMatch = line.match(/^([A-Za-z][A-Za-z0-9_-]*):$/);
-      if (typeMatch) {
-        currentEntity = { kind: typeMatch[1], fields: {} };
-        entities.push(currentEntity);
-        continue;
-      }
-      const fieldMatch = line.match(/^([A-Za-z][A-Za-z0-9_-]*):\s*(.+)$/);
-      if (fieldMatch && currentEntity) {
-        currentEntity.fields[fieldMatch[1]] = fieldMatch[2];
-        continue;
-      }
-      // Silently skip unrecognised lines so authors can add free-form annotations.
-      continue;
-    }
-
-    if (section === 'conversation') {
-      const utteranceMatch = line.match(/^([^:]+):\s*(.+)$/);
-      if (utteranceMatch) {
-        const speaker = utteranceMatch[1].trim();
-        const text = utteranceMatch[2].trim();
-        if (text.length > 0) {
-          utterances.push({ speaker, text });
-          speakerOrder.add(speaker);
-        }
-      }
-    }
-  }
-
-  if (utterances.length === 0) {
-    throw new Error('parseConversation: no utterances found');
-  }
-
-  return { entities, utterances, speakers: Array.from(speakerOrder) };
-};

package/src/enrichment/stream-simulator.ts

@@ -1,125 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-// Turns a parsed conversation into a chronologically-ordered sequence of Whisper-shaped
-// transcript batches, one per (speaker, utterance) pair. Each batch carries a virtual
-// timestamp (ms since the conversation start) and the speaker label so consumers can fan
-// out per-stream — analogous to two microphones being transcribed in parallel.
-//
-// The simulator does NOT do any wall-clock waiting. Tests drive virtual time via
-// `vi.useFakeTimers()` or step through the batches directly; production code wouldn't use
-// this module at all.
-
-import { type ContentBlock } from '@dxos/types';
-
-import type { ConversationUtterance, ParsedConversation } from './parser';
-
-export type StreamSimulatorOptions = {
-  /**
-   * Average speaking rate, words per second. Used to assign a virtual duration to each
-   * utterance so consecutive utterances land at realistic offsets.
-   * @default 2.5 (≈150 wpm — typical conversational pace)
-   */
-  wordsPerSecond?: number;
-
-  /**
-   * Gap between consecutive utterances in seconds. Models a brief pause for turn-taking
-   * even when speakers don't actually overlap.
-   * @default 0.5
-   */
-  gapSeconds?: number;
-
-  /**
-   * Whisper batches typically arrive in chunks of N seconds rather than per-utterance.
-   * When set, utterances shorter than `batchSeconds` are merged into the same batch as
-   * the following utterance from the same speaker. `0` disables merging — every utterance
-   * is its own batch.
-   * @default 0
-   */
-  batchSeconds?: number;
-};
-
-export type SimulatedBatch = {
-  /** Speaker label from the fixture (`A`, `B`, …). Maps to a virtual `identityDid`. */
-  speaker: string;
-  /** ms offset from the conversation start when this batch is delivered. */
-  offsetMs: number;
-  /** Whisper-shaped transcript blocks (raw text, no enrichment). */
-  blocks: ContentBlock.Transcript[];
-};
-
-/**
- * Build the chronological batch schedule for a parsed conversation. The result is a
- * pure data structure — consumers iterate and drive virtual time themselves.
- */
-export const simulateStream = (
-  conversation: ParsedConversation,
-  options: StreamSimulatorOptions = {},
-): SimulatedBatch[] => {
-  const wordsPerSecond = options.wordsPerSecond ?? 2.5;
-  const gapSeconds = options.gapSeconds ?? 0.5;
-  const batchSeconds = options.batchSeconds ?? 0;
-
-  // Guard against divide-by-zero / NaN / negative inputs — `wordsPerSecond` divides into a
-  // duration below, and the other knobs feed into offsets that must be finite.
-  if (!Number.isFinite(wordsPerSecond) || wordsPerSecond <= 0) {
-    throw new Error(`simulateStream: wordsPerSecond must be a finite number > 0 (got ${wordsPerSecond}).`);
-  }
-  if (!Number.isFinite(gapSeconds) || gapSeconds < 0) {
-    throw new Error(`simulateStream: gapSeconds must be a finite number >= 0 (got ${gapSeconds}).`);
-  }
-  if (!Number.isFinite(batchSeconds) || batchSeconds < 0) {
-    throw new Error(`simulateStream: batchSeconds must be a finite number >= 0 (got ${batchSeconds}).`);
-  }
-
-  const conversationStart = Date.UTC(2024, 0, 1, 12, 0, 0); // deterministic, ignored by callers
-  let cursorMs = 0;
-  const batches: SimulatedBatch[] = [];
-
-  // Optionally merge same-speaker consecutive utterances until they reach `batchSeconds`.
-  const grouped: { speaker: string; utterances: ConversationUtterance[] }[] = [];
-  if (batchSeconds === 0) {
-    for (const u of conversation.utterances) {
-      grouped.push({ speaker: u.speaker, utterances: [u] });
-    }
-  } else {
-    let pending: { speaker: string; utterances: ConversationUtterance[]; words: number } | undefined;
-    for (const u of conversation.utterances) {
-      const words = utteranceWordCount(u.text);
-      if (pending && pending.speaker === u.speaker && pending.words / wordsPerSecond < batchSeconds) {
-        pending.utterances.push(u);
-        pending.words += words;
-      } else {
-        if (pending) {
-          grouped.push({ speaker: pending.speaker, utterances: pending.utterances });
-        }
-        pending = { speaker: u.speaker, utterances: [u], words };
-      }
-    }
-    if (pending) {
-      grouped.push({ speaker: pending.speaker, utterances: pending.utterances });
-    }
-  }
-
-  for (const group of grouped) {
-    const startMs = cursorMs;
-    const blocks: ContentBlock.Transcript[] = [];
-    let segmentCursorMs = cursorMs;
-    for (const u of group.utterances) {
-      const durationMs = Math.max(500, (utteranceWordCount(u.text) / wordsPerSecond) * 1_000);
-      blocks.push({
-        _tag: 'transcript',
-        started: new Date(conversationStart + segmentCursorMs).toISOString(),
-        text: u.text,
-      });
-      segmentCursorMs += durationMs;
-    }
-    batches.push({ speaker: group.speaker, offsetMs: startMs, blocks });
-    cursorMs = segmentCursorMs + gapSeconds * 1_000;
-  }
-
-  return batches;
-};
-
-const utteranceWordCount = (text: string): number => text.trim().split(/\s+/).filter(Boolean).length;

package/src/hooks/useAudioFile.ts

@@ -1,88 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { useEffect, useState } from 'react';
-
-import { scheduleTask } from '@dxos/async';
-import { Context } from '@dxos/context';
-import { log } from '@dxos/log';
-
-export type UseAudioState = {
-  audio?: HTMLAudioElement;
-  stream?: MediaStream;
-  track?: MediaStreamTrack;
-};
-
-export const useAudioFile = (audioUrl: string, constraints?: MediaTrackConstraints): UseAudioState => {
-  const [{ audio, stream, track }, setStream] = useState<UseAudioState>({});
-  useEffect(() => {
-    const ctx = new Context();
-
-    scheduleTask(ctx, async () => {
-      // TODO(wittjosiah): Fetch to external url fails in headless storybook test.
-      try {
-        const response = await fetch(audioUrl);
-        const blob = await response.blob();
-        const objectUrl = URL.createObjectURL(blob);
-        ctx.onDispose(() => {
-          URL.revokeObjectURL(objectUrl);
-        });
-
-        const audio = new Audio();
-        audio.src = objectUrl;
-
-        await new Promise<void>((resolve, reject) => {
-          audio.addEventListener(
-            'error',
-            (err) => {
-              log.error('error', { err });
-              reject(err);
-            },
-            { once: true },
-          );
-          audio.addEventListener(
-            'canplay',
-            async () => {
-              resolve();
-            },
-            { once: true },
-          );
-          audio.load();
-        });
-
-        const audioCtx = new AudioContext();
-
-        // Resume AudioContext if it's suspended.
-        if (audioCtx.state === 'suspended') {
-          await audioCtx.resume();
-        }
-
-        const destination = audioCtx.createMediaStreamDestination();
-        destination.channelCount = 1;
-
-        const source = audioCtx.createMediaElementSource(audio);
-        source.connect(destination);
-        const track = destination.stream.getAudioTracks()[0];
-        if (constraints) {
-          await track.applyConstraints(constraints).catch((err) => log.catch(err));
-        }
-
-        // Also connect to speakers so audio is audible.
-        setStream({
-          audio,
-          stream: destination.stream,
-          track: destination.stream.getAudioTracks()[0],
-        });
-      } catch (err) {
-        log.catch(err);
-      }
-    });
-
-    return () => {
-      void ctx.dispose();
-    };
-  }, [audioUrl]);
-
-  return { audio, stream, track };
-};

package/src/hooks/useAudioTrack.ts

@@ -1,35 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { useEffect, useState } from 'react';
-
-/**
- *
- */
-// TODO(burdon): Reconcile with react-ui-experimental and plugin-calls.
-export const useAudioTrack = (active?: boolean, constraints?: MediaTrackConstraints): MediaStreamTrack | undefined => {
-  const [track, setTrack] = useState<MediaStreamTrack>();
-  useEffect(() => {
-    if (!active) {
-      track?.stop();
-      setTrack(undefined);
-      return;
-    }
-
-    const initAudio = async () => {
-      const audio = new Audio();
-      audio.srcObject = await navigator.mediaDevices.getUserMedia({ audio: constraints ?? true });
-      const [track] = audio.srcObject.getAudioTracks();
-      if (track) {
-        setTrack(track);
-      }
-    };
-
-    if (active) {
-      void initAudio();
-    }
-  }, [active]);
-
-  return track;
-};

package/src/hooks/useFeedModelAdapter.ts

@@ -1,46 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { useEffect, useMemo } from 'react';
-
-import type { Obj } from '@dxos/echo';
-
-import { type ChunkRenderer, SerializationModel } from '../model';
-
-/**
- * Feeds an array of objects (typically from `useQuery(db, Query.from(feed))`) into a
- * `SerializationModel`, appending each new tail item as the array grows.
- *
- * @param renderer - chunk renderer.
- * @param objects - current snapshot of feed items.
- * @param initialChunks - chunks to seed the model with on first render.
- */
-export const useFeedModelAdapter = <T extends Obj.Unknown>(
-  renderer: ChunkRenderer<T>,
-  objects: readonly T[],
-  initialChunks: T[] = [],
-): SerializationModel<T> => {
-  const model = useMemo(() => new SerializationModel<T>(renderer, initialChunks), [renderer]);
-
-  // Seed the model with the current snapshot on first render and whenever the
-  // identity of `objects` changes (e.g. feed swapped).
-  useEffect(() => {
-    for (const chunk of objects) {
-      model.appendChunk(chunk);
-    }
-  }, [model, objects]);
-
-  // Append new tail items as the array grows.
-  useEffect(() => {
-    if (model.chunks.length === objects.length) {
-      return;
-    }
-    const chunk = objects.at(-1);
-    if (chunk) {
-      model.appendChunk(chunk);
-    }
-  }, [model, objects.length]);
-
-  return model;
-};

package/src/hooks/useTranscriber.ts

@@ -1,47 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { useEffect, useMemo } from 'react';
-
-import { useCapabilities } from '@dxos/app-framework/ui';
-
-import { TranscriptionCapabilities } from '#types';
-
-import { type Transcriber } from '../transcriber';
-
-/**
- * Records audio while user is speaking and transcribes it after user is done speaking.
- */
-export const useTranscriber = ({
-  audioStreamTrack,
-  recorderConfig,
-  transcriberConfig,
-  onSegments,
-  transcribe,
-}: Partial<TranscriptionCapabilities.TranscriberProviderProps>) => {
-  const [transcriberProvider] = useCapabilities(TranscriptionCapabilities.TranscriberProvider);
-
-  // Initialize audio transcription.
-  const transcriber = useMemo<Transcriber | undefined>(() => {
-    if (!transcriberProvider || !audioStreamTrack || !onSegments) {
-      return undefined;
-    }
-
-    return transcriberProvider({
-      audioStreamTrack,
-      recorderConfig,
-      transcriberConfig,
-      onSegments,
-      transcribe,
-    });
-  }, [transcriberProvider, audioStreamTrack, recorderConfig, transcriberConfig, onSegments, transcribe]);
-
-  useEffect(() => {
-    return () => {
-      void transcriber?.close();
-    };
-  }, [transcriber]);
-
-  return transcriber;
-};

package/src/hooks/useVoiceInput.ts

@@ -1,66 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { useCallback, useEffect, useState } from 'react';
-
-import { scheduleMicroTask } from '@dxos/async';
-import { Context } from '@dxos/context';
-import { log } from '@dxos/log';
-import { useSoundEffect } from '@dxos/react-ui-audio';
-
-import { type TranscriberProps } from '../transcriber';
-import { useAudioTrack } from './useAudioTrack';
-import { useTranscriber } from './useTranscriber';
-
-export type UseVoiceInputProps = {
-  active?: boolean;
-  onUpdate: (text: string) => void;
-};
-
-/**
- * Hook for voice input.
- */
-export const useVoiceInput = ({ active, onUpdate }: UseVoiceInputProps) => {
-  const soundStart = useSoundEffect('StartRecording');
-  const soundStop = useSoundEffect('StopRecording');
-
-  const handleSegments = useCallback<TranscriberProps['onSegments']>(async (segments) => {
-    const text = segments.map((str) => str.text.trim().replace(/[^\w\s]/g, '')).join(' ');
-    onUpdate(text);
-  }, []);
-
-  // Audio/transcription.
-  const [recording, setRecording] = useState(false);
-  const track = useAudioTrack(active);
-  const transcriber = useTranscriber({
-    audioStreamTrack: track,
-    onSegments: handleSegments,
-  });
-
-  // Start/stop transcription.
-  useEffect(() => {
-    const ctx = new Context();
-    scheduleMicroTask(ctx, async () => {
-      if (active && transcriber) {
-        await transcriber.open();
-        log.info('starting...');
-        setRecording(true);
-        void soundStart.play();
-        transcriber.startChunksRecording();
-      } else if (!active && transcriber?.isOpen) {
-        transcriber?.stopChunksRecording();
-        await transcriber?.close();
-        log.info('stopped');
-        setRecording(false);
-        void soundStop.play();
-      }
-    });
-
-    return () => {
-      void ctx.dispose();
-    };
-  }, [active, transcriber]);
-
-  return { recording };
-};

package/src/model/index.ts

@@ -1,5 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-export * from './model';

package/src/model/model.test.ts

@@ -1,175 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { EditorView } from '@codemirror/view';
-import { describe, test } from 'vitest';
-
-import { Obj } from '@dxos/echo';
-import type { EntityId } from '@dxos/keys';
-import { Message } from '@dxos/types';
-
-import { type ChunkRenderer, DocumentAdapter, SerializationModel } from './model';
-
-const blockToMarkdown: ChunkRenderer<Message.Message> = (
-  message: Message.Message,
-  index: number,
-  debug = true,
-): string[] => {
-  return [
-    `###### ${message.sender.name}`,
-    ...message.blocks.filter((block) => block._tag === 'transcript').map((block) => block.text),
-    '',
-  ];
-};
-
-const createDate = () => new Date().toISOString();
-
-describe('SerializationModel', () => {
-  test('basic', ({ expect }) => {
-    const model = new SerializationModel<Message.Message>(blockToMarkdown);
-    expect(model.chunks.length).to.eq(0);
-    expect(model.doc.toString()).to.eq('');
-
-    // Create message.
-    const message = Obj.make(Message.Message, {
-      created: createDate(),
-      sender: { name: 'Alice' },
-      blocks: [
-        {
-          _tag: 'transcript',
-          started: createDate(),
-          text: 'Hello world!',
-        },
-      ],
-    });
-    model.appendChunk(message);
-    {
-      const text = model.doc.toString();
-      expect(text).to.eq('###### Alice\nHello world!\n\n');
-      expect(model.chunks.length).to.eq(1);
-    }
-
-    // Update message.
-    Obj.update(message, (message) => {
-      message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
-    });
-    model.updateChunk(message);
-    {
-      const text = model.doc.toString();
-      expect(text).to.eq('###### Alice\nHello world!\nHello again!\n\n');
-      expect(model.chunks.length).to.eq(1);
-    }
-  });
-
-  test('sync - append', ({ expect }) => {
-    const view = new EditorView({ extensions: [], doc: '' });
-    const model = new SerializationModel<Message.Message>(blockToMarkdown);
-    const adapter = new DocumentAdapter(view);
-    expect(adapter.lineCount()).to.eq(0);
-    expect(view.state.doc.toString()).to.eq('');
-
-    // Append message.
-    {
-      const message = Obj.make(Message.Message, {
-        created: createDate(),
-        sender: { name: 'Alice' },
-        blocks: [
-          {
-            _tag: 'transcript',
-            started: createDate(),
-            text: 'Hello world!',
-          },
-        ],
-      });
-      model.appendChunk(message);
-      model.sync(adapter);
-      expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n');
-    }
-
-    // Append message.
-    {
-      const message = Obj.make(Message.Message, {
-        created: createDate(),
-        sender: { name: 'Bob' },
-        blocks: [
-          {
-            _tag: 'transcript',
-            started: createDate(),
-            text: 'Hello world!',
-          },
-        ],
-      });
-      model.appendChunk(message);
-      model.sync(adapter);
-      expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n###### Bob\nHello world!\n\n');
-    }
-  });
-
-  test('sync - append, update, delete', ({ expect }) => {
-    const view = new EditorView({ extensions: [], doc: '' });
-    const model = new SerializationModel<Message.Message>(blockToMarkdown);
-    const adapter = new DocumentAdapter(view);
-    expect(adapter.lineCount()).to.eq(0);
-
-    let msgId: EntityId;
-
-    // Append message.
-    {
-      const message = Obj.make(Message.Message, {
-        created: createDate(),
-        sender: { name: 'Alice' },
-        blocks: [
-          {
-            _tag: 'transcript',
-            started: createDate(),
-            text: 'Hello world!',
-          },
-        ],
-      });
-      msgId = message.id;
-      model.appendChunk(message);
-      model.sync(adapter);
-      expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n');
-
-      // Update message (add block).
-      Obj.update(message, (message) => {
-        message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
-      });
-      model.updateChunk(message);
-      model.sync(adapter);
-      expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\nHello again!\n\n');
-
-      // Update message (remove block).
-      Obj.update(message, (message) => {
-        message.blocks.splice(0, 1);
-      });
-      model.updateChunk(message);
-      model.sync(adapter);
-      expect(view.state.doc.toString()).to.eq('###### Alice\nHello again!\n\n');
-
-      expect(model.getChunkAtLine(1)).to.eq(message);
-    }
-
-    // Append message.
-    {
-      const message = Obj.make(Message.Message, {
-        created: createDate(),
-        sender: { name: 'Bob' },
-        blocks: [{ _tag: 'transcript', started: createDate(), text: 'Hello again!' }],
-      });
-      model.appendChunk(message);
-      model.sync(adapter);
-      expect(view.state.doc.toString()).to.eq('###### Alice\nHello again!\n\n###### Bob\nHello again!\n\n');
-
-      expect(model.getChunkAtLine(4)).to.eq(message);
-    }
-
-    // Delete block.
-    {
-      model.deleteBlock(msgId);
-      model.sync(adapter);
-      expect(view.state.doc.toString()).to.eq('###### Bob\nHello again!\n\n');
-    }
-  });
-});