@dxos/plugin-transcription 0.8.4-main.ae835ea → 0.8.4-main.bc2380dfbc

package/src/enrichment/parser.ts

@@ -0,0 +1,101 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+// Parser for the human-authored conversation fixture (see `conversation.txt`).
+// Captures both the seed entities (Persons, Organizations, …) and the speaker-tagged
+// dialogue used by `stream-simulator.ts` to feed the enrichment pipeline.
+
+export type ConversationEntity = {
+  /** Type name as written in the fixture, e.g. `Person`, `Organization`. */
+  kind: string;
+  /** Free-form field values; consumers map these to the relevant ECHO schema. */
+  fields: Record<string, string>;
+};
+
+export type ConversationUtterance = {
+  /** Speaker label as written in the fixture (e.g. `A`, `B`, `Rich`). */
+  speaker: string;
+  /** Raw utterance text. */
+  text: string;
+};
+
+export type ParsedConversation = {
+  entities: ConversationEntity[];
+  utterances: ConversationUtterance[];
+  /** Distinct speaker labels in first-seen order. */
+  speakers: string[];
+};
+
+/**
+ * Parse the on-disk conversation format into structured data.
+ *
+ * Tolerates leading whitespace inside blocks and ignores blank / `#` comment lines.
+ * Throws if the input doesn't contain a `Conversation:` block — `Entities:` is optional.
+ */
+export const parseConversation = (input: string): ParsedConversation => {
+  const lines = input.split('\n').map((line) => line.replace(/\s+$/, ''));
+
+  const entities: ConversationEntity[] = [];
+  const utterances: ConversationUtterance[] = [];
+  const speakerOrder = new Set<string>();
+
+  type Section = 'none' | 'entities' | 'conversation';
+  let section: Section = 'none';
+  let currentEntity: ConversationEntity | undefined;
+
+  for (const raw of lines) {
+    const line = raw.trim();
+    if (line.length === 0 || line.startsWith('#')) {
+      continue;
+    }
+
+    // Section headers (must be left-aligned to switch sections).
+    if (raw.startsWith('Entities:') || raw.startsWith('Entitites:')) {
+      // tolerate the typo present in earlier fixtures
+      section = 'entities';
+      currentEntity = undefined;
+      continue;
+    }
+    if (raw.startsWith('Conversation:')) {
+      section = 'conversation';
+      currentEntity = undefined;
+      continue;
+    }
+
+    if (section === 'entities') {
+      // `<TypeName>:` starts a new entity; nested `<field>: <value>` lines fill it.
+      const typeMatch = line.match(/^([A-Za-z][A-Za-z0-9_-]*):$/);
+      if (typeMatch) {
+        currentEntity = { kind: typeMatch[1], fields: {} };
+        entities.push(currentEntity);
+        continue;
+      }
+      const fieldMatch = line.match(/^([A-Za-z][A-Za-z0-9_-]*):\s*(.+)$/);
+      if (fieldMatch && currentEntity) {
+        currentEntity.fields[fieldMatch[1]] = fieldMatch[2];
+        continue;
+      }
+      // Silently skip unrecognised lines so authors can add free-form annotations.
+      continue;
+    }
+
+    if (section === 'conversation') {
+      const utteranceMatch = line.match(/^([^:]+):\s*(.+)$/);
+      if (utteranceMatch) {
+        const speaker = utteranceMatch[1].trim();
+        const text = utteranceMatch[2].trim();
+        if (text.length > 0) {
+          utterances.push({ speaker, text });
+          speakerOrder.add(speaker);
+        }
+      }
+    }
+  }
+
+  if (utterances.length === 0) {
+    throw new Error('parseConversation: no utterances found');
+  }
+
+  return { entities, utterances, speakers: Array.from(speakerOrder) };
+};

package/src/enrichment/stream-simulator.ts

@@ -0,0 +1,125 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+// Turns a parsed conversation into a chronologically-ordered sequence of Whisper-shaped
+// transcript batches, one per (speaker, utterance) pair. Each batch carries a virtual
+// timestamp (ms since the conversation start) and the speaker label so consumers can fan
+// out per-stream — analogous to two microphones being transcribed in parallel.
+//
+// The simulator does NOT do any wall-clock waiting. Tests drive virtual time via
+// `vi.useFakeTimers()` or step through the batches directly; production code wouldn't use
+// this module at all.
+
+import { type ContentBlock } from '@dxos/types';
+
+import type { ConversationUtterance, ParsedConversation } from './parser';
+
+export type StreamSimulatorOptions = {
+  /**
+   * Average speaking rate, words per second. Used to assign a virtual duration to each
+   * utterance so consecutive utterances land at realistic offsets.
+   * @default 2.5 (≈150 wpm — typical conversational pace)
+   */
+  wordsPerSecond?: number;
+
+  /**
+   * Gap between consecutive utterances in seconds. Models a brief pause for turn-taking
+   * even when speakers don't actually overlap.
+   * @default 0.5
+   */
+  gapSeconds?: number;
+
+  /**
+   * Whisper batches typically arrive in chunks of N seconds rather than per-utterance.
+   * When set, utterances shorter than `batchSeconds` are merged into the same batch as
+   * the following utterance from the same speaker. `0` disables merging — every utterance
+   * is its own batch.
+   * @default 0
+   */
+  batchSeconds?: number;
+};
+
+export type SimulatedBatch = {
+  /** Speaker label from the fixture (`A`, `B`, …). Maps to a virtual `identityDid`. */
+  speaker: string;
+  /** ms offset from the conversation start when this batch is delivered. */
+  offsetMs: number;
+  /** Whisper-shaped transcript blocks (raw text, no enrichment). */
+  blocks: ContentBlock.Transcript[];
+};
+
+/**
+ * Build the chronological batch schedule for a parsed conversation. The result is a
+ * pure data structure — consumers iterate and drive virtual time themselves.
+ */
+export const simulateStream = (
+  conversation: ParsedConversation,
+  options: StreamSimulatorOptions = {},
+): SimulatedBatch[] => {
+  const wordsPerSecond = options.wordsPerSecond ?? 2.5;
+  const gapSeconds = options.gapSeconds ?? 0.5;
+  const batchSeconds = options.batchSeconds ?? 0;
+
+  // Guard against divide-by-zero / NaN / negative inputs — `wordsPerSecond` divides into a
+  // duration below, and the other knobs feed into offsets that must be finite.
+  if (!Number.isFinite(wordsPerSecond) || wordsPerSecond <= 0) {
+    throw new Error(`simulateStream: wordsPerSecond must be a finite number > 0 (got ${wordsPerSecond}).`);
+  }
+  if (!Number.isFinite(gapSeconds) || gapSeconds < 0) {
+    throw new Error(`simulateStream: gapSeconds must be a finite number >= 0 (got ${gapSeconds}).`);
+  }
+  if (!Number.isFinite(batchSeconds) || batchSeconds < 0) {
+    throw new Error(`simulateStream: batchSeconds must be a finite number >= 0 (got ${batchSeconds}).`);
+  }
+
+  const conversationStart = Date.UTC(2024, 0, 1, 12, 0, 0); // deterministic, ignored by callers
+  let cursorMs = 0;
+  const batches: SimulatedBatch[] = [];
+
+  // Optionally merge same-speaker consecutive utterances until they reach `batchSeconds`.
+  const grouped: { speaker: string; utterances: ConversationUtterance[] }[] = [];
+  if (batchSeconds === 0) {
+    for (const u of conversation.utterances) {
+      grouped.push({ speaker: u.speaker, utterances: [u] });
+    }
+  } else {
+    let pending: { speaker: string; utterances: ConversationUtterance[]; words: number } | undefined;
+    for (const u of conversation.utterances) {
+      const words = utteranceWordCount(u.text);
+      if (pending && pending.speaker === u.speaker && pending.words / wordsPerSecond < batchSeconds) {
+        pending.utterances.push(u);
+        pending.words += words;
+      } else {
+        if (pending) {
+          grouped.push({ speaker: pending.speaker, utterances: pending.utterances });
+        }
+        pending = { speaker: u.speaker, utterances: [u], words };
+      }
+    }
+    if (pending) {
+      grouped.push({ speaker: pending.speaker, utterances: pending.utterances });
+    }
+  }
+
+  for (const group of grouped) {
+    const startMs = cursorMs;
+    const blocks: ContentBlock.Transcript[] = [];
+    let segmentCursorMs = cursorMs;
+    for (const u of group.utterances) {
+      const durationMs = Math.max(500, (utteranceWordCount(u.text) / wordsPerSecond) * 1_000);
+      blocks.push({
+        _tag: 'transcript',
+        started: new Date(conversationStart + segmentCursorMs).toISOString(),
+        text: u.text,
+      });
+      segmentCursorMs += durationMs;
+    }
+    batches.push({ speaker: group.speaker, offsetMs: startMs, blocks });
+    cursorMs = segmentCursorMs + gapSeconds * 1_000;
+  }
+
+  return batches;
+};
+
+const utteranceWordCount = (text: string): number => text.trim().split(/\s+/).filter(Boolean).length;

package/src/hooks/index.ts

@@ -4,6 +4,7 @@
 
 export * from './useAudioFile';
 export * from './useAudioTrack';
-export * from './useQueueModelAdapter';
+export * from './useFeedModelAdapter';
 export * from './useTranscriber';
+export * from './useTranscriptionRecording';
 export * from './useVoiceInput';

package/src/hooks/useFeedModelAdapter.ts

@@ -0,0 +1,46 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useEffect, useMemo } from 'react';
+
+import type { Obj } from '@dxos/echo';
+
+import { type ChunkRenderer, SerializationModel } from '../model';
+
+/**
+ * Feeds an array of objects (typically from `useQuery(db, Query.from(feed))`) into a
+ * `SerializationModel`, appending each new tail item as the array grows.
+ *
+ * @param renderer - chunk renderer.
+ * @param objects - current snapshot of feed items.
+ * @param initialChunks - chunks to seed the model with on first render.
+ */
+export const useFeedModelAdapter = <T extends Obj.Unknown>(
+  renderer: ChunkRenderer<T>,
+  objects: readonly T[],
+  initialChunks: T[] = [],
+): SerializationModel<T> => {
+  const model = useMemo(() => new SerializationModel<T>(renderer, initialChunks), [renderer]);
+
+  // Seed the model with the current snapshot on first render and whenever the
+  // identity of `objects` changes (e.g. feed swapped).
+  useEffect(() => {
+    for (const chunk of objects) {
+      model.appendChunk(chunk);
+    }
+  }, [model, objects]);
+
+  // Append new tail items as the array grows.
+  useEffect(() => {
+    if (model.chunks.length === objects.length) {
+      return;
+    }
+    const chunk = objects.at(-1);
+    if (chunk) {
+      model.appendChunk(chunk);
+    }
+  }, [model, objects.length]);
+
+  return model;
+};

package/src/hooks/useTranscriber.ts

@@ -4,9 +4,10 @@
 
 import { useEffect, useMemo } from 'react';
 
-import { useCapabilities } from '@dxos/app-framework';
+import { useCapabilities } from '@dxos/app-framework/ui';
+
+import { TranscriptionCapabilities } from '#types';
 
-import { TranscriptionCapabilities } from '../capabilities';
 import { type Transcriber } from '../transcriber';
 
 /**
@@ -17,6 +18,7 @@ export const useTranscriber = ({
   recorderConfig,
   transcriberConfig,
   onSegments,
+  transcribe,
 }: Partial<TranscriptionCapabilities.GetTranscriberProps>) => {
   const [getTranscriber] = useCapabilities(TranscriptionCapabilities.Transcriber);
 
@@ -26,8 +28,14 @@ export const useTranscriber = ({
       return undefined;
     }
 
-    return getTranscriber({ audioStreamTrack, recorderConfig, transcriberConfig, onSegments });
-  }, [getTranscriber, audioStreamTrack, recorderConfig, transcriberConfig, onSegments]);
+    return getTranscriber({
+      audioStreamTrack,
+      recorderConfig,
+      transcriberConfig,
+      onSegments,
+      transcribe,
+    });
+  }, [getTranscriber, audioStreamTrack, recorderConfig, transcriberConfig, onSegments, transcribe]);
 
   useEffect(() => {
     return () => {

package/src/hooks/useTranscriptionRecording.ts

@@ -0,0 +1,87 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import { useCallback, useEffect, useState } from 'react';
+
+import { createFeedServiceLayer } from '@dxos/client/echo';
+import { Feed, Obj } from '@dxos/echo';
+import { runAndForwardErrors } from '@dxos/effect';
+import { log } from '@dxos/log';
+import { getSpace } from '@dxos/react-client/echo';
+import { useIdentity } from '@dxos/react-client/halo';
+import { Message, type Transcript } from '@dxos/types';
+
+import { useAudioTrack } from './useAudioTrack';
+import { useTranscriber } from './useTranscriber';
+
+export type TranscriptionRecording = {
+  recording: boolean;
+  toggleRecording: () => void;
+};
+
+/**
+ * Drives the recording lifecycle for a transcript: opens an audio track, feeds it through the
+ * transcriber, and appends transcribed message batches to the transcript's feed. Returns the
+ * recording flag and a toggle callback for the caller to wire to UI.
+ */
+export const useTranscriptionRecording = (transcript: Transcript.Transcript): TranscriptionRecording => {
+  const space = getSpace(transcript);
+  const identity = useIdentity();
+  const feed = transcript.feed.target;
+
+  const [recording, setRecording] = useState(false);
+  const track = useAudioTrack(recording);
+
+  // Append a transcribed message batch to the transcript's feed.
+  const handleSegments = useCallback(
+    async (blocks: Message.Message['blocks']) => {
+      if (!space || !feed || blocks.length === 0) {
+        return;
+      }
+      const sender = identity ? { identityDid: identity.did, name: identity.profile?.displayName } : {};
+      const message = Obj.make(Message.Message, {
+        sender,
+        created: new Date().toISOString(),
+        blocks,
+      });
+      const feedServiceLayer = createFeedServiceLayer(space.queues);
+      await Feed.append(feed, [message]).pipe(Effect.provide(feedServiceLayer), runAndForwardErrors);
+    },
+    [space, feed, identity],
+  );
+
+  // Drive the transcriber lifecycle off the recording flag + audio track.
+  const transcriber = useTranscriber({ audioStreamTrack: track, onSegments: handleSegments });
+  useEffect(() => {
+    if (!transcriber) {
+      return;
+    }
+    let cancelled = false;
+    // If `open()` rejects (e.g. mic permission denied, device init failure) we must
+    // clear `recording` — otherwise the toolbar button reports "stop" forever while
+    // nothing is actually being captured.
+    void transcriber.open().then(
+      () => {
+        if (!cancelled && recording) {
+          transcriber.startChunksRecording();
+        }
+      },
+      (err) => {
+        if (!cancelled) {
+          log.catch(err);
+          setRecording(false);
+        }
+      },
+    );
+    return () => {
+      cancelled = true;
+      transcriber.stopChunksRecording();
+    };
+  }, [transcriber, recording]);
+
+  const toggleRecording = useCallback(() => setRecording((value) => !value), []);
+
+  return { recording, toggleRecording };
+};

package/src/hooks/useVoiceInput.ts

@@ -9,8 +9,7 @@ import { Context } from '@dxos/context';
 import { log } from '@dxos/log';
 import { useSoundEffect } from '@dxos/react-ui-sfx';
 
-import { type TranscriberParams } from '../transcriber';
-
+import { type TranscriberProps } from '../transcriber';
 import { useAudioTrack } from './useAudioTrack';
 import { useTranscriber } from './useTranscriber';
 
@@ -26,7 +25,7 @@ export const useVoiceInput = ({ active, onUpdate }: UseVoiceInputProps) => {
   const soundStart = useSoundEffect('StartRecording');
   const soundStop = useSoundEffect('StopRecording');
 
-  const handleSegments = useCallback<TranscriberParams['onSegments']>(async (segments) => {
+  const handleSegments = useCallback<TranscriberProps['onSegments']>(async (segments) => {
     const text = segments.map((str) => str.text.trim().replace(/[^\w\s]/g, '')).join(' ');
     onUpdate(text);
   }, []);

package/src/index.ts

@@ -2,11 +2,10 @@
 // Copyright 2025 DXOS.org
 //
 
-export { TranscriptionCapabilities } from './capabilities';
-
-export * from './components';
+// TODO(wittjosiah): Hooks should not be exported from the plugin package at all.
+//   Either refactor callers to not need them or factor them out to a shared package.
+export * from './blueprints';
 export * from './hooks';
 export * from './meta';
 export * from './transcriber';
-
-export * from './TranscriptionPlugin';
+export * from './types';

package/src/meta.ts

@@ -2,11 +2,11 @@
 // Copyright 2023 DXOS.org
 //
 
-import { type PluginMeta } from '@dxos/app-framework';
+import { type Plugin } from '@dxos/app-framework';
 import { trim } from '@dxos/util';
 
-export const meta: PluginMeta = {
-  id: 'dxos.org/plugin/transcription',
+export const meta: Plugin.Meta = {
+  id: 'org.dxos.plugin.transcription',
   name: 'Transcription',
   description: trim`
     Real-time voice-to-text transcription service for capturing spoken content.

package/src/model/model.test.ts

@@ -7,12 +7,12 @@ import { describe, test } from 'vitest';
 
 import { Obj } from '@dxos/echo';
 import type { ObjectId } from '@dxos/keys';
-import { DataType } from '@dxos/schema';
+import { Message } from '@dxos/types';
 
 import { type ChunkRenderer, DocumentAdapter, SerializationModel } from './model';
 
-const blockToMarkdown: ChunkRenderer<DataType.Message> = (
-  message: DataType.Message,
+const blockToMarkdown: ChunkRenderer<Message.Message> = (
+  message: Message.Message,
   index: number,
   debug = true,
 ): string[] => {
@@ -27,12 +27,12 @@ const createDate = () => new Date().toISOString();
 
 describe('SerializationModel', () => {
   test('basic', ({ expect }) => {
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     expect(model.chunks.length).to.eq(0);
     expect(model.doc.toString()).to.eq('');
 
     // Create message.
-    const message = Obj.make(DataType.Message, {
+    const message = Obj.make(Message.Message, {
       created: createDate(),
       sender: { name: 'Alice' },
       blocks: [
@@ -51,7 +51,9 @@ describe('SerializationModel', () => {
     }
 
     // Update message.
-    message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
+    Obj.update(message, (message) => {
+      message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
+    });
     model.updateChunk(message);
     {
       const text = model.doc.toString();
@@ -62,14 +64,14 @@ describe('SerializationModel', () => {
 
   test('sync - append', ({ expect }) => {
     const view = new EditorView({ extensions: [], doc: '' });
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     const adapter = new DocumentAdapter(view);
     expect(adapter.lineCount()).to.eq(0);
     expect(view.state.doc.toString()).to.eq('');
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Alice' },
         blocks: [
@@ -87,7 +89,7 @@ describe('SerializationModel', () => {
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Bob' },
         blocks: [
@@ -106,7 +108,7 @@ describe('SerializationModel', () => {
 
   test('sync - append, update, delete', ({ expect }) => {
     const view = new EditorView({ extensions: [], doc: '' });
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     const adapter = new DocumentAdapter(view);
     expect(adapter.lineCount()).to.eq(0);
 
@@ -114,7 +116,7 @@ describe('SerializationModel', () => {
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Alice' },
         blocks: [
@@ -131,13 +133,17 @@ describe('SerializationModel', () => {
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n');
 
       // Update message (add block).
-      message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
+      Obj.update(message, (message) => {
+        message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
+      });
       model.updateChunk(message);
       model.sync(adapter);
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\nHello again!\n\n');
 
       // Update message (remove block).
-      message.blocks.splice(0, 1);
+      Obj.update(message, (message) => {
+        message.blocks.splice(0, 1);
+      });
       model.updateChunk(message);
       model.sync(adapter);
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello again!\n\n');
@@ -147,7 +153,7 @@ describe('SerializationModel', () => {
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Bob' },
         blocks: [{ _tag: 'transcript', started: createDate(), text: 'Hello again!' }],

package/src/model/model.ts

@@ -80,6 +80,9 @@ type ChunkChange<T extends Chunk> =
  * Instead this model tracks changes and syncs them with the ChunkDocument.
  */
 export class SerializationModel<T extends Chunk> {
+  /** Emits when the document is updated. */
+  public readonly update = new Event<void>();
+
   /** Ordered set of chunks. */
   private readonly _chunks: T[] = [];
 
@@ -92,9 +95,6 @@ export class SerializationModel<T extends Chunk> {
   /** Track chunk changes since last sync. */
   private readonly _changes: Array<ChunkChange<T>> = [];
 
-  /** Emits when the document is updated. */
-  public readonly update = new Event<void>();
-
   constructor(
     private readonly _renderer: ChunkRenderer<T>,
     initialChunks: T[] = [],