@dxos/plugin-transcription 0.8.3 → 0.8.4-main.1c7ec43d41

package/src/hooks/useAudioFile.ts

@@ -20,67 +20,63 @@ export const useAudioFile = (audioUrl: string, constraints?: MediaTrackConstrain
     const ctx = new Context();
 
     scheduleTask(ctx, async () => {
-      const response = await fetch(audioUrl);
-      const blob = await response.blob();
-      const objectUrl = URL.createObjectURL(blob);
-      ctx.onDispose(() => {
-        URL.revokeObjectURL(objectUrl);
-      });
+      // TODO(wittjosiah): Fetch to external url fails in headless storybook test.
+      try {
+        const response = await fetch(audioUrl);
+        const blob = await response.blob();
+        const objectUrl = URL.createObjectURL(blob);
+        ctx.onDispose(() => {
+          URL.revokeObjectURL(objectUrl);
+        });
 
-      const audio = new Audio();
-      audio.src = objectUrl;
+        const audio = new Audio();
+        audio.src = objectUrl;
 
-      await new Promise<void>((resolve, reject) => {
-        audio.addEventListener(
-          'error',
-          (err) => {
-            log.error('error', { err });
-            reject(err);
-          },
-          { once: true },
-        );
-        audio.addEventListener(
-          'canplay',
-          async () => {
-            log.info('starting...');
-            try {
-              // Try to play the audio
-              await audio.play();
+        await new Promise<void>((resolve, reject) => {
+          audio.addEventListener(
+            'error',
+            (err) => {
+              log.error('error', { err });
+              reject(err);
+            },
+            { once: true },
+          );
+          audio.addEventListener(
+            'canplay',
+            async () => {
               resolve();
-            } catch (playError) {
-              log.error('Play failed', { playError });
-              // Still resolve as the audio is ready, even if autoplay failed.
-              resolve();
-            }
-          },
-          { once: true },
-        );
-        audio.load();
-      });
+            },
+            { once: true },
+          );
+          audio.load();
+        });
 
-      const audioCtx = new AudioContext();
+        const audioCtx = new AudioContext();
 
-      // Resume AudioContext if it's suspended.
-      if (audioCtx.state === 'suspended') {
-        await audioCtx.resume();
-      }
+        // Resume AudioContext if it's suspended.
+        if (audioCtx.state === 'suspended') {
+          await audioCtx.resume();
+        }
 
-      const destination = audioCtx.createMediaStreamDestination();
-      destination.channelCount = 1;
+        const destination = audioCtx.createMediaStreamDestination();
+        destination.channelCount = 1;
 
-      const source = audioCtx.createMediaElementSource(audio);
-      source.connect(destination);
-      const track = destination.stream.getAudioTracks()[0];
-      if (constraints) {
-        await track.applyConstraints(constraints).catch((err) => log.catch(err));
-      }
+        const source = audioCtx.createMediaElementSource(audio);
+        source.connect(destination);
+        const track = destination.stream.getAudioTracks()[0];
+        if (constraints) {
+          await track.applyConstraints(constraints).catch((err) => log.catch(err));
+        }
 
-      // Also connect to speakers so audio is audible.
-      setStream({
-        audio,
-        stream: destination.stream,
-        track: destination.stream.getAudioTracks()[0],
-      });
+        // Also connect to speakers so audio is audible.
+        setStream({
+          audio,
+          stream: destination.stream,
+          track: destination.stream.getAudioTracks()[0],
+        });
+      } catch (err) {
+        log.catch(err);
+      }
     });
 
     return () => {

package/src/hooks/useQueueModelAdapter.ts

@@ -12,7 +12,7 @@ import { type ChunkRenderer, SerializationModel } from '../model';
 /**
  * Model adapter for a queue.
  */
-export const useQueueModelAdapter = <T extends Obj.Any>(
+export const useQueueModelAdapter = <T extends Obj.Unknown>(
   renderer: ChunkRenderer<T>,
   queue: Queue<T> | undefined,
   initialChunks: T[] = [],

package/src/hooks/useTranscriber.ts

@@ -4,9 +4,10 @@
 
 import { useEffect, useMemo } from 'react';
 
-import { useCapabilities } from '@dxos/app-framework';
+import { useCapabilities } from '@dxos/app-framework/ui';
+
+import { TranscriptionCapabilities } from '#types';
 
-import { TranscriptionCapabilities } from '../capabilities';
 import { type Transcriber } from '../transcriber';
 
 /**
@@ -14,20 +15,27 @@ import { type Transcriber } from '../transcriber';
  */
 export const useTranscriber = ({
   audioStreamTrack,
-  onSegments,
-  transcriberConfig,
   recorderConfig,
+  transcriberConfig,
+  onSegments,
+  transcribe,
 }: Partial<TranscriptionCapabilities.GetTranscriberProps>) => {
   const [getTranscriber] = useCapabilities(TranscriptionCapabilities.Transcriber);
 
   // Initialize audio transcription.
   const transcriber = useMemo<Transcriber | undefined>(() => {
-    if (!onSegments || !audioStreamTrack || !getTranscriber) {
+    if (!getTranscriber || !audioStreamTrack || !onSegments) {
       return undefined;
     }
 
-    return getTranscriber({ audioStreamTrack, onSegments, transcriberConfig, recorderConfig });
-  }, [audioStreamTrack, onSegments, getTranscriber, transcriberConfig, recorderConfig]);
+    return getTranscriber({
+      audioStreamTrack,
+      recorderConfig,
+      transcriberConfig,
+      onSegments,
+      transcribe,
+    });
+  }, [getTranscriber, audioStreamTrack, recorderConfig, transcriberConfig, onSegments, transcribe]);
 
   useEffect(() => {
     return () => {

package/src/hooks/useVoiceInput.ts

@@ -2,16 +2,16 @@
 // Copyright 2025 DXOS.org
 //
 
-import { useState, useEffect, useCallback } from 'react';
+import { useCallback, useEffect, useState } from 'react';
 
 import { scheduleMicroTask } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { log } from '@dxos/log';
 import { useSoundEffect } from '@dxos/react-ui-sfx';
 
+import { type TranscriberProps } from '../transcriber';
 import { useAudioTrack } from './useAudioTrack';
 import { useTranscriber } from './useTranscriber';
-import { type TranscriberParams } from '../transcriber';
 
 export type UseVoiceInputProps = {
   active?: boolean;
@@ -25,7 +25,7 @@ export const useVoiceInput = ({ active, onUpdate }: UseVoiceInputProps) => {
   const soundStart = useSoundEffect('StartRecording');
   const soundStop = useSoundEffect('StopRecording');
 
-  const handleSegments = useCallback<TranscriberParams['onSegments']>(async (segments) => {
+  const handleSegments = useCallback<TranscriberProps['onSegments']>(async (segments) => {
     const text = segments.map((str) => str.text.trim().replace(/[^\w\s]/g, '')).join(' ');
     onUpdate(text);
   }, []);

package/src/index.ts

@@ -2,11 +2,16 @@
 // Copyright 2025 DXOS.org
 //
 
-export { TranscriptionCapabilities } from './capabilities';
+import { Plugin } from '@dxos/app-framework';
 
-export * from './components';
-export * from './hooks';
+import { meta } from './meta';
+
+export const TranscriptionPlugin = Plugin.lazy(meta, () => import('#plugin'));
+
+export { TranscriptionCapabilities } from './types';
+
+// TODO(wittjosiah): Hooks should not be exported from the plugin package at all.
+//   Either refactor callers to not need them or factor them out to a shared package.
 export * from './meta';
+export * from './hooks';
 export * from './transcriber';
-
-export * from './TranscriptionPlugin';

package/src/meta.ts

@@ -2,15 +2,18 @@
 // Copyright 2023 DXOS.org
 //
 
-import { type PluginMeta } from '@dxos/app-framework';
+import { type Plugin } from '@dxos/app-framework';
+import { trim } from '@dxos/util';
 
-export const TRANSCRIPTION_PLUGIN = 'dxos.org/plugin/transcription';
-
-export const meta: PluginMeta = {
-  id: TRANSCRIPTION_PLUGIN,
+export const meta: Plugin.Meta = {
+  id: 'org.dxos.plugin.transcription',
   name: 'Transcription',
-  description: 'Voice transcription.',
+  description: trim`
+    Real-time voice-to-text transcription service for capturing spoken content.
+    Convert audio input into searchable text that integrates seamlessly with notes and documents.
+  `,
   icon: 'ph--microphone--regular',
+  iconHue: 'sky',
   source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transcription',
   tags: ['labs'],
   screenshots: [],

package/src/model/model.test.ts

@@ -7,18 +7,18 @@ import { describe, test } from 'vitest';
 
 import { Obj } from '@dxos/echo';
 import type { ObjectId } from '@dxos/keys';
-import { DataType } from '@dxos/schema';
+import { Message } from '@dxos/types';
 
-import { SerializationModel, DocumentAdapter, type ChunkRenderer } from './model';
+import { type ChunkRenderer, DocumentAdapter, SerializationModel } from './model';
 
-const blockToMarkdown: ChunkRenderer<DataType.Message> = (
-  message: DataType.Message,
+const blockToMarkdown: ChunkRenderer<Message.Message> = (
+  message: Message.Message,
   index: number,
   debug = true,
 ): string[] => {
   return [
     `###### ${message.sender.name}`,
-    ...message.blocks.filter((block) => block.type === 'transcription').map((block) => block.text),
+    ...message.blocks.filter((block) => block._tag === 'transcript').map((block) => block.text),
     '',
   ];
 };
@@ -27,17 +27,17 @@ const createDate = () => new Date().toISOString();
 
 describe('SerializationModel', () => {
   test('basic', ({ expect }) => {
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     expect(model.chunks.length).to.eq(0);
     expect(model.doc.toString()).to.eq('');
 
     // Create message.
-    const message = Obj.make(DataType.Message, {
+    const message = Obj.make(Message.Message, {
       created: createDate(),
       sender: { name: 'Alice' },
       blocks: [
         {
-          type: 'transcription',
+          _tag: 'transcript',
           started: createDate(),
           text: 'Hello world!',
         },
@@ -51,7 +51,9 @@ describe('SerializationModel', () => {
     }
 
     // Update message.
-    message.blocks.push({ type: 'transcription', started: createDate(), text: 'Hello again!' });
+    Obj.update(message, (message) => {
+      message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
+    });
     model.updateChunk(message);
     {
       const text = model.doc.toString();
@@ -62,19 +64,19 @@ describe('SerializationModel', () => {
 
   test('sync - append', ({ expect }) => {
     const view = new EditorView({ extensions: [], doc: '' });
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     const adapter = new DocumentAdapter(view);
     expect(adapter.lineCount()).to.eq(0);
     expect(view.state.doc.toString()).to.eq('');
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Alice' },
         blocks: [
           {
-            type: 'transcription',
+            _tag: 'transcript',
             started: createDate(),
             text: 'Hello world!',
           },
@@ -87,12 +89,12 @@ describe('SerializationModel', () => {
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Bob' },
         blocks: [
           {
-            type: 'transcription',
+            _tag: 'transcript',
             started: createDate(),
             text: 'Hello world!',
           },
@@ -106,7 +108,7 @@ describe('SerializationModel', () => {
 
   test('sync - append, update, delete', ({ expect }) => {
     const view = new EditorView({ extensions: [], doc: '' });
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     const adapter = new DocumentAdapter(view);
     expect(adapter.lineCount()).to.eq(0);
 
@@ -114,12 +116,12 @@ describe('SerializationModel', () => {
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Alice' },
         blocks: [
           {
-            type: 'transcription',
+            _tag: 'transcript',
             started: createDate(),
             text: 'Hello world!',
           },
@@ -131,13 +133,17 @@ describe('SerializationModel', () => {
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n');
 
       // Update message (add block).
-      message.blocks.push({ type: 'transcription', started: createDate(), text: 'Hello again!' });
+      Obj.update(message, (message) => {
+        message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
+      });
       model.updateChunk(message);
       model.sync(adapter);
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\nHello again!\n\n');
 
       // Update message (remove block).
-      message.blocks.splice(0, 1);
+      Obj.update(message, (message) => {
+        message.blocks.splice(0, 1);
+      });
       model.updateChunk(message);
       model.sync(adapter);
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello again!\n\n');
@@ -147,10 +153,10 @@ describe('SerializationModel', () => {
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Bob' },
-        blocks: [{ type: 'transcription', started: createDate(), text: 'Hello again!' }],
+        blocks: [{ _tag: 'transcript', started: createDate(), text: 'Hello again!' }],
       });
       model.appendChunk(message);
       model.sync(adapter);

package/src/model/model.ts

@@ -80,6 +80,9 @@ type ChunkChange<T extends Chunk> =
  * Instead this model tracks changes and syncs them with the ChunkDocument.
  */
 export class SerializationModel<T extends Chunk> {
+  /** Emits when the document is updated. */
+  public readonly update = new Event<void>();
+
   /** Ordered set of chunks. */
   private readonly _chunks: T[] = [];
 
@@ -92,9 +95,6 @@ export class SerializationModel<T extends Chunk> {
   /** Track chunk changes since last sync. */
   private readonly _changes: Array<ChunkChange<T>> = [];
 
-  /** Emits when the document is updated. */
-  public readonly update = new Event<void>();
-
   constructor(
     private readonly _renderer: ChunkRenderer<T>,
     initialChunks: T[] = [],

package/src/{segments-normalization → normalization}/message-normalizer.ts

@@ -2,28 +2,22 @@
 // Copyright 2025 DXOS.org
 //
 
-//
-// Copyright 2025 DXOS.org
-//
-
-import { effect } from '@preact/signals-core';
-
-import { asyncTimeout, DeferredTask } from '@dxos/async';
+import { DeferredTask, asyncTimeout } from '@dxos/async';
 import { LifecycleState, Resource } from '@dxos/context';
 import { type Queue } from '@dxos/echo-db';
-import { type FunctionExecutor } from '@dxos/functions';
+import { type FunctionExecutor } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
-import { type DataType } from '@dxos/schema';
+import { type Message } from '@dxos/types';
 
-import { type MessageWithRangeId, sentenceNormalization } from './normalization';
+import { type MessageWithRangeId, type NormalizationOutput, sentenceNormalization } from './normalization';
 import { getActorId } from './utils';
 
 const PROCESSING_TIMEOUT = 20_000; // ms
 const MAX_RANGE_ID_COUNT = 10;
 
-export type SegmentsNormalizerParams = {
+export type SegmentsNormalizerProps = {
   functionExecutor: FunctionExecutor;
-  queue: Queue<DataType.Message>;
+  queue: Queue<Message.Message>;
   startingCursor: QueueCursor;
 };
 
@@ -33,16 +27,16 @@ export type QueueCursor = {
   timestamp: string;
 };
 
-// TODO(mykola): .
 export class MessageNormalizer extends Resource {
   private readonly _functionExecutor: FunctionExecutor;
-  private _queue: Queue<DataType.Message>;
+
+  private _queue: Queue<Message.Message>;
   private _cursor: QueueCursor;
   private _messagesToProcess: MessageWithRangeId[] = [];
   private _normalizationTask?: DeferredTask;
   private _lastProcessedMessageIds?: string[];
 
-  constructor({ functionExecutor, queue, startingCursor }: SegmentsNormalizerParams) {
+  constructor({ functionExecutor, queue, startingCursor }: SegmentsNormalizerProps) {
     super();
     this._functionExecutor = functionExecutor;
     this._queue = queue;
@@ -51,7 +45,8 @@ export class MessageNormalizer extends Resource {
 
   protected override async _open(): Promise<void> {
     this._normalizationTask = new DeferredTask(this._ctx, () => this._processMessages());
-    const unsubscribe = effect(() => {
+
+    const updateMessages = () => {
       if (this._lifecycleState !== LifecycleState.OPEN) {
         return;
       }
@@ -62,8 +57,14 @@ export class MessageNormalizer extends Resource {
       });
 
       this._normalizationTask!.schedule();
-    });
-    this._ctx.onDispose(() => unsubscribe());
+    };
+
+    // Initial update.
+    updateMessages();
+
+    // Subscribe to queue changes.
+    const unsubscribe = this._queue.subscribe(updateMessages);
+    this._ctx.onDispose(unsubscribe);
   }
 
   // Need to unpack strings from blocks from messages run them through the function and then pack them back into blocks into messages.
@@ -81,7 +82,7 @@ export class MessageNormalizer extends Resource {
 
     try {
       // TODO(mykola): Executor should support timeout.
-      const response = await asyncTimeout(
+      const response: NormalizationOutput = await asyncTimeout(
         this._functionExecutor.invoke(sentenceNormalization, { messages }),
         PROCESSING_TIMEOUT,
       );

package/src/{segments-normalization → normalization}/normalization.test.ts

@@ -2,25 +2,22 @@
 // Copyright 2025 DXOS.org
 //
 
-import { effect } from '@preact/signals-core';
 import { describe, test } from 'vitest';
 
-import { EdgeAiServiceClient, OllamaAiServiceClient } from '@dxos/ai';
-import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
 import { scheduleTaskInterval } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { Obj } from '@dxos/echo';
 import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo-schema';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { createQueueDXN } from '@dxos/echo/internal';
+import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
-import { DataType } from '@dxos/schema';
+import { type Actor, Message } from '@dxos/types';
 
 import { MessageNormalizer } from './message-normalizer';
 import { type MessageWithRangeId, sentenceNormalization } from './normalization';
 import { getActorId } from './utils';
 
-const sender: DataType.Actor = {
+const sender: Actor.Actor = {
   identityDid: 'did:key:123',
 };
 
@@ -50,34 +47,33 @@ const messages: MessageWithRangeId[] = [
   // No punctuation.
   'in classical physics objects have well-defined properties such as position speed and momentum',
 ].map((string, index) =>
-  Obj.make(DataType.Message, {
-    created: new Date(Date.now() + 1000 * index).toISOString(),
+  Obj.make(Message.Message, {
+    created: new Date(Date.now() + 1_000 * index).toISOString(),
     sender,
-    blocks: [{ type: 'transcription', started: new Date(Date.now() + 1000 * index).toISOString(), text: string }],
-    rangeId: [],
-  } as any),
+    blocks: [{ _tag: 'transcript', started: new Date(Date.now() + 1_000 * index).toISOString(), text: string }],
+  }),
 );
 
-const REMOTE_AI = true;
+// const REMOTE_AI = true;
 
 describe.skip('SentenceNormalization', () => {
   const getExecutor = () => {
     return new FunctionExecutor(
       new ServiceContainer().setServices({
-        ai: {
-          client: REMOTE_AI
-            ? new EdgeAiServiceClient({
-                endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-                defaultGenerationOptions: {
-                  model: '@anthropic/claude-3-5-sonnet-20241022',
-                },
-              })
-            : new OllamaAiServiceClient({
-                overrides: {
-                  model: 'llama3.1:8b',
-                },
-              }),
-        },
+        // ai: {
+        //   client: REMOTE_AI
+        //     ? new Edge AiServiceClient({
+        //         endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+        //         defaultGenerationOptions: {
+        //           model: '@anthropic/claude-3-5-sonnet-20241022',
+        //         },
+        //       })
+        //     : new Ollama AiServiceClient({
+        //         overrides: {
+        //           model: 'llama3.1:8b',
+        //         },
+        //       }),
+        // },
       }),
     );
   };
@@ -100,8 +96,9 @@ describe.skip('SentenceNormalization', () => {
         buffer = sentences.slice(activeSentenceIndex);
       }
     }
+
     log.info('sentences', {
-      originalMessages: JSON.stringify(messages, null, 2),
+      messages: JSON.stringify(messages, null, 2),
       sentences: JSON.stringify(sentences, null, 2),
     });
     throw new Error('test');
@@ -110,7 +107,7 @@ describe.skip('SentenceNormalization', () => {
   test.only('queue', { timeout: 120_000 }, async () => {
     // Create queue.
     // TODO(dmaretskyi): Use space.queues.create() instead.
-    const queue = new MemoryQueue<DataType.Message>(createQueueDXN());
+    const queue = new MemoryQueue<Message.Message>(createQueueDXN());
     const ctx = new Context();
     let idx = 0;
     scheduleTaskInterval(
@@ -135,10 +132,6 @@ describe.skip('SentenceNormalization', () => {
 
     // Start normalizer.
     await normalizer.open();
-    effect(() => {
-      log.info('normalizer');
-      log.info(JSON.stringify(queue.objects, null, 2));
-    });
 
     await new Promise(() => {});
   });