@dxos/plugin-transcription 0.8.2-staging.7ac8446 → 0.8.2

package/src/components/stories/MicrophoneTranscription.stories.tsx

@@ -0,0 +1,274 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import { type Meta, type StoryObj } from '@storybook/react';
+import React, { useCallback, useEffect, useMemo, useState } from 'react';
+
+import { AIServiceEdgeClient } from '@dxos/ai';
+import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
+import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
+import { withPluginManager } from '@dxos/app-framework/testing';
+import {
+  extractionAnthropicFn,
+  type ExtractionFunction,
+  extractionNerFn,
+  processTranscriptMessage,
+  getNer,
+} from '@dxos/assistant';
+import { Filter, MemoryQueue } from '@dxos/echo-db';
+import { create, createQueueDxn, type Expando } from '@dxos/echo-schema';
+import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+import { ClientPlugin } from '@dxos/plugin-client';
+import { PreviewPlugin } from '@dxos/plugin-preview';
+import { SpacePlugin } from '@dxos/plugin-space';
+import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
+import { ThemePlugin } from '@dxos/plugin-theme';
+import { IndexKind, useSpace } from '@dxos/react-client/echo';
+import { defaultTx } from '@dxos/react-ui-theme';
+import { DataType } from '@dxos/schema';
+import { seedTestData, Testing } from '@dxos/schema/testing';
+import { withLayout, withTheme } from '@dxos/storybook-utils';
+
+import { TranscriptionStory } from './TranscriptionStory';
+import { useIsSpeaking } from './useIsSpeaking';
+import { TranscriptionPlugin } from '../../TranscriptionPlugin';
+import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
+import { TestItem } from '../../testing';
+import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
+import { renderMarkdown } from '../Transcript';
+
+const TRANSCRIBER_CONFIG = {
+  transcribeAfterChunksAmount: 50,
+  prefixBufferChunksAmount: 10,
+};
+
+const RECORDER_CONFIG = {
+  interval: 200,
+};
+
+type StoryProps = {
+  detectSpeaking?: boolean;
+  transcriberConfig: TranscriberParams['config'];
+  recorderConfig: MediaStreamRecorderParams['config'];
+  audioConstraints?: MediaTrackConstraints;
+  entityExtraction: 'none' | 'ner' | 'llm';
+};
+
+const DefaultStory = ({
+  detectSpeaking,
+  entityExtraction = 'none',
+  transcriberConfig,
+  recorderConfig,
+  audioConstraints,
+}: StoryProps) => {
+  const [running, setRunning] = useState(false);
+
+  // Audio.
+  const track = useAudioTrack(running, audioConstraints);
+
+  // Speaking.
+  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
+
+  // Queue.
+  const queueDxn = useMemo(() => createQueueDxn(), []);
+  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
+  const model = useQueueModelAdapter(renderMarkdown([]), queue);
+  const space = useSpace();
+
+  useEffect(() => {
+    if (!space) {
+      log.warn('no space');
+    }
+  }, [space]);
+
+  // Entity extraction.
+  const { extractionFunction, executor, objects } = useMemo(() => {
+    if (!space) {
+      log.warn('no space');
+      return {};
+    }
+
+    let executor: FunctionExecutor | undefined;
+    let extractionFunction: ExtractionFunction | undefined;
+    let objects: Promise<Expando[]> | undefined;
+
+    if (entityExtraction === 'ner') {
+      // Init model loading. Takes time.
+      void getNer();
+      extractionFunction = extractionNerFn;
+    } else if (entityExtraction === 'llm') {
+      extractionFunction = extractionAnthropicFn;
+      objects = space.db
+        .query(
+          Filter.or(
+            Filter.type(DataType.Person),
+            Filter.type(DataType.Organization),
+            Filter.type(Testing.DocumentType),
+          ),
+        )
+        .run()
+        .then((result) => result.objects);
+    }
+    if (entityExtraction !== 'none') {
+      const aiService = new AIServiceEdgeClient({
+        endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+      });
+      executor = new FunctionExecutor(
+        new ServiceContainer().setServices({ ai: { client: aiService }, database: { db: space!.db } }),
+      );
+    }
+
+    return { extractionFunction, executor, objects };
+  }, [entityExtraction, space]);
+
+  // Transcriber.
+  const handleSegments = useCallback<TranscriberParams['onSegments']>(
+    async (blocks) => {
+      const message = create(DataType.Message, { sender: { name: 'You' }, created: new Date().toISOString(), blocks });
+      if (!space) {
+        void queue.append([message]);
+        return;
+      }
+
+      if (entityExtraction !== 'none') {
+        invariant(extractionFunction, 'extractionFunction is required');
+        invariant(executor, 'executor is required');
+        const result = await processTranscriptMessage({
+          executor,
+          function: extractionFunction,
+          input: {
+            message,
+            objects: await objects,
+          },
+          options: {
+            fallbackToRaw: true,
+            timeout: 30_000,
+          },
+        });
+        void queue.append([result.message]);
+      } else {
+        void queue.append([message]);
+      }
+    },
+    [queue, space],
+  );
+
+  const transcriber = useTranscriber({
+    audioStreamTrack: track,
+    onSegments: handleSegments,
+    transcriberConfig,
+    recorderConfig,
+  });
+  const [isOpen, setIsOpen] = useState(false);
+  useEffect(() => {
+    void transcriber?.open().then(() => setIsOpen(true));
+    return () => {
+      void transcriber?.close().then(() => setIsOpen(false));
+    };
+  }, [transcriber]);
+
+  // Manage transcription state.
+  useEffect(() => {
+    if (running && transcriber?.isOpen && isSpeaking) {
+      transcriber?.startChunksRecording();
+    } else if (!running || !isSpeaking) {
+      transcriber?.stopChunksRecording();
+    }
+  }, [transcriber, running, isOpen, isSpeaking]);
+
+  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
+};
+
+const meta: Meta<typeof DefaultStory> = {
+  title: 'plugins/plugin-transcription/MicrophoneTranscription',
+  render: DefaultStory,
+  decorators: [
+    withPluginManager({
+      plugins: [
+        ThemePlugin({ tx: defaultTx }),
+        StorybookLayoutPlugin(),
+        ClientPlugin({
+          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
+          onClientInitialized: async (_, client) => {
+            await client.halo.createIdentity();
+            await client.spaces.waitUntilReady();
+            await client.spaces.default.waitUntilReady();
+            // TODO(mykola): Make API easier to use.
+            // TODO(mykola): Delete after enabling vector indexing by default.
+            // Enable vector indexing.
+            await client.services.services.QueryService!.setConfig({
+              enabled: true,
+              indexes: [
+                //
+                { kind: IndexKind.Kind.SCHEMA_MATCH },
+                { kind: IndexKind.Kind.GRAPH },
+                { kind: IndexKind.Kind.VECTOR },
+              ],
+            });
+            await client.services.services.QueryService!.reindex();
+            await seedTestData(client.spaces.default);
+          },
+        }),
+        SpacePlugin(),
+        SettingsPlugin(),
+        PreviewPlugin(),
+        IntentPlugin(),
+        TranscriptionPlugin(),
+      ],
+      fireEvents: [Events.SetupAppGraph],
+    }),
+    withTheme,
+    withLayout({ fullscreen: true, classNames: 'justify-center' }),
+  ],
+};
+
+export default meta;
+
+type Story = StoryObj<typeof DefaultStory>;
+
+export const Default: Story = {
+  args: {
+    detectSpeaking: false,
+    entityExtraction: 'none',
+    transcriberConfig: TRANSCRIBER_CONFIG,
+    recorderConfig: RECORDER_CONFIG,
+    audioConstraints: {
+      echoCancellation: false,
+      noiseSuppression: false,
+      autoGainControl: false,
+    },
+  },
+};
+
+export const EntityExtraction: Story = {
+  args: {
+    detectSpeaking: true,
+    entityExtraction: 'ner',
+    transcriberConfig: TRANSCRIBER_CONFIG,
+    recorderConfig: RECORDER_CONFIG,
+    audioConstraints: {
+      echoCancellation: true,
+      noiseSuppression: true,
+      autoGainControl: true,
+    },
+  },
+};
+
+export const SpeechDetection: Story = {
+  args: {
+    detectSpeaking: true,
+    entityExtraction: 'none',
+    transcriberConfig: TRANSCRIBER_CONFIG,
+    recorderConfig: RECORDER_CONFIG,
+    audioConstraints: {
+      echoCancellation: true,
+      noiseSuppression: true,
+      autoGainControl: true,
+    },
+  },
+};

package/src/components/stories/TranscriptionStory.tsx

@@ -0,0 +1,41 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import React, { type Dispatch, type FC, type RefObject, type SetStateAction } from 'react';
+
+import { useSpace } from '@dxos/react-client/echo';
+import { IconButton, Toolbar } from '@dxos/react-ui';
+import { ScrollContainer } from '@dxos/react-ui-components';
+import { type DataType } from '@dxos/schema';
+
+import { type SerializationModel } from '../../model';
+import { Transcript } from '../Transcript';
+
+export const TranscriptionStory: FC<{
+  model: SerializationModel<DataType.Message>;
+  running: boolean;
+  onRunningChange: Dispatch<SetStateAction<boolean>>;
+  audioRef?: RefObject<HTMLAudioElement>;
+}> = ({ model, running, onRunningChange, audioRef }) => {
+  const space = useSpace();
+
+  return (
+    <div className='flex flex-col w-[30rem]'>
+      {audioRef && <audio ref={audioRef} autoPlay />}
+      <Toolbar.Root>
+        <IconButton
+          iconOnly
+          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+          label={running ? 'Pause' : 'Play'}
+          onClick={() => onRunningChange((running) => !running)}
+        />
+      </Toolbar.Root>
+      <ScrollContainer>
+        <Transcript space={space} model={model} attendableId='story' />
+      </ScrollContainer>
+    </div>
+  );
+};

package/src/components/stories/useIsSpeaking.tsx

@@ -0,0 +1,40 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import { useEffect, useMemo, useState } from 'react';
+
+import { scheduleTask } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { SpeakingMonitor } from '@dxos/plugin-thread';
+
+export const useIsSpeaking = (track?: MediaStreamTrack) => {
+  const [isSpeaking, setIsSpeaking] = useState(false);
+  const speakingMonitor = useMemo(() => {
+    if (!track) {
+      return;
+    }
+    return new SpeakingMonitor(track);
+  }, [track]);
+
+  useEffect(() => {
+    if (!speakingMonitor) {
+      return;
+    }
+
+    const ctx = new Context();
+    scheduleTask(ctx, async () => {
+      speakingMonitor.speakingChanged.on(ctx, () => setIsSpeaking(speakingMonitor.isSpeaking));
+      await speakingMonitor.open();
+      ctx.onDispose(() => speakingMonitor.close());
+    });
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [speakingMonitor]);
+
+  return isSpeaking;
+};

package/src/hooks/index.ts

@@ -4,5 +4,6 @@
 
 export * from './useAudioFile';
 export * from './useAudioTrack';
+export * from './useQueueModelAdapter';
 export * from './useTranscriber';
 export * from './useVoiceInput';

package/src/hooks/useAudioFile.ts

@@ -4,6 +4,8 @@
 
 import { useEffect, useState } from 'react';
 
+import { scheduleTask } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { log } from '@dxos/log';
 
 export type UseAudioState = {
@@ -12,14 +14,22 @@ export type UseAudioState = {
   track?: MediaStreamTrack;
 };
 
-export const useAudioFile = (audioUrl: string): UseAudioState => {
+export const useAudioFile = (audioUrl: string, constraints?: MediaTrackConstraints): UseAudioState => {
   const [{ audio, stream, track }, setStream] = useState<UseAudioState>({});
   useEffect(() => {
-    const t = setTimeout(async () => {
+    const ctx = new Context();
+
+    scheduleTask(ctx, async () => {
       const response = await fetch(audioUrl);
       const blob = await response.blob();
+      const objectUrl = URL.createObjectURL(blob);
+      ctx.onDispose(() => {
+        URL.revokeObjectURL(objectUrl);
+      });
+
       const audio = new Audio();
-      audio.src = URL.createObjectURL(blob);
+      audio.src = objectUrl;
+
       await new Promise<void>((resolve, reject) => {
         audio.addEventListener(
           'error',
@@ -31,23 +41,41 @@ export const useAudioFile = (audioUrl: string): UseAudioState => {
         );
         audio.addEventListener(
           'canplay',
-          () => {
+          async () => {
             log.info('starting...');
-            resolve();
+            try {
+              // Try to play the audio
+              await audio.play();
+              resolve();
+            } catch (playError) {
+              log.error('Play failed', { playError });
+              // Still resolve as the audio is ready, even if autoplay failed.
+              resolve();
+            }
           },
           { once: true },
         );
         audio.load();
       });
 
-      const ctx = new AudioContext();
+      const audioCtx = new AudioContext();
+
+      // Resume AudioContext if it's suspended.
+      if (audioCtx.state === 'suspended') {
+        await audioCtx.resume();
+      }
 
-      const destination = ctx.createMediaStreamDestination();
+      const destination = audioCtx.createMediaStreamDestination();
       destination.channelCount = 1;
 
-      const source = ctx.createMediaElementSource(audio);
+      const source = audioCtx.createMediaElementSource(audio);
       source.connect(destination);
+      const track = destination.stream.getAudioTracks()[0];
+      if (constraints) {
+        await track.applyConstraints(constraints).catch((err) => log.catch(err));
+      }
 
+      // Also connect to speakers so audio is audible.
       setStream({
         audio,
         stream: destination.stream,
@@ -55,7 +83,9 @@ export const useAudioFile = (audioUrl: string): UseAudioState => {
       });
     });
 
-    return () => clearTimeout(t);
+    return () => {
+      void ctx.dispose();
+    };
   }, [audioUrl]);
 
   return { audio, stream, track };

package/src/hooks/useAudioTrack.ts

@@ -8,7 +8,7 @@ import { useEffect, useState } from 'react';
  *
  */
 // TODO(burdon): Reconcile with react-ui-sfx and plugin-calls.
-export const useAudioTrack = (active?: boolean): MediaStreamTrack | undefined => {
+export const useAudioTrack = (active?: boolean, constraints?: MediaTrackConstraints): MediaStreamTrack | undefined => {
   const [track, setTrack] = useState<MediaStreamTrack>();
   useEffect(() => {
     if (!active) {
@@ -19,7 +19,7 @@ export const useAudioTrack = (active?: boolean): MediaStreamTrack | undefined =>
 
     const initAudio = async () => {
       const audio = new Audio();
-      audio.srcObject = await navigator.mediaDevices.getUserMedia({ audio: true });
+      audio.srcObject = await navigator.mediaDevices.getUserMedia({ audio: constraints ?? true });
       const [track] = audio.srcObject.getAudioTracks();
       if (track) {
         setTrack(track);

package/src/hooks/useQueueModelAdapter.ts

@@ -0,0 +1,65 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useMemo, useEffect, useState } from 'react';
+
+import { type Queue } from '@dxos/react-client/echo';
+
+import { type Chunk, type ChunkRenderer, SerializationModel } from '../model';
+
+/**
+ * Model adapter for a queue.
+ */
+export const useQueueModelAdapter = <T extends Chunk>(
+  renderer: ChunkRenderer<T>,
+  queue: Queue<T> | undefined,
+  initialChunks: T[] = [],
+): SerializationModel<T> => {
+  const model = useMemo(() => new SerializationModel<T>(renderer, initialChunks), [queue]);
+  const [loaded, setLoaded] = useState(false);
+
+  // Set initial blocks.
+  useEffect(() => {
+    if (!queue) {
+      return;
+    }
+
+    const update = () => {
+      for (const block of queue?.objects ?? []) {
+        model.appendChunk(block);
+      }
+
+      setLoaded(true);
+    };
+
+    // TODO(burdon): This is a hack to ensure the queue is loaded.
+    let i: NodeJS.Timeout | undefined;
+    if (queue.isLoading) {
+      i = setInterval(() => {
+        if (!queue?.isLoading) {
+          clearInterval(i);
+          update();
+        }
+      }, 1_000);
+    } else {
+      update();
+    }
+
+    return () => clearTimeout(i);
+  }, [model, queue]);
+
+  // TODO(burdon): Can we listen for queue events?
+  useEffect(() => {
+    if (!loaded || !queue || model.chunks.length === queue.objects.length) {
+      return;
+    }
+
+    const chunk = queue.objects.at(-1);
+    if (chunk) {
+      model.appendChunk(chunk);
+    }
+  }, [model, loaded, queue?.objects.length]);
+
+  return model;
+};

package/src/hooks/useTranscriber.ts

@@ -15,6 +15,8 @@ import { type Transcriber } from '../transcriber';
 export const useTranscriber = ({
   audioStreamTrack,
   onSegments,
+  transcriberConfig,
+  recorderConfig,
 }: Partial<TranscriptionCapabilities.GetTranscriberProps>) => {
   const [getTranscriber] = useCapabilities(TranscriptionCapabilities.Transcriber);
 
@@ -24,8 +26,8 @@ export const useTranscriber = ({
       return undefined;
     }
 
-    return getTranscriber({ audioStreamTrack, onSegments });
-  }, [audioStreamTrack, onSegments, getTranscriber]);
+    return getTranscriber({ audioStreamTrack, onSegments, transcriberConfig, recorderConfig });
+  }, [audioStreamTrack, onSegments, getTranscriber, transcriberConfig, recorderConfig]);
 
   useEffect(() => {
     return () => {

package/src/meta.ts

@@ -11,7 +11,7 @@ export const meta: PluginMeta = {
   name: 'Transcription',
   description: 'Voice transcription.',
   icon: 'ph--microphone--regular',
-  source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/experimental/plugin-transcription',
-  tags: ['experimental'],
+  source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transcription',
+  tags: ['labs'],
   screenshots: [],
 };

package/src/model/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './model';