@dxos/plugin-transcription 0.8.4-main.abd8ff62ef → 0.8.4-main.bc2380dfbc

package/src/stories/FileTranscription.stories.tsx

@@ -3,57 +3,50 @@
 //
 
 import { type Meta, type StoryObj } from '@storybook/react-vite';
-import * as Effect from 'effect/Effect';
-import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import React, { useCallback, useEffect, useRef, useState } from 'react';
 
-import { withPluginManager } from '@dxos/app-framework/testing';
-import { AppActivationEvents } from '@dxos/app-toolkit';
-import { scheduleTask } from '@dxos/async';
-import { Context } from '@dxos/context';
-import { Obj } from '@dxos/echo';
-import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo/internal';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
-import { log } from '@dxos/log';
-import { ClientPlugin } from '@dxos/plugin-client';
-import { initializeIdentity } from '@dxos/plugin-client/testing';
-import { PreviewPlugin } from '@dxos/plugin-preview';
-import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
-import { withLayout } from '@dxos/react-ui/testing';
-import { TestSchema } from '@dxos/schema/testing';
-import { type Actor, Message, Organization, Person } from '@dxos/types';
-import { seedTestData } from '@dxos/types/testing';
+import { useAudioFile } from '#hooks';
 
-import { useAudioFile, useQueueModelAdapter, useTranscriber } from '#hooks';
-import { TestItem } from '#testing';
-
-import { MessageNormalizer, getActorId } from '../normalization';
 import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
-import { TranscriptionPlugin } from '../TranscriptionPlugin';
-import { renderByline } from '../util';
+import {
+  useIsSpeaking,
+  createStoryDecorators,
+  useStoryAppendSegments,
+  useStoryMessageModel,
+  useStoryTranscriber,
+} from './common';
 import { TranscriptionStory } from './TranscriptionStory';
-import { useIsSpeaking } from './useIsSpeaking';
 
-const AudioFile = ({
-  detectSpeaking,
-  audioUrl,
-  normalizeSentences,
-  transcriberConfig,
-  recorderConfig,
-  audioConstraints,
-}: {
+const DEFAULT_TRANSCRIBER_CONFIG = {
+  transcribeAfterChunksAmount: 100,
+  prefixBufferChunksAmount: 50,
+  normalizeSentences: true,
+};
+
+const DEFAULT_RECORDER_CONFIG = {
+  interval: 200,
+};
+
+type DefaultStoryProps = {
   detectSpeaking?: boolean;
   normalizeSentences?: boolean;
   audioUrl: string;
   transcriberConfig?: TranscriberProps['config'];
   recorderConfig?: MediaStreamRecorderProps['config'];
   audioConstraints?: MediaTrackConstraints;
-}) => {
+};
+
+const DefaultStory = ({
+  detectSpeaking,
+  audioUrl,
+  normalizeSentences,
+  transcriberConfig = DEFAULT_TRANSCRIBER_CONFIG,
+  recorderConfig = DEFAULT_RECORDER_CONFIG,
+  audioConstraints,
+}: DefaultStoryProps) => {
   const [running, setRunning] = useState(false);
-  const actor: Actor.Actor = useMemo(() => ({ name: 'You' }), []);
 
-  // Optional uploaded file overrides the default URL.
-  // The useEffect below revokes the previous URL whenever it changes (and on unmount).
+  // Optional uploaded file overrides the default URL; revoke the previous URL on change/unmount.
   const [uploadedUrl, setUploadedUrl] = useState<string>();
   useEffect(() => {
     return () => {
@@ -69,12 +62,10 @@ const AudioFile = ({
   }, []);
 
   const sourceUrl = uploadedUrl ?? audioUrl;
-
-  // Audio.
   const { audio, track, stream } = useAudioFile(sourceUrl, audioConstraints);
-
-  // Speaking.
   const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
+
+  // Pipe the decoded stream into the <audio> element so the user hears playback.
   const ref = useRef<HTMLAudioElement>(null);
   useEffect(() => {
     if (ref.current && stream) {
@@ -82,6 +73,7 @@ const AudioFile = ({
     }
   }, [stream]);
 
+  // Mirror the toolbar's running flag to the underlying HTMLAudioElement.
   useEffect(() => {
     if (!audio) {
       return;
@@ -94,176 +86,58 @@ const AudioFile = ({
     }
   }, [audio, running]);
 
-  // Transcriber.
-  // TODO(dmaretskyi): Use space.queues.create() instead.
-  const queueDxn = useMemo(() => createQueueDXN(), []);
-  const queue = useMemo(() => new MemoryQueue<Message.Message>(queueDxn), [queueDxn]);
-
-  const model = useQueueModelAdapter(renderByline([]), queue);
-  const handleSegments = useCallback<TranscriberProps['onSegments']>(
-    async (blocks) => {
-      void queue?.append([
-        Obj.make(Message.Message, {
-          created: new Date().toISOString(),
-          sender: actor,
-          blocks,
-        }),
-      ]);
-    },
-    [queue],
-  );
+  const { model, appendMessage } = useStoryMessageModel();
+  const handleSegments = useStoryAppendSegments(appendMessage);
 
-  const transcriber = useTranscriber({
+  useStoryTranscriber({
     audioStreamTrack: track,
-    onSegments: handleSegments,
+    running,
+    isSpeaking,
     transcriberConfig,
     recorderConfig,
+    onSegments: handleSegments,
   });
 
-  // Normalize sentences.
-  const normalizer = useMemo(() => {
-    if (!normalizeSentences) {
-      return;
-    }
-    const executor = new FunctionExecutor(
-      new ServiceContainer().setServices({
-        // ai: {
-        //   client: new Edge AiServiceClient({
-        //     endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-        //     defaultGenerationOptions: {
-        //       model: '@anthropic/claude-3-5-sonnet-20241022',
-        //     },
-        //   }),
-        // },
-      }),
-    );
-
-    return new MessageNormalizer({
-      functionExecutor: executor,
-      queue,
-      startingCursor: { actorId: getActorId(actor), timestamp: new Date().toISOString() },
-    });
-  }, [normalizeSentences, queue, actor]);
-
-  useEffect(() => {
-    if (!normalizer) {
-      return;
-    }
-    const ctx = new Context();
-    scheduleTask(ctx, async () => {
-      await normalizer.open();
-      ctx.onDispose(async () => {
-        await normalizer.close();
-      });
-    });
-    return () => {
-      void ctx.dispose();
-    };
-  }, [normalizer]);
-
-  const manageChunkRecording = () => {
-    if (running && isSpeaking && transcriber) {
-      log.info('starting transcription');
-      transcriber?.startChunksRecording();
-    } else if ((!isSpeaking || !running) && transcriber) {
-      log.info('stopping transcription');
-      transcriber?.stopChunksRecording();
-    }
-  };
-
-  useEffect(() => {
-    const ctx = new Context();
-    scheduleTask(ctx, async () => {
-      if (transcriber && running) {
-        await transcriber.open();
-      } else if (!running) {
-        await transcriber?.close();
-      }
-      manageChunkRecording();
-    });
-
-    return () => {
-      void ctx.dispose();
-    };
-  }, [transcriber, running]);
-
-  useEffect(() => {
-    manageChunkRecording();
-  }, [isSpeaking, stream]);
+  // TODO(burdon): Sentence normalization moved to require a real space-backed Feed + FeedService runtime.
+  //  Re-enable here once the story has access to one.
+  void normalizeSentences;
 
   return (
     <TranscriptionStory
+      audioRef={ref}
       disabled={!stream}
       model={model}
       running={running}
       onRunningChange={setRunning}
-      audioRef={ref}
       onUpload={handleUpload}
     />
   );
 };
 
 const meta = {
-  title: 'plugins/plugin-transcription/components/FileTranscription',
-  decorators: [
-    withLayout({ layout: 'column' }),
-    withPluginManager({
-      plugins: [
-        ...corePlugins(),
-        StorybookPlugin({}),
-        ClientPlugin({
-          types: [TestItem, Person.Person, Organization.Organization, TestSchema.DocumentType],
-          onClientInitialized: ({ client }) =>
-            Effect.gen(function* () {
-              const { personalSpace } = yield* initializeIdentity(client);
-              yield* Effect.promise(() => seedTestData(personalSpace));
-            }),
-        }),
-
-        PreviewPlugin(),
-        TranscriptionPlugin(),
-      ],
-      fireEvents: [AppActivationEvents.SetupAppGraph],
-    }),
-  ],
-} satisfies Meta;
+  title: 'plugins/plugin-transcription/stories/FileTranscription',
+  render: DefaultStory,
+  decorators: createStoryDecorators(),
+} satisfies Meta<DefaultStoryProps>;
 
 export default meta;
 
 type Story = StoryObj<typeof meta>;
 
-const TRANSCRIBER_CONFIG = {
-  transcribeAfterChunksAmount: 100,
-  prefixBufferChunksAmount: 50,
-  normalizeSentences: true,
-};
-
-const RECORDER_CONFIG = {
-  interval: 200,
-};
-
-export const Default: StoryObj<typeof AudioFile> = {
-  render: AudioFile,
+export const Default: Story = {
   args: {
     detectSpeaking: true,
     // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
     audioUrl: 'https://dxos.network/audio-london.m4a',
     // textUrl: 'https://dxos.network/audio-london.txt',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
   },
 };
 
 // TODO(mykola): Fix sentence normalization.
 // export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
-//   render: AudioFile,
 //   args: {
 //     detectSpeaking: true,
 //     normalizeSentences: true,
-//     // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
 //     audioUrl: 'https://dxos.network/audio-london.m4a',
-//     // textUrl: 'https://dxos.network/audio-london.txt',
-//     transcriberConfig: TRANSCRIBER_CONFIG,
-//     recorderConfig: RECORDER_CONFIG,
 //   },
 // };

package/src/stories/KeyWordRecognition.stories.tsx

@@ -23,7 +23,6 @@ const DefaultStory = ({ keywords }: DefaultStoryProps) => {
   const [running, setRunning] = useState(false);
   const [matchingWords, setMatchingWords] = useState<Word[]>(keywords.map((word) => ({ text: word, matched: false })));
 
-  const [transcript, setTranscript] = useState('');
   const recognition = useMemo(() => {
     // TODO(mykola): Fix types
     const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
@@ -38,16 +37,16 @@ const DefaultStory = ({ keywords }: DefaultStoryProps) => {
     recognition.interimResults = true;
     recognition.lang = 'en-US';
     recognition.maxAlternatives = 1;
-    // Add this line to enable punctuation
+    // Add this line to enable punctuation.
     recognition.grammars = new SpeechGrammarList();
 
-    // TODO(mykola): Fix types
+    // TODO(mykola): Fix types.
     recognition.onresult = (event: any) => {
       log.info('recognition result', { event });
       const current = event.resultIndex;
       const transcript = event.results[current][0].transcript;
 
-      // Remove punctuation and normalize whitespace for comparison
+      // Remove punctuation and normalize whitespace for comparison.
       const normalizeText = (text: string) =>
         text
           .toLowerCase()
@@ -57,7 +56,7 @@ const DefaultStory = ({ keywords }: DefaultStoryProps) => {
 
       const normalizedTranscript = normalizeText(transcript);
 
-      // Check each keyword and update matching words
+      // Check each keyword and update matching words.
       const updatedWords = matchingWords.map((word) => {
         const normalizedKeyword = normalizeText(word.text);
         const matched = normalizedTranscript.includes(normalizedKeyword);
@@ -65,10 +64,9 @@ const DefaultStory = ({ keywords }: DefaultStoryProps) => {
       });
 
       setMatchingWords(updatedWords);
-      setTranscript(transcript);
     };
 
-    // TODO(mykola): Fix types
+    // TODO(mykola): Fix types.
     recognition.onerror = (event: any) => {
       log.error('Speech recognition error:', { error: event.error });
       setRunning(false);
@@ -99,7 +97,7 @@ const DefaultStory = ({ keywords }: DefaultStoryProps) => {
       </Panel.Toolbar>
 
       <Panel.Content>
-        <div className='p-4 grid grid-cols-[repeat(auto-fill,minmax(6rem,1fr))] gap-2'>
+        <div className='p-2 grid grid-cols-[repeat(auto-fill,minmax(6rem,1fr))] gap-2'>
           {matchingWords.map((word, index) => (
             <span
               key={index}
@@ -118,7 +116,7 @@ const DefaultStory = ({ keywords }: DefaultStoryProps) => {
 };
 
 const meta = {
-  title: 'plugins/plugin-transcription/components/KeyWordDetection',
+  title: 'plugins/plugin-transcription/stories/KeyWordDetection',
   render: DefaultStory,
   decorators: [withTheme(), withLayout({ layout: 'column' })],
 } satisfies Meta<typeof DefaultStory>;

package/src/stories/LiveTranscription.stories.tsx

@@ -0,0 +1,153 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import React, { useCallback, useMemo, useState } from 'react';
+
+import {
+  type ExtractionFunction,
+  extractionAnthropicFunction,
+  extractionNerFunction,
+  getNer,
+  processTranscriptMessage,
+} from '@dxos/assistant/extraction';
+import { Filter, type Obj } from '@dxos/echo';
+import { invariant } from '@dxos/invariant';
+import { useSpaces } from '@dxos/react-client/echo';
+import { Message, Organization, Person } from '@dxos/types';
+
+import { useAudioTrack } from '#hooks';
+
+import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
+import { useIsSpeaking, createStoryDecorators, useStoryMessageModel, useStoryTranscriber } from './common';
+import { TranscriptionStory } from './TranscriptionStory';
+
+const DEFAULT_TRANSCRIBER_CONFIG = {
+  transcribeAfterChunksAmount: 50,
+  prefixBufferChunksAmount: 10,
+};
+
+const DEFAULT_RECORDER_CONFIG = {
+  interval: 200,
+};
+
+type DefaultStoryProps = {
+  detectSpeaking?: boolean;
+  transcriberConfig?: TranscriberProps['config'];
+  recorderConfig?: MediaStreamRecorderProps['config'];
+  audioConstraints?: MediaTrackConstraints;
+  entityExtraction?: 'ner' | 'llm';
+};
+
+const DefaultStory = ({
+  detectSpeaking,
+  entityExtraction,
+  transcriberConfig = DEFAULT_TRANSCRIBER_CONFIG,
+  recorderConfig = DEFAULT_RECORDER_CONFIG,
+  audioConstraints,
+}: DefaultStoryProps) => {
+  const [running, setRunning] = useState(false);
+  const track = useAudioTrack(running, audioConstraints);
+  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
+  const { model, appendMessage } = useStoryMessageModel();
+  const [space] = useSpaces();
+
+  // Resolve the entity-extraction function + nearby objects once per `entityExtraction` toggle.
+  const { extractionFunction, objects } = useMemo(() => {
+    if (!space || !entityExtraction) {
+      return {};
+    }
+
+    let extractionFunction: ExtractionFunction | undefined;
+    let objects: Promise<Obj.Unknown[]> | undefined;
+    if (entityExtraction === 'ner') {
+      void getNer(); // Init model loading. Takes time.
+      extractionFunction = extractionNerFunction;
+    } else if (entityExtraction === 'llm') {
+      extractionFunction = extractionAnthropicFunction;
+      objects = space.db.query(Filter.or(Filter.type(Person.Person), Filter.type(Organization.Organization))).run();
+    }
+    return { extractionFunction, objects };
+  }, [entityExtraction, space]);
+
+  // Build a Message from each transcribed batch; optionally enrich via entity extraction.
+  const handleSegments = useCallback<TranscriberProps['onSegments']>(
+    async (blocks) => {
+      const message = Message.make({
+        sender: { name: 'You' },
+        created: new Date().toISOString(),
+        blocks,
+      });
+
+      if (entityExtraction && space) {
+        invariant(extractionFunction, 'extractionFunction is required');
+        const result = await processTranscriptMessage({
+          function: extractionFunction,
+          input: { message, objects: await objects },
+          options: { fallbackToRaw: true, timeout: 30_000 },
+        });
+        appendMessage(result.message);
+      } else {
+        appendMessage(message);
+      }
+    },
+    [appendMessage, space, entityExtraction, extractionFunction, objects],
+  );
+
+  useStoryTranscriber({
+    audioStreamTrack: track,
+    running,
+    isSpeaking,
+    transcriberConfig,
+    recorderConfig,
+    onSegments: handleSegments,
+  });
+
+  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
+};
+
+const meta = {
+  title: 'plugins/plugin-transcription/stories/LiveTranscription',
+  render: DefaultStory,
+  decorators: createStoryDecorators({ enableVectorIndex: true }),
+} satisfies Meta<typeof DefaultStory>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {
+  args: {
+    detectSpeaking: false,
+    audioConstraints: {
+      echoCancellation: false,
+      noiseSuppression: false,
+      autoGainControl: false,
+    },
+  },
+};
+
+export const SpeechDetection: Story = {
+  args: {
+    detectSpeaking: true,
+    audioConstraints: {
+      echoCancellation: true,
+      noiseSuppression: true,
+      autoGainControl: true,
+    },
+  },
+};
+
+// TODO(mykola): Fix hugging face quota issues.
+// export const EntityExtraction: Story = {
+//   args: {
+//     detectSpeaking: true,
+//     entityExtraction: 'ner',
+//     audioConstraints: {
+//       echoCancellation: true,
+//       noiseSuppression: true,
+//       autoGainControl: true,
+//     },
+//   },
+// };

package/src/stories/TranscriptionStory.tsx

@@ -2,15 +2,12 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
 import React, {
   type ChangeEvent,
   type Dispatch,
-  type FC,
-  type ReactNode,
   type RefObject,
   type SetStateAction,
+  useCallback,
   useRef,
 } from 'react';
 
@@ -20,34 +17,46 @@ import { type Message } from '@dxos/types';
 import { Transcription } from '../components';
 import { type SerializationModel } from '../model';
 
+export type TranscriptionStoryProps = {
+  audioRef?: RefObject<HTMLAudioElement | null>;
+  model: SerializationModel<Message.Message>;
+  running: boolean;
+  disabled?: boolean;
+  uploadAccept?: string;
+  onRunningChange: Dispatch<SetStateAction<boolean>>;
+  onUpload?: (file: File) => void;
+};
+
 /**
  * Story wrapper that renders a transcript with playback controls, an optional audio file
  * upload affordance, and a toolbar slot for additional controls.
  *
  * @param onUpload - Callback fired when the user picks a local audio file via the upload button.
  * @param uploadAccept - HTML `accept` filter for the hidden file input. Defaults to `audio/*`.
- * @param toolbarSlot - Extra content rendered after the upload button inside the toolbar.
  */
-export const TranscriptionStory: FC<{
-  model: SerializationModel<Message.Message>;
-  disabled?: boolean;
-  running: boolean;
-  onRunningChange: Dispatch<SetStateAction<boolean>>;
-  audioRef?: RefObject<HTMLAudioElement | null>;
-  onUpload?: (file: File) => void;
-  uploadAccept?: string;
-  toolbarSlot?: ReactNode;
-}> = ({ model, running, onRunningChange, audioRef, disabled, onUpload, uploadAccept = 'audio/*', toolbarSlot }) => {
+export const TranscriptionStory = ({
+  audioRef,
+  model,
+  running,
+  disabled,
+  uploadAccept = 'audio/*',
+  onRunningChange,
+  onUpload,
+}: TranscriptionStoryProps) => {
   const fileInputRef = useRef<HTMLInputElement>(null);
 
-  const handleFileChange = (event: ChangeEvent<HTMLInputElement>) => {
-    const file = event.target.files?.[0];
-    if (file && onUpload) {
-      onUpload(file);
-    }
-    // Reset so the same file can be re-selected.
-    event.target.value = '';
-  };
+  const handleFileChange = useCallback(
+    (event: ChangeEvent<HTMLInputElement>) => {
+      const file = event.target.files?.[0];
+      if (file && onUpload) {
+        onUpload(file);
+      }
+
+      // Reset so the same file can be re-selected.
+      event.target.value = '';
+    },
+    [onUpload],
+  );
 
   return (
     <>
@@ -79,7 +88,6 @@ export const TranscriptionStory: FC<{
             />
           </>
         )}
-        {toolbarSlot}
       </Toolbar.Root>
       <ScrollContainer.Root pin>
         <ScrollContainer.Content>