@dxos/plugin-transcription 0.8.2-main.fbd8ed0 → 0.8.2-staging.4d6ad0f

package/src/components/stories/FileTranscription.stories.tsx

@@ -0,0 +1,240 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import { type Meta, type StoryObj } from '@storybook/react';
+import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+
+import { AIServiceEdgeClient } from '@dxos/ai';
+import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
+import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { scheduleTask } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { MemoryQueue } from '@dxos/echo-db';
+import { create, createQueueDxn } from '@dxos/echo-schema';
+import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { log } from '@dxos/log';
+import { ClientPlugin } from '@dxos/plugin-client';
+import { PreviewPlugin } from '@dxos/plugin-preview';
+import { SpacePlugin } from '@dxos/plugin-space';
+import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
+import { ThemePlugin } from '@dxos/plugin-theme';
+import { defaultTx } from '@dxos/react-ui-theme';
+import { DataType } from '@dxos/schema';
+import { seedTestData, Testing } from '@dxos/schema/testing';
+import { withLayout, withTheme } from '@dxos/storybook-utils';
+
+import { TranscriptionStory } from './TranscriptionStory';
+import { useIsSpeaking } from './useIsSpeaking';
+import { TranscriptionPlugin } from '../../TranscriptionPlugin';
+import { useAudioFile, useQueueModelAdapter, useTranscriber } from '../../hooks';
+import { getActorId, MessageNormalizer } from '../../segments-normalization';
+import { TestItem } from '../../testing';
+import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
+import { renderMarkdown } from '../Transcript';
+
+const AudioFile = ({
+  detectSpeaking,
+  audioUrl,
+  normalizeSentences,
+  transcriberConfig,
+  recorderConfig,
+  audioConstraints,
+}: {
+  detectSpeaking?: boolean;
+  normalizeSentences?: boolean;
+  audioUrl: string;
+  transcriberConfig?: TranscriberParams['config'];
+  recorderConfig?: MediaStreamRecorderParams['config'];
+  audioConstraints?: MediaTrackConstraints;
+}) => {
+  const [running, setRunning] = useState(false);
+  const actor: DataType.Actor = useMemo(() => ({ name: 'You' }), []);
+
+  // Audio.
+  const { audio, track, stream } = useAudioFile(audioUrl, audioConstraints);
+
+  // Speaking.
+  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
+  const ref = useRef<HTMLAudioElement>(null);
+  useEffect(() => {
+    if (ref.current && stream) {
+      ref.current.srcObject = stream;
+    }
+  }, [stream]);
+
+  useEffect(() => {
+    if (!audio) {
+      log.warn('no audio');
+      return;
+    }
+
+    if (running) {
+      void audio.play();
+    } else {
+      void audio.pause();
+    }
+  }, [audio, running]);
+
+  // Transcriber.
+  const queueDxn = useMemo(() => createQueueDxn(), []);
+  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
+
+  const model = useQueueModelAdapter(renderMarkdown([]), queue);
+  const handleSegments = useCallback<TranscriberParams['onSegments']>(
+    async (blocks) => {
+      const message = create(DataType.Message, { sender: actor, created: new Date().toISOString(), blocks });
+      void queue?.append([message]);
+    },
+    [queue],
+  );
+
+  const transcriber = useTranscriber({
+    audioStreamTrack: track,
+    onSegments: handleSegments,
+    transcriberConfig,
+    recorderConfig,
+  });
+
+  // Normalize sentences.
+  const normalizer = useMemo(() => {
+    if (!normalizeSentences) {
+      return;
+    }
+    const executor = new FunctionExecutor(
+      new ServiceContainer().setServices({
+        ai: {
+          client: new AIServiceEdgeClient({
+            endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+            defaultGenerationOptions: {
+              model: '@anthropic/claude-3-5-sonnet-20241022',
+            },
+          }),
+        },
+      }),
+    );
+
+    return new MessageNormalizer({
+      functionExecutor: executor,
+      queue,
+      startingCursor: { actorId: getActorId(actor), timestamp: new Date().toISOString() },
+    });
+  }, [normalizeSentences, queue, actor]);
+
+  useEffect(() => {
+    if (!normalizer) {
+      return;
+    }
+    const ctx = new Context();
+    scheduleTask(ctx, async () => {
+      await normalizer.open();
+      ctx.onDispose(async () => {
+        await normalizer.close();
+      });
+    });
+    return () => {
+      void ctx.dispose();
+    };
+  }, [normalizer]);
+
+  const manageChunkRecording = () => {
+    if (running && isSpeaking) {
+      log.info('starting transcription');
+      transcriber?.startChunksRecording();
+    } else if (!isSpeaking || !running) {
+      log.info('stopping transcription');
+      transcriber?.stopChunksRecording();
+    }
+  };
+
+  useEffect(() => {
+    const ctx = new Context();
+    scheduleTask(ctx, async () => {
+      if (transcriber && running) {
+        await transcriber.open();
+      } else if (!running) {
+        await transcriber?.close();
+      }
+      manageChunkRecording();
+    });
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [transcriber, running]);
+
+  useEffect(() => {
+    manageChunkRecording();
+  }, [isSpeaking, stream]);
+
+  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} audioRef={ref} />;
+};
+
+const meta: Meta<typeof AudioFile> = {
+  title: 'plugins/plugin-transcription/FileTranscription',
+  decorators: [
+    withPluginManager({
+      plugins: [
+        ThemePlugin({ tx: defaultTx }),
+        StorybookLayoutPlugin(),
+        ClientPlugin({
+          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
+          onClientInitialized: async (_, client) => {
+            await client.halo.createIdentity();
+            await client.spaces.waitUntilReady();
+            await client.spaces.default.waitUntilReady();
+            await seedTestData(client.spaces.default);
+          },
+        }),
+        SpacePlugin(),
+        SettingsPlugin(),
+        PreviewPlugin(),
+        IntentPlugin(),
+        TranscriptionPlugin(),
+      ],
+      fireEvents: [Events.SetupAppGraph],
+    }),
+    withTheme,
+    withLayout({ fullscreen: true, classNames: 'justify-center' }),
+  ],
+};
+
+export default meta;
+
+const TRANSCRIBER_CONFIG = {
+  transcribeAfterChunksAmount: 100,
+  prefixBufferChunksAmount: 50,
+  normalizeSentences: true,
+};
+
+const RECORDER_CONFIG = {
+  interval: 200,
+};
+
+export const Default: StoryObj<typeof AudioFile> = {
+  render: AudioFile,
+  args: {
+    detectSpeaking: true,
+    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
+    audioUrl: 'https://dxos.network/audio-london.m4a',
+    // textUrl: 'https://dxos.network/audio-london.txt',
+    transcriberConfig: TRANSCRIBER_CONFIG,
+    recorderConfig: RECORDER_CONFIG,
+  },
+};
+
+export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
+  render: AudioFile,
+  args: {
+    detectSpeaking: true,
+    normalizeSentences: true,
+    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
+    audioUrl: 'https://dxos.network/audio-london.m4a',
+    // textUrl: 'https://dxos.network/audio-london.txt',
+    transcriberConfig: TRANSCRIBER_CONFIG,
+    recorderConfig: RECORDER_CONFIG,
+  },
+};

package/src/components/stories/KeyWordRecognition.stories.tsx

@@ -0,0 +1,128 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import { type StoryObj, type Meta } from '@storybook/react';
+import React, { useEffect, useMemo, useState } from 'react';
+
+import { log } from '@dxos/log';
+import { Toolbar } from '@dxos/react-ui';
+import { mx } from '@dxos/react-ui-theme';
+import { withLayout, withTheme } from '@dxos/storybook-utils';
+
+type Word = {
+  text: string;
+  matched: boolean;
+};
+
+const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
+  const [running, setRunning] = useState(false);
+  const [matchingWords, setMatchingWords] = useState<Word[]>(keywords.map((word) => ({ text: word, matched: false })));
+
+  const [transcript, setTranscript] = useState('');
+  const recognition = useMemo(() => {
+    // TODO(mykola): Fix types
+    const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
+    const SpeechGrammarList = (window as any).SpeechGrammarList || (window as any).webkitSpeechGrammarList;
+    if (!SpeechRecognition || !SpeechGrammarList) {
+      log.error('Speech recognition not supported in this browser');
+      return;
+    }
+
+    const recognition = new SpeechRecognition();
+    recognition.continuous = true;
+    recognition.interimResults = true;
+    recognition.lang = 'en-US';
+    recognition.maxAlternatives = 1;
+    // Add this line to enable punctuation
+    recognition.grammars = new SpeechGrammarList();
+
+    // TODO(mykola): Fix types
+    recognition.onresult = (event: any) => {
+      log.info('recognition result', { event });
+      const current = event.resultIndex;
+      const transcript = event.results[current][0].transcript;
+
+      // Remove punctuation and normalize whitespace for comparison
+      const normalizeText = (text: string) =>
+        text
+          .toLowerCase()
+          .replace(/[.,!?]/g, '')
+          .replace(/\s+/g, ' ')
+          .trim();
+
+      const normalizedTranscript = normalizeText(transcript);
+
+      // Check each keyword and update matching words
+      const updatedWords = matchingWords.map((word) => {
+        const normalizedKeyword = normalizeText(word.text);
+        const matched = normalizedTranscript.includes(normalizedKeyword);
+        return { ...word, matched };
+      });
+
+      setMatchingWords(updatedWords);
+      setTranscript(transcript);
+    };
+
+    // TODO(mykola): Fix types
+    recognition.onerror = (event: any) => {
+      log.error('Speech recognition error:', { error: event.error });
+      setRunning(false);
+    };
+
+    return recognition;
+  }, []);
+
+  useEffect(() => {
+    if (running) {
+      recognition?.start();
+    } else {
+      recognition?.stop();
+    }
+  }, [running, recognition]);
+
+  return (
+    <div className={mx('flex flex-col gap-2 w-[30rem]')}>
+      <Toolbar.Root>
+        <Toolbar.IconButton
+          iconOnly
+          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+          label={running ? 'Pause' : 'Play'}
+          onClick={() => setRunning((running) => !running)}
+        />
+      </Toolbar.Root>
+
+      <div className='flex flex-wrap'>
+        {matchingWords.map((word, index) => (
+          <span
+            key={index}
+            className={mx(
+              'px-3 py-1 m-2 rounded border',
+              word.matched ? 'bg-red-200 text-red-800 border-red-300' : 'bg-gray-200 text-gray-600 border-gray-300',
+            )}
+          >
+            {word.text}
+          </span>
+        ))}
+      </div>
+
+      <span>{transcript || (running ? 'Listening...' : 'Click play to start speech recognition')}</span>
+    </div>
+  );
+};
+
+const meta: Meta<typeof SpeechRecognition> = {
+  title: 'plugins/plugin-transcription/KeyWordDetection',
+  decorators: [withTheme, withLayout({ fullscreen: true, classNames: 'justify-center' })],
+};
+
+export default meta;
+
+export const SpeechRecognitionAPI: StoryObj<typeof SpeechRecognition> = {
+  render: SpeechRecognition,
+  args: {
+    keywords: ['kai', 'computer', 'hello'],
+  },
+};

package/src/components/stories/MicrophoneTranscription.stories.tsx

@@ -0,0 +1,274 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import { type Meta, type StoryObj } from '@storybook/react';
+import React, { useCallback, useEffect, useMemo, useState } from 'react';
+
+import { AIServiceEdgeClient } from '@dxos/ai';
+import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
+import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
+import { withPluginManager } from '@dxos/app-framework/testing';
+import {
+  extractionAnthropicFn,
+  type ExtractionFunction,
+  extractionNerFn,
+  processTranscriptMessage,
+  getNer,
+} from '@dxos/assistant';
+import { Filter, MemoryQueue } from '@dxos/echo-db';
+import { create, createQueueDxn, type Expando } from '@dxos/echo-schema';
+import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+import { ClientPlugin } from '@dxos/plugin-client';
+import { PreviewPlugin } from '@dxos/plugin-preview';
+import { SpacePlugin } from '@dxos/plugin-space';
+import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
+import { ThemePlugin } from '@dxos/plugin-theme';
+import { IndexKind, useSpace } from '@dxos/react-client/echo';
+import { defaultTx } from '@dxos/react-ui-theme';
+import { DataType } from '@dxos/schema';
+import { seedTestData, Testing } from '@dxos/schema/testing';
+import { withLayout, withTheme } from '@dxos/storybook-utils';
+
+import { TranscriptionStory } from './TranscriptionStory';
+import { useIsSpeaking } from './useIsSpeaking';
+import { TranscriptionPlugin } from '../../TranscriptionPlugin';
+import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
+import { TestItem } from '../../testing';
+import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
+import { renderMarkdown } from '../Transcript';
+
+const TRANSCRIBER_CONFIG = {
+  transcribeAfterChunksAmount: 50,
+  prefixBufferChunksAmount: 10,
+};
+
+const RECORDER_CONFIG = {
+  interval: 200,
+};
+
+type StoryProps = {
+  detectSpeaking?: boolean;
+  transcriberConfig: TranscriberParams['config'];
+  recorderConfig: MediaStreamRecorderParams['config'];
+  audioConstraints?: MediaTrackConstraints;
+  entityExtraction: 'none' | 'ner' | 'llm';
+};
+
+const DefaultStory = ({
+  detectSpeaking,
+  entityExtraction = 'none',
+  transcriberConfig,
+  recorderConfig,
+  audioConstraints,
+}: StoryProps) => {
+  const [running, setRunning] = useState(false);
+
+  // Audio.
+  const track = useAudioTrack(running, audioConstraints);
+
+  // Speaking.
+  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
+
+  // Queue.
+  const queueDxn = useMemo(() => createQueueDxn(), []);
+  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
+  const model = useQueueModelAdapter(renderMarkdown([]), queue);
+  const space = useSpace();
+
+  useEffect(() => {
+    if (!space) {
+      log.warn('no space');
+    }
+  }, [space]);
+
+  // Entity extraction.
+  const { extractionFunction, executor, objects } = useMemo(() => {
+    if (!space) {
+      log.warn('no space');
+      return {};
+    }
+
+    let executor: FunctionExecutor | undefined;
+    let extractionFunction: ExtractionFunction | undefined;
+    let objects: Promise<Expando[]> | undefined;
+
+    if (entityExtraction === 'ner') {
+      // Init model loading. Takes time.
+      void getNer();
+      extractionFunction = extractionNerFn;
+    } else if (entityExtraction === 'llm') {
+      extractionFunction = extractionAnthropicFn;
+      objects = space.db
+        .query(
+          Filter.or(
+            Filter.type(DataType.Person),
+            Filter.type(DataType.Organization),
+            Filter.type(Testing.DocumentType),
+          ),
+        )
+        .run()
+        .then((result) => result.objects);
+    }
+    if (entityExtraction !== 'none') {
+      const aiService = new AIServiceEdgeClient({
+        endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+      });
+      executor = new FunctionExecutor(
+        new ServiceContainer().setServices({ ai: { client: aiService }, database: { db: space!.db } }),
+      );
+    }
+
+    return { extractionFunction, executor, objects };
+  }, [entityExtraction, space]);
+
+  // Transcriber.
+  const handleSegments = useCallback<TranscriberParams['onSegments']>(
+    async (blocks) => {
+      const message = create(DataType.Message, { sender: { name: 'You' }, created: new Date().toISOString(), blocks });
+      if (!space) {
+        void queue.append([message]);
+        return;
+      }
+
+      if (entityExtraction !== 'none') {
+        invariant(extractionFunction, 'extractionFunction is required');
+        invariant(executor, 'executor is required');
+        const result = await processTranscriptMessage({
+          executor,
+          function: extractionFunction,
+          input: {
+            message,
+            objects: await objects,
+          },
+          options: {
+            fallbackToRaw: true,
+            timeout: 30_000,
+          },
+        });
+        void queue.append([result.message]);
+      } else {
+        void queue.append([message]);
+      }
+    },
+    [queue, space],
+  );
+
+  const transcriber = useTranscriber({
+    audioStreamTrack: track,
+    onSegments: handleSegments,
+    transcriberConfig,
+    recorderConfig,
+  });
+  const [isOpen, setIsOpen] = useState(false);
+  useEffect(() => {
+    void transcriber?.open().then(() => setIsOpen(true));
+    return () => {
+      void transcriber?.close().then(() => setIsOpen(false));
+    };
+  }, [transcriber]);
+
+  // Manage transcription state.
+  useEffect(() => {
+    if (running && transcriber?.isOpen && isSpeaking) {
+      transcriber?.startChunksRecording();
+    } else if (!running || !isSpeaking) {
+      transcriber?.stopChunksRecording();
+    }
+  }, [transcriber, running, isOpen, isSpeaking]);
+
+  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
+};
+
+const meta: Meta<typeof DefaultStory> = {
+  title: 'plugins/plugin-transcription/MicrophoneTranscription',
+  render: DefaultStory,
+  decorators: [
+    withPluginManager({
+      plugins: [
+        ThemePlugin({ tx: defaultTx }),
+        StorybookLayoutPlugin(),
+        ClientPlugin({
+          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
+          onClientInitialized: async (_, client) => {
+            await client.halo.createIdentity();
+            await client.spaces.waitUntilReady();
+            await client.spaces.default.waitUntilReady();
+            // TODO(mykola): Make API easier to use.
+            // TODO(mykola): Delete after enabling vector indexing by default.
+            // Enable vector indexing.
+            await client.services.services.QueryService!.setConfig({
+              enabled: true,
+              indexes: [
+                //
+                { kind: IndexKind.Kind.SCHEMA_MATCH },
+                { kind: IndexKind.Kind.GRAPH },
+                { kind: IndexKind.Kind.VECTOR },
+              ],
+            });
+            await client.services.services.QueryService!.reindex();
+            await seedTestData(client.spaces.default);
+          },
+        }),
+        SpacePlugin(),
+        SettingsPlugin(),
+        PreviewPlugin(),
+        IntentPlugin(),
+        TranscriptionPlugin(),
+      ],
+      fireEvents: [Events.SetupAppGraph],
+    }),
+    withTheme,
+    withLayout({ fullscreen: true, classNames: 'justify-center' }),
+  ],
+};
+
+export default meta;
+
+type Story = StoryObj<typeof DefaultStory>;
+
+export const Default: Story = {
+  args: {
+    detectSpeaking: false,
+    entityExtraction: 'none',
+    transcriberConfig: TRANSCRIBER_CONFIG,
+    recorderConfig: RECORDER_CONFIG,
+    audioConstraints: {
+      echoCancellation: false,
+      noiseSuppression: false,
+      autoGainControl: false,
+    },
+  },
+};
+
+export const EntityExtraction: Story = {
+  args: {
+    detectSpeaking: true,
+    entityExtraction: 'ner',
+    transcriberConfig: TRANSCRIBER_CONFIG,
+    recorderConfig: RECORDER_CONFIG,
+    audioConstraints: {
+      echoCancellation: true,
+      noiseSuppression: true,
+      autoGainControl: true,
+    },
+  },
+};
+
+export const SpeechDetection: Story = {
+  args: {
+    detectSpeaking: true,
+    entityExtraction: 'none',
+    transcriberConfig: TRANSCRIBER_CONFIG,
+    recorderConfig: RECORDER_CONFIG,
+    audioConstraints: {
+      echoCancellation: true,
+      noiseSuppression: true,
+      autoGainControl: true,
+    },
+  },
+};

package/src/components/stories/TranscriptionStory.tsx

@@ -0,0 +1,41 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import React, { type Dispatch, type FC, type RefObject, type SetStateAction } from 'react';
+
+import { useSpace } from '@dxos/react-client/echo';
+import { IconButton, Toolbar } from '@dxos/react-ui';
+import { ScrollContainer } from '@dxos/react-ui-components';
+import { type DataType } from '@dxos/schema';
+
+import { type SerializationModel } from '../../model';
+import { Transcript } from '../Transcript';
+
+export const TranscriptionStory: FC<{
+  model: SerializationModel<DataType.Message>;
+  running: boolean;
+  onRunningChange: Dispatch<SetStateAction<boolean>>;
+  audioRef?: RefObject<HTMLAudioElement>;
+}> = ({ model, running, onRunningChange, audioRef }) => {
+  const space = useSpace();
+
+  return (
+    <div className='flex flex-col w-[30rem]'>
+      {audioRef && <audio ref={audioRef} autoPlay />}
+      <Toolbar.Root>
+        <IconButton
+          iconOnly
+          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+          label={running ? 'Pause' : 'Play'}
+          onClick={() => onRunningChange((running) => !running)}
+        />
+      </Toolbar.Root>
+      <ScrollContainer>
+        <Transcript space={space} model={model} attendableId='story' />
+      </ScrollContainer>
+    </div>
+  );
+};