@dxos/plugin-transcription 0.8.4-main.fd6878d → 0.9.0

package/src/components/stories/FileTranscription.stories.tsx

@@ -1,246 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import '@dxos-theme';
-
-import { type Meta, type StoryObj } from '@storybook/react-vite';
-import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
-
-import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
-import { withPluginManager } from '@dxos/app-framework/testing';
-import { scheduleTask } from '@dxos/async';
-import { Context } from '@dxos/context';
-import { Obj } from '@dxos/echo';
-import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo-schema';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
-import { log } from '@dxos/log';
-import { ClientPlugin } from '@dxos/plugin-client';
-import { PreviewPlugin } from '@dxos/plugin-preview';
-import { SpacePlugin } from '@dxos/plugin-space';
-import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
-import { ThemePlugin } from '@dxos/plugin-theme';
-import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { Testing, seedTestData } from '@dxos/schema/testing';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
-
-import { useAudioFile, useQueueModelAdapter, useTranscriber } from '../../hooks';
-import { MessageNormalizer, getActorId } from '../../segments-normalization';
-import { TestItem } from '../../testing';
-import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
-import { TranscriptionPlugin } from '../../TranscriptionPlugin';
-import { renderMarkdown } from '../Transcript';
-
-import { TranscriptionStory } from './TranscriptionStory';
-import { useIsSpeaking } from './useIsSpeaking';
-
-const AudioFile = ({
-  detectSpeaking,
-  audioUrl,
-  normalizeSentences,
-  transcriberConfig,
-  recorderConfig,
-  audioConstraints,
-}: {
-  detectSpeaking?: boolean;
-  normalizeSentences?: boolean;
-  audioUrl: string;
-  transcriberConfig?: TranscriberParams['config'];
-  recorderConfig?: MediaStreamRecorderParams['config'];
-  audioConstraints?: MediaTrackConstraints;
-}) => {
-  const [running, setRunning] = useState(false);
-  const actor: DataType.Actor = useMemo(() => ({ name: 'You' }), []);
-
-  // Audio.
-  const { audio, track, stream } = useAudioFile(audioUrl, audioConstraints);
-
-  // Speaking.
-  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
-  const ref = useRef<HTMLAudioElement>(null);
-  useEffect(() => {
-    if (ref.current && stream) {
-      ref.current.srcObject = stream;
-    }
-  }, [stream]);
-
-  useEffect(() => {
-    if (!audio) {
-      log.warn('no audio');
-      return;
-    }
-
-    if (running) {
-      void audio.play();
-    } else {
-      void audio.pause();
-    }
-  }, [audio, running]);
-
-  // Transcriber.
-  // TODO(dmaretskyi): Use space.queues.create() instead.
-  const queueDxn = useMemo(() => createQueueDXN(), []);
-  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
-
-  const model = useQueueModelAdapter(renderMarkdown([]), queue);
-  const handleSegments = useCallback<TranscriberParams['onSegments']>(
-    async (blocks) => {
-      void queue?.append([
-        Obj.make(DataType.Message, {
-          created: new Date().toISOString(),
-          sender: actor,
-          blocks,
-        }),
-      ]);
-    },
-    [queue],
-  );
-
-  const transcriber = useTranscriber({
-    audioStreamTrack: track,
-    onSegments: handleSegments,
-    transcriberConfig,
-    recorderConfig,
-  });
-
-  // Normalize sentences.
-  const normalizer = useMemo(() => {
-    if (!normalizeSentences) {
-      return;
-    }
-    const executor = new FunctionExecutor(
-      new ServiceContainer().setServices({
-        // ai: {
-        //   client: new Edge AiServiceClient({
-        //     endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-        //     defaultGenerationOptions: {
-        //       model: '@anthropic/claude-3-5-sonnet-20241022',
-        //     },
-        //   }),
-        // },
-      }),
-    );
-
-    return new MessageNormalizer({
-      functionExecutor: executor,
-      queue,
-      startingCursor: { actorId: getActorId(actor), timestamp: new Date().toISOString() },
-    });
-  }, [normalizeSentences, queue, actor]);
-
-  useEffect(() => {
-    if (!normalizer) {
-      return;
-    }
-    const ctx = new Context();
-    scheduleTask(ctx, async () => {
-      await normalizer.open();
-      ctx.onDispose(async () => {
-        await normalizer.close();
-      });
-    });
-    return () => {
-      void ctx.dispose();
-    };
-  }, [normalizer]);
-
-  const manageChunkRecording = () => {
-    if (running && isSpeaking) {
-      log.info('starting transcription');
-      transcriber?.startChunksRecording();
-    } else if (!isSpeaking || !running) {
-      log.info('stopping transcription');
-      transcriber?.stopChunksRecording();
-    }
-  };
-
-  useEffect(() => {
-    const ctx = new Context();
-    scheduleTask(ctx, async () => {
-      if (transcriber && running) {
-        await transcriber.open();
-      } else if (!running) {
-        await transcriber?.close();
-      }
-      manageChunkRecording();
-    });
-
-    return () => {
-      void ctx.dispose();
-    };
-  }, [transcriber, running]);
-
-  useEffect(() => {
-    manageChunkRecording();
-  }, [isSpeaking, stream]);
-
-  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} audioRef={ref} />;
-};
-
-const meta: Meta<typeof AudioFile> = {
-  title: 'plugins/plugin-transcription/FileTranscription',
-  decorators: [
-    withPluginManager({
-      plugins: [
-        ThemePlugin({ tx: defaultTx }),
-        StorybookLayoutPlugin(),
-        ClientPlugin({
-          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
-          onClientInitialized: async ({ client }) => {
-            await client.halo.createIdentity();
-            await client.spaces.waitUntilReady();
-            await client.spaces.default.waitUntilReady();
-            await seedTestData(client.spaces.default);
-          },
-        }),
-        SpacePlugin(),
-        SettingsPlugin(),
-        PreviewPlugin(),
-        IntentPlugin(),
-        TranscriptionPlugin(),
-      ],
-      fireEvents: [Events.SetupAppGraph],
-    }),
-    withTheme,
-    withLayout({ fullscreen: true, classNames: 'justify-center' }),
-  ],
-};
-
-export default meta;
-
-const TRANSCRIBER_CONFIG = {
-  transcribeAfterChunksAmount: 100,
-  prefixBufferChunksAmount: 50,
-  normalizeSentences: true,
-};
-
-const RECORDER_CONFIG = {
-  interval: 200,
-};
-
-export const Default: StoryObj<typeof AudioFile> = {
-  render: AudioFile,
-  args: {
-    detectSpeaking: true,
-    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
-    audioUrl: 'https://dxos.network/audio-london.m4a',
-    // textUrl: 'https://dxos.network/audio-london.txt',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-  },
-};
-
-export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
-  render: AudioFile,
-  args: {
-    detectSpeaking: true,
-    normalizeSentences: true,
-    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
-    audioUrl: 'https://dxos.network/audio-london.m4a',
-    // textUrl: 'https://dxos.network/audio-london.txt',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-  },
-};

package/src/components/stories/MicrophoneTranscription.stories.tsx

@@ -1,283 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import '@dxos-theme';
-
-import { type Meta, type StoryObj } from '@storybook/react-vite';
-import React, { useCallback, useEffect, useMemo, useState } from 'react';
-
-import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
-import { withPluginManager } from '@dxos/app-framework/testing';
-import {
-  type ExtractionFunction,
-  extractionAnthropicFn,
-  extractionNerFn,
-  getNer,
-  processTranscriptMessage,
-} from '@dxos/assistant/extraction';
-import { Filter, Obj, type Type } from '@dxos/echo';
-import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo-schema';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
-import { invariant } from '@dxos/invariant';
-import { log } from '@dxos/log';
-import { ClientPlugin } from '@dxos/plugin-client';
-import { PreviewPlugin } from '@dxos/plugin-preview';
-import { SpacePlugin } from '@dxos/plugin-space';
-import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
-import { ThemePlugin } from '@dxos/plugin-theme';
-import { IndexKind, useSpace } from '@dxos/react-client/echo';
-import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { Testing, seedTestData } from '@dxos/schema/testing';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
-
-import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
-import { TestItem } from '../../testing';
-import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
-import { TranscriptionPlugin } from '../../TranscriptionPlugin';
-import { renderMarkdown } from '../Transcript';
-
-import { TranscriptionStory } from './TranscriptionStory';
-import { useIsSpeaking } from './useIsSpeaking';
-
-const TRANSCRIBER_CONFIG = {
-  transcribeAfterChunksAmount: 50,
-  prefixBufferChunksAmount: 10,
-};
-
-const RECORDER_CONFIG = {
-  interval: 200,
-};
-
-type StoryProps = {
-  detectSpeaking?: boolean;
-  transcriberConfig: TranscriberParams['config'];
-  recorderConfig: MediaStreamRecorderParams['config'];
-  audioConstraints?: MediaTrackConstraints;
-  entityExtraction: 'none' | 'ner' | 'llm';
-};
-
-const DefaultStory = ({
-  detectSpeaking,
-  entityExtraction = 'none',
-  transcriberConfig,
-  recorderConfig,
-  audioConstraints,
-}: StoryProps) => {
-  const [running, setRunning] = useState(false);
-
-  // Audio.
-  const track = useAudioTrack(running, audioConstraints);
-
-  // Speaking.
-  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
-
-  // Queue.
-  // TODO(dmaretskyi): Use space.queues.create() instead.
-  const queueDxn = useMemo(() => createQueueDXN(), []);
-  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
-  const model = useQueueModelAdapter(renderMarkdown([]), queue);
-  const space = useSpace();
-
-  useEffect(() => {
-    if (!space) {
-      log.warn('no space');
-    }
-  }, [space]);
-
-  // Entity extraction.
-  const { extractionFunction, executor, objects } = useMemo(() => {
-    if (!space) {
-      log.warn('no space');
-      return {};
-    }
-
-    let executor: FunctionExecutor | undefined;
-    let extractionFunction: ExtractionFunction | undefined;
-    let objects: Promise<Type.Expando[]> | undefined;
-
-    if (entityExtraction === 'ner') {
-      // Init model loading. Takes time.
-      void getNer();
-      extractionFunction = extractionNerFn;
-    } else if (entityExtraction === 'llm') {
-      extractionFunction = extractionAnthropicFn;
-      objects = space.db
-        .query(
-          Filter.or(
-            Filter.type(DataType.Person),
-            Filter.type(DataType.Organization),
-            Filter.type(Testing.DocumentType),
-          ),
-        )
-        .run()
-        .then((result) => result.objects);
-    }
-    if (entityExtraction !== 'none') {
-      executor = new FunctionExecutor(
-        new ServiceContainer().setServices({
-          // ai: {
-          //   client: new Edge AiServiceClient({
-          //     endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-          //   }),
-          // },
-          // database: { db: space!.db },
-        }),
-      );
-    }
-
-    return { extractionFunction, executor, objects };
-  }, [entityExtraction, space]);
-
-  // Transcriber.
-  const handleSegments = useCallback<TranscriberParams['onSegments']>(
-    async (blocks) => {
-      const message = Obj.make(DataType.Message, {
-        sender: { name: 'You' },
-        created: new Date().toISOString(),
-        blocks,
-      });
-      if (!space) {
-        void queue.append([message]);
-        return;
-      }
-
-      if (entityExtraction !== 'none') {
-        invariant(extractionFunction, 'extractionFunction is required');
-        invariant(executor, 'executor is required');
-        const result = await processTranscriptMessage({
-          executor,
-          function: extractionFunction,
-          input: {
-            message,
-            objects: await objects,
-          },
-          options: {
-            fallbackToRaw: true,
-            timeout: 30_000,
-          },
-        });
-        void queue.append([result.message]);
-      } else {
-        void queue.append([message]);
-      }
-    },
-    [queue, space],
-  );
-
-  const transcriber = useTranscriber({
-    audioStreamTrack: track,
-    onSegments: handleSegments,
-    transcriberConfig,
-    recorderConfig,
-  });
-  const [isOpen, setIsOpen] = useState(false);
-  useEffect(() => {
-    void transcriber?.open().then(() => setIsOpen(true));
-    return () => {
-      void transcriber?.close().then(() => setIsOpen(false));
-    };
-  }, [transcriber]);
-
-  // Manage transcription state.
-  useEffect(() => {
-    if (running && transcriber?.isOpen && isSpeaking) {
-      transcriber?.startChunksRecording();
-    } else if (!running || !isSpeaking) {
-      transcriber?.stopChunksRecording();
-    }
-  }, [transcriber, running, isOpen, isSpeaking]);
-
-  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
-};
-
-const meta: Meta<typeof DefaultStory> = {
-  title: 'plugins/plugin-transcription/MicrophoneTranscription',
-  render: DefaultStory,
-  decorators: [
-    withPluginManager({
-      plugins: [
-        ThemePlugin({ tx: defaultTx }),
-        StorybookLayoutPlugin(),
-        ClientPlugin({
-          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
-          onClientInitialized: async ({ client }) => {
-            await client.halo.createIdentity();
-            await client.spaces.waitUntilReady();
-            await client.spaces.default.waitUntilReady();
-            // TODO(mykola): Make API easier to use.
-            // TODO(mykola): Delete after enabling vector indexing by default.
-            // Enable vector indexing.
-            await client.services.services.QueryService!.setConfig({
-              enabled: true,
-              indexes: [
-                //
-                { kind: IndexKind.Kind.SCHEMA_MATCH },
-                { kind: IndexKind.Kind.GRAPH },
-                { kind: IndexKind.Kind.VECTOR },
-              ],
-            });
-            await client.services.services.QueryService!.reindex();
-            await seedTestData(client.spaces.default);
-          },
-        }),
-        SpacePlugin(),
-        SettingsPlugin(),
-        PreviewPlugin(),
-        IntentPlugin(),
-        TranscriptionPlugin(),
-      ],
-      fireEvents: [Events.SetupAppGraph],
-    }),
-    withTheme,
-    withLayout({ fullscreen: true, classNames: 'justify-center' }),
-  ],
-};
-
-export default meta;
-
-type Story = StoryObj<typeof DefaultStory>;
-
-export const Default: Story = {
-  args: {
-    detectSpeaking: false,
-    entityExtraction: 'none',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-    audioConstraints: {
-      echoCancellation: false,
-      noiseSuppression: false,
-      autoGainControl: false,
-    },
-  },
-};
-
-export const EntityExtraction: Story = {
-  args: {
-    detectSpeaking: true,
-    entityExtraction: 'ner',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-    audioConstraints: {
-      echoCancellation: true,
-      noiseSuppression: true,
-      autoGainControl: true,
-    },
-  },
-};
-
-export const SpeechDetection: Story = {
-  args: {
-    detectSpeaking: true,
-    entityExtraction: 'none',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-    audioConstraints: {
-      echoCancellation: true,
-      noiseSuppression: true,
-      autoGainControl: true,
-    },
-  },
-};

package/src/components/stories/TranscriptionStory.tsx

@@ -1,41 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import '@dxos-theme';
-
-import React, { type Dispatch, type FC, type RefObject, type SetStateAction } from 'react';
-
-import { useSpace } from '@dxos/react-client/echo';
-import { IconButton, Toolbar } from '@dxos/react-ui';
-import { ScrollContainer } from '@dxos/react-ui-components';
-import { type DataType } from '@dxos/schema';
-
-import { type SerializationModel } from '../../model';
-import { Transcript } from '../Transcript';
-
-export const TranscriptionStory: FC<{
-  model: SerializationModel<DataType.Message>;
-  running: boolean;
-  onRunningChange: Dispatch<SetStateAction<boolean>>;
-  audioRef?: RefObject<HTMLAudioElement>;
-}> = ({ model, running, onRunningChange, audioRef }) => {
-  const space = useSpace();
-
-  return (
-    <div className='flex flex-col w-[30rem]'>
-      {audioRef && <audio ref={audioRef} autoPlay />}
-      <Toolbar.Root>
-        <IconButton
-          iconOnly
-          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
-          label={running ? 'Pause' : 'Play'}
-          onClick={() => onRunningChange((running) => !running)}
-        />
-      </Toolbar.Root>
-      <ScrollContainer>
-        <Transcript space={space} model={model} attendableId='story' />
-      </ScrollContainer>
-    </div>
-  );
-};

package/src/components/stories/useIsSpeaking.tsx

@@ -1,40 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import '@dxos-theme';
-
-import { useEffect, useMemo, useState } from 'react';
-
-import { scheduleTask } from '@dxos/async';
-import { Context } from '@dxos/context';
-import { SpeakingMonitor } from '@dxos/plugin-thread';
-
-export const useIsSpeaking = (track?: MediaStreamTrack) => {
-  const [isSpeaking, setIsSpeaking] = useState(false);
-  const speakingMonitor = useMemo(() => {
-    if (!track) {
-      return;
-    }
-    return new SpeakingMonitor(track);
-  }, [track]);
-
-  useEffect(() => {
-    if (!speakingMonitor) {
-      return;
-    }
-
-    const ctx = new Context();
-    scheduleTask(ctx, async () => {
-      speakingMonitor.speakingChanged.on(ctx, () => setIsSpeaking(speakingMonitor.isSpeaking));
-      await speakingMonitor.open();
-      ctx.onDispose(() => speakingMonitor.close());
-    });
-
-    return () => {
-      void ctx.dispose();
-    };
-  }, [speakingMonitor]);
-
-  return isSpeaking;
-};

package/src/hooks/useQueueModelAdapter.ts

@@ -1,66 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { useEffect, useMemo, useState } from 'react';
-
-import type { Obj } from '@dxos/echo';
-import { type Queue } from '@dxos/react-client/echo';
-
-import { type ChunkRenderer, SerializationModel } from '../model';
-
-/**
- * Model adapter for a queue.
- */
-export const useQueueModelAdapter = <T extends Obj.Any>(
-  renderer: ChunkRenderer<T>,
-  queue: Queue<T> | undefined,
-  initialChunks: T[] = [],
-): SerializationModel<T> => {
-  const model = useMemo(() => new SerializationModel<T>(renderer, initialChunks), [queue]);
-  const [loaded, setLoaded] = useState(false);
-
-  // Set initial blocks.
-  useEffect(() => {
-    if (!queue) {
-      return;
-    }
-
-    const update = () => {
-      for (const block of queue?.objects ?? []) {
-        model.appendChunk(block);
-      }
-
-      setLoaded(true);
-    };
-
-    // TODO(burdon): This is a hack to ensure the queue is loaded.
-    let i: NodeJS.Timeout | undefined;
-    if (queue.isLoading) {
-      i = setInterval(() => {
-        if (!queue?.isLoading) {
-          clearInterval(i);
-          update();
-        }
-      }, 1_000);
-    } else {
-      update();
-    }
-
-    return () => clearTimeout(i);
-  }, [model, queue]);
-
-  // TODO(burdon): Can we listen for queue events?
-  useEffect(() => {
-    if (!loaded || !queue || model.chunks.length === queue.objects.length) {
-      return;
-    }
-
-    const chunk = queue.objects.at(-1);
-    if (chunk) {
-      model.appendChunk(chunk);
-    }
-  }, [model, loaded, queue?.objects.length]);
-
-  return model;
-};